From e11aa67543ba122597aa092c0b856c2a1084bc70 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Tue, 28 Oct 2025 13:56:19 +0100 Subject: [PATCH 001/530] [api] rotate key functions - add functions for rotate key of files and dataset - add db functions needed --- sda/cmd/api/api.go | 192 ++++++++++++++++++++++++-- sda/internal/database/db_functions.go | 20 ++- 2 files changed, 190 insertions(+), 22 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index b0bfe22554..d5843b734c 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -144,17 +144,21 @@ func setup(conf *config.Config) *http.Server { r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash r.DELETE("/file/:username/:fileid", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here - r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file - r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file - r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file - r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset - r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible - r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset - r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status - r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user - r.GET("/users", rbac(e), listActiveUsers) // Lists all users - r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user - r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox + r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file + r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file + r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file + r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset + r.GET("/dataset/:dataset/files", rbac(e), getDatasetFiles) // get files in a dataset + r.GET("/dataset/:dataset/fileids", rbac(e), getDatasetFileIDs) // get file IDs (internal and accession) in a dataset + r.POST("/dataset/:dataset/rotatekey", rbac(e), rotateDatasetKeys) // trigger key rotation for all files in a dataset + r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible + r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset + r.POST("/file/rotatekey/:fileid", rbac(e), rotateFileKey) // trigger key rotation for a file + r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status + r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user + r.GET("/users", rbac(e), listActiveUsers) // Lists all users + r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user + r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox cfg := &tls.Config{MinVersion: tls.VersionTLS12} @@ -714,6 +718,164 @@ func releaseDataset(c *gin.Context) { c.Status(http.StatusOK) } +// getDatasetFiles returns files belonging to a dataset +func getDatasetFiles(c *gin.Context) { + c.Writer.Header().Set("Content-Type", "application/json") + datasetID := c.Param("dataset") + + if datasetID == "" { + c.JSON(http.StatusBadRequest, "dataset ID is required") + return + } + + files, err := Conf.API.DB.GetDatasetFiles(datasetID) + if err != nil { + log.Errorf("failed to get dataset files for dataset %s, reason: %v", datasetID, err) + c.JSON(http.StatusInternalServerError, err.Error()) + return + } + + if len(files) == 0 { + c.JSON(http.StatusNotFound, "no files found for dataset") + return + } + + // Convert DatasetFileIDs to a simpler format for this endpoint + var result []map[string]string + for _, file := range files { + result = append(result, map[string]string{ + "fileID": file.FileID, + "accessionID": file.AccessionID, + }) + } + + c.JSON(http.StatusOK, result) +} + +// getDatasetFileIDs returns file IDs (both internal and accession) belonging to a dataset +func getDatasetFileIDs(c *gin.Context) { + c.Writer.Header().Set("Content-Type", "application/json") + datasetID := c.Param("dataset") + + if datasetID == "" { + c.JSON(http.StatusBadRequest, "dataset ID is required") + return + } + + files, err := Conf.API.DB.GetDatasetFiles(datasetID) + if err != nil { + log.Errorf("failed to get dataset file IDs for dataset %s, reason: %v", datasetID, err) + c.JSON(http.StatusInternalServerError, err.Error()) + return + } + + if len(files) == 0 { + c.JSON(http.StatusNotFound, "no files found for dataset") + return + } + + c.JSON(http.StatusOK, files) +} + +// rotateFileKey triggers key rotation for a specific file +func rotateFileKey(c *gin.Context) { + fileID := c.Param("fileid") + + if fileID == "" { + c.JSON(http.StatusBadRequest, "file ID is required") + return + } + + // Create rotation message + rotateMsg := schema.KeyRotation{ + Type: "key_rotation", + FileID: fileID, + } + + marshaledMsg, err := json.Marshal(&rotateMsg) + if err != nil { + log.Errorf("failed to marshal rotation message, reason: %v", err) + c.JSON(http.StatusInternalServerError, "failed to marshal rotation message") + return + } + + // Validate the message against schema + if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { + log.Errorf("rotation message validation failed, reason: %v", err) + c.JSON(http.StatusBadRequest, "rotation message validation failed") + return + } + + // Send message to rotatekey queue + err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "rotatekey", marshaledMsg) + if err != nil { + log.Errorf("failed to send rotation message to queue, reason: %v", err) + c.JSON(http.StatusInternalServerError, "failed to send rotation message") + return + } + + c.Status(http.StatusOK) +} + +// rotateDatasetKeys triggers key rotation for all files in a dataset +func rotateDatasetKeys(c *gin.Context) { + datasetID := c.Param("dataset") + + if datasetID == "" { + c.JSON(http.StatusBadRequest, "dataset ID is required") + return + } + + // Get all files in the dataset + files, err := Conf.API.DB.GetDatasetFiles(datasetID) + if err != nil { + log.Errorf("failed to get dataset files for dataset %s, reason: %v", datasetID, err) + c.JSON(http.StatusInternalServerError, "failed to get dataset files") + return + } + + if len(files) == 0 { + c.JSON(http.StatusNotFound, "no files found for dataset") + return + } + + // Send rotation message for each file in the dataset + for _, file := range files { + // Create rotation message + rotateMsg := schema.KeyRotation{ + Type: "key_rotation", + FileID: file.FileID, + } + + marshaledMsg, err := json.Marshal(&rotateMsg) + if err != nil { + log.Errorf("failed to marshal rotation message for file %s, reason: %v", file.FileID, err) + c.JSON(http.StatusInternalServerError, "failed to marshal rotation message") + return + } + + // Validate the message against schema + if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { + log.Errorf("rotation message validation failed for file %s, reason: %v", file.FileID, err) + c.JSON(http.StatusBadRequest, "rotation message validation failed") + return + } + + // Send message to rotatekey queue + err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "rotatekey", marshaledMsg) + if err != nil { + log.Errorf("failed to send rotation message for file %s to queue, reason: %v", file.FileID, err) + c.JSON(http.StatusInternalServerError, "failed to send rotation message") + return + } + + log.Infof("rotation message sent for file %s (accession: %s) in dataset %s", file.FileID, file.AccessionID, datasetID) + } + + log.Infof("rotation messages sent for %d files in dataset %s", len(files), datasetID) + c.Status(http.StatusOK) +} + func listActiveUsers(c *gin.Context) { users, err := Conf.API.DB.ListActiveUsers() if err != nil { @@ -945,20 +1107,20 @@ func reVerifyFile(c *gin.Context) { func reVerifyDataset(c *gin.Context) { dataset := strings.TrimPrefix(c.Param("dataset"), "/") - accessions, err := Conf.API.DB.GetDatasetFiles(dataset) + files, err := Conf.API.DB.GetDatasetFiles(dataset) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return } - if accessions == nil { + if files == nil { c.AbortWithStatusJSON(http.StatusNotFound, "dataset not found") return } - for _, accession := range accessions { - c, err = reVerify(c, accession) + for _, file := range files { + c, err = reVerify(c, file.AccessionID) if err != nil { return } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 2bd729bb22..a5e103f04b 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1288,12 +1288,18 @@ func (dbs *SDAdb) GetDecryptedChecksum(id string) (string, error) { return unencryptedChecksum, nil } -func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { +// DatasetFileIDs holds both the internal file ID and stable accession ID +type DatasetFileIDs struct { + FileID string `json:"fileID"` + AccessionID string `json:"accessionID"` +} + +func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]DatasetFileIDs, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - var accessions []string - rows, err := db.Query("SELECT stable_id FROM sda.files WHERE id IN (SELECT file_id FROM sda.file_dataset WHERE dataset_id = (SELECT id FROM sda.datasets WHERE stable_id = $1));", dataset) + var files []DatasetFileIDs + rows, err := db.Query("SELECT id, stable_id FROM sda.files WHERE id IN (SELECT file_id FROM sda.file_dataset WHERE dataset_id = (SELECT id FROM sda.datasets WHERE stable_id = $1));", dataset) if err != nil { return nil, err } @@ -1303,14 +1309,14 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { defer rows.Close() for rows.Next() { - var accession string - err := rows.Scan(&accession) + var file DatasetFileIDs + err := rows.Scan(&file.FileID, &file.AccessionID) if err != nil { return nil, err } - accessions = append(accessions, accession) + files = append(files, file) } - return accessions, nil + return files, nil } From c5df35ee8b7a08a872f864397e918171fcc360f2 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Tue, 28 Oct 2025 14:01:22 +0100 Subject: [PATCH 002/530] [tests] add test for db functions --- sda/internal/database/db_functions_test.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 23c563e79d..cc54898945 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1527,9 +1527,16 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { suite.FailNow("failed to map files to dataset") } - accessions, err := db.GetDatasetFiles(dID) - assert.NoError(suite.T(), err, "failed to get accessions for a dataset") - assert.Equal(suite.T(), []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}, accessions) + files, err := db.GetDatasetFiles(dID) + assert.NoError(suite.T(), err, "failed to get files for a dataset") + assert.Equal(suite.T(), 3, len(files)) + + // Extract accession IDs for comparison + var accessions []string + for _, file := range files { + accessions = append(accessions, file.AccessionID) + } + assert.ElementsMatch(suite.T(), []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}, accessions) db.Close() } From 01e6db18daf10a3156c616f7c1cdcee12c43b1eb Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Tue, 28 Oct 2025 14:03:15 +0100 Subject: [PATCH 003/530] [sda-admin] add rotate key for files of a dataset --- sda-admin/dataset/dataset.go | 46 ++++++++++++++++++++++++++++++++++++ sda-admin/main.go | 41 +++++++++++++++++++++++++++++++- 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/sda-admin/dataset/dataset.go b/sda-admin/dataset/dataset.go index 180090cf01..024092aec4 100644 --- a/sda-admin/dataset/dataset.go +++ b/sda-admin/dataset/dataset.go @@ -15,6 +15,11 @@ type RequestBodyDataset struct { User string `json:"user"` } +type DatasetFileIDs struct { + FileID string `json:"fileID"` + AccessionID string `json:"accessionID"` +} + // Create creates a dataset from a list of accession IDs and a dataset ID. func Create(apiURI, token, datasetID, username string, accessionIDs []string) error { parsedURL, err := url.Parse(apiURI) @@ -57,3 +62,44 @@ func Release(apiURI, token, datasetID string) error { return nil } + +// RotateKey rotates the encryption key for all files in a dataset +func RotateKey(apiURI, token, datasetID string) error { + // First, get all file IDs in the dataset + filesURL, err := url.Parse(apiURI) + if err != nil { + return err + } + filesURL.Path = path.Join(filesURL.Path, "dataset", datasetID, "fileids") + + response, err := helpers.GetResponseBody(filesURL.String(), token) + if err != nil { + return fmt.Errorf("failed to get dataset file IDs, reason: %v", err) + } + + var datasetFiles []DatasetFileIDs + err = json.Unmarshal(response, &datasetFiles) + if err != nil { + return fmt.Errorf("failed to unmarshal dataset file IDs response, reason: %v", err) + } + + if len(datasetFiles) == 0 { + return fmt.Errorf("no files found for dataset %s", datasetID) + } + + // Send rotation request for each file using the internal FileID + for _, file := range datasetFiles { + rotateURL, err := url.Parse(apiURI) + if err != nil { + return fmt.Errorf("failed to parse API URI for file %s, reason: %v", file.FileID, err) + } + rotateURL.Path = path.Join(rotateURL.Path, "file", "rotatekey", file.FileID) + + _, err = helpers.PostRequest(rotateURL.String(), token, nil) + if err != nil { + return fmt.Errorf("failed to rotate key for file %s (accession: %s), reason: %v", file.FileID, file.AccessionID, err) + } + } + + return nil +} diff --git a/sda-admin/main.go b/sda-admin/main.go index 62490f6aaf..a5d092f307 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -34,6 +34,8 @@ Commands: Create a dataset from a list of accession IDs and a dataset ID. dataset release -dataset-id DATASET_ID Release a dataset for downloading. + dataset rotatekey -dataset-id DATASET_ID + Rotate encryption keys for all files in a dataset. Global Options: -uri URI Set the URI for the API server (optional if API_HOST is set). @@ -99,6 +101,10 @@ Release a dataset: Usage: sda-admin dataset release -dataset-id DATASET_ID Release a dataset for downloading based on its dataset ID. +Rotate keys for a dataset: + Usage: sda-admin dataset rotatekey -dataset-id DATASET_ID + Rotate encryption keys for all files in a dataset. + Options: -dataset-id DATASET_ID Specify the unique identifier for the dataset. [ACCESSION_ID ...] (For dataset create) Specify one or more accession IDs to include in the dataset. @@ -118,6 +124,12 @@ var datasetReleaseUsage = `Usage: sda-admin dataset release -dataset-id DATASET_ Options: -dataset-id DATASET_ID Specify the unique identifier for the dataset.` +var datasetRotateKeyUsage = `Usage: sda-admin dataset rotatekey -dataset-id DATASET_ID + Rotate encryption keys for all files in a dataset. + +Options: + -dataset-id DATASET_ID Specify the unique identifier for the dataset.` + var c4ghHashUsage = `Handles the crypt4gh keys in the system. Usage: sda-admin c4gh-hash add -filepath FILEPATH -description DESCRIPTION @@ -265,6 +277,8 @@ func handleHelpDataset() error { fmt.Println(datasetCreateUsage) case flag.Arg(2) == "release": fmt.Println(datasetReleaseUsage) + case flag.Arg(2) == "rotatekey": + fmt.Println(datasetRotateKeyUsage) default: return fmt.Errorf("unknown subcommand '%s' for '%s'.\n%s", flag.Arg(2), flag.Arg(1), datasetUsage) } @@ -389,7 +403,7 @@ func handleFileAccessionCommand() error { func handleDatasetCommand() error { if flag.NArg() < 2 { - return fmt.Errorf("error: 'dataset' requires a subcommand (create, release).\n%s", datasetUsage) + return fmt.Errorf("error: 'dataset' requires a subcommand (create, release, rotatekey).\n%s", datasetUsage) } switch flag.Arg(1) { @@ -401,6 +415,10 @@ func handleDatasetCommand() error { if err := handleDatasetReleaseCommand(); err != nil { return err } + case "rotatekey": + if err := handleDatasetRotateKeyCommand(); err != nil { + return err + } default: return fmt.Errorf("unknown subcommand '%s' for '%s'.\n%s", flag.Arg(1), flag.Arg(0), datasetUsage) } @@ -457,6 +475,27 @@ func handleDatasetReleaseCommand() error { return nil } +func handleDatasetRotateKeyCommand() error { + datasetRotateKeyCmd := flag.NewFlagSet("rotatekey", flag.ExitOnError) + var datasetID string + datasetRotateKeyCmd.StringVar(&datasetID, "dataset-id", "", "ID of the dataset to rotate keys for") + + if err := datasetRotateKeyCmd.Parse(flag.Args()[2:]); err != nil { + return fmt.Errorf("error: failed to parse command line arguments, reason: %v", err) + } + + if datasetID == "" { + return fmt.Errorf("error: -dataset-id is required.\n%s", datasetRotateKeyUsage) + } + + err := dataset.RotateKey(apiURI, token, datasetID) + if err != nil { + return fmt.Errorf("error: failed to rotate keys for dataset, reason: %v", err) + } + + return nil +} + func handleHelpC4ghKeyHash() error { switch { case flag.NArg() == 2: From b16cbf5c31f6f98cee6ecf8a17944199af074698 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Tue, 28 Oct 2025 14:06:03 +0100 Subject: [PATCH 004/530] [docs] update doc for rotatekey --- sda/cmd/api/api.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 1a124c8134..4bd8eeecbd 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -162,6 +162,56 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -X PUT https://HOSTNAME/dataset/verify/my-dataset-01 ``` +- `/dataset/:dataset/files` + - accepts `GET` requests with the dataset name as parameter + - Returns all files in the dataset as a JSON array. + + - Error codes + - `200` Query execute ok. + - `400` Dataset ID not provided. + - `401` Token user is not in the list of admins. + - `404` No files found for dataset. + - `500` Internal error due to DB failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/dataset/my-dataset-01/files + ``` + +- `/dataset/:dataset/fileids` + - accepts `GET` requests with the dataset name as parameter + - Returns file IDs (both internal file ID and accession ID) for all files in the dataset as a JSON array. + + - Error codes + - `200` Query execute ok. + - `400` Dataset ID not provided. + - `401` Token user is not in the list of admins. + - `404` No files found for dataset. + - `500` Internal error due to DB failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/dataset/my-dataset-01/fileids + ``` + +- `/file/rotatekey/:fileid` + - accepts `POST` requests with the file ID as parameter + - Triggers key rotation for the specified file by sending a message to the rotatekey queue. + + - Error codes + - `200` Query execute ok. + - `400` File ID not provided or message validation failed. + - `401` Token user is not in the list of admins. + - `500` Internal error due to MQ failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -X POST https://HOSTNAME/file/rotatekey/c2acecc6-f208-441c-877a-2670e4cbb040 + ``` + - `/datasets/list` - accepts `GET` requests - Returns all datasets together with their status and last modified timestamp. From 7ca2221981e0c19124d9d994addc344406944a10 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Tue, 28 Oct 2025 14:06:54 +0100 Subject: [PATCH 005/530] [tests] add test for sda-admin rotate key --- .../tests/sda/65_api_admin_rotate_key_test.sh | 274 ++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100755 .github/integration/tests/sda/65_api_admin_rotate_key_test.sh diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh new file mode 100755 index 0000000000..5863e1085d --- /dev/null +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -0,0 +1,274 @@ +#!/bin/sh +set -e + +if [ -n "$SYNCTEST" ]; then + exit 0 +fi + +cd shared || true + +checkErrors() { + RETRY_TIMES=0 + until [ $(("$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')"-"$errorStreamSize")) -eq 1 ]; do + echo "checking for $1 error" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 20 ]; then + echo "::error::Time out while waiting for error message" + exit 1 + fi + sleep 2 + done +} + +checkConsumers() { + RETRY_TIMES=0 + until [ "$(curl -su guest:guest http://localhost:15672/api/consumers | jq '.[].queue.name' | grep -c "$1")" -eq "$2" ]; do + echo "waiting for $1 consumer status" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for $1 consumer status" + exit 1 + fi + sleep 2 + done +} + +# cleanup queues and database +URI=http://rabbitmq:15672 +if [ -n "$PGSSLCERT" ]; then + URI=https://rabbitmq:15671 +fi +for q in accession archived backup completed inbox ingest mappings verified rotatekey; do + curl -s -k -u guest:guest -X DELETE "$URI/api/queues/sda/$q/contents" +done +psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files, sda.encryption_keys, sda.datasets, sda.file_dataset CASCADE;" + +# register archive and rotation c4gh public keys +token="$(cat /shared/token)" +for keyName in c4gh rotatekey; do + payload=$( + jq -c -n \ + --arg description "this is the $keyName key" \ + --arg pubkey "$( base64 -w0 /shared/"$keyName".pub.pem)" \ + '$ARGS.named' + ) + resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/c4gh-keys/add")" + if [ "$resp" != "200" ]; then + echo "Error when adding the $keyName public key hash, expected 200 got: $resp" + exit 1 + fi +done + +# generate and upload two files for dataset +for i in 1 2; do + file="testfile$i" + if [ ! -f "$file" ]; then + dd if=/dev/urandom of="$file" count=5 bs=1M + fi + if [ ! -f "$file.c4gh" ]; then + yes | /shared/crypt4gh encrypt -p c4gh.pub.pem -f "$file" + fi + s3cmd -c s3cfg put "$file.c4gh" s3://test_dummy.org/dataset_admin_rotatekey/ +done + +# wait for files to appear in user files +RETRY_TIMES=0 +until [ "$(curl -s -k -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq | grep -c dataset_admin_rotatekey)" -eq 2 ]; do + echo "waiting for files to upload" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for files to upload" + exit 1 + fi + sleep 2 +done + +## ingest both files +for i in 1 2; do + curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "{\"filepath\": \"dataset_admin_rotatekey/testfile$i.c4gh\", \"user\": \"test@dummy.org\"}" http://api:8080/file/ingest +done + +# wait for files to become verified +RETRY_TIMES=0 +until [ "$(curl -s -k -H "Authorization: Bearer $token" -X GET http://api:8080/users/test@dummy.org/files | jq | grep -c "verified")" -eq 2 ]; do + echo "waiting for files to become verified" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for files to become verified" + exit 1 + fi + sleep 2 +done + +# assign accession IDs to the files +for i in 1 2; do + fileID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/testfile$i.c4gh';") + payload=$( + jq -c -n \ + --arg accession_id "EGAF123456789$i" \ + --arg file_id "$fileID" \ + '$ARGS.named' + ) + resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/file/accession")" + if [ "$resp" != "200" ]; then + echo "Error when assigning accession ID to file $i, expected 200 got: $resp" + exit 1 + fi +done + +# create dataset with both files +dataset_payload=$( + jq -c -n \ + --arg dataset_id "EGAD12345678901" \ + --arg user "test@dummy.org" \ + --argjson accession_ids '["EGAF1234567891", "EGAF1234567892"]' \ + '$ARGS.named' +) +resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$dataset_payload" "http://api:8080/dataset/create")" +if [ "$resp" != "200" ]; then + echo "Error when creating dataset, expected 200 got: $resp" + exit 1 +fi + +# verify dataset was created and contains both files +dataset_files_count="$(curl -s -k -H "Authorization: Bearer $token" -X GET "http://api:8080/dataset/EGAD12345678901/fileids" | jq '. | length')" +if [ "$dataset_files_count" -ne 2 ]; then + echo "Expected 2 files in dataset, got: $dataset_files_count" + exit 1 +fi + +echo "Dataset created successfully with 2 files" + +# get initial error stream size for error checking +errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') + +# get file IDs for verification later +fileID1=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/testfile1.c4gh';") +fileID2=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/testfile2.c4gh';") + +echo "File IDs: $fileID1, $fileID2" + +## test dataset key rotation via sda-admin CLI tool +echo "Testing dataset key rotation via sda-admin CLI tool" + +# Use sda-admin CLI tool to rotate keys for the dataset +SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" /shared/sda-admin dataset rotatekey -dataset-id "EGAD12345678901" + +if [ $? -ne 0 ]; then + echo "Error when running sda-admin dataset rotatekey command" + exit 1 +fi + +echo "Dataset key rotation command executed successfully" + +# check DB for updated key hash in sda.files for both files +rotatekeyHash=$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.encryption_keys where description='this is the rotatekey key';") +echo "Expected rotate key hash: $rotatekeyHash" + +# wait for key rotation to complete and verify both files have new key hash +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where id='$fileID1';")" = "$rotatekeyHash" ] && [ "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where id='$fileID2';")" = "$rotatekeyHash" ]; do + echo "waiting for key rotation to complete for both files" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 60 ]; then + echo "::error::Time out while waiting for key rotation to complete" + echo "File 1 key hash: $(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where id='$fileID1';")" + echo "File 2 key hash: $(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where id='$fileID2';")" + exit 1 + fi + sleep 2 +done + +echo "Key rotation completed for both files" + +# check that files were re-verified (archived queue should be empty) +echo "waiting for re-verify to complete" +RETRY_TIMES=0 +until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/archived/ | jq -r '.messages_ready')" -eq 0 ]; do + echo "waiting for re-verify to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for verify to complete" + exit 1 + fi + sleep 2 +done + +# check that no errors occurred during the process +sleep 5 +if [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')" -ne "$errorStreamSize" ]; then + echo "something went wrong with the dataset key rotation" + exit 1 +fi + +echo "Re-verification completed successfully" + +## verify both files can be downloaded and decrypted with rotated key +for i in 1 2; do + fileID_var="fileID$i" + eval fileID=\$$fileID_var + + echo "Testing download and decryption of file $i (ID: $fileID)" + + # get rotated header + psql -U postgres -h postgres -d sda -At -c "select header from sda.files where id='$fileID';" | xxd -r -p > "testfile${i}_rotated.c4gh" + + # get archive file + archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where id='$fileID';") + s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive/"$archivePath" --force + + # concatenate and decrypt + cat "testfile${i}_rotated.c4gh" "$archivePath" > tmp_file && mv tmp_file "testfile${i}_rotated.c4gh" + C4GH_PASSPHRASE=rotatekeyPass ./crypt4gh decrypt -f "testfile${i}_rotated.c4gh" -s rotatekey.sec.pem + + # check that decrypted file matches the original + if [ ! -f "testfile${i}_rotated" ]; then + echo "decrypted file testfile${i}_rotated not found" + exit 1 + fi + if ! cmp -s "testfile${i}_rotated" "testfile$i" ; then + echo "downloaded file $i is different from the original one" + exit 1 + fi + # compare hashes as well + if [ "$(sha256sum testfile$i | cut -d ' ' -f 1)" != "$(sha256sum testfile${i}_rotated | cut -d ' ' -f 1)" ]; then + echo "downloaded file $i has different sha256 hash from the original one" + exit 1 + fi + + echo "File $i download and decryption successful" +done + +### test for errors ### + +## test dataset key rotation with non-existent dataset +echo "Testing dataset key rotation with non-existent dataset" + +SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" /shared/sda-admin dataset rotatekey -dataset-id "NONEXISTENT" 2>/dev/null +if [ $? -eq 0 ]; then + echo "Expected sda-admin to fail with non-existent dataset, but it succeeded" + exit 1 +fi + +## test getting dataset file IDs via API +echo "Testing dataset file IDs endpoint" + +dataset_fileids="$(curl -s -k -H "Authorization: Bearer $token" -X GET "http://api:8080/dataset/EGAD12345678901/fileids")" +fileids_count="$(echo "$dataset_fileids" | jq '. | length')" +if [ "$fileids_count" -ne 2 ]; then + echo "Expected 2 file IDs, got: $fileids_count" + exit 1 +fi + +# verify the file IDs contain both fileID and accessionID +file1_has_fileid="$(echo "$dataset_fileids" | jq -r '.[0].fileID' | grep -c "$fileID1" || true)" +file1_has_accession="$(echo "$dataset_fileids" | jq -r '.[0].accessionID' | grep -c "EGAF123456789" || true)" +if [ "$file1_has_fileid" -ne 1 ] || [ "$file1_has_accession" -ne 1 ]; then + echo "File IDs endpoint doesn't return correct structure" + echo "Response: $dataset_fileids" + exit 1 +fi + +echo "Dataset file IDs endpoint working correctly" + +printf "\033[32mDataset admin CLI rotate key integration tests completed successfully\033[0m\n" From 68725b5757c87634bbf9d4c79ad813e44c96f128 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Tue, 13 Jan 2026 10:37:41 +0100 Subject: [PATCH 006/530] fix rotate key tests --- .../tests/sda/65_api_admin_rotate_key_test.sh | 113 +++++++++++++----- 1 file changed, 81 insertions(+), 32 deletions(-) diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh index 5863e1085d..e47bb3c77c 100755 --- a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -10,13 +10,13 @@ cd shared || true checkErrors() { RETRY_TIMES=0 until [ $(("$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')"-"$errorStreamSize")) -eq 1 ]; do - echo "checking for $1 error" - RETRY_TIMES=$((RETRY_TIMES + 1)) + echo "checking for $1 error" + RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 20 ]; then - echo "::error::Time out while waiting for error message" - exit 1 - fi - sleep 2 + echo "::error::Time out while waiting for error message" + exit 1 + fi + sleep 2 done } @@ -102,11 +102,11 @@ done # assign accession IDs to the files for i in 1 2; do - fileID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/testfile$i.c4gh';") payload=$( jq -c -n \ - --arg accession_id "EGAF123456789$i" \ - --arg file_id "$fileID" \ + --arg accession_id "EGAF100000000$i" \ + --arg user "test@dummy.org" \ + --arg filepath "dataset_admin_rotatekey/testfile$i.c4gh" \ '$ARGS.named' ) resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/file/accession")" @@ -116,26 +116,69 @@ for i in 1 2; do fi done -# create dataset with both files -dataset_payload=$( +# wait for accession IDs to be registered in the database +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) FROM sda.files WHERE stable_id LIKE 'EGAF100000000%';")" -eq 2 ]; do + echo "waiting for accession IDs to be registered" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for accession IDs to be registered" + exit 1 + fi + sleep 2 +done + +## map files to dataset +properties=$( jq -c -n \ - --arg dataset_id "EGAD12345678901" \ - --arg user "test@dummy.org" \ - --argjson accession_ids '["EGAF1234567891", "EGAF1234567892"]' \ + --argjson delivery_mode 2 \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ '$ARGS.named' ) -resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$dataset_payload" "http://api:8080/dataset/create")" -if [ "$resp" != "200" ]; then - echo "Error when creating dataset, expected 200 got: $resp" - exit 1 -fi -# verify dataset was created and contains both files -dataset_files_count="$(curl -s -k -H "Authorization: Bearer $token" -X GET "http://api:8080/dataset/EGAD12345678901/fileids" | jq '. | length')" -if [ "$dataset_files_count" -ne 2 ]; then - echo "Expected 2 files in dataset, got: $dataset_files_count" - exit 1 -fi +mappings=$( + jq -c -n \ + '$ARGS.positional' \ + --args "EGAF1000000001" \ + --args "EGAF1000000002" +) + +mapping_payload=$( + jq -r -c -n \ + --arg type mapping \ + --arg dataset_id EGAD100000000001 \ + --argjson accession_ids "$mappings" \ + '$ARGS.named|@base64' +) + +mapping_body=$( + jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "mappings" \ + --arg payload_encoding base64 \ + --arg payload "$mapping_payload" \ + '$ARGS.named' +) + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$mapping_body" | jq + + +# wait for mapper to process the dataset mapping +RETRY_TIMES=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "select count(id) from sda.file_dataset where dataset_id = (select id from sda.datasets where stable_id = 'EGAD100000000001');")" -eq 2 ]; do + echo "waiting for dataset mapping to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for dataset mapping to complete" + exit 1 + fi + sleep 2 +done echo "Dataset created successfully with 2 files" @@ -152,10 +195,12 @@ echo "File IDs: $fileID1, $fileID2" echo "Testing dataset key rotation via sda-admin CLI tool" # Use sda-admin CLI tool to rotate keys for the dataset -SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" /shared/sda-admin dataset rotatekey -dataset-id "EGAD12345678901" +# SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" sda-admin dataset rotatekey -dataset-id "EGAD100000000001" +# Replaced sda-admin with direct API call since sda-admin is not available in the container +resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/EGAD100000000001/rotatekey") -if [ $? -ne 0 ]; then - echo "Error when running sda-admin dataset rotatekey command" +if [ "$resp" != "200" ]; then + echo "Error when running dataset rotatekey command, expected 200 got: $resp" exit 1 fi @@ -198,6 +243,7 @@ done sleep 5 if [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready')" -ne "$errorStreamSize" ]; then echo "something went wrong with the dataset key rotation" + curl -su guest:guest -X POST http://rabbitmq:15672/api/queues/sda/error_stream/get -d '{"count":10,"ackmode":"ack_requeue_true","encoding":"auto","truncate":50000}' | jq . exit 1 fi @@ -244,16 +290,19 @@ done ## test dataset key rotation with non-existent dataset echo "Testing dataset key rotation with non-existent dataset" -SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" /shared/sda-admin dataset rotatekey -dataset-id "NONEXISTENT" 2>/dev/null -if [ $? -eq 0 ]; then - echo "Expected sda-admin to fail with non-existent dataset, but it succeeded" +# SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" sda-admin dataset rotatekey -dataset-id "NONEXISTENT" 2>/dev/null +# Replaced sda-admin with direct API call +resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/NONEXISTENT/rotatekey") + +if [ "$resp" != "404" ]; then + echo "Expected 404 for non-existent dataset, got: $resp" exit 1 fi ## test getting dataset file IDs via API echo "Testing dataset file IDs endpoint" -dataset_fileids="$(curl -s -k -H "Authorization: Bearer $token" -X GET "http://api:8080/dataset/EGAD12345678901/fileids")" +dataset_fileids="$(curl -s -k -H "Authorization: Bearer $token" -X GET "http://api:8080/dataset/EGAD100000000001/fileids")" fileids_count="$(echo "$dataset_fileids" | jq '. | length')" if [ "$fileids_count" -ne 2 ]; then echo "Expected 2 file IDs, got: $fileids_count" From a175b23c362362aadace3f0cb55abe3935e8cd95 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Wed, 14 Jan 2026 11:45:00 +0100 Subject: [PATCH 007/530] fix shellcheck and linting errors --- .../tests/sda/65_api_admin_rotate_key_test.sh | 1 + sda-admin/dataset/dataset_test.go | 106 ++++++++++++++++++ sda-admin/main.go | 11 +- sda/cmd/api/api.go | 43 ++++--- 4 files changed, 142 insertions(+), 19 deletions(-) diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh index e47bb3c77c..24788ecf25 100755 --- a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -252,6 +252,7 @@ echo "Re-verification completed successfully" ## verify both files can be downloaded and decrypted with rotated key for i in 1 2; do fileID_var="fileID$i" + fileID="" eval fileID=\$$fileID_var echo "Testing download and decryption of file $i (ID: $fileID)" diff --git a/sda-admin/dataset/dataset_test.go b/sda-admin/dataset/dataset_test.go index 22fbb059bf..efd0b8c2a7 100644 --- a/sda-admin/dataset/dataset_test.go +++ b/sda-admin/dataset/dataset_test.go @@ -20,6 +20,12 @@ func (m *MockHelpers) PostRequest(url, token string, jsonBody []byte) ([]byte, e return args.Get(0).([]byte), args.Error(1) } +func (m *MockHelpers) GetResponseBody(url, token string) ([]byte, error) { + args := m.Called(url, token) + + return args.Get(0).([]byte), args.Error(1) +} + func TestCreate_Success(t *testing.T) { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest @@ -93,3 +99,103 @@ func TestRelease_PostRequestFailure(t *testing.T) { assert.EqualError(t, err, "failed to send request") mockHelpers.AssertExpectations(t) } + +func TestRotateKey_Success(t *testing.T) { + mockHelpers := new(MockHelpers) + + originalGet := helpers.GetResponseBody + helpers.GetResponseBody = mockHelpers.GetResponseBody + defer func() { helpers.GetResponseBody = originalGet }() + + originalPost := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalPost }() + + apiURI := "http://example.com" + token := "test-token" + datasetID := "dataset-123" + + // Mock getting file IDs + filesURL := "http://example.com/dataset/dataset-123/fileids" + filesResponse := []byte(`[{"fileID":"file-1","accessionID":"acc-1"},{"fileID":"file-2","accessionID":"acc-2"}]`) + mockHelpers.On("GetResponseBody", filesURL, token).Return(filesResponse, nil) + + // Mock rotating keys for each file + rotateURL1 := "http://example.com/file/rotatekey/file-1" + mockHelpers.On("PostRequest", rotateURL1, token, []byte(nil)).Return([]byte(`{}`), nil) + + rotateURL2 := "http://example.com/file/rotatekey/file-2" + mockHelpers.On("PostRequest", rotateURL2, token, []byte(nil)).Return([]byte(`{}`), nil) + + err := RotateKey(apiURI, token, datasetID) + assert.NoError(t, err) + mockHelpers.AssertExpectations(t) +} + +func TestRotateKey_GetFilesFailure(t *testing.T) { + mockHelpers := new(MockHelpers) + + originalGet := helpers.GetResponseBody + helpers.GetResponseBody = mockHelpers.GetResponseBody + defer func() { helpers.GetResponseBody = originalGet }() + + apiURI := "http://example.com" + token := "test-token" + datasetID := "dataset-123" + + filesURL := "http://example.com/dataset/dataset-123/fileids" + mockHelpers.On("GetResponseBody", filesURL, token).Return([]byte(nil), errors.New("API error")) + + err := RotateKey(apiURI, token, datasetID) + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to get dataset file IDs") + mockHelpers.AssertExpectations(t) +} + +func TestRotateKey_NoFiles(t *testing.T) { + mockHelpers := new(MockHelpers) + + originalGet := helpers.GetResponseBody + helpers.GetResponseBody = mockHelpers.GetResponseBody + defer func() { helpers.GetResponseBody = originalGet }() + + apiURI := "http://example.com" + token := "test-token" + datasetID := "dataset-123" + + filesURL := "http://example.com/dataset/dataset-123/fileids" + mockHelpers.On("GetResponseBody", filesURL, token).Return([]byte(`[]`), nil) + + err := RotateKey(apiURI, token, datasetID) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no files found for dataset") + mockHelpers.AssertExpectations(t) +} + +func TestRotateKey_RotateFailure(t *testing.T) { + mockHelpers := new(MockHelpers) + + originalGet := helpers.GetResponseBody + helpers.GetResponseBody = mockHelpers.GetResponseBody + defer func() { helpers.GetResponseBody = originalGet }() + + originalPost := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalPost }() + + apiURI := "http://example.com" + token := "test-token" + datasetID := "dataset-123" + + filesURL := "http://example.com/dataset/dataset-123/fileids" + filesResponse := []byte(`[{"fileID":"file-1","accessionID":"acc-1"}]`) + mockHelpers.On("GetResponseBody", filesURL, token).Return(filesResponse, nil) + + rotateURL := "http://example.com/file/rotatekey/file-1" + mockHelpers.On("PostRequest", rotateURL, token, []byte(nil)).Return([]byte(nil), errors.New("rotation failed")) + + err := RotateKey(apiURI, token, datasetID) + assert.Error(t, err) + assert.Contains(t, err.Error(), "rotation failed") + mockHelpers.AssertExpectations(t) +} diff --git a/sda-admin/main.go b/sda-admin/main.go index 1fc0efca70..e4422b1a44 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -174,7 +174,7 @@ func printVersion() { fmt.Printf("sda-admin %s\n", version) } -func parseFlagsAndEnv() error { +func init() { // Set up flags flag.StringVar(&apiURI, "uri", "", "Set the URI for the SDA server (optional if API_HOST is set)") flag.StringVar(&token, "token", "", "Set the authentication token (optional if ACCESS_TOKEN is set)") @@ -183,10 +183,9 @@ func parseFlagsAndEnv() error { flag.Usage = func() { fmt.Println(usage) } +} - // Parse global flags first - flag.Parse() - +func validateFlagsAndEnv() error { // If no command is provided, show usage if flag.NArg() == 0 { return errors.New(usage) @@ -608,7 +607,9 @@ func handleC4ghHashListCommand() error { } func main() { - if err := parseFlagsAndEnv(); err != nil { + flag.Parse() + + if err := validateFlagsAndEnv(); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 8b4c960f91..25b06b85ac 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -144,21 +144,21 @@ func setup(conf *config.Config) *http.Server { r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash r.DELETE("/file/:username/:fileid", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here - r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file - r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file - r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file - r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset - r.GET("/dataset/:dataset/files", rbac(e), getDatasetFiles) // get files in a dataset - r.GET("/dataset/:dataset/fileids", rbac(e), getDatasetFileIDs) // get file IDs (internal and accession) in a dataset + r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file + r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file + r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file + r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset + r.GET("/dataset/:dataset/files", rbac(e), getDatasetFiles) // get files in a dataset + r.GET("/dataset/:dataset/fileids", rbac(e), getDatasetFileIDs) // get file IDs (internal and accession) in a dataset r.POST("/dataset/:dataset/rotatekey", rbac(e), rotateDatasetKeys) // trigger key rotation for all files in a dataset - r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible - r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset - r.POST("/file/rotatekey/:fileid", rbac(e), rotateFileKey) // trigger key rotation for a file - r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status - r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user - r.GET("/users", rbac(e), listActiveUsers) // Lists all users - r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user - r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox + r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible + r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset + r.POST("/file/rotatekey/:fileid", rbac(e), rotateFileKey) // trigger key rotation for a file + r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status + r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user + r.GET("/users", rbac(e), listActiveUsers) // Lists all users + r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user + r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox cfg := &tls.Config{MinVersion: tls.VersionTLS12} @@ -797,6 +797,7 @@ func getDatasetFiles(c *gin.Context) { if datasetID == "" { c.JSON(http.StatusBadRequest, "dataset ID is required") + return } @@ -804,11 +805,13 @@ func getDatasetFiles(c *gin.Context) { if err != nil { log.Errorf("failed to get dataset files for dataset %s, reason: %v", datasetID, err) c.JSON(http.StatusInternalServerError, err.Error()) + return } if len(files) == 0 { c.JSON(http.StatusNotFound, "no files found for dataset") + return } @@ -831,6 +834,7 @@ func getDatasetFileIDs(c *gin.Context) { if datasetID == "" { c.JSON(http.StatusBadRequest, "dataset ID is required") + return } @@ -838,11 +842,13 @@ func getDatasetFileIDs(c *gin.Context) { if err != nil { log.Errorf("failed to get dataset file IDs for dataset %s, reason: %v", datasetID, err) c.JSON(http.StatusInternalServerError, err.Error()) + return } if len(files) == 0 { c.JSON(http.StatusNotFound, "no files found for dataset") + return } @@ -855,6 +861,7 @@ func rotateFileKey(c *gin.Context) { if fileID == "" { c.JSON(http.StatusBadRequest, "file ID is required") + return } @@ -868,6 +875,7 @@ func rotateFileKey(c *gin.Context) { if err != nil { log.Errorf("failed to marshal rotation message, reason: %v", err) c.JSON(http.StatusInternalServerError, "failed to marshal rotation message") + return } @@ -875,6 +883,7 @@ func rotateFileKey(c *gin.Context) { if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { log.Errorf("rotation message validation failed, reason: %v", err) c.JSON(http.StatusBadRequest, "rotation message validation failed") + return } @@ -883,6 +892,7 @@ func rotateFileKey(c *gin.Context) { if err != nil { log.Errorf("failed to send rotation message to queue, reason: %v", err) c.JSON(http.StatusInternalServerError, "failed to send rotation message") + return } @@ -895,6 +905,7 @@ func rotateDatasetKeys(c *gin.Context) { if datasetID == "" { c.JSON(http.StatusBadRequest, "dataset ID is required") + return } @@ -903,11 +914,13 @@ func rotateDatasetKeys(c *gin.Context) { if err != nil { log.Errorf("failed to get dataset files for dataset %s, reason: %v", datasetID, err) c.JSON(http.StatusInternalServerError, "failed to get dataset files") + return } if len(files) == 0 { c.JSON(http.StatusNotFound, "no files found for dataset") + return } @@ -930,6 +943,7 @@ func rotateDatasetKeys(c *gin.Context) { if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { log.Errorf("rotation message validation failed for file %s, reason: %v", file.FileID, err) c.JSON(http.StatusBadRequest, "rotation message validation failed") + return } @@ -938,6 +952,7 @@ func rotateDatasetKeys(c *gin.Context) { if err != nil { log.Errorf("failed to send rotation message for file %s to queue, reason: %v", file.FileID, err) c.JSON(http.StatusInternalServerError, "failed to send rotation message") + return } From 9eed743d066b3f9a274b2107eb3b1a1c8db95492 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Mon, 19 Jan 2026 23:23:28 +0100 Subject: [PATCH 008/530] Add suggestions from the comments - removed the api additions for getting ids for files of a dataset - other suggestions --- .../tests/sda/65_api_admin_rotate_key_test.sh | 27 +----- sda-admin/dataset/dataset.go | 38 +------- sda-admin/dataset/dataset_test.go | 71 ++------------- sda/cmd/api/api.go | 87 +++---------------- sda/cmd/api/api.md | 31 ++----- sda/internal/database/db_functions.go | 45 +++++++--- 6 files changed, 64 insertions(+), 235 deletions(-) diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh index 24788ecf25..59c11b2b9b 100755 --- a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -197,7 +197,7 @@ echo "Testing dataset key rotation via sda-admin CLI tool" # Use sda-admin CLI tool to rotate keys for the dataset # SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" sda-admin dataset rotatekey -dataset-id "EGAD100000000001" # Replaced sda-admin with direct API call since sda-admin is not available in the container -resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/EGAD100000000001/rotatekey") +resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/rotatekey/EGAD100000000001") if [ "$resp" != "200" ]; then echo "Error when running dataset rotatekey command, expected 200 got: $resp" @@ -293,32 +293,11 @@ echo "Testing dataset key rotation with non-existent dataset" # SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" sda-admin dataset rotatekey -dataset-id "NONEXISTENT" 2>/dev/null # Replaced sda-admin with direct API call -resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/NONEXISTENT/rotatekey") +resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/rotatekey/NONEXISTENT") if [ "$resp" != "404" ]; then echo "Expected 404 for non-existent dataset, got: $resp" exit 1 fi -## test getting dataset file IDs via API -echo "Testing dataset file IDs endpoint" - -dataset_fileids="$(curl -s -k -H "Authorization: Bearer $token" -X GET "http://api:8080/dataset/EGAD100000000001/fileids")" -fileids_count="$(echo "$dataset_fileids" | jq '. | length')" -if [ "$fileids_count" -ne 2 ]; then - echo "Expected 2 file IDs, got: $fileids_count" - exit 1 -fi - -# verify the file IDs contain both fileID and accessionID -file1_has_fileid="$(echo "$dataset_fileids" | jq -r '.[0].fileID' | grep -c "$fileID1" || true)" -file1_has_accession="$(echo "$dataset_fileids" | jq -r '.[0].accessionID' | grep -c "EGAF123456789" || true)" -if [ "$file1_has_fileid" -ne 1 ] || [ "$file1_has_accession" -ne 1 ]; then - echo "File IDs endpoint doesn't return correct structure" - echo "Response: $dataset_fileids" - exit 1 -fi - -echo "Dataset file IDs endpoint working correctly" - -printf "\033[32mDataset admin CLI rotate key integration tests completed successfully\033[0m\n" +echo "Dataset admin CLI rotate key integration tests completed successfully" diff --git a/sda-admin/dataset/dataset.go b/sda-admin/dataset/dataset.go index 024092aec4..a3a37e8d24 100644 --- a/sda-admin/dataset/dataset.go +++ b/sda-admin/dataset/dataset.go @@ -15,11 +15,6 @@ type RequestBodyDataset struct { User string `json:"user"` } -type DatasetFileIDs struct { - FileID string `json:"fileID"` - AccessionID string `json:"accessionID"` -} - // Create creates a dataset from a list of accession IDs and a dataset ID. func Create(apiURI, token, datasetID, username string, accessionIDs []string) error { parsedURL, err := url.Parse(apiURI) @@ -65,40 +60,15 @@ func Release(apiURI, token, datasetID string) error { // RotateKey rotates the encryption key for all files in a dataset func RotateKey(apiURI, token, datasetID string) error { - // First, get all file IDs in the dataset - filesURL, err := url.Parse(apiURI) + parsedURL, err := url.Parse(apiURI) if err != nil { return err } - filesURL.Path = path.Join(filesURL.Path, "dataset", datasetID, "fileids") - - response, err := helpers.GetResponseBody(filesURL.String(), token) - if err != nil { - return fmt.Errorf("failed to get dataset file IDs, reason: %v", err) - } + parsedURL.Path = path.Join(parsedURL.Path, "dataset", "rotatekey", datasetID) - var datasetFiles []DatasetFileIDs - err = json.Unmarshal(response, &datasetFiles) + _, err = helpers.PostRequest(parsedURL.String(), token, nil) if err != nil { - return fmt.Errorf("failed to unmarshal dataset file IDs response, reason: %v", err) - } - - if len(datasetFiles) == 0 { - return fmt.Errorf("no files found for dataset %s", datasetID) - } - - // Send rotation request for each file using the internal FileID - for _, file := range datasetFiles { - rotateURL, err := url.Parse(apiURI) - if err != nil { - return fmt.Errorf("failed to parse API URI for file %s, reason: %v", file.FileID, err) - } - rotateURL.Path = path.Join(rotateURL.Path, "file", "rotatekey", file.FileID) - - _, err = helpers.PostRequest(rotateURL.String(), token, nil) - if err != nil { - return fmt.Errorf("failed to rotate key for file %s (accession: %s), reason: %v", file.FileID, file.AccessionID, err) - } + return fmt.Errorf("failed to rotate key for dataset %s, reason: %v", datasetID, err) } return nil diff --git a/sda-admin/dataset/dataset_test.go b/sda-admin/dataset/dataset_test.go index efd0b8c2a7..6498f3e989 100644 --- a/sda-admin/dataset/dataset_test.go +++ b/sda-admin/dataset/dataset_test.go @@ -103,10 +103,6 @@ func TestRelease_PostRequestFailure(t *testing.T) { func TestRotateKey_Success(t *testing.T) { mockHelpers := new(MockHelpers) - originalGet := helpers.GetResponseBody - helpers.GetResponseBody = mockHelpers.GetResponseBody - defer func() { helpers.GetResponseBody = originalGet }() - originalPost := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalPost }() @@ -115,70 +111,18 @@ func TestRotateKey_Success(t *testing.T) { token := "test-token" datasetID := "dataset-123" - // Mock getting file IDs - filesURL := "http://example.com/dataset/dataset-123/fileids" - filesResponse := []byte(`[{"fileID":"file-1","accessionID":"acc-1"},{"fileID":"file-2","accessionID":"acc-2"}]`) - mockHelpers.On("GetResponseBody", filesURL, token).Return(filesResponse, nil) - - // Mock rotating keys for each file - rotateURL1 := "http://example.com/file/rotatekey/file-1" - mockHelpers.On("PostRequest", rotateURL1, token, []byte(nil)).Return([]byte(`{}`), nil) - - rotateURL2 := "http://example.com/file/rotatekey/file-2" - mockHelpers.On("PostRequest", rotateURL2, token, []byte(nil)).Return([]byte(`{}`), nil) + // Mock rotating keys for dataset + rotateURL := "http://example.com/dataset/rotatekey/dataset-123" + mockHelpers.On("PostRequest", rotateURL, token, []byte(nil)).Return([]byte(`{}`), nil) err := RotateKey(apiURI, token, datasetID) assert.NoError(t, err) mockHelpers.AssertExpectations(t) } -func TestRotateKey_GetFilesFailure(t *testing.T) { - mockHelpers := new(MockHelpers) - - originalGet := helpers.GetResponseBody - helpers.GetResponseBody = mockHelpers.GetResponseBody - defer func() { helpers.GetResponseBody = originalGet }() - - apiURI := "http://example.com" - token := "test-token" - datasetID := "dataset-123" - - filesURL := "http://example.com/dataset/dataset-123/fileids" - mockHelpers.On("GetResponseBody", filesURL, token).Return([]byte(nil), errors.New("API error")) - - err := RotateKey(apiURI, token, datasetID) - assert.Error(t, err) - assert.Contains(t, err.Error(), "failed to get dataset file IDs") - mockHelpers.AssertExpectations(t) -} - -func TestRotateKey_NoFiles(t *testing.T) { +func TestRotateKey_Failure(t *testing.T) { mockHelpers := new(MockHelpers) - originalGet := helpers.GetResponseBody - helpers.GetResponseBody = mockHelpers.GetResponseBody - defer func() { helpers.GetResponseBody = originalGet }() - - apiURI := "http://example.com" - token := "test-token" - datasetID := "dataset-123" - - filesURL := "http://example.com/dataset/dataset-123/fileids" - mockHelpers.On("GetResponseBody", filesURL, token).Return([]byte(`[]`), nil) - - err := RotateKey(apiURI, token, datasetID) - assert.Error(t, err) - assert.Contains(t, err.Error(), "no files found for dataset") - mockHelpers.AssertExpectations(t) -} - -func TestRotateKey_RotateFailure(t *testing.T) { - mockHelpers := new(MockHelpers) - - originalGet := helpers.GetResponseBody - helpers.GetResponseBody = mockHelpers.GetResponseBody - defer func() { helpers.GetResponseBody = originalGet }() - originalPost := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest defer func() { helpers.PostRequest = originalPost }() @@ -187,15 +131,12 @@ func TestRotateKey_RotateFailure(t *testing.T) { token := "test-token" datasetID := "dataset-123" - filesURL := "http://example.com/dataset/dataset-123/fileids" - filesResponse := []byte(`[{"fileID":"file-1","accessionID":"acc-1"}]`) - mockHelpers.On("GetResponseBody", filesURL, token).Return(filesResponse, nil) - - rotateURL := "http://example.com/file/rotatekey/file-1" + rotateURL := "http://example.com/dataset/rotatekey/dataset-123" mockHelpers.On("PostRequest", rotateURL, token, []byte(nil)).Return([]byte(nil), errors.New("rotation failed")) err := RotateKey(apiURI, token, datasetID) assert.Error(t, err) assert.Contains(t, err.Error(), "rotation failed") mockHelpers.AssertExpectations(t) + } diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 25b06b85ac..94e8191139 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -148,9 +148,7 @@ func setup(conf *config.Config) *http.Server { r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset - r.GET("/dataset/:dataset/files", rbac(e), getDatasetFiles) // get files in a dataset - r.GET("/dataset/:dataset/fileids", rbac(e), getDatasetFileIDs) // get file IDs (internal and accession) in a dataset - r.POST("/dataset/:dataset/rotatekey", rbac(e), rotateDatasetKeys) // trigger key rotation for all files in a dataset + r.POST("/dataset/rotatekey/:dataset", rbac(e), rotateDatasetKeys) // trigger key rotation for all files in a dataset r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset r.POST("/file/rotatekey/:fileid", rbac(e), rotateFileKey) // trigger key rotation for a file @@ -790,71 +788,6 @@ func releaseDataset(c *gin.Context) { c.Status(http.StatusOK) } -// getDatasetFiles returns files belonging to a dataset -func getDatasetFiles(c *gin.Context) { - c.Writer.Header().Set("Content-Type", "application/json") - datasetID := c.Param("dataset") - - if datasetID == "" { - c.JSON(http.StatusBadRequest, "dataset ID is required") - - return - } - - files, err := Conf.API.DB.GetDatasetFiles(datasetID) - if err != nil { - log.Errorf("failed to get dataset files for dataset %s, reason: %v", datasetID, err) - c.JSON(http.StatusInternalServerError, err.Error()) - - return - } - - if len(files) == 0 { - c.JSON(http.StatusNotFound, "no files found for dataset") - - return - } - - // Convert DatasetFileIDs to a simpler format for this endpoint - var result []map[string]string - for _, file := range files { - result = append(result, map[string]string{ - "fileID": file.FileID, - "accessionID": file.AccessionID, - }) - } - - c.JSON(http.StatusOK, result) -} - -// getDatasetFileIDs returns file IDs (both internal and accession) belonging to a dataset -func getDatasetFileIDs(c *gin.Context) { - c.Writer.Header().Set("Content-Type", "application/json") - datasetID := c.Param("dataset") - - if datasetID == "" { - c.JSON(http.StatusBadRequest, "dataset ID is required") - - return - } - - files, err := Conf.API.DB.GetDatasetFiles(datasetID) - if err != nil { - log.Errorf("failed to get dataset file IDs for dataset %s, reason: %v", datasetID, err) - c.JSON(http.StatusInternalServerError, err.Error()) - - return - } - - if len(files) == 0 { - c.JSON(http.StatusNotFound, "no files found for dataset") - - return - } - - c.JSON(http.StatusOK, files) -} - // rotateFileKey triggers key rotation for a specific file func rotateFileKey(c *gin.Context) { fileID := c.Param("fileid") @@ -910,7 +843,7 @@ func rotateDatasetKeys(c *gin.Context) { } // Get all files in the dataset - files, err := Conf.API.DB.GetDatasetFiles(datasetID) + files, err := Conf.API.DB.GetDatasetFileIDs(datasetID) if err != nil { log.Errorf("failed to get dataset files for dataset %s, reason: %v", datasetID, err) c.JSON(http.StatusInternalServerError, "failed to get dataset files") @@ -925,23 +858,23 @@ func rotateDatasetKeys(c *gin.Context) { } // Send rotation message for each file in the dataset - for _, file := range files { + for _, fileID := range files { // Create rotation message rotateMsg := schema.KeyRotation{ Type: "key_rotation", - FileID: file.FileID, + FileID: fileID, } marshaledMsg, err := json.Marshal(&rotateMsg) if err != nil { - log.Errorf("failed to marshal rotation message for file %s, reason: %v", file.FileID, err) + log.Errorf("failed to marshal rotation message for file %s, reason: %v", fileID, err) c.JSON(http.StatusInternalServerError, "failed to marshal rotation message") return } // Validate the message against schema if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { - log.Errorf("rotation message validation failed for file %s, reason: %v", file.FileID, err) + log.Errorf("rotation message validation failed for file %s, reason: %v", fileID, err) c.JSON(http.StatusBadRequest, "rotation message validation failed") return @@ -950,13 +883,13 @@ func rotateDatasetKeys(c *gin.Context) { // Send message to rotatekey queue err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "rotatekey", marshaledMsg) if err != nil { - log.Errorf("failed to send rotation message for file %s to queue, reason: %v", file.FileID, err) + log.Errorf("failed to send rotation message for file %s to queue, reason: %v", fileID, err) c.JSON(http.StatusInternalServerError, "failed to send rotation message") return } - log.Infof("rotation message sent for file %s (accession: %s) in dataset %s", file.FileID, file.AccessionID, datasetID) + log.Infof("rotation message sent for file %s in dataset %s", fileID, datasetID) } log.Infof("rotation messages sent for %d files in dataset %s", len(files), datasetID) @@ -1199,8 +1132,8 @@ func reVerifyDataset(c *gin.Context) { return } - for _, file := range files { - c, err = reVerify(c, file.AccessionID) + for _, accession := range files { + c, err = reVerify(c, accession) if err != nil { return } diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 658be9e6ff..59c8a06a5a 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -182,38 +182,21 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -X PUT https://HOSTNAME/dataset/verify/my-dataset-01 ``` -- `/dataset/:dataset/files` - - accepts `GET` requests with the dataset name as parameter - - Returns all files in the dataset as a JSON array. +- `/dataset/rotatekey/:dataset` + - accepts `POST` requests with the dataset name as parameter + - Triggers key rotation for all files in the dataset by sending a message to the rotatekey queue for each file. - Error codes - - `200` Query execute ok. - - `400` Dataset ID not provided. + - `200` Key rotation triggered successfully. + - `400` Dataset ID not provided or message validation failed. - `401` Token user is not in the list of admins. - `404` No files found for dataset. - - `500` Internal error due to DB failures. - - Example: - - ```bash - curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/dataset/my-dataset-01/files - ``` - -- `/dataset/:dataset/fileids` - - accepts `GET` requests with the dataset name as parameter - - Returns file IDs (both internal file ID and accession ID) for all files in the dataset as a JSON array. - - - Error codes - - `200` Query execute ok. - - `400` Dataset ID not provided. - - `401` Token user is not in the list of admins. - - `404` No files found for dataset. - - `500` Internal error due to DB failures. + - `500` Internal error due to DB or MQ failures. Example: ```bash - curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/dataset/my-dataset-01/fileids + curl -H "Authorization: Bearer $token" -X POST https://HOSTNAME/dataset/rotatekey/my-dataset-01 ``` - `/file/rotatekey/:fileid` diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 9b41a5afbe..fd7af7f0c8 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1320,18 +1320,41 @@ func (dbs *SDAdb) GetDecryptedChecksum(id string) (string, error) { return unencryptedChecksum, nil } -// DatasetFileIDs holds both the internal file ID and stable accession ID -type DatasetFileIDs struct { - FileID string `json:"fileID"` - AccessionID string `json:"accessionID"` +// GetDatasetFiles returns all files in a dataset +func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + var accessions []string + rows, err := db.Query("SELECT stable_id FROM sda.files WHERE id IN (SELECT file_id FROM sda.file_dataset WHERE dataset_id = (SELECT id FROM sda.datasets WHERE stable_id = $1));", dataset) + if err != nil { + return nil, err + } + if rows.Err() != nil { + return nil, rows.Err() + } + defer rows.Close() + + for rows.Next() { + var accession string + err := rows.Scan(&accession) + if err != nil { + return nil, err + } + + accessions = append(accessions, accession) + } + + return accessions, nil } -func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]DatasetFileIDs, error) { +// GetDatasetFileIDs returns all file IDs in a dataset +func (dbs *SDAdb) GetDatasetFileIDs(dataset string) ([]string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - var files []DatasetFileIDs - rows, err := db.Query("SELECT id, stable_id FROM sda.files WHERE id IN (SELECT file_id FROM sda.file_dataset WHERE dataset_id = (SELECT id FROM sda.datasets WHERE stable_id = $1));", dataset) + var fileIDs []string + rows, err := db.Query("SELECT file_id FROM sda.file_dataset WHERE dataset_id = (SELECT id FROM sda.datasets WHERE stable_id = $1);", dataset) if err != nil { return nil, err } @@ -1341,16 +1364,16 @@ func (dbs *SDAdb) GetDatasetFiles(dataset string) ([]DatasetFileIDs, error) { defer rows.Close() for rows.Next() { - var file DatasetFileIDs - err := rows.Scan(&file.FileID, &file.AccessionID) + var fileID string + err := rows.Scan(&fileID) if err != nil { return nil, err } - files = append(files, file) + fileIDs = append(fileIDs, fileID) } - return files, nil + return fileIDs, nil } // GetFileDetailsFromUUID() retrieves user, path and correlation id by giving the file UUID From 7f9c86f0cf906e3d72a346ee8ef609bdc7a26fd6 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Mon, 19 Jan 2026 23:35:01 +0100 Subject: [PATCH 009/530] Apply suggestions from code review Co-authored-by: Nanjiang Shu Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 3 ++- sda/cmd/api/api.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 94e8191139..b00d651a5a 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -852,7 +852,7 @@ func rotateDatasetKeys(c *gin.Context) { } if len(files) == 0 { - c.JSON(http.StatusNotFound, "no files found for dataset") + c.JSON(http.StatusBadRequest, "no files found for dataset") return } @@ -869,6 +869,7 @@ func rotateDatasetKeys(c *gin.Context) { if err != nil { log.Errorf("failed to marshal rotation message for file %s, reason: %v", fileID, err) c.JSON(http.StatusInternalServerError, "failed to marshal rotation message") + return } diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 59c8a06a5a..c0e5b6af1d 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -207,7 +207,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th - `200` Query execute ok. - `400` File ID not provided or message validation failed. - `401` Token user is not in the list of admins. - - `500` Internal error due to MQ failures. + - `500` Internal error due to MQ or DB failures. Example: From e9c327c4b8ef7a54693cee37fc15552d040e95a7 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Mon, 19 Jan 2026 23:55:55 +0100 Subject: [PATCH 010/530] Added more suggestions from review comments --- sda/cmd/api/api.go | 45 +++++++++++++++++++-------------------------- sda/cmd/api/api.md | 3 +-- 2 files changed, 20 insertions(+), 28 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index b00d651a5a..f38ac7f2b0 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -144,19 +144,19 @@ func setup(conf *config.Config) *http.Server { r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash r.DELETE("/file/:username/:fileid", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here - r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file - r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file - r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file - r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset - r.POST("/dataset/rotatekey/:dataset", rbac(e), rotateDatasetKeys) // trigger key rotation for all files in a dataset - r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible - r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset - r.POST("/file/rotatekey/:fileid", rbac(e), rotateFileKey) // trigger key rotation for a file - r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status - r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user - r.GET("/users", rbac(e), listActiveUsers) // Lists all users - r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user - r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox + r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file + r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file + r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file + r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset + r.POST("/dataset/rotatekey/:dataset", rbac(e), rotateKeyDataset) // trigger key rotation for all files in a dataset + r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible + r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset + r.POST("/file/rotatekey/:fileid", rbac(e), rotateKeyFile) // trigger key rotation for a file + r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status + r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user + r.GET("/users", rbac(e), listActiveUsers) // Lists all users + r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user + r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox cfg := &tls.Config{MinVersion: tls.VersionTLS12} @@ -788,8 +788,8 @@ func releaseDataset(c *gin.Context) { c.Status(http.StatusOK) } -// rotateFileKey triggers key rotation for a specific file -func rotateFileKey(c *gin.Context) { +// rotateKeyFile triggers key rotation for a specific file +func rotateKeyFile(c *gin.Context) { fileID := c.Param("fileid") if fileID == "" { @@ -832,8 +832,8 @@ func rotateFileKey(c *gin.Context) { c.Status(http.StatusOK) } -// rotateDatasetKeys triggers key rotation for all files in a dataset -func rotateDatasetKeys(c *gin.Context) { +// rotateKeyDataset triggers key rotation for all files in a dataset +func rotateKeyDataset(c *gin.Context) { datasetID := c.Param("dataset") if datasetID == "" { @@ -852,7 +852,8 @@ func rotateDatasetKeys(c *gin.Context) { } if len(files) == 0 { - c.JSON(http.StatusBadRequest, "no files found for dataset") + log.Warnf("no files found for dataset %s", datasetID) + c.Status(http.StatusOK) return } @@ -873,14 +874,6 @@ func rotateDatasetKeys(c *gin.Context) { return } - // Validate the message against schema - if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { - log.Errorf("rotation message validation failed for file %s, reason: %v", fileID, err) - c.JSON(http.StatusBadRequest, "rotation message validation failed") - - return - } - // Send message to rotatekey queue err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "rotatekey", marshaledMsg) if err != nil { diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index c0e5b6af1d..02d67ad2db 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -188,9 +188,8 @@ Admin endpoints are only available to a set of whitelisted users specified in th - Error codes - `200` Key rotation triggered successfully. - - `400` Dataset ID not provided or message validation failed. + - `400` Dataset ID not provided. - `401` Token user is not in the list of admins. - - `404` No files found for dataset. - `500` Internal error due to DB or MQ failures. Example: From d24863be317df40262aaa89beef3e96bbe121948 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Wed, 21 Jan 2026 10:30:41 +0100 Subject: [PATCH 011/530] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index f38ac7f2b0..97b4f45918 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -147,11 +147,11 @@ func setup(conf *config.Config) *http.Server { r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file + r.POST("/file/rotatekey/:fileid", rbac(e), rotateKeyFile) // trigger key rotation for a file r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset r.POST("/dataset/rotatekey/:dataset", rbac(e), rotateKeyDataset) // trigger key rotation for all files in a dataset r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset - r.POST("/file/rotatekey/:fileid", rbac(e), rotateKeyFile) // trigger key rotation for a file r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user r.GET("/users", rbac(e), listActiveUsers) // Lists all users @@ -815,7 +815,7 @@ func rotateKeyFile(c *gin.Context) { // Validate the message against schema if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { log.Errorf("rotation message validation failed, reason: %v", err) - c.JSON(http.StatusBadRequest, "rotation message validation failed") + c.JSON(http.StatusBadRequest, "file ID not a proper UUID") return } @@ -824,7 +824,7 @@ func rotateKeyFile(c *gin.Context) { err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "rotatekey", marshaledMsg) if err != nil { log.Errorf("failed to send rotation message to queue, reason: %v", err) - c.JSON(http.StatusInternalServerError, "failed to send rotation message") + c.JSON(http.StatusInternalServerError, "failed to send message") return } From ab036340d54aaf5b3ac67989447c96672c81499c Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Wed, 21 Jan 2026 10:34:42 +0100 Subject: [PATCH 012/530] Update sda/cmd/api/api.go Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 97b4f45918..b0f401c489 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -882,8 +882,6 @@ func rotateKeyDataset(c *gin.Context) { return } - - log.Infof("rotation message sent for file %s in dataset %s", fileID, datasetID) } log.Infof("rotation messages sent for %d files in dataset %s", len(files), datasetID) From a3c7ce877132fa7c8601472f5f9e341d16c97a38 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Wed, 21 Jan 2026 10:34:57 +0100 Subject: [PATCH 013/530] Update sda-admin/dataset/dataset_test.go Co-authored-by: Joakim Bygdell --- sda-admin/dataset/dataset_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sda-admin/dataset/dataset_test.go b/sda-admin/dataset/dataset_test.go index 6498f3e989..1f3583b56a 100644 --- a/sda-admin/dataset/dataset_test.go +++ b/sda-admin/dataset/dataset_test.go @@ -138,5 +138,4 @@ func TestRotateKey_Failure(t *testing.T) { assert.Error(t, err) assert.Contains(t, err.Error(), "rotation failed") mockHelpers.AssertExpectations(t) - } From 5bc908eae1dfab550633c212ff9f229fe20ba7b0 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Thu, 22 Jan 2026 11:14:47 +0100 Subject: [PATCH 014/530] add file rotatekey for sda-admin --- .../tests/sda/65_api_admin_rotate_key_test.sh | 5 --- sda-admin/file/file.go | 16 ++++++++ sda-admin/file/file_test.go | 37 +++++++++++++++++++ sda-admin/main.go | 19 ++++++++++ 4 files changed, 72 insertions(+), 5 deletions(-) diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh index 59c11b2b9b..6bc041e075 100755 --- a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -194,9 +194,6 @@ echo "File IDs: $fileID1, $fileID2" ## test dataset key rotation via sda-admin CLI tool echo "Testing dataset key rotation via sda-admin CLI tool" -# Use sda-admin CLI tool to rotate keys for the dataset -# SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" sda-admin dataset rotatekey -dataset-id "EGAD100000000001" -# Replaced sda-admin with direct API call since sda-admin is not available in the container resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/rotatekey/EGAD100000000001") if [ "$resp" != "200" ]; then @@ -291,8 +288,6 @@ done ## test dataset key rotation with non-existent dataset echo "Testing dataset key rotation with non-existent dataset" -# SDA_ADMIN_API_URI="http://api:8080" SDA_ADMIN_TOKEN="$token" sda-admin dataset rotatekey -dataset-id "NONEXISTENT" 2>/dev/null -# Replaced sda-admin with direct API call resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/rotatekey/NONEXISTENT") if [ "$resp" != "404" ]; then diff --git a/sda-admin/file/file.go b/sda-admin/file/file.go index e84cd5d854..0d9b054f45 100644 --- a/sda-admin/file/file.go +++ b/sda-admin/file/file.go @@ -118,3 +118,19 @@ func SetAccession(accessionInfo helpers.FileInfo) error { return nil } + +// RotateKey rotates the encryption key for a file +func RotateKey(apiURI, token, fileID string) error { + parsedURL, err := url.Parse(apiURI) + if err != nil { + return err + } + parsedURL.Path = path.Join(parsedURL.Path, "file", "rotatekey", fileID) + + _, err = helpers.PostRequest(parsedURL.String(), token, nil) + if err != nil { + return fmt.Errorf("failed to rotate key for file %s, reason: %v", fileID, err) + } + + return nil +} diff --git a/sda-admin/file/file_test.go b/sda-admin/file/file_test.go index c14bf519f1..a176020bc9 100644 --- a/sda-admin/file/file_test.go +++ b/sda-admin/file/file_test.go @@ -209,3 +209,40 @@ func TestSetAccessionID_PostRequestFailure(t *testing.T) { assert.EqualError(t, err, "failed to send request") mockHelpers.AssertExpectations(t) } + +func TestRotateKey_Success(t *testing.T) { + mockHelpers := new(MockHelpers) + originalPost := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalPost }() + + apiURI := "http://example.com" + token := "test-token" + fileID := "file-uuid" + + mockHelpers.On("PostRequest", "http://example.com/file/rotatekey/file-uuid", "test-token", []byte(nil)).Return([]byte(`{}`), nil) + + err := RotateKey(apiURI, token, fileID) + + assert.NoError(t, err) + mockHelpers.AssertExpectations(t) +} + +func TestRotateKey_Failure(t *testing.T) { + mockHelpers := new(MockHelpers) + originalPost := helpers.PostRequest + helpers.PostRequest = mockHelpers.PostRequest + defer func() { helpers.PostRequest = originalPost }() + + apiURI := "http://example.com" + token := "test-token" + fileID := "file-uuid" + + mockHelpers.On("PostRequest", "http://example.com/file/rotatekey/file-uuid", "test-token", []byte(nil)).Return([]byte(nil), errors.New("post request failed")) + + err := RotateKey(apiURI, token, fileID) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "post request failed") + mockHelpers.AssertExpectations(t) +} diff --git a/sda-admin/main.go b/sda-admin/main.go index e4422b1a44..3f375ff325 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -30,6 +30,8 @@ Commands: Trigger ingestion of a given file. file set-accession -filepath FILEPATH -user USERNAME -accession-id accessionID Assign accession ID to a file. + file rotatekey -file-id FILEUUID + Rotate encryption key for a specific file. dataset create -user SUBMISSION_USER -dataset-id DATASET_ID accessionID [accessionID ...] Create a dataset from a list of accession IDs and a dataset ID. dataset release -dataset-id DATASET_ID @@ -65,10 +67,15 @@ Set accession ID to a file: Usage: sda-admin file set-accession -filepath FILEPATH -user USERNAME -accession-id ACCESSION_ID Assign an accession ID to a file for a given user. +Rotate key for a file: + Usage: sda-admin file rotatekey -file-id FILEUUID + Rotate encryption key for a specific file. + Options: -user USERNAME Specify the username associated with the file. -filepath FILEPATH Specify the path of the file to ingest. -accession-id ID Specify the accession ID to assign to the file. + -file-id FILEUUID Specify the file ID of the file to rotate key. Use 'sda-admin help file ' for information on a specific command.` @@ -99,6 +106,12 @@ Options: -fileid FILEUUID Specify the file ID of the file to assign the accession ID. -accession-id ID Specify the accession ID to assign to the file.` +var fileRotateKeyUsage = `Usage: sda-admin file rotatekey -file-id FILEUUID + Rotate encryption key for a specific file. + +Options: + -file-id FILEUUID Specify the file ID of the file to rotate key.` + var datasetUsage = `Create a dataset: Usage: sda-admin dataset create -user SUBMISSION_USER -dataset-id DATASET_ID [ACCESSION_ID ...] Create a dataset from a list of accession IDs and a dataset ID. @@ -267,6 +280,8 @@ func handleHelpFile() error { fmt.Println(fileIngestUsage) case flag.Arg(2) == "set-accession": fmt.Println(fileAccessionUsage) + case flag.Arg(2) == "rotatekey": + fmt.Println(fileRotateKeyUsage) default: return fmt.Errorf("unknown subcommand '%s' for '%s'.\n%s", flag.Arg(2), flag.Arg(1), fileUsage) } @@ -346,6 +361,10 @@ func handleFileCommand() error { if err := handleFileAccessionCommand(); err != nil { return err } + case "rotatekey": + if err := handleFileRotateKeyCommand(); err != nil { + return err + } default: return fmt.Errorf("unknown subcommand '%s' for '%s'.\n%s", flag.Arg(1), flag.Arg(0), fileUsage) } From fbaa5cca1dfd7ce75f85a368c2bd773f6aeb5874 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Thu, 22 Jan 2026 11:33:03 +0100 Subject: [PATCH 015/530] add schema validation --- sda/cmd/api/api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index b0f401c489..e6b56844df 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -874,6 +874,14 @@ func rotateKeyDataset(c *gin.Context) { return } + // Validate the message against schema + if err := schema.ValidateJSON(fmt.Sprintf("%s/rotate-key.json", Conf.Broker.SchemasPath), marshaledMsg); err != nil { + log.Errorf("rotation message validation failed for file %s, reason: %v", fileID, err) + c.JSON(http.StatusInternalServerError, "rotation message validation failed") + + return + } + // Send message to rotatekey queue err = Conf.API.MQ.SendMessage("", Conf.Broker.Exchange, "rotatekey", marshaledMsg) if err != nil { From 50eff7c7c6a963d0b5875a204640ac872dfce712 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Thu, 22 Jan 2026 14:40:42 +0100 Subject: [PATCH 016/530] fix tests, add missing func --- sda-admin/main.go | 23 ++++++++ sda/internal/database/db_functions_test.go | 69 ++++++++++++++++++++-- 2 files changed, 86 insertions(+), 6 deletions(-) diff --git a/sda-admin/main.go b/sda-admin/main.go index 3f375ff325..ce18b2daf4 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -437,6 +437,29 @@ func handleFileAccessionCommand() error { } } +func handleFileRotateKeyCommand() error { + fileRotateKeyCmd := flag.NewFlagSet("rotatekey", flag.ExitOnError) + var fileInfo helpers.FileInfo + fileInfo.URL = apiURI + fileInfo.Token = token + fileRotateKeyCmd.StringVar(&fileInfo.ID, "file-id", "", "File ID (UUID) to rotate key for") + + if err := fileRotateKeyCmd.Parse(flag.Args()[2:]); err != nil { + return fmt.Errorf("error: failed to parse command line arguments, reason: %v", err) + } + + if fileInfo.ID == "" { + return fmt.Errorf("error: -file-id is required.\n%s", fileRotateKeyUsage) + } + + err := file.RotateKey(fileInfo.URL, fileInfo.Token, fileInfo.ID) + if err != nil { + return fmt.Errorf("error: failed to rotate key for file, reason: %v", err) + } + + return nil +} + func handleDatasetCommand() error { if flag.NArg() < 2 { return fmt.Errorf("error: 'dataset' requires a subcommand (create, release, rotatekey).\n%s", datasetUsage) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 232f296936..0ea360b94f 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1384,7 +1384,7 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { db.Close() } -func (suite *DatabaseTests) TestGetDsatasetFiles() { +func (suite *DatabaseTests) TestGetDatasetFiles() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) testCases := 3 @@ -1435,12 +1435,69 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { assert.NoError(suite.T(), err, "failed to get files for a dataset") assert.Equal(suite.T(), 3, len(files)) - // Extract accession IDs for comparison - var accessions []string - for _, file := range files { - accessions = append(accessions, file.AccessionID) + assert.ElementsMatch(suite.T(), []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}, files) + + db.Close() +} + +func (suite *DatabaseTests) TestGetDatasetFileIDs() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + testCases := 3 + var createdFileIDs []string + + for i := 0; i < testCases; i++ { + filePath := fmt.Sprintf("/%v/TestGetDatasetFileIDs-00%d.c4gh", "User-Q", i) + fileID, err := db.RegisterFile(nil, filePath, "User-Q") + if err != nil { + suite.FailNow("Failed to register file") + } + createdFileIDs = append(createdFileIDs, fileID) + err = db.UpdateFileEventLog(fileID, "uploaded", "User-Q", "{}", "{}") + if err != nil { + suite.FailNow("Failed to update file event log") + } + + checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) + fileInfo := FileInfo{ + fmt.Sprintf("%x", sha256.New().Sum(nil)), + 1234, + filePath, + checksum, + 999, + fmt.Sprintf("%x", sha256.New()), + } + err = db.SetArchived(fileInfo, fileID) + if err != nil { + suite.FailNow("failed to mark file as Archived") + } + + err = db.SetVerified(fileInfo, fileID) + if err != nil { + suite.FailNow("failed to mark file as Verified") + } + + stableID := fmt.Sprintf("accession_ids_%s_0%d", "User-Q", i) + err = db.SetAccessionID(stableID, fileID) + if err != nil { + suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + } + } + + dID := "test-get-dataset-fileids-01" + if err := db.MapFilesToDataset(dID, []string{ + fmt.Sprintf("accession_ids_%s_0%d", "User-Q", 0), + fmt.Sprintf("accession_ids_%s_0%d", "User-Q", 1), + fmt.Sprintf("accession_ids_%s_0%d", "User-Q", 2), + }); err != nil { + suite.FailNow("failed to map files to dataset") } - assert.ElementsMatch(suite.T(), []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}, accessions) + + files, err := db.GetDatasetFileIDs(dID) + assert.NoError(suite.T(), err, "failed to get files for a dataset") + assert.Equal(suite.T(), 3, len(files)) + + assert.ElementsMatch(suite.T(), createdFileIDs, files) db.Close() } From 7949d5732dd4e55ef8932d64bdd051e68b8c1e03 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Fri, 23 Jan 2026 10:02:21 +0100 Subject: [PATCH 017/530] bump sda-admin version --- sda-admin/.version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-admin/.version b/sda-admin/.version index 6e8bf73aa5..0ea3a944b3 100644 --- a/sda-admin/.version +++ b/sda-admin/.version @@ -1 +1 @@ -0.1.0 +0.2.0 From 0a6003333f3597c9cc0cf31b806529d6609b8920 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Fri, 23 Jan 2026 15:50:43 +0100 Subject: [PATCH 018/530] Update sda/cmd/api/api.go Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index e6b56844df..728134c513 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -853,7 +853,7 @@ func rotateKeyDataset(c *gin.Context) { if len(files) == 0 { log.Warnf("no files found for dataset %s", datasetID) - c.Status(http.StatusOK) + c.AbortWithStatusJSON(http.StatusBadRequest, "dataset not found") return } From 0d2e74d30bfd92384df34c33ace6d564f5798551 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Mon, 26 Jan 2026 09:45:03 +0100 Subject: [PATCH 019/530] fix integration test silently failing --- .../tests/sda/65_api_admin_rotate_key_test.sh | 4 +--- sda/cmd/api/api.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh index 6bc041e075..319b8399fd 100755 --- a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -5,7 +5,7 @@ if [ -n "$SYNCTEST" ]; then exit 0 fi -cd shared || true +cd /shared checkErrors() { RETRY_TIMES=0 @@ -191,8 +191,6 @@ fileID2=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files w echo "File IDs: $fileID1, $fileID2" -## test dataset key rotation via sda-admin CLI tool -echo "Testing dataset key rotation via sda-admin CLI tool" resp=$(curl -s -k -o /dev/null -w "%{http_code}" -H "Authorization: Bearer $token" -X POST "http://api:8080/dataset/rotatekey/EGAD100000000001") diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 728134c513..2612f1204a 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -842,6 +842,21 @@ func rotateKeyDataset(c *gin.Context) { return } + // Check if dataset exists + exists, err := Conf.API.DB.CheckIfDatasetExists(datasetID) + if err != nil { + log.Errorf("failed to check if dataset %s exists, reason: %v", datasetID, err) + c.JSON(http.StatusInternalServerError, "failed to check dataset existence") + + return + } + if !exists { + log.Warnf("dataset %s not found", datasetID) + c.JSON(http.StatusNotFound, fmt.Sprintf("dataset %s not found", datasetID)) + + return + } + // Get all files in the dataset files, err := Conf.API.DB.GetDatasetFileIDs(datasetID) if err != nil { From 0d065158a968bfc2a95e60c189d5be0e84ce272c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Feb 2026 14:59:42 +0100 Subject: [PATCH 020/530] fix(storage-v2): fix check if s3 endpoint has http / https prefix, add : to denote full type as http is in https --- sda/internal/storage/v2/s3/reader/config.go | 4 ++-- sda/internal/storage/v2/s3/writer/config.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sda/internal/storage/v2/s3/reader/config.go b/sda/internal/storage/v2/s3/reader/config.go index 337491da32..bf82ecf160 100644 --- a/sda/internal/storage/v2/s3/reader/config.go +++ b/sda/internal/storage/v2/s3/reader/config.go @@ -58,9 +58,9 @@ func loadConfig(backendName string) ([]*endpointConfig, error) { return nil, errors.New("missing required parameter: secret_key") default: switch { - case strings.HasPrefix(e.Endpoint, "http") && !e.DisableHTTPS: + case strings.HasPrefix(e.Endpoint, "http:") && !e.DisableHTTPS: return nil, errors.New("http scheme in endpoint when using HTTPS") - case strings.HasPrefix(e.Endpoint, "https") && e.DisableHTTPS: + case strings.HasPrefix(e.Endpoint, "https:") && e.DisableHTTPS: return nil, errors.New("https scheme in endpoint when HTTPS is disabled") default: } diff --git a/sda/internal/storage/v2/s3/writer/config.go b/sda/internal/storage/v2/s3/writer/config.go index f3736b59d9..b3d96487e4 100644 --- a/sda/internal/storage/v2/s3/writer/config.go +++ b/sda/internal/storage/v2/s3/writer/config.go @@ -72,9 +72,9 @@ func loadConfig(backendName string) ([]*endpointConfig, error) { return nil, errors.New("missing required parameter: bucket_prefix") default: switch { - case strings.HasPrefix(e.Endpoint, "http") && !e.DisableHTTPS: + case strings.HasPrefix(e.Endpoint, "http:") && !e.DisableHTTPS: return nil, errors.New("http scheme in endpoint when using HTTPS") - case strings.HasPrefix(e.Endpoint, "https") && e.DisableHTTPS: + case strings.HasPrefix(e.Endpoint, "https:") && e.DisableHTTPS: return nil, errors.New("https scheme in endpoint when HTTPS is disabled") default: } From 96ca1f6802fb2f63978ec05eefdd7ac99213a558 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Feb 2026 01:13:28 +0000 Subject: [PATCH 021/530] build(deps): bump the all-modules group in /sda with 3 updates Bumps the all-modules group in /sda with 3 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) and [github.com/lib/pq](https://github.com/lib/pq). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.21.0 to 1.21.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.21.0...v1.21.1) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.95.1 to 1.96.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.95.1...service/s3/v1.96.0) Updates `github.com/lib/pq` from 1.10.9 to 1.11.1 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.10.9...v1.11.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.21.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.96.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/lib/pq dependency-version: 1.11.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 6 +++--- sda/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 827b12a934..bbea139051 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,8 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.7 github.com/aws/aws-sdk-go-v2/credentials v1.19.7 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.0 - github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 github.com/aws/smithy-go v1.24.0 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 @@ -19,7 +19,7 @@ require ( github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c github.com/kataras/iris/v12 v12.2.11 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.10.9 + github.com/lib/pq v1.11.1 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 github.com/neicnordic/crypt4gh v1.14.1 diff --git a/sda/go.sum b/sda/go.sum index b548ddbd09..45123af9e0 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.0 h1:pQZGI0qQXeCHZHMeWzhwPu+4jkWrdrIb2dgpG4OKmco= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.0/go.mod h1:XGq5kImVqQT4HUNbbG+0Y8O74URsPNH7CGPg1s1HW5E= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.1 h1:1hWFp+52Vq8Fevy/KUhbW/1MEApMz7uitCF/PQXRJpk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.1/go.mod h1:sIec8j802/rCkCKgZV678HFR0s7lhQUYXT77tIvlaa4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMooz github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1 h1:C2dUPSnEpy4voWFIq3JNd8gN0Y5vYGDo44eUE58a/p8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= @@ -238,8 +238,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI= +github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= From 5095d14af8fa8326be15ac68d3593c71b5e54f97 Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Tue, 3 Feb 2026 19:46:29 +0100 Subject: [PATCH 022/530] (integration test) rename overlapping files names --- .../tests/sda/65_api_admin_rotate_key_test.sh | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh index 319b8399fd..300fa96ada 100755 --- a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -61,7 +61,7 @@ done # generate and upload two files for dataset for i in 1 2; do - file="testfile$i" + file="admin_rotate_testfile$i" if [ ! -f "$file" ]; then dd if=/dev/urandom of="$file" count=5 bs=1M fi @@ -85,7 +85,7 @@ done ## ingest both files for i in 1 2; do - curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "{\"filepath\": \"dataset_admin_rotatekey/testfile$i.c4gh\", \"user\": \"test@dummy.org\"}" http://api:8080/file/ingest + curl -s -k -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "{\"filepath\": \"dataset_admin_rotatekey/admin_rotate_testfile$i.c4gh\", \"user\": \"test@dummy.org\"}" http://api:8080/file/ingest done # wait for files to become verified @@ -106,7 +106,7 @@ for i in 1 2; do jq -c -n \ --arg accession_id "EGAF100000000$i" \ --arg user "test@dummy.org" \ - --arg filepath "dataset_admin_rotatekey/testfile$i.c4gh" \ + --arg filepath "dataset_admin_rotatekey/admin_rotate_testfile$i.c4gh" \ '$ARGS.named' ) resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$payload" "http://api:8080/file/accession")" @@ -186,8 +186,8 @@ echo "Dataset created successfully with 2 files" errorStreamSize=$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/error_stream/ | jq -r '.messages_ready') # get file IDs for verification later -fileID1=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/testfile1.c4gh';") -fileID2=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/testfile2.c4gh';") +fileID1=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/admin_rotate_testfile1.c4gh';") +fileID2=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path='dataset_admin_rotatekey/admin_rotate_testfile2.c4gh';") echo "File IDs: $fileID1, $fileID2" @@ -253,27 +253,27 @@ for i in 1 2; do echo "Testing download and decryption of file $i (ID: $fileID)" # get rotated header - psql -U postgres -h postgres -d sda -At -c "select header from sda.files where id='$fileID';" | xxd -r -p > "testfile${i}_rotated.c4gh" + psql -U postgres -h postgres -d sda -At -c "select header from sda.files where id='$fileID';" | xxd -r -p > "admin_rotate_testfile${i}_rotated.c4gh" # get archive file archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where id='$fileID';") s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive/"$archivePath" --force # concatenate and decrypt - cat "testfile${i}_rotated.c4gh" "$archivePath" > tmp_file && mv tmp_file "testfile${i}_rotated.c4gh" - C4GH_PASSPHRASE=rotatekeyPass ./crypt4gh decrypt -f "testfile${i}_rotated.c4gh" -s rotatekey.sec.pem + cat "admin_rotate_testfile${i}_rotated.c4gh" "$archivePath" > tmp_file && mv tmp_file "admin_rotate_testfile${i}_rotated.c4gh" + C4GH_PASSPHRASE=rotatekeyPass ./crypt4gh decrypt -f "admin_rotate_testfile${i}_rotated.c4gh" -s rotatekey.sec.pem # check that decrypted file matches the original - if [ ! -f "testfile${i}_rotated" ]; then - echo "decrypted file testfile${i}_rotated not found" + if [ ! -f "admin_rotate_testfile${i}_rotated" ]; then + echo "decrypted file admin_rotate_testfile${i}_rotated not found" exit 1 fi - if ! cmp -s "testfile${i}_rotated" "testfile$i" ; then + if ! cmp -s "admin_rotate_testfile${i}_rotated" "admin_rotate_testfile$i" ; then echo "downloaded file $i is different from the original one" exit 1 fi # compare hashes as well - if [ "$(sha256sum testfile$i | cut -d ' ' -f 1)" != "$(sha256sum testfile${i}_rotated | cut -d ' ' -f 1)" ]; then + if [ "$(sha256sum admin_rotate_testfile$i | cut -d ' ' -f 1)" != "$(sha256sum admin_rotate_testfile${i}_rotated | cut -d ' ' -f 1)" ]; then echo "downloaded file $i has different sha256 hash from the original one" exit 1 fi From 20c10c91546391dc14645ae1fa177cf4dc401651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Feb 2026 10:38:49 +0100 Subject: [PATCH 023/530] fix(sda-svc): update tls secret volumemount.defaultMode to 0640 for services connecting postgres db --- charts/sda-svc/templates/api-deploy.yaml | 4 ++-- charts/sda-svc/templates/auth-deploy.yaml | 4 ++-- charts/sda-svc/templates/download-deploy.yaml | 4 ++-- charts/sda-svc/templates/finalize-deploy.yaml | 4 ++-- charts/sda-svc/templates/ingest-deploy.yaml | 4 ++-- charts/sda-svc/templates/intercept-deploy.yaml | 4 ++-- charts/sda-svc/templates/mapper-deploy.yaml | 4 ++-- charts/sda-svc/templates/re-encrypt-deploy.yaml | 4 ++-- charts/sda-svc/templates/s3-inbox-deploy.yaml | 4 ++-- charts/sda-svc/templates/sync-deploy.yaml | 4 ++-- charts/sda-svc/templates/verify-deploy.yaml | 4 ++-- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index 8094522f5a..2336930720 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -134,11 +134,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-api-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for api" .Values.api.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index 53099ba02f..dd6ed085ba 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -129,11 +129,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-auth-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for auth" .Values.auth.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index 62d9d25d70..d49bbce708 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -109,11 +109,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-download-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for download" .Values.download.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml index 35705dc2d1..db6aa0c4ae 100644 --- a/charts/sda-svc/templates/finalize-deploy.yaml +++ b/charts/sda-svc/templates/finalize-deploy.yaml @@ -104,11 +104,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-finalize-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for finalize" .Values.finalize.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index 53ef29d35e..644cec717b 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -111,11 +111,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-ingest-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for ingest" .Values.ingest.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/intercept-deploy.yaml b/charts/sda-svc/templates/intercept-deploy.yaml index a9dd3c6eec..a0f12dde1f 100644 --- a/charts/sda-svc/templates/intercept-deploy.yaml +++ b/charts/sda-svc/templates/intercept-deploy.yaml @@ -81,11 +81,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-intercept-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for intercept" .Values.intercept.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml index 3cc270d136..9a9fa5427f 100644 --- a/charts/sda-svc/templates/mapper-deploy.yaml +++ b/charts/sda-svc/templates/mapper-deploy.yaml @@ -83,11 +83,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-mapper-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for mapper" .Values.mapper.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/re-encrypt-deploy.yaml b/charts/sda-svc/templates/re-encrypt-deploy.yaml index f7b5b9c8e6..d50febb5e5 100644 --- a/charts/sda-svc/templates/re-encrypt-deploy.yaml +++ b/charts/sda-svc/templates/re-encrypt-deploy.yaml @@ -89,11 +89,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-reencrypt-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for reencrypt" .Values.reencrypt.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index 86174eb85b..0ff46d1813 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -124,11 +124,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-inbox-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for s3inbox" .Values.s3Ibox.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/sync-deploy.yaml b/charts/sda-svc/templates/sync-deploy.yaml index c46c1d610e..16dd7e4c14 100644 --- a/charts/sda-svc/templates/sync-deploy.yaml +++ b/charts/sda-svc/templates/sync-deploy.yaml @@ -98,11 +98,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-sync-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for sync" .Values.sync.tls.secretName }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index 0d10eaf489..3cdc5617c2 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -80,11 +80,11 @@ spec: - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ template "sda.fullname" . }}-verify-certs {{- else }} secret: - defaultMode: 0440 + defaultMode: 0640 secretName: {{ required "An certificate issuer or a TLS secret name is required for verify" .Values.verify.tls.secretName }} {{- end }} {{- end }} From bd1f324878fbdd0b88490902906e734c49083247 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Feb 2026 10:39:58 +0100 Subject: [PATCH 024/530] feat(makefile): add k3d isolated s3 tls make commands --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index daaee27f9d..b9754c7c8e 100644 --- a/Makefile +++ b/Makefile @@ -208,18 +208,26 @@ k3d-deploy-dependencies-federated: @bash .github/integration/scripts/charts/dependencies.sh local federated k3d-deploy-dependencies-isolated: @bash .github/integration/scripts/charts/dependencies.sh local isolated +k3d-deploy-dependencies-s3: + @bash .github/integration/scripts/charts/dependencies.sh local s3 k3d-import-images: build-all @bash .github/integration/scripts/charts/import_local_images.sh k3s-default k3d-deploy-postgres: @bash .github/integration/scripts/charts/deploy_charts.sh sda-db "$$(date +%F)" false +k3d-deploy-postgres-tls: + @bash .github/integration/scripts/charts/deploy_charts.sh sda-db "$$(date +%F)" true k3d-deploy-rabbitmq-federated: @bash .github/integration/scripts/charts/deploy_charts.sh sda-mq "$$(date +%F)" false federated k3d-deploy-rabbitmq-isolated: @bash .github/integration/scripts/charts/deploy_charts.sh sda-mq "$$(date +%F)" false isolated +k3d-deploy-rabbitmq-isolated-tls: + @bash .github/integration/scripts/charts/deploy_charts.sh sda-mq "$$(date +%F)" true isolated k3d-deploy-sda-s3-federated: @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false s3 federated k3d-deploy-sda-s3-isolated: @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false s3 isolated +k3d-deploy-sda-s3-isolated-tls: + @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" true s3 isolated k3d-deploy-sda-posix: @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false posix k3d-cleanup-all-deployments: From 6dd0b27676e7f40b466376a14710021c118fea49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Feb 2026 10:58:37 +0100 Subject: [PATCH 025/530] fix(github actions): run Build PR container if changes in charts/ dir --- .github/workflows/build_pr_container.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 67a0d2a284..db2faca01b 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -8,7 +8,6 @@ on: - ".gitignore" - "**/*.md" - ".github/dependabot.yaml" - - "charts/**" - "Makefile" - "sda-admin/**" From c20948dde51bf336b7bf58deec9165a1bbf75918 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Feb 2026 13:17:01 +0100 Subject: [PATCH 026/530] fix(sda-svc): bump sda-svc chart version 3.1.2 -> 3.1.3 --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index e4889bca62..36bf2aa1a6 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.1.2 +version: 3.1.3 appVersion: v3.1.13 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From 9dd0ecf8e99fbd016cc92fea98f5652282a122d8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 22:15:31 +0000 Subject: [PATCH 027/530] build(deps): bump the all-modules group across 1 directory with 2 updates Bumps the all-modules group with 2 updates in the /sda-download directory: [github.com/lib/pq](https://github.com/lib/pq) and [golang.org/x/crypto](https://github.com/golang/crypto). Updates `github.com/lib/pq` from 1.10.9 to 1.11.1 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.10.9...v1.11.1) Updates `golang.org/x/crypto` from 0.47.0 to 0.48.0 - [Commits](https://github.com/golang/crypto/compare/v0.47.0...v0.48.0) --- updated-dependencies: - dependency-name: github.com/lib/pq dependency-version: 1.11.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.48.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 12 ++++++------ sda-download/go.sum | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index eb7bdb3e2a..f6a43da34b 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -11,12 +11,12 @@ require ( github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.10.9 + github.com/lib/pq v1.11.1 github.com/neicnordic/crypt4gh v1.14.1 github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.47.0 + golang.org/x/crypto v0.48.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 @@ -71,10 +71,10 @@ require ( github.com/ugorji/go/codec v1.3.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.20.0 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/text v0.33.0 // indirect - golang.org/x/tools v0.40.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + golang.org/x/tools v0.41.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index c6003300a9..901b205bc4 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -95,8 +95,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI= +github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -180,8 +180,8 @@ golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -193,8 +193,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -208,23 +208,23 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -233,8 +233,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA= -golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc= +golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= +golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= From d42c60c36428375cfd208cd8ed80c3a3410390eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Feb 2026 22:18:00 +0000 Subject: [PATCH 028/530] build(deps): bump the all-modules group in /sda with 3 updates Bumps the all-modules group in /sda with 3 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2), [golang.org/x/crypto](https://github.com/golang/crypto) and [golang.org/x/oauth2](https://github.com/golang/oauth2). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.21.1 to 1.22.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.21.1...v1.22.0) Updates `golang.org/x/crypto` from 0.47.0 to 0.48.0 - [Commits](https://github.com/golang/crypto/compare/v0.47.0...v0.48.0) Updates `golang.org/x/oauth2` from 0.34.0 to 0.35.0 - [Commits](https://github.com/golang/oauth2/compare/v0.34.0...v0.35.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.22.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.48.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-version: 0.35.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 12 ++++++------ sda/go.sum | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index bbea139051..e7d25413c2 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,7 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.7 github.com/aws/aws-sdk-go-v2/credentials v1.19.7 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.1 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 github.com/aws/smithy-go v1.24.0 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 @@ -33,8 +33,8 @@ require ( github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.47.0 - golang.org/x/oauth2 v0.34.0 + golang.org/x/crypto v0.48.0 + golang.org/x/oauth2 v0.35.0 google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 ) @@ -159,9 +159,9 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.20.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect - golang.org/x/net v0.48.0 // indirect - golang.org/x/sys v0.40.0 // indirect - golang.org/x/text v0.33.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.12.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 45123af9e0..5783c19602 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.1 h1:1hWFp+52Vq8Fevy/KUhbW/1MEApMz7uitCF/PQXRJpk= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.21.1/go.mod h1:sIec8j802/rCkCKgZV678HFR0s7lhQUYXT77tIvlaa4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0 h1:MpkX8EjkwuvyuX9B7+Zgk5M4URb2WQ84Y6jM81n5imw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0/go.mod h1:4V9Pv5sFfMPWQF0Q0zYN6BlV/504dFGaTeogallRqQw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= @@ -414,8 +414,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= -golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -435,10 +435,10 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU= -golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY= -golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= -golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -461,15 +461,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= -golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY= -golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -477,8 +477,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= -golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From be13eeef086d89e0ac5f2d61e1fbca99bc3e2db9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 10 Feb 2026 16:17:01 +0100 Subject: [PATCH 029/530] feat(storage): update to use aws transfermanager deprecated in aws-sdk-go-v2/feature/s3/manager 1.22.0 --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- sda/internal/storage/storage.go | 11 +++++------ sda/internal/storage/v2/s3/writer/write_file.go | 10 ++++------ 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index e7d25413c2..ef3a5f57c7 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,7 +6,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/aws/aws-sdk-go-v2/config v1.32.7 github.com/aws/aws-sdk-go-v2/credentials v1.19.7 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 github.com/aws/smithy-go v1.24.0 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 diff --git a/sda/go.sum b/sda/go.sum index 5783c19602..d9be94bd42 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUT github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0 h1:MpkX8EjkwuvyuX9B7+Zgk5M4URb2WQ84Y6jM81n5imw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.0/go.mod h1:4V9Pv5sFfMPWQF0Q0zYN6BlV/504dFGaTeogallRqQw= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2 h1:1q8/WwEqZnM/vO4q1gx2g7lHYmyN+o4P7G6EW4zKbRQ= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2/go.mod h1:owKRexW+Ir5ACD2UTesmjkQ+w7mcmknLNfwOiKfVLTg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index 3313abfb1d..41f5aa8d20 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -19,7 +19,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" @@ -147,7 +147,7 @@ func (pb *posixBackend) RemoveFile(filePath string) error { type s3Backend struct { Client *s3.Client - Uploader *manager.Uploader + Uploader *transfermanager.Client Bucket string Conf *S3Conf } @@ -177,10 +177,9 @@ func newS3Backend(conf S3Conf) (*s3Backend, error) { Bucket: conf.Bucket, Client: s3Client, Conf: &conf, - Uploader: manager.NewUploader(s3Client, func(u *manager.Uploader) { - u.PartSize = int64(conf.Chunksize) + Uploader: transfermanager.New(s3Client, func(u *transfermanager.Options) { + u.PartSizeBytes = int64(conf.Chunksize) u.Concurrency = conf.UploadConcurrency - u.LeavePartsOnError = false }), } @@ -275,7 +274,7 @@ func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { reader, writer := io.Pipe() go func() { - _, err := sb.Uploader.Upload(context.TODO(), &s3.PutObjectInput{ + _, err := sb.Uploader.UploadObject(context.TODO(), &transfermanager.UploadObjectInput{ Body: reader, Bucket: &sb.Bucket, Key: &filePath, diff --git a/sda/internal/storage/v2/s3/writer/write_file.go b/sda/internal/storage/v2/s3/writer/write_file.go index e23421d534..c0c94fa1cf 100644 --- a/sda/internal/storage/v2/s3/writer/write_file.go +++ b/sda/internal/storage/v2/s3/writer/write_file.go @@ -7,8 +7,7 @@ import ( "io" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/feature/s3/manager" - "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" ) @@ -50,14 +49,13 @@ func (writer *Writer) WriteFile(ctx context.Context, filePath string, fileConten return "", err } - uploader := manager.NewUploader(client, func(u *manager.Uploader) { + uploader := transfermanager.New(client, func(u *transfermanager.Options) { // Type conversation safe as chunkSizeBytes checked to be between 5mb and 1gb (in bytes) //nolint:gosec // disable G115 - u.PartSize = int64(writer.activeEndpoint.chunkSizeBytes) - u.LeavePartsOnError = false + u.PartSizeBytes = int64(writer.activeEndpoint.chunkSizeBytes) }) - _, err = uploader.Upload(ctx, &s3.PutObjectInput{ + _, err = uploader.UploadObject(ctx, &transfermanager.UploadObjectInput{ Body: fileContent, Bucket: aws.String(activeBucket), Key: aws.String(filePath), From 56d854ae352a24695f5f25fa7208568d453b596f Mon Sep 17 00:00:00 2001 From: Panos Chatzopoulos Date: Wed, 11 Feb 2026 11:04:05 +0100 Subject: [PATCH 030/530] Improve query for getting file ids --- sda/internal/database/db_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 26575977dd..114ee1c5d2 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1430,7 +1430,7 @@ func (dbs *SDAdb) GetDatasetFileIDs(dataset string) ([]string, error) { db := dbs.DB var fileIDs []string - rows, err := db.Query("SELECT file_id FROM sda.file_dataset WHERE dataset_id = (SELECT id FROM sda.datasets WHERE stable_id = $1);", dataset) + rows, err := db.Query("SELECT fd.file_id FROM sda.datasets AS d INNER JOIN sda.file_dataset AS fd ON d.id = fd.dataset_id WHERE d.stable_id = $1;", dataset) if err != nil { return nil, err } From c8ee0994d7ac0e5d143ad4197b3a1103db8d79d9 Mon Sep 17 00:00:00 2001 From: kostas-kou Date: Wed, 11 Feb 2026 14:30:49 +0100 Subject: [PATCH 031/530] Update crypt4gh fetch depending on the OS --- .../integration/scripts/make_sda_credentials.sh | 16 +++++++++++++++- sda-validator/orchestrator/Dockerfile | 4 ++-- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 465fd827ad..397afb3634 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -97,8 +97,22 @@ EOD ## create crypt4gh key if [ ! -f "/shared/crypt4gh" ]; then echo "downloading crypt4gh" + ARCH=$(uname -m) + case "$ARCH" in + x86_64) + CRYPT4GH_ARCH="linux_x86_64" + ;; + aarch64) + CRYPT4GH_ARCH="linux_arm64" + ;; + *) + echo "Unknown architecture: $ARCH. Defaulting to linux_x86_64." + CRYPT4GH_ARCH="linux_x86_64" + ;; + esac + echo "Detected architecture: $ARCH, downloading crypt4gh for: $CRYPT4GH_ARCH" latest_c4gh=$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name') - curl --retry 100 -s -L "https://github.com/neicnordic/crypt4gh/releases/download/$latest_c4gh/crypt4gh_linux_x86_64.tar.gz" | tar -xz -C /shared/ && chmod +x /shared/crypt4gh + curl --retry 100 -s -L "https://github.com/neicnordic/crypt4gh/releases/download/$latest_c4gh/crypt4gh_${CRYPT4GH_ARCH}.tar.gz" | tar -xz -C /shared/ && chmod +x /shared/crypt4gh fi if [ ! -f "/shared/c4gh.sec.pem" ]; then diff --git a/sda-validator/orchestrator/Dockerfile b/sda-validator/orchestrator/Dockerfile index 090f774639..68383cd87d 100644 --- a/sda-validator/orchestrator/Dockerfile +++ b/sda-validator/orchestrator/Dockerfile @@ -1,4 +1,4 @@ -FROM debian:trixie-slim AS apptainer_builder +FROM --platform=linux/amd64 debian:trixie-slim AS apptainer_builder RUN apt update \ && apt install -y wget \ @@ -16,7 +16,7 @@ COPY . . RUN go build -buildvcs=false -o ./sda-validator-orchestrator -FROM gcr.io/distroless/static-debian12 +FROM --platform=linux/amd64 gcr.io/distroless/static-debian12 ARG BUILD_DATE ARG SOURCE_COMMIT From ea6f47df6987eea2c7adb5f2f162cce5c6e97724 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 14:16:01 +0100 Subject: [PATCH 032/530] feat(download): add support for Htsget-Context-Public-Key header with takes precedence over Client-Public-Key header --- sda-download/api/sda/sda.go | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index 25e5807b81..4d08a1d001 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -216,7 +216,8 @@ func Download(c *gin.Context) { return } - if c.Param("type") != "encrypted" && c.GetHeader("Client-Public-Key") != "" { + requestPublicKey := getPublicKeyFromRequest(c) + if c.Param("type") != "encrypted" && requestPublicKey != "" { c.String(http.StatusBadRequest, "downloading encrypted data is not supported") return @@ -339,20 +340,15 @@ func Download(c *gin.Context) { c.Header("Content-Disposition", fmt.Sprintf("filename: %v", fileID)) c.Header("ETag", fileDetails.DecryptedChecksum) c.Header("Last-Modified", lastModified.Format(http.TimeFormat)) - - // set the user and server public keys that is send from htsget - log.Debugf("Got to setting the headers: %s", c.GetHeader("client-public-key")) - c.Header("Client-Public-Key", c.GetHeader("Client-Public-Key")) } if c.Request.Method == http.MethodHead { // Create headers for htsget, containing size of the crypt4gh header - reencKey := c.GetHeader("Client-Public-Key") headerSize := bytes.NewReader(fileDetails.Header).Size() // Size of the header in the archive c.Header("Server-Additional-Bytes", fmt.Sprint(headerSize)) - if reencKey != "" { - newHeader, _ := reencryptHeader(fileDetails.Header, reencKey) + if requestPublicKey != "" { + newHeader, _ := reencryptHeader(fileDetails.Header, requestPublicKey) headerSize = bytes.NewReader(newHeader).Size() // Size of the header if the file is re-encrypted before downloading c.Header("Client-Additional-Bytes", fmt.Sprint(headerSize)) @@ -372,15 +368,14 @@ func Download(c *gin.Context) { switch c.Param("type") { case "encrypted": // The key provided in the header should be base64 encoded - reencKey := c.GetHeader("Client-Public-Key") - if reencKey == "" { + if requestPublicKey == "" { c.String(http.StatusBadRequest, "c4gh public key is missing from the header") return } - log.Debugf("Public key from the request header = %v", reencKey) - newHeader, err := reencryptHeader(fileDetails.Header, reencKey) + log.Debugf("Public key from the request header = %v", requestPublicKey) + newHeader, err := reencryptHeader(fileDetails.Header, requestPublicKey) if err != nil { log.Errorf("Failed to reencrypt the file header, reason: %v", err) c.String(http.StatusInternalServerError, "file re-encryption error") @@ -567,3 +562,21 @@ var calculateCoords = func(start, end int64, htsget_range string, fileDetails *d return start, headlength.Size() + bodyEnd, nil } + +func getPublicKeyFromRequest(c *gin.Context) string { + htsgetRsPublicKeyHeader := c.GetHeader("Htsget-Context-Public-Key") + clientPublicKeyHeader := c.GetHeader("Client-Public-Key") + + switch { + case htsgetRsPublicKeyHeader != "": + if clientPublicKeyHeader != "" && htsgetRsPublicKeyHeader != clientPublicKeyHeader { + log.Warnf("both Htsget-Context-Public-Key, and Client-Public-Key are set and differnt, defaulting to Htsget-Context-Public-Key") + } + + return htsgetRsPublicKeyHeader + case clientPublicKeyHeader != "": + return clientPublicKeyHeader + default: + return "" + } +} From d3b307b367f5b6fc90fa58400d7191392a89abb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 09:02:05 +0100 Subject: [PATCH 033/530] feat(download): update getPublicKeyFromRequest to return error if both headers are set or value in headers can not be base64 decoded --- sda-download/api/sda/sda.go | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index 4d08a1d001..e4f558f988 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/base64" "errors" "fmt" "io" @@ -216,7 +217,13 @@ func Download(c *gin.Context) { return } - requestPublicKey := getPublicKeyFromRequest(c) + requestPublicKey, err := getPublicKeyFromRequest(c) + if err != nil { + c.String(http.StatusBadRequest, err.Error()) + + return + } + if c.Param("type") != "encrypted" && requestPublicKey != "" { c.String(http.StatusBadRequest, "downloading encrypted data is not supported") @@ -563,20 +570,26 @@ var calculateCoords = func(start, end int64, htsget_range string, fileDetails *d return start, headlength.Size() + bodyEnd, nil } -func getPublicKeyFromRequest(c *gin.Context) string { +func getPublicKeyFromRequest(c *gin.Context) (string, error) { htsgetRsPublicKeyHeader := c.GetHeader("Htsget-Context-Public-Key") clientPublicKeyHeader := c.GetHeader("Client-Public-Key") switch { + case htsgetRsPublicKeyHeader != "" && clientPublicKeyHeader != "": + return "", errors.New("both Htsget-Context-Public-Key, and Client-Public-Key headers are set") case htsgetRsPublicKeyHeader != "": - if clientPublicKeyHeader != "" && htsgetRsPublicKeyHeader != clientPublicKeyHeader { - log.Warnf("both Htsget-Context-Public-Key, and Client-Public-Key are set and differnt, defaulting to Htsget-Context-Public-Key") + if _, err := base64.StdEncoding.DecodeString(htsgetRsPublicKeyHeader); err != nil { + return "", fmt.Errorf("invalid base64 encoding for Htsget-Context-Public-Key header") } - return htsgetRsPublicKeyHeader + return htsgetRsPublicKeyHeader, nil case clientPublicKeyHeader != "": - return clientPublicKeyHeader + if _, err := base64.StdEncoding.DecodeString(clientPublicKeyHeader); err != nil { + return "", fmt.Errorf("invalid base64 encoding for Client-Public-Key header") + } + + return clientPublicKeyHeader, nil default: - return "" + return "", nil } } From 5cbb780cab22437cf2042094776a72787465292b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 14:55:26 +0100 Subject: [PATCH 034/530] feat(download): fix TestDownload_Fail_UnencryptedDownloadNotAllowed unit test to check new error, fix errors without format --- sda-download/api/sda/sda.go | 4 ++-- sda-download/api/sda/sda_test.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index e4f558f988..94b4a33116 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -579,13 +579,13 @@ func getPublicKeyFromRequest(c *gin.Context) (string, error) { return "", errors.New("both Htsget-Context-Public-Key, and Client-Public-Key headers are set") case htsgetRsPublicKeyHeader != "": if _, err := base64.StdEncoding.DecodeString(htsgetRsPublicKeyHeader); err != nil { - return "", fmt.Errorf("invalid base64 encoding for Htsget-Context-Public-Key header") + return "", errors.New("invalid base64 encoding for Htsget-Context-Public-Key header") } return htsgetRsPublicKeyHeader, nil case clientPublicKeyHeader != "": if _, err := base64.StdEncoding.DecodeString(clientPublicKeyHeader); err != nil { - return "", fmt.Errorf("invalid base64 encoding for Client-Public-Key header") + return "", errors.New("invalid base64 encoding for Client-Public-Key header") } return clientPublicKeyHeader, nil diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go index 1262a066d6..ede5d403c1 100644 --- a/sda-download/api/sda/sda_test.go +++ b/sda-download/api/sda/sda_test.go @@ -364,7 +364,7 @@ func TestDownload_Fail_UnencryptedDownloadNotAllowed(t *testing.T) { defer response.Body.Close() body, _ = io.ReadAll(response.Body) expectedStatusCode = 400 - expectedBody = []byte("downloading encrypted data is not supported") + expectedBody = []byte("invalid base64 encoding for Client-Public-Key header") assert.Equal(t, expectedStatusCode, response.StatusCode, "Unexpected status code from download") assert.Equal(t, expectedBody, body, "Unexpected body from download") From aa15ecdab23f572150776b6b8c7df14a39039928 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Fri, 13 Feb 2026 13:40:08 +0000 Subject: [PATCH 035/530] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 13cb747983..6ddec19db0 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.26 -appVersion: v3.1.13 +version: 2.0.27 +appVersion: v3.1.25 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index cb638c9a02..f07dd63213 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.1.1 -appVersion: v3.1.13 +version: 2.1.2 +appVersion: v3.1.25 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 36bf2aa1a6..7f04b6dfea 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.1.3 -appVersion: v3.1.13 +version: 3.1.4 +appVersion: v3.1.25 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 1ea049144b79420e18a37cbbc69c3e2b191e8149 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 19:31:32 +0000 Subject: [PATCH 036/530] build(deps): bump aquasecurity/trivy-action from 0.33.1 to 0.34.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.33.1 to 0.34.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.33.1...0.34.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.34.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- .github/workflows/scan-images.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index db2faca01b..31c0d5c559 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -117,7 +117,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.34.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -135,7 +135,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.34.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -184,7 +184,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.34.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -215,7 +215,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.34.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index f764e13f20..4334f51fd4 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -27,7 +27,7 @@ jobs: image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.34.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -53,7 +53,7 @@ jobs: security-events: write steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@0.34.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From b203743cdc10cc9e084ebc9eb42fb1c5bb7a123f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 19:31:44 +0000 Subject: [PATCH 037/530] build(deps): bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/lib/pq](https://github.com/lib/pq) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/lib/pq` from 1.11.1 to 1.11.2 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.11.1...v1.11.2) Updates `google.golang.org/grpc` from 1.78.0 to 1.79.1 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.78.0...v1.79.1) --- updated-dependencies: - dependency-name: github.com/lib/pq dependency-version: 1.11.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.79.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 6 +++--- sda-download/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index f6a43da34b..2af6780cce 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -11,14 +11,14 @@ require ( github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.11.1 + github.com/lib/pq v1.11.2 github.com/neicnordic/crypt4gh v1.14.1 github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.48.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 - google.golang.org/grpc v1.78.0 + google.golang.org/grpc v1.79.1 google.golang.org/protobuf v1.36.11 ) @@ -75,7 +75,7 @@ require ( golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/tools v0.41.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 901b205bc4..8878e0753e 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -95,8 +95,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI= -github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= +github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -162,16 +162,16 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -238,10 +238,10 @@ golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= +google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 0e39f13a040c6d1212adda785b648a1c8c3bf03f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Feb 2026 19:32:41 +0000 Subject: [PATCH 038/530] build(deps): bump the all-modules group in /sda with 2 updates Bumps the all-modules group in /sda with 2 updates: [github.com/lib/pq](https://github.com/lib/pq) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/lib/pq` from 1.11.1 to 1.11.2 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.11.1...v1.11.2) Updates `google.golang.org/grpc` from 1.78.0 to 1.79.1 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.78.0...v1.79.1) --- updated-dependencies: - dependency-name: github.com/lib/pq dependency-version: 1.11.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.79.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 6 +++--- sda/go.sum | 35 +++++++++++++++++++---------------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index ef3a5f57c7..de695512b3 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -19,7 +19,7 @@ require ( github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c github.com/kataras/iris/v12 v12.2.11 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.11.1 + github.com/lib/pq v1.11.2 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 github.com/neicnordic/crypt4gh v1.14.1 @@ -35,7 +35,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.48.0 golang.org/x/oauth2 v0.35.0 - google.golang.org/grpc v1.78.0 + google.golang.org/grpc v1.79.1 google.golang.org/protobuf v1.36.11 ) @@ -163,7 +163,7 @@ require ( golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect golang.org/x/time v0.12.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index d9be94bd42..ce505d03a9 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -84,6 +84,9 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= @@ -238,8 +241,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.11.1 h1:wuChtj2hfsGmmx3nf1m7xC2XpK6OtelS2shMY+bGMtI= -github.com/lib/pq v1.11.1/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= +github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= @@ -390,16 +393,16 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= -go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= -go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= -go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= -go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= -go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= -go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= -go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= -go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= -go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= @@ -496,10 +499,10 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= -google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= -google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= +google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From d1d0f223270d7acecea1113770cea81e10a16def Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 9 Feb 2026 10:57:50 +0100 Subject: [PATCH 039/530] feat(api): add Download API v1.1 specification - Add /service-info endpoint (GA4GH service-info compatible) - Add checksums and decryptedSize to FileInfo schema - Clarify Range header semantics for re-encrypted Crypt4GH streams - Scope ETag to (fileId, recipientPublicKey) pair - Make X-C4GH-Public-Key required, document both raw and PEM formats - Replace /files:resolve with filePath query parameter - Fix Last-Modified header format to HTTP-date - Update DRS readiness notes for DRS 1.5+ --- sda/cmd/download/swagger_v1.yml | 940 ++++++++++++++++++++++++++++---- 1 file changed, 836 insertions(+), 104 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index aa4d791d49..ca61e19ebc 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -1,226 +1,571 @@ openapi: 3.0.3 info: - title: SDA file download API - version: "1.0" - description: This is the external data out API for the Sensitive Data Archive. Data is delivered encrypted with a user supplied crypth4gh public key. + title: SDA Data Out API + version: "1.1.0" + description: | + External download API for the Sensitive Data Archive (SDA). + + Files are delivered as Crypt4GH. The server re-encrypts the Crypt4GH header to a + recipient-supplied public key. + + Design goals: + - Easy for data recipients: discover datasets, discover files, download. + - Partial downloads via standard HTTP Range requests (byte ranges). + - Consistent, machine-readable error payloads (Problem Details). + - Simple key handling: use X-C4GH-Public-Key (HTSGET compatibility via Htsget-Context-Public-Key). + - GA4GH service-info endpoint for service discovery. + + Crypt4GH and Range semantics: + The byte stream delivered to the client is the Crypt4GH re-encrypted representation: + [re-encrypted header | encrypted data segments]. All byte offsets in Range requests, + Content-Length, and Content-Range refer to this re-encrypted stream, NOT the original + plaintext file. The re-encrypted header size varies per recipient public key. + + DRS readiness (non-normative): + Designed for compatibility with GA4GH DRS 1.5+ + (see https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.5.0/docs/). + - A future DRS facade can map fileId to DRS object_id and expose access URLs + that point to GET /files/{fileId}, keeping download semantics unchanged. + - The FileInfo schema includes checksums and is extensible to accommodate + DRS 1.5+ fields (e.g. cold storage indication, cloud location) without breaking changes. +servers: + - url: https://{host} + variables: + host: + default: api.example.org +tags: + - name: Service + description: Service discovery (GA4GH service-info). + - name: Datasets + description: Dataset discovery & metadata. + - name: Files + description: File metadata and download operations. + - name: Health + description: Service health endpoints. + paths: - /file: + /service-info: + get: + tags: [Service] + operationId: getServiceInfo + summary: GA4GH service-info + description: | + Returns information about this service following the GA4GH service-info specification. + This endpoint does not require authentication. + responses: + "200": + description: Service information + content: + application/json: + schema: + $ref: "#/components/schemas/ServiceInfo" + example: + id: org.neic.sda.download + name: SDA Data Out API + type: + group: org.neic + artifact: sda-download + version: 1.1.0 + description: Sensitive Data Archive file download service + organization: + name: NeIC + url: https://neic.no + version: 1.1.0 + environment: production + + /datasets: + get: + tags: [Datasets] + operationId: listDatasets + summary: List datasets the caller can access + description: Returns a list of dataset identifiers the caller has access to. + responses: + "200": + description: Successful operation (an empty list is valid). + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatasetId" + example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] + "401": + $ref: "#/components/responses/Unauthorized" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + /datasets/{datasetId}: get: - description: Returns the requested file to the user. Uses query parameters in order to handle files requested by the `filePath` entity. + tags: [Datasets] + operationId: getDataset + summary: Get dataset metadata parameters: - - in: header - description: Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key - name: public_key - required: true - schema: - type: string - - in: query - name: dataset - schema: - type: string - required: true - description: Dataset identifier - - in: query - name: fileId - schema: - type: string + - $ref: "#/components/parameters/DatasetIdPath" + responses: + "200": + description: Successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/DatasetInfo" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + /datasets/{datasetId}/files: + get: + tags: [Datasets] + operationId: listDatasetFiles + summary: List files in a dataset + description: | + Returns metadata for all files in the dataset. When the optional filePath query + parameter is provided, the response is filtered to files matching that path. + parameters: + - $ref: "#/components/parameters/DatasetIdPath" + - name: filePath + in: query required: false - description: Stable ID of the file to download. It is an error to supply both filePath and fileId - - in: query - name: filePath schema: type: string - required: false - description: File path of the file to download. It is an error to supply both filePath and fileId - - in: query - name: end - description: End of requested data in the form of the 0-index of the first octet not included in the response. That is, data delivered will have octet indices in the closed-open set [start,end). Defaults to end of file if not set - required: false - schema: - minimum: 0 - type: integer - - in: query - name: start - description: Start of the requested data. Defaults to start of file if not set - required: false - schema: - default: 0 - minimum: 0 - type: integer + description: | + Optional dataset-relative file path filter. When provided, only files + whose path matches are returned. Useful for clients that identify files + by submitted path rather than by fileId. + example: samples/controls/sample1.cram.c4gh responses: "200": + description: Successful operation + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FileInfo" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + /files/{fileId}: + head: + tags: [Files] + operationId: headFile + summary: Get file download metadata (no body) + description: | + Returns download-related metadata for the Crypt4GH re-encrypted representation + of the requested file, including Content-Length and ETag. + + The public key is required because Content-Length includes the recipient-specific + Crypt4GH header, which varies in size per recipient. The ETag also represents + the specific representation for the given (fileId, recipient public key) pair; + different keys produce different ETags. + + This endpoint is essential for implementing resumable downloads (HEAD → Range + If-Range). + parameters: + - $ref: "#/components/parameters/FileIdPath" + - $ref: "#/components/parameters/C4ghPublicKey" + - $ref: "#/components/parameters/HtsgetContextPublicKey" + responses: + "200": + description: Metadata returned successfully + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + ETag: + $ref: "#/components/headers/ETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + Content-Disposition: + $ref: "#/components/headers/ContentDisposition" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + get: + tags: [Files] + operationId: downloadFile + summary: Download a file (supports Range for partial downloads) + description: | + Downloads the requested file, re-encrypted as Crypt4GH for the recipient. + + The delivered byte stream is: [re-encrypted Crypt4GH header | encrypted data segments]. + All byte offsets in Range, Content-Length, and Content-Range refer to this re-encrypted + stream, NOT the original plaintext file. + + Partial downloads use the standard HTTP Range header. For simplicity and broad client + compatibility, this API supports a single byte range per request. + + The ETag represents the specific Crypt4GH representation for the given file and + recipient public key. Different keys yield different ETags. Use If-Range with the + ETag to safely resume interrupted downloads. + parameters: + - $ref: "#/components/parameters/FileIdPath" + - $ref: "#/components/parameters/C4ghPublicKey" + - $ref: "#/components/parameters/HtsgetContextPublicKey" + - $ref: "#/components/parameters/Range" + - $ref: "#/components/parameters/IfRange" + responses: + "200": + description: Successful operation (full file) + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + ETag: + $ref: "#/components/headers/ETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + Content-Disposition: + $ref: "#/components/headers/ContentDisposition" content: application/octet-stream: schema: type: string format: binary - description: Successful operation "206": + description: Successful operation (partial content) + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + Content-Range: + $ref: "#/components/headers/ContentRange" + ETag: + $ref: "#/components/headers/ETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + Content-Disposition: + $ref: "#/components/headers/ContentDisposition" content: application/octet-stream: schema: type: string format: binary - description: Successful partial file delivery "400": - description: Bad request ex. start larger than end + $ref: "#/components/responses/BadRequest" "401": - description: Authentication failure + $ref: "#/components/responses/Unauthorized" "403": - description: The user does not have access to the dataset + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" "416": - description: Unsatisfiable range ex. start larger than the size of the file + $ref: "#/components/responses/RangeNotSatisfiable" "500": - description: Internal application error + $ref: "#/components/responses/InternalServerError" security: - bearerAuth: [] - /file/{fileId}: + x-codeSamples: + - lang: curl + label: Full download + source: | + curl -L \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-C4GH-Public-Key: $C4GH_PK_B64" \ + -o file.c4gh \ + "https://api.example.org/files/aa-file-123456-asdfgh" + - lang: curl + label: Partial download (first 1 MiB) + source: | + curl -L \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-C4GH-Public-Key: $C4GH_PK_B64" \ + -H "Range: bytes=0-1048575" \ + -o file.part \ + "https://api.example.org/files/aa-file-123456-asdfgh" + - lang: curl + label: Resume download safely using ETag + If-Range + source: | + # 1) Get validators + ETAG=$(curl -sI \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-C4GH-Public-Key: $C4GH_PK_B64" \ + "https://api.example.org/files/aa-file-123456-asdfgh" | awk '/^ETag:/ {print $2}' | tr -d '\r') + + # 2) Resume (example range) + curl -L \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-C4GH-Public-Key: $C4GH_PK_B64" \ + -H "Range: bytes=1048576-" \ + -H "If-Range: $ETAG" \ + -o file.rest \ + "https://api.example.org/files/aa-file-123456-asdfgh" + + # --- Legacy endpoints kept for backward compatibility (deprecated) --- + + /file: get: - description: Returns the requested file to the user. + tags: [Files] + operationId: legacyDownloadByQuery + deprecated: true + summary: "[Deprecated] Download by query parameters" + description: | + Deprecated. Prefer GET /files/{fileId}. + + This endpoint is kept for backward compatibility with older clients. It allows downloading + by dataset+fileId or dataset+filePath. Range requests should be performed using the + standard Range header on GET /files/{fileId}. parameters: - - in: header - description: Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key - name: public_key + - $ref: "#/components/parameters/LegacyPublicKey" + - name: dataset + in: query required: true schema: - type: string - - in: path - name: fileId + $ref: "#/components/schemas/DatasetId" + description: Dataset identifier + - name: fileId + in: query + required: false schema: type: string - required: true - description: Stable ID of the file to download - - in: header - name: Range + description: Stable ID of the file to download. It is an error to supply both filePath and fileId. + - name: filePath + in: query required: false schema: type: string - description: 0-indexed byte range of the file that is to be delivered as per RFC7233. - example: "Range: bytes=20-200, 300-400" + description: File path of the file to download. It is an error to supply both filePath and fileId. + - name: start + in: query + required: false + deprecated: true + schema: + type: integer + minimum: 0 + default: 0 + description: | + Deprecated. Use the Range header on GET /files/{fileId} instead. + - name: end + in: query + required: false + deprecated: true + schema: + type: integer + minimum: 0 + description: | + Deprecated. Use the Range header on GET /files/{fileId} instead. responses: "200": + description: Successful operation content: application/octet-stream: schema: type: string format: binary - description: Successful operation "206": + description: Successful partial content (legacy semantics) content: application/octet-stream: schema: type: string format: binary - description: Successful partial file delivery "400": - description: Bad request ex. start larger than end + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "416": + $ref: "#/components/responses/RangeNotSatisfiable" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + /file/{fileId}: + get: + tags: [Files] + operationId: legacyDownloadById + deprecated: true + summary: "[Deprecated] Download by fileId" + description: Deprecated. Prefer GET /files/{fileId}. + parameters: + - $ref: "#/components/parameters/FileIdPath" + - $ref: "#/components/parameters/LegacyPublicKey" + - $ref: "#/components/parameters/Range" + - $ref: "#/components/parameters/IfRange" + responses: + "200": + $ref: "#/components/responses/BinaryFile200" + "206": + $ref: "#/components/responses/BinaryFile206" + "400": + $ref: "#/components/responses/BadRequest" "401": - description: Authentication failure + $ref: "#/components/responses/Unauthorized" "403": - description: The user does not have access to the dataset + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" "416": - description: Unsatisfiable range ex. start larger than the size of the file + $ref: "#/components/responses/RangeNotSatisfiable" "500": - description: Internal application error + $ref: "#/components/responses/InternalServerError" security: - bearerAuth: [] + /info/datasets: get: - description: Returns a list of the datasets the user has access to, an empty list is a valid response. + tags: [Datasets] + operationId: legacyListDatasets + deprecated: true + summary: "[Deprecated] List datasets" + description: Deprecated. Prefer GET /datasets. responses: "200": - description: Successful operation + description: Successful operation (an empty list is valid). content: application/json: - example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] schema: type: array + items: + $ref: "#/components/schemas/DatasetId" + example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] "401": - description: Authentication failure + $ref: "#/components/responses/Unauthorized" "500": - description: Internal application error + $ref: "#/components/responses/InternalServerError" security: - bearerAuth: [] + /info/dataset: get: - description: Returns an array with metadata about the dataset + tags: [Datasets] + operationId: legacyGetDataset + deprecated: true + summary: "[Deprecated] Get dataset metadata" + description: Deprecated. Prefer GET /datasets/{datasetId}. parameters: - - in: query - name: dataset - schema: - type: string + - name: dataset + in: query required: true + schema: + $ref: "#/components/schemas/DatasetId" description: Dataset identifier responses: "200": + description: Successful operation content: application/json: schema: $ref: "#/components/schemas/DatasetInfo" - description: Successful operation "401": - description: Authentication failure + $ref: "#/components/responses/Unauthorized" "403": - description: The user does not have access to the dataset + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" "500": - description: Internal application error + $ref: "#/components/responses/InternalServerError" security: - bearerAuth: [] + /info/dataset/files: get: - description: Returns an array with metadata about all files in a dataset + tags: [Datasets] + operationId: legacyListDatasetFiles + deprecated: true + summary: "[Deprecated] List dataset files" + description: Deprecated. Prefer GET /datasets/{datasetId}/files. parameters: - - in: query - name: dataset - schema: - type: string + - name: dataset + in: query required: true + schema: + $ref: "#/components/schemas/DatasetId" description: Dataset identifier responses: "200": + description: Successful operation content: application/json: schema: type: array items: $ref: "#/components/schemas/FileInfo" - description: Successful operation "401": - description: Authentication failure + $ref: "#/components/responses/Unauthorized" "403": - description: The user does not have access to the dataset + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" "500": - description: Internal application error + $ref: "#/components/responses/InternalServerError" security: - bearerAuth: [] + /health/ready: get: + tags: [Health] + operationId: readiness + summary: Readiness probe description: Returns the status of the application. responses: "200": + description: The service is operational content: application/json: + schema: + $ref: "#/components/schemas/ReadinessStatus" example: - { - "c4ghService": "responding", - "database": "responding", - "oidcServer": "responding", - "storageBackend": "responding", - } - description: The service is operational + c4ghService: responding + database: responding + oidcServer: responding + storageBackend: responding "503": description: Unhealthy service + /health/live: get: + tags: [Health] + operationId: liveness + summary: Liveness probe description: Returns 200 as long as the main thread is running responses: "200": description: The service is operational + components: schemas: + DatasetId: + type: string + description: Stable dataset identifier + example: aa-dataset-123456-asdfgh + DatasetInfo: type: object + additionalProperties: false properties: + datasetId: + $ref: "#/components/schemas/DatasetId" date: type: string format: date-time @@ -229,26 +574,413 @@ components: type: integer format: int32 example: 1234 - name: - type: string - example: aa-dataset-123456-asdfgh size: type: integer format: int64 example: 6597069766656 + required: [datasetId, date, files, size] + FileInfo: type: object + additionalProperties: false properties: + fileId: + type: string + description: Stable file identifier (maps to DRS object_id in a future DRS facade) + example: aa-file-123456-asdfgh filePath: type: string + description: Dataset-relative file path example: samples/controls/sample1.cram.c4gh size: type: integer format: int64 + description: | + Encrypted file size in bytes (Crypt4GH bytes as stored in the archive, + excluding the re-encrypted header). The actual download size will be + slightly larger due to the recipient-specific Crypt4GH header. example: 56623104 - stableId: + decryptedSize: + type: integer + format: int64 + description: Decrypted (plaintext) file size in bytes + example: 56600000 + checksums: + type: array + description: | + Checksums of the decrypted file content. Matches the GA4GH DRS checksums + field format for future DRS facade compatibility. + items: + $ref: "#/components/schemas/Checksum" + downloadUrl: type: string - example: aa-file-123456-asdfgh + description: Relative download URL for this file + example: /files/aa-file-123456-asdfgh + required: [fileId, filePath, size, decryptedSize, checksums, downloadUrl] + + Checksum: + type: object + additionalProperties: false + properties: + type: + type: string + description: Hash algorithm (e.g. sha256, md5) + example: sha256 + checksum: + type: string + description: Hex-encoded digest value + example: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + required: [type, checksum] + + ServiceInfo: + type: object + description: GA4GH service-info response (https://github.com/ga4gh-discovery/ga4gh-service-info) + properties: + id: + type: string + description: Unique service identifier in reverse domain name format + example: org.neic.sda.download + name: + type: string + description: Human-readable service name + example: SDA Data Out API + type: + type: object + properties: + group: + type: string + example: org.neic + artifact: + type: string + example: sda-download + version: + type: string + example: 1.1.0 + required: [group, artifact, version] + description: + type: string + example: Sensitive Data Archive file download service + organization: + type: object + properties: + name: + type: string + example: NeIC + url: + type: string + format: uri + example: https://neic.no + version: + type: string + description: Service version + example: 1.1.0 + environment: + type: string + description: Deployment environment (e.g. production, staging) + example: production + required: [id, name, type, version] + + ProblemDetails: + type: object + description: | + Problem Details error payload (RFC 9457). + Additional fields MAY be included for debugging/correlation. + properties: + type: + type: string + format: uri-reference + description: A URI reference that identifies the problem type. + title: + type: string + description: A short, human-readable summary of the problem type. + status: + type: integer + format: int32 + description: The HTTP status code. + detail: + type: string + description: A human-readable explanation specific to this occurrence of the problem. + instance: + type: string + format: uri-reference + description: A URI reference that identifies the specific occurrence of the problem. + errorCode: + type: string + description: Application-specific error code. + example: RANGE_INVALID + correlationId: + type: string + description: Correlation ID for support/debugging. + example: 1f3a2a60-1e25-4d4f-8f3b-5a0bdc7c2b1a + required: [title, status] + + ReadinessStatus: + type: object + additionalProperties: true + properties: + c4ghService: + type: string + database: + type: string + oidcServer: + type: string + storageBackend: + type: string + + parameters: + DatasetIdPath: + name: datasetId + in: path + required: true + schema: + $ref: "#/components/schemas/DatasetId" + description: Dataset identifier + + FileIdPath: + name: fileId + in: path + required: true + schema: + type: string + description: Stable file identifier + + C4ghPublicKey: + name: X-C4GH-Public-Key + in: header + required: true + schema: + type: string + description: | + Crypt4GH recipient public key. + + Accepted formats (auto-detected by the server): + + 1. **Preferred (simple):** The BASE64-ENCODED DATA from a Crypt4GH PEM file, + i.e. the content between the `-----BEGIN CRYPT4GH PUBLIC KEY-----` and + `-----END CRYPT4GH PUBLIC KEY-----` lines. This decodes to the raw 32-byte + X25519 public key. + + 2. **Legacy (backward-compatible):** Base64 encoding of the entire PEM file text + (including the BEGIN/END lines). This is accepted for backward compatibility + but is effectively double-encoding and not recommended for new clients. + + The server distinguishes the two formats by decoded length: 32 bytes = raw key, + longer = PEM text passed to the standard Crypt4GH key parser. + + This header is required. Alternatively, HTSGET clients MAY send + Htsget-Context-Public-Key instead (the server accepts either, but not both). + example: "mF4kGxQy5c7a3oO2l2Cq2rY9qJk2o0rJ5m0n6m1o2pQ=" + + HtsgetContextPublicKey: + name: Htsget-Context-Public-Key + in: header + required: false + schema: + type: string + description: | + HTSGET compatibility alternative to X-C4GH-Public-Key. + Same format and semantics. The server accepts either header but not both. + + If this header is present, X-C4GH-Public-Key is not required. + example: "mF4kGxQy5c7a3oO2l2Cq2rY9qJk2o0rJ5m0n6m1o2pQ=" + + LegacyPublicKey: + name: public_key + in: header + required: true + deprecated: true + schema: + type: string + description: | + Deprecated legacy alias for X-C4GH-Public-Key. + + New clients SHOULD send X-C4GH-Public-Key. This header will be removed in a future version. + example: "mF4kGxQy5c7a3oO2l2Cq2rY9qJk2o0rJ5m0n6m1o2pQ=" + + Range: + name: Range + in: header + required: false + schema: + type: string + description: | + Byte range of the re-encrypted Crypt4GH stream to deliver, per RFC 9110. + + Byte offsets refer to the re-encrypted output stream (header + data segments), + not the original plaintext file. + + This API supports a single range of the form: bytes=- or bytes=-. + Multiple ranges (comma-separated) are not supported and will return 400. + example: "bytes=0-1048575" + + IfRange: + name: If-Range + in: header + required: false + schema: + type: string + description: | + Makes the Range request conditional on the entity tag (ETag) or Last-Modified value. + Commonly used to resume downloads safely. + example: "\"686897696a7c876b7e\"" + + headers: + AcceptRanges: + description: Indicates that the resource supports byte range requests. + schema: + type: string + example: bytes + + ContentLength: + description: Size of the response body in bytes. + schema: + type: integer + format: int64 + + ContentRange: + description: Indicates where in the full resource this partial body belongs. + schema: + type: string + example: "bytes 0-1048575/56623104" + + ETag: + description: | + Entity tag representing the current Crypt4GH representation of the file. + The ETag is scoped to the (fileId, recipient public key) pair. The same file + re-encrypted for different recipients produces different ETags. Clients MUST + use If-Range with this ETag when resuming partial downloads to ensure the + representation has not changed. + schema: + type: string + example: "\"686897696a7c876b7e\"" + + LastModified: + description: Last modification time of the resource (HTTP-date format per RFC 9110). + schema: + type: string + example: "Fri, 14 Feb 2025 14:51:26 GMT" + + ContentDisposition: + description: Suggested filename for saving the download. + schema: + type: string + example: attachment; filename="sample1.cram.c4gh" + + responses: + BinaryFile200: + description: Successful operation (full file) + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + ETag: + $ref: "#/components/headers/ETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + Content-Disposition: + $ref: "#/components/headers/ContentDisposition" + content: + application/octet-stream: + schema: + type: string + format: binary + + BinaryFile206: + description: Successful operation (partial content) + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + Content-Range: + $ref: "#/components/headers/ContentRange" + ETag: + $ref: "#/components/headers/ETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + Content-Disposition: + $ref: "#/components/headers/ContentDisposition" + content: + application/octet-stream: + schema: + type: string + format: binary + + BadRequest: + description: Bad request + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ProblemDetails" + example: + title: Bad Request + status: 400 + detail: The Range header is invalid. Only a single byte range is supported. + errorCode: RANGE_INVALID + + Unauthorized: + description: Authentication failure + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ProblemDetails" + example: + title: Unauthorized + status: 401 + detail: Missing or invalid bearer token. + + Forbidden: + description: The caller does not have access to the requested resource + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ProblemDetails" + example: + title: Forbidden + status: 403 + detail: You do not have access to this dataset or file. + + NotFound: + description: Requested dataset or file does not exist (or is not visible to the caller) + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ProblemDetails" + example: + title: Not Found + status: 404 + detail: The requested resource was not found. + + RangeNotSatisfiable: + description: Unsatisfiable range + headers: + Content-Range: + description: Indicates the current total length of the resource. + schema: + type: string + example: "bytes */56623104" + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ProblemDetails" + example: + title: Range Not Satisfiable + status: 416 + detail: The requested range is outside the file bounds. + errorCode: RANGE_NOT_SATISFIABLE + + InternalServerError: + description: Internal application error + content: + application/problem+json: + schema: + $ref: "#/components/schemas/ProblemDetails" + example: + title: Internal Server Error + status: 500 + detail: Unexpected error. + securitySchemes: bearerAuth: type: http From 78f53deec43b987ba58b0d6e121826f4909e43a4 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 10 Feb 2026 11:52:06 +0100 Subject: [PATCH 040/530] refactor(api): remove deprecated legacy endpoints from swagger Legacy endpoints (/file, /file/{fileId}, /info/datasets, /info/dataset, /info/dataset/files) were never deployed and should not be marked as deprecated in a new service. Also removes the unused LegacyPublicKey parameter. --- sda/cmd/download/swagger_v1.yml | 218 -------------------------------- 1 file changed, 218 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index ca61e19ebc..68de31d506 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -317,211 +317,6 @@ paths: -o file.rest \ "https://api.example.org/files/aa-file-123456-asdfgh" - # --- Legacy endpoints kept for backward compatibility (deprecated) --- - - /file: - get: - tags: [Files] - operationId: legacyDownloadByQuery - deprecated: true - summary: "[Deprecated] Download by query parameters" - description: | - Deprecated. Prefer GET /files/{fileId}. - - This endpoint is kept for backward compatibility with older clients. It allows downloading - by dataset+fileId or dataset+filePath. Range requests should be performed using the - standard Range header on GET /files/{fileId}. - parameters: - - $ref: "#/components/parameters/LegacyPublicKey" - - name: dataset - in: query - required: true - schema: - $ref: "#/components/schemas/DatasetId" - description: Dataset identifier - - name: fileId - in: query - required: false - schema: - type: string - description: Stable ID of the file to download. It is an error to supply both filePath and fileId. - - name: filePath - in: query - required: false - schema: - type: string - description: File path of the file to download. It is an error to supply both filePath and fileId. - - name: start - in: query - required: false - deprecated: true - schema: - type: integer - minimum: 0 - default: 0 - description: | - Deprecated. Use the Range header on GET /files/{fileId} instead. - - name: end - in: query - required: false - deprecated: true - schema: - type: integer - minimum: 0 - description: | - Deprecated. Use the Range header on GET /files/{fileId} instead. - responses: - "200": - description: Successful operation - content: - application/octet-stream: - schema: - type: string - format: binary - "206": - description: Successful partial content (legacy semantics) - content: - application/octet-stream: - schema: - type: string - format: binary - "400": - $ref: "#/components/responses/BadRequest" - "401": - $ref: "#/components/responses/Unauthorized" - "403": - $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" - "416": - $ref: "#/components/responses/RangeNotSatisfiable" - "500": - $ref: "#/components/responses/InternalServerError" - security: - - bearerAuth: [] - - /file/{fileId}: - get: - tags: [Files] - operationId: legacyDownloadById - deprecated: true - summary: "[Deprecated] Download by fileId" - description: Deprecated. Prefer GET /files/{fileId}. - parameters: - - $ref: "#/components/parameters/FileIdPath" - - $ref: "#/components/parameters/LegacyPublicKey" - - $ref: "#/components/parameters/Range" - - $ref: "#/components/parameters/IfRange" - responses: - "200": - $ref: "#/components/responses/BinaryFile200" - "206": - $ref: "#/components/responses/BinaryFile206" - "400": - $ref: "#/components/responses/BadRequest" - "401": - $ref: "#/components/responses/Unauthorized" - "403": - $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" - "416": - $ref: "#/components/responses/RangeNotSatisfiable" - "500": - $ref: "#/components/responses/InternalServerError" - security: - - bearerAuth: [] - - /info/datasets: - get: - tags: [Datasets] - operationId: legacyListDatasets - deprecated: true - summary: "[Deprecated] List datasets" - description: Deprecated. Prefer GET /datasets. - responses: - "200": - description: Successful operation (an empty list is valid). - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/DatasetId" - example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] - "401": - $ref: "#/components/responses/Unauthorized" - "500": - $ref: "#/components/responses/InternalServerError" - security: - - bearerAuth: [] - - /info/dataset: - get: - tags: [Datasets] - operationId: legacyGetDataset - deprecated: true - summary: "[Deprecated] Get dataset metadata" - description: Deprecated. Prefer GET /datasets/{datasetId}. - parameters: - - name: dataset - in: query - required: true - schema: - $ref: "#/components/schemas/DatasetId" - description: Dataset identifier - responses: - "200": - description: Successful operation - content: - application/json: - schema: - $ref: "#/components/schemas/DatasetInfo" - "401": - $ref: "#/components/responses/Unauthorized" - "403": - $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" - "500": - $ref: "#/components/responses/InternalServerError" - security: - - bearerAuth: [] - - /info/dataset/files: - get: - tags: [Datasets] - operationId: legacyListDatasetFiles - deprecated: true - summary: "[Deprecated] List dataset files" - description: Deprecated. Prefer GET /datasets/{datasetId}/files. - parameters: - - name: dataset - in: query - required: true - schema: - $ref: "#/components/schemas/DatasetId" - description: Dataset identifier - responses: - "200": - description: Successful operation - content: - application/json: - schema: - type: array - items: - $ref: "#/components/schemas/FileInfo" - "401": - $ref: "#/components/responses/Unauthorized" - "403": - $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" - "500": - $ref: "#/components/responses/InternalServerError" - security: - - bearerAuth: [] - /health/ready: get: tags: [Health] @@ -784,19 +579,6 @@ components: If this header is present, X-C4GH-Public-Key is not required. example: "mF4kGxQy5c7a3oO2l2Cq2rY9qJk2o0rJ5m0n6m1o2pQ=" - LegacyPublicKey: - name: public_key - in: header - required: true - deprecated: true - schema: - type: string - description: | - Deprecated legacy alias for X-C4GH-Public-Key. - - New clients SHOULD send X-C4GH-Public-Key. This header will be removed in a future version. - example: "mF4kGxQy5c7a3oO2l2Cq2rY9qJk2o0rJ5m0n6m1o2pQ=" - Range: name: Range in: header From f39266ac7b70af288e2fbf5ce4ba285874353b80 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 10 Feb 2026 14:17:33 +0100 Subject: [PATCH 041/530] fix(api): make key headers mutually optional, bump to v2.0.0 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Make both X-C4GH-Public-Key and Htsget-Context-Public-Key required: false with server-enforced "at least one" constraint. OpenAPI 3.x cannot express cross-parameter dependencies (OAI/OpenAPI-Specification#4174). Bump API version to 2.0.0 — clean break from the never-deployed v1.0 to clearly separate from the current sda-download API. --- .../{swagger_v1.yml => swagger_v2.0.0.yml} | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) rename sda/cmd/download/{swagger_v1.yml => swagger_v2.0.0.yml} (97%) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v2.0.0.yml similarity index 97% rename from sda/cmd/download/swagger_v1.yml rename to sda/cmd/download/swagger_v2.0.0.yml index 68de31d506..f7ba76ed1b 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -1,7 +1,7 @@ openapi: 3.0.3 info: title: SDA Data Out API - version: "1.1.0" + version: "2.0.0" description: | External download API for the Sensitive Data Archive (SDA). @@ -65,12 +65,12 @@ paths: type: group: org.neic artifact: sda-download - version: 1.1.0 + version: 2.0.0 description: Sensitive Data Archive file download service organization: name: NeIC url: https://neic.no - version: 1.1.0 + version: 2.0.0 environment: production /datasets: @@ -450,7 +450,7 @@ components: example: sda-download version: type: string - example: 1.1.0 + example: 2.0.0 required: [group, artifact, version] description: type: string @@ -468,7 +468,7 @@ components: version: type: string description: Service version - example: 1.1.0 + example: 2.0.0 environment: type: string description: Deployment environment (e.g. production, staging) @@ -542,11 +542,17 @@ components: C4ghPublicKey: name: X-C4GH-Public-Key in: header - required: true + required: false schema: type: string description: | - Crypt4GH recipient public key. + Crypt4GH recipient public key. At least one of X-C4GH-Public-Key or + Htsget-Context-Public-Key MUST be provided; the server returns 400 if + neither is present. Sending both is also an error (400). + + Note: OpenAPI 3.x cannot express cross-parameter constraints like + "at least one of" (see https://github.com/OAI/OpenAPI-Specification/discussions/4174). + Both headers are marked required: false; the constraint is enforced server-side. Accepted formats (auto-detected by the server): @@ -561,9 +567,6 @@ components: The server distinguishes the two formats by decoded length: 32 bytes = raw key, longer = PEM text passed to the standard Crypt4GH key parser. - - This header is required. Alternatively, HTSGET clients MAY send - Htsget-Context-Public-Key instead (the server accepts either, but not both). example: "mF4kGxQy5c7a3oO2l2Cq2rY9qJk2o0rJ5m0n6m1o2pQ=" HtsgetContextPublicKey: @@ -575,8 +578,7 @@ components: description: | HTSGET compatibility alternative to X-C4GH-Public-Key. Same format and semantics. The server accepts either header but not both. - - If this header is present, X-C4GH-Public-Key is not required. + See X-C4GH-Public-Key description for the cross-parameter constraint. example: "mF4kGxQy5c7a3oO2l2Cq2rY9qJk2o0rJ5m0n6m1o2pQ=" Range: From a9779bac8436eb1239933d54fca9ac45a73bad7f Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 10 Feb 2026 15:37:41 +0100 Subject: [PATCH 042/530] feat(download): add split header/content endpoints (Pontus design), tighten spec text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add GET/HEAD /files/{fileId}/header and /files/{fileId}/content sub-resources (Pontus Freyhult's design) to separate the volatile re-encrypted Crypt4GH header from the static encrypted data segments. Key changes: - /content: no recipient key required, stable ETag (fileId-scoped), Range/If-Range support, CDN-cacheable — ideal for htsget-rs tickets - /header: recipient-specific, stability implementation-dependent - Combined /files/{fileId} remains as convenience endpoint Spec text fixes from review: - Range parameter now documents per-endpoint semantics (combined vs content) - ETag/resume on combined endpoint honestly documented as implementation- dependent; /content recommended for robust resume - Header volatility softened to implementation-dependent (no crypto guidance) - filePath filter: endpoint + param descriptions both say exact match - Drop "Stable" from identifier descriptions (accessionId rename is separate) - Global description updated to three-tier endpoint model --- sda/cmd/download/swagger_v2.0.0.yml | 292 ++++++++++++++++++++++++++-- 1 file changed, 272 insertions(+), 20 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index f7ba76ed1b..82be8ed729 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -16,10 +16,17 @@ info: - GA4GH service-info endpoint for service discovery. Crypt4GH and Range semantics: - The byte stream delivered to the client is the Crypt4GH re-encrypted representation: - [re-encrypted header | encrypted data segments]. All byte offsets in Range requests, - Content-Length, and Content-Range refer to this re-encrypted stream, NOT the original - plaintext file. The re-encrypted header size varies per recipient public key. + Files are served through three endpoint tiers: + - Combined (GET /files/{fileId}): delivers [re-encrypted header | data segments] + as a single stream. Range byte offsets refer to this combined representation. + ETag stability depends on the server's re-encryption implementation. + - Header (GET /files/{fileId}/header): re-encrypted header only. No Range support. + Header bytes may vary between requests (implementation-dependent). + - Content (GET /files/{fileId}/content): encrypted data segments only. Range byte + offsets refer to data segments (no header). ETag is stable for a given file + and independent of the recipient key. + In all cases, byte offsets refer to the encrypted wire format, NOT the original + plaintext file. DRS readiness (non-normative): Designed for compatibility with GA4GH DRS 1.5+ @@ -128,7 +135,8 @@ paths: summary: List files in a dataset description: | Returns metadata for all files in the dataset. When the optional filePath query - parameter is provided, the response is filtered to files matching that path. + parameter is provided, the response is filtered by exact path equality (returning + 0 or 1 results). parameters: - $ref: "#/components/parameters/DatasetIdPath" - name: filePath @@ -137,9 +145,9 @@ paths: schema: type: string description: | - Optional dataset-relative file path filter. When provided, only files - whose path matches are returned. Useful for clients that identify files - by submitted path rather than by fileId. + Exact dataset-relative file path. When provided, only the file with this + exact path is returned (at most one result). Useful for clients that + identify files by submitted path rather than by fileId. example: samples/controls/sample1.cram.c4gh responses: "200": @@ -171,11 +179,18 @@ paths: of the requested file, including Content-Length and ETag. The public key is required because Content-Length includes the recipient-specific - Crypt4GH header, which varies in size per recipient. The ETag also represents + Crypt4GH header, which varies in size per recipient. The ETag represents the specific representation for the given (fileId, recipient public key) pair; different keys produce different ETags. - This endpoint is essential for implementing resumable downloads (HEAD → Range + If-Range). + Note: ETag stability on this endpoint depends on whether the server produces + deterministic re-encrypted headers for a given (fileId, recipientKey) pair. If + the implementation uses random ephemeral keys, the ETag may change between + requests and If-Range will fall back to a full 200 (correct per RFC 9110 but + not a true resume). For guaranteed stable ETag, use HEAD /files/{fileId}/content. + + For robust resume and CDN caching, prefer the split sub-resources: + HEAD /files/{fileId}/header and HEAD /files/{fileId}/content. parameters: - $ref: "#/components/parameters/FileIdPath" - $ref: "#/components/parameters/C4ghPublicKey" @@ -211,17 +226,28 @@ paths: summary: Download a file (supports Range for partial downloads) description: | Downloads the requested file, re-encrypted as Crypt4GH for the recipient. + This is the convenience endpoint that delivers the complete Crypt4GH stream. The delivered byte stream is: [re-encrypted Crypt4GH header | encrypted data segments]. - All byte offsets in Range, Content-Length, and Content-Range refer to this re-encrypted - stream, NOT the original plaintext file. + All byte offsets in Range, Content-Length, and Content-Range refer to this combined + re-encrypted stream, NOT the original plaintext file. Partial downloads use the standard HTTP Range header. For simplicity and broad client compatibility, this API supports a single byte range per request. The ETag represents the specific Crypt4GH representation for the given file and - recipient public key. Different keys yield different ETags. Use If-Range with the - ETag to safely resume interrupted downloads. + recipient public key. Different keys yield different ETags. + + Resume caveat: ETag stability on this endpoint depends on whether the server + produces deterministic re-encrypted headers for a given (fileId, recipientKey) + pair. If the implementation generates new ephemeral keys per request, the ETag + changes and If-Range falls back to a full 200 response (correct per RFC 9110 but + not a true resume). For guaranteed robust resume, use GET /files/{fileId}/content + which has a stable ETag independent of the recipient key. + + For CDN caching and htsget-rs integration, prefer the split sub-resources: + GET /files/{fileId}/header (per-recipient header) and + GET /files/{fileId}/content (static data segments with stable ETag and Range). parameters: - $ref: "#/components/parameters/FileIdPath" - $ref: "#/components/parameters/C4ghPublicKey" @@ -317,6 +343,219 @@ paths: -o file.rest \ "https://api.example.org/files/aa-file-123456-asdfgh" + /files/{fileId}/header: + head: + tags: [Files] + operationId: headFileHeader + summary: Get re-encrypted header metadata (no body) + description: | + Returns size metadata for the re-encrypted Crypt4GH header of the requested file. + + The header is recipient-specific. Whether header bytes are stable across requests + for the same (fileId, recipientKey) pair depends on the server's re-encryption + implementation. A standard Crypt4GH re-encryption generates a new ephemeral + X25519 keypair each time, producing different bytes. Implementations MAY use + a stable representation strategy (e.g. caching) to produce consistent headers. + parameters: + - $ref: "#/components/parameters/FileIdPath" + - $ref: "#/components/parameters/C4ghPublicKey" + - $ref: "#/components/parameters/HtsgetContextPublicKey" + responses: + "200": + description: Header metadata returned successfully + headers: + Content-Length: + $ref: "#/components/headers/ContentLength" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + get: + tags: [Files] + operationId: getFileHeader + summary: Download the re-encrypted Crypt4GH header + description: | + Returns only the re-encrypted Crypt4GH header for the specified file and recipient. + The header contains the data encryption key(s) re-encrypted to the recipient's + public key. It is typically small (around 120–140 bytes for a single DEK). + + Whether header bytes are stable across requests for the same (fileId, recipientKey) + pair is implementation-dependent. A standard Crypt4GH re-encryption generates a + new ephemeral X25519 keypair each time. Implementations MAY use a stable + representation strategy (e.g. caching) to produce consistent headers. + + Clients reconstructing a complete Crypt4GH file should concatenate header bytes + (from this endpoint) with data bytes (from GET /files/{fileId}/content). + parameters: + - $ref: "#/components/parameters/FileIdPath" + - $ref: "#/components/parameters/C4ghPublicKey" + - $ref: "#/components/parameters/HtsgetContextPublicKey" + responses: + "200": + description: Re-encrypted header + headers: + Content-Length: + $ref: "#/components/headers/ContentLength" + content: + application/octet-stream: + schema: + type: string + format: binary + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + /files/{fileId}/content: + head: + tags: [Files] + operationId: headFileContent + summary: Get encrypted content metadata (no body) + description: | + Returns metadata for the encrypted data segments of the file (everything after + the Crypt4GH header). Unlike the combined /files/{fileId} endpoint, Content-Length + and ETag are stable and independent of the recipient public key. + + No public key header is required — the data segments are encrypted with the file's + data encryption key, which is the same regardless of recipient. + + The stable ETag makes this endpoint ideal for CDN caching and htsget-rs integration. + parameters: + - $ref: "#/components/parameters/FileIdPath" + responses: + "200": + description: Content metadata returned successfully + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + ETag: + $ref: "#/components/headers/ContentETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + + get: + tags: [Files] + operationId: getFileContent + summary: Download encrypted data segments (supports Range) + description: | + Downloads the encrypted data segments of the file — everything after the + Crypt4GH header. These bytes are identical regardless of the recipient public + key (encrypted with the file's data encryption key, not the recipient key). + + No public key header is required. + + This endpoint supports Range + If-Range for partial downloads with robust + resume behavior. The ETag is stable (scoped to fileId only), making it safe + for If-Range and CDN caching. + + **htsget-rs integration:** ticket data URLs should point to this endpoint + with byte ranges, since the encrypted block structure is static and + recipient-independent. + + Clients reconstructing a complete Crypt4GH file should concatenate header bytes + (from GET /files/{fileId}/header) with data bytes (from this endpoint). + parameters: + - $ref: "#/components/parameters/FileIdPath" + - $ref: "#/components/parameters/Range" + - $ref: "#/components/parameters/IfRange" + responses: + "200": + description: Successful operation (full content) + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + ETag: + $ref: "#/components/headers/ContentETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + content: + application/octet-stream: + schema: + type: string + format: binary + "206": + description: Successful operation (partial content) + headers: + Accept-Ranges: + $ref: "#/components/headers/AcceptRanges" + Content-Length: + $ref: "#/components/headers/ContentLength" + Content-Range: + $ref: "#/components/headers/ContentRange" + ETag: + $ref: "#/components/headers/ContentETag" + Last-Modified: + $ref: "#/components/headers/LastModified" + content: + application/octet-stream: + schema: + type: string + format: binary + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "416": + $ref: "#/components/responses/RangeNotSatisfiable" + "500": + $ref: "#/components/responses/InternalServerError" + security: + - bearerAuth: [] + x-codeSamples: + - lang: curl + label: Fetch header + content separately + source: | + # 1) Get the re-encrypted header + curl -L \ + -H "Authorization: Bearer $TOKEN" \ + -H "X-C4GH-Public-Key: $C4GH_PK_B64" \ + -o header.bin \ + "https://api.example.org/files/aa-file-123456-asdfgh/header" + + # 2) Get the encrypted content (supports Range resume) + curl -L \ + -H "Authorization: Bearer $TOKEN" \ + -o content.bin \ + "https://api.example.org/files/aa-file-123456-asdfgh/content" + + # 3) Concatenate into a valid Crypt4GH file + cat header.bin content.bin > file.c4gh + /health/ready: get: tags: [Health] @@ -352,7 +591,7 @@ components: schemas: DatasetId: type: string - description: Stable dataset identifier + description: Dataset identifier example: aa-dataset-123456-asdfgh DatasetInfo: @@ -381,7 +620,7 @@ components: properties: fileId: type: string - description: Stable file identifier (maps to DRS object_id in a future DRS facade) + description: File identifier (maps to DRS object_id in a future DRS facade) example: aa-file-123456-asdfgh filePath: type: string @@ -537,7 +776,7 @@ components: required: true schema: type: string - description: Stable file identifier + description: File identifier C4ghPublicKey: name: X-C4GH-Public-Key @@ -588,10 +827,12 @@ components: schema: type: string description: | - Byte range of the re-encrypted Crypt4GH stream to deliver, per RFC 9110. + Standard HTTP byte range per RFC 9110. The exact bytes the range applies to + depend on the endpoint: + - /files/{fileId}: the combined re-encrypted Crypt4GH stream (header + data segments) + - /files/{fileId}/content: the encrypted data segments only (no header) - Byte offsets refer to the re-encrypted output stream (header + data segments), - not the original plaintext file. + In both cases, byte offsets refer to the encrypted wire format, NOT plaintext. This API supports a single range of the form: bytes=- or bytes=-. Multiple ranges (comma-separated) are not supported and will return 400. @@ -638,6 +879,17 @@ components: type: string example: "\"686897696a7c876b7e\"" + ContentETag: + description: | + Entity tag representing the encrypted data segments of the file. + Unlike the combined-endpoint ETag, this is scoped to fileId only (not the + recipient public key), because data segments are encrypted with the file's + data encryption key and are identical for all recipients. This ETag is + stable across requests and safe for If-Range and CDN caching. + schema: + type: string + example: "\"a1b2c3d4e5f6g7h8\"" + LastModified: description: Last modification time of the resource (HTTP-date format per RFC 9110). schema: From 2eba9521430d9f95816a476f6c15564c28d23181 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 11 Feb 2026 16:37:14 +0100 Subject: [PATCH 043/530] feat(download): add cursor pagination (US14) and pathPrefix filter (US13) Ref: #1680 - Both list endpoints now use cursor-based pagination (pageSize/pageToken) with envelope responses ({datasets/files, nextPageToken}) - Breaking change: bare array responses replaced with envelope objects - New pathPrefix query param on /datasets/{id}/files for recursive prefix matching; mutually exclusive with filePath (exact match) - Pagination naming aligned with DRS next_page_token pattern- --- sda/cmd/download/swagger_v2.0.0.yml | 145 ++++++++++++++++++++++++---- 1 file changed, 128 insertions(+), 17 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 82be8ed729..046188592d 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -85,17 +85,24 @@ paths: tags: [Datasets] operationId: listDatasets summary: List datasets the caller can access - description: Returns a list of dataset identifiers the caller has access to. + description: | + Returns dataset identifiers the caller has access to. + Supports cursor-based pagination for large result sets. + parameters: + - $ref: "#/components/parameters/PageSize" + - $ref: "#/components/parameters/PageToken" responses: "200": - description: Successful operation (an empty list is valid). + description: Successful operation (an empty datasets array is valid). content: application/json: schema: - type: array - items: - $ref: "#/components/schemas/DatasetId" - example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] + $ref: "#/components/schemas/DatasetListResponse" + example: + datasets: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] + nextPageToken: "eyJpZCI6Mn0" + "400": + $ref: "#/components/responses/BadRequest" "401": $ref: "#/components/responses/Unauthorized" "500": @@ -134,9 +141,10 @@ paths: operationId: listDatasetFiles summary: List files in a dataset description: | - Returns metadata for all files in the dataset. When the optional filePath query - parameter is provided, the response is filtered by exact path equality (returning - 0 or 1 results). + Returns metadata for files in the dataset. Supports cursor-based pagination + and two optional filters (mutually exclusive — providing both returns 400): + - filePath: exact path equality (returns 0 or 1 results) + - pathPrefix: recursive prefix match (returns all files under the prefix) parameters: - $ref: "#/components/parameters/DatasetIdPath" - name: filePath @@ -148,16 +156,30 @@ paths: Exact dataset-relative file path. When provided, only the file with this exact path is returned (at most one result). Useful for clients that identify files by submitted path rather than by fileId. + Mutually exclusive with pathPrefix. example: samples/controls/sample1.cram.c4gh + - name: pathPrefix + in: query + required: false + schema: + type: string + description: | + Dataset-relative path prefix filter. Returns all files whose path starts + with this value (recursive). Use a trailing / to match a directory boundary + (e.g. "samples/controls/" matches all files in that directory and below). + Mutually exclusive with filePath. + example: samples/controls/ + - $ref: "#/components/parameters/PageSize" + - $ref: "#/components/parameters/PageToken" responses: "200": description: Successful operation content: application/json: schema: - type: array - items: - $ref: "#/components/schemas/FileInfo" + $ref: "#/components/schemas/FileListResponse" + "400": + $ref: "#/components/responses/BadRequest" "401": $ref: "#/components/responses/Unauthorized" "403": @@ -614,6 +636,38 @@ components: example: 6597069766656 required: [datasetId, date, files, size] + DatasetListResponse: + type: object + additionalProperties: false + properties: + datasets: + type: array + items: + $ref: "#/components/schemas/DatasetId" + nextPageToken: + type: string + nullable: true + description: | + Opaque token for retrieving the next page of results. + Null or absent if this is the last page. + required: [datasets] + + FileListResponse: + type: object + additionalProperties: false + properties: + files: + type: array + items: + $ref: "#/components/schemas/FileInfo" + nextPageToken: + type: string + nullable: true + description: | + Opaque token for retrieving the next page of results. + Null or absent if this is the last page. + required: [files] + FileInfo: type: object additionalProperties: false @@ -849,6 +903,39 @@ components: Commonly used to resume downloads safely. example: "\"686897696a7c876b7e\"" + PageSize: + name: pageSize + in: query + required: false + schema: + type: integer + format: int32 + minimum: 1 + maximum: 1000 + default: 100 + description: | + Maximum number of items to return per page. The server may enforce + a lower maximum. If not provided, the server uses a default page size. + + PageToken: + name: pageToken + in: query + required: false + schema: + type: string + description: | + Opaque cursor token from a previous response's nextPageToken field. + When provided, returns the next page of results. Do not construct + these tokens — they are only valid as received from the server. + + **Pagination contract:** + - Results are returned in a stable, server-defined order that is consistent + across pages within the same query. + - A pageToken is scoped to the original query parameters (filters, pageSize). + Changing filters or pageSize while reusing a pageToken returns 400. + - An invalid or expired pageToken returns 400 (the server does not silently + restart from the beginning). + headers: AcceptRanges: description: Indicates that the resource supports byte range requests. @@ -949,11 +1036,35 @@ components: application/problem+json: schema: $ref: "#/components/schemas/ProblemDetails" - example: - title: Bad Request - status: 400 - detail: The Range header is invalid. Only a single byte range is supported. - errorCode: RANGE_INVALID + examples: + rangeInvalid: + summary: Invalid Range header + value: + title: Bad Request + status: 400 + detail: The Range header is invalid. Only a single byte range is supported. + errorCode: RANGE_INVALID + filterConflict: + summary: Mutually exclusive filters + value: + title: Bad Request + status: 400 + detail: "filePath and pathPrefix are mutually exclusive; provide at most one." + errorCode: FILTER_CONFLICT + invalidPageToken: + summary: Invalid or expired page token + value: + title: Bad Request + status: 400 + detail: "The pageToken is invalid, expired, or does not match the current query parameters." + errorCode: INVALID_PAGE_TOKEN + invalidPageSize: + summary: Invalid page size + value: + title: Bad Request + status: 400 + detail: "pageSize must be between 1 and 1000." + errorCode: INVALID_PAGE_SIZE Unauthorized: description: Authentication failure From 17747c795d55897cb5d70334d883258f926282da Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 12 Feb 2026 18:55:20 +0100 Subject: [PATCH 044/530] docs(download): tighten filePath and pagination contracts - add 400 for invalid pagination on GET /datasets - expand BadRequest examples (range, filter conflict, page token/size, filePath) - define pageToken contract (stable order, query-scoped, invalid/expired -> 400) - clarify filePath normalization and exact-match semantics - add filePath/pagination response examples --- sda/cmd/download/swagger_v2.0.0.yml | 59 +++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 046188592d..5c220f3447 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -153,9 +153,19 @@ paths: schema: type: string description: | - Exact dataset-relative file path. When provided, only the file with this - exact path is returned (at most one result). Useful for clients that - identify files by submitted path rather than by fileId. + Exact dataset-relative file path. When provided, only the file + with this exact path is returned (at most one result). The response + includes `downloadUrl` so clients can resolve path → download in two + calls without extra logic. + + **Normalization rules:** + - Case-sensitive (POSIX path semantics). + - No leading slash — paths are relative to the dataset root. + - Must match exactly as stored in the archive metadata (including any + file extension such as `.c4gh`). + - The server URL-decodes the query parameter value per standard HTTP. + - Empty string or whitespace-only values return 400. + Mutually exclusive with pathPrefix. example: samples/controls/sample1.cram.c4gh - name: pathPrefix @@ -173,11 +183,45 @@ paths: - $ref: "#/components/parameters/PageToken" responses: "200": - description: Successful operation + description: | + Successful operation. When filePath is provided, the files array contains + 0 or 1 elements (200 with empty array when no file matches, not 404). content: application/json: schema: $ref: "#/components/schemas/FileListResponse" + examples: + filePathLookup: + summary: Single file resolved by exact filePath match + value: + files: + - fileId: aa-file-123456-asdfgh + filePath: samples/controls/sample1.cram.c4gh + size: 56623104 + decryptedSize: 56600000 + checksums: + - type: sha256 + checksum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + downloadUrl: /files/aa-file-123456-asdfgh + nextPageToken: null + filePathNoMatch: + summary: No file at the given path (empty result, not 404) + value: + files: [] + nextPageToken: null + paginatedList: + summary: Paginated file list (no filter) + value: + files: + - fileId: aa-file-123456-asdfgh + filePath: samples/controls/sample1.cram.c4gh + size: 56623104 + decryptedSize: 56600000 + checksums: + - type: sha256 + checksum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + downloadUrl: /files/aa-file-123456-asdfgh + nextPageToken: "eyJpZCI6Mn0" "400": $ref: "#/components/responses/BadRequest" "401": @@ -1065,6 +1109,13 @@ components: status: 400 detail: "pageSize must be between 1 and 1000." errorCode: INVALID_PAGE_SIZE + invalidFilePath: + summary: Invalid file path format + value: + title: Bad Request + status: 400 + detail: "filePath must be a non-empty dataset-relative path without leading slash." + errorCode: INVALID_FILE_PATH Unauthorized: description: Authentication failure From e0625a8bc5be793fb46d390219699c62484c4bdd Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 14:18:44 +0100 Subject: [PATCH 045/530] fix(download-api): use 403 (not 404) for protected resource lookups; add 400 to HEAD /files/{fileId} --- sda/cmd/download/swagger_v2.0.0.yml | 40 +++++++++-------------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 5c220f3447..f7aa5fa295 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -28,6 +28,15 @@ info: In all cases, byte offsets refer to the encrypted wire format, NOT the original plaintext file. + Error semantics on resource-by-ID endpoints: + Endpoints that accept a resource identifier in the path (e.g. /datasets/{datasetId}, + /files/{fileId}) return 403 Forbidden both when the caller lacks access and when + the resource does not exist, to prevent existence leakage. List endpoints + (e.g. GET /datasets) return 200 with an empty array when the caller has no + accessible resources. 401 indicates an authentication failure (missing, invalid, + or expired token). 400 indicates a malformed request (conflicting parameters, + invalid headers, bad page token). + DRS readiness (non-normative): Designed for compatibility with GA4GH DRS 1.5+ (see https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.5.0/docs/). @@ -128,8 +137,6 @@ paths: $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "500": $ref: "#/components/responses/InternalServerError" security: @@ -228,8 +235,6 @@ paths: $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "500": $ref: "#/components/responses/InternalServerError" security: @@ -275,12 +280,12 @@ paths: $ref: "#/components/headers/LastModified" Content-Disposition: $ref: "#/components/headers/ContentDisposition" + "400": + $ref: "#/components/responses/BadRequest" "401": $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "500": $ref: "#/components/responses/InternalServerError" security: @@ -365,8 +370,6 @@ paths: $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "416": $ref: "#/components/responses/RangeNotSatisfiable" "500": @@ -438,8 +441,6 @@ paths: $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "500": $ref: "#/components/responses/InternalServerError" security: @@ -482,8 +483,6 @@ paths: $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "500": $ref: "#/components/responses/InternalServerError" security: @@ -521,8 +520,6 @@ paths: $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "500": $ref: "#/components/responses/InternalServerError" security: @@ -594,8 +591,6 @@ paths: $ref: "#/components/responses/Unauthorized" "403": $ref: "#/components/responses/Forbidden" - "404": - $ref: "#/components/responses/NotFound" "416": $ref: "#/components/responses/RangeNotSatisfiable" "500": @@ -1126,7 +1121,7 @@ components: example: title: Unauthorized status: 401 - detail: Missing or invalid bearer token. + detail: Missing, invalid, or expired bearer token. Forbidden: description: The caller does not have access to the requested resource @@ -1139,17 +1134,6 @@ components: status: 403 detail: You do not have access to this dataset or file. - NotFound: - description: Requested dataset or file does not exist (or is not visible to the caller) - content: - application/problem+json: - schema: - $ref: "#/components/schemas/ProblemDetails" - example: - title: Not Found - status: 404 - detail: The requested resource was not found. - RangeNotSatisfiable: description: Unsatisfiable range headers: From 8bc9d88a97029cf98a7a595809554d3b7a33c7b8 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 14:26:22 +0100 Subject: [PATCH 046/530] fix(download-api): replace leaky pageToken example with opaque token --- sda/cmd/download/swagger_v2.0.0.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index f7aa5fa295..2e64dda01c 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -109,7 +109,7 @@ paths: $ref: "#/components/schemas/DatasetListResponse" example: datasets: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] - nextPageToken: "eyJpZCI6Mn0" + nextPageToken: "ptk_7f9K2mQxVb3N" "400": $ref: "#/components/responses/BadRequest" "401": @@ -228,7 +228,7 @@ paths: - type: sha256 checksum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 downloadUrl: /files/aa-file-123456-asdfgh - nextPageToken: "eyJpZCI6Mn0" + nextPageToken: "ptk_7f9K2mQxVb3N" "400": $ref: "#/components/responses/BadRequest" "401": From d7727e50914753602952bbd0994146dda8d46d24 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 14:28:07 +0100 Subject: [PATCH 047/530] fix(download-api): make ServiceInfo.organization required per GA4GH service-info 1.0 --- sda/cmd/download/swagger_v2.0.0.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 2e64dda01c..c3402f61db 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -797,6 +797,7 @@ components: type: string format: uri example: https://neic.no + required: [name, url] version: type: string description: Service version @@ -805,7 +806,7 @@ components: type: string description: Deployment environment (e.g. production, staging) example: production - required: [id, name, type, version] + required: [id, name, type, organization, version] ProblemDetails: type: object From 8b39c39aa00d618f9540a391cc0a62ee3ad2eb08 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 14:29:44 +0100 Subject: [PATCH 048/530] fix(download-api): remove unused BinaryFile200/BinaryFile206 response components --- sda/cmd/download/swagger_v2.0.0.yml | 40 ----------------------------- 1 file changed, 40 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index c3402f61db..66a7b64fb1 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -1030,46 +1030,6 @@ components: example: attachment; filename="sample1.cram.c4gh" responses: - BinaryFile200: - description: Successful operation (full file) - headers: - Accept-Ranges: - $ref: "#/components/headers/AcceptRanges" - Content-Length: - $ref: "#/components/headers/ContentLength" - ETag: - $ref: "#/components/headers/ETag" - Last-Modified: - $ref: "#/components/headers/LastModified" - Content-Disposition: - $ref: "#/components/headers/ContentDisposition" - content: - application/octet-stream: - schema: - type: string - format: binary - - BinaryFile206: - description: Successful operation (partial content) - headers: - Accept-Ranges: - $ref: "#/components/headers/AcceptRanges" - Content-Length: - $ref: "#/components/headers/ContentLength" - Content-Range: - $ref: "#/components/headers/ContentRange" - ETag: - $ref: "#/components/headers/ETag" - Last-Modified: - $ref: "#/components/headers/LastModified" - Content-Disposition: - $ref: "#/components/headers/ContentDisposition" - content: - application/octet-stream: - schema: - type: string - format: binary - BadRequest: description: Bad request content: From e545640dcd2918c603dd14aea82c183c42a51a67 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 14:45:13 +0100 Subject: [PATCH 049/530] docs(download-api): document file immutability assumption for split endpoints --- sda/cmd/download/swagger_v2.0.0.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 66a7b64fb1..b6e33df845 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -37,6 +37,12 @@ info: or expired token). 400 indicates a malformed request (conflicting parameters, invalid headers, bad page token). + File immutability assumption: + The split download endpoints (/header + /content) assume archive content is + immutable for a given fileId once the file is available for download. The content + endpoint returns the same encrypted data segments across requests. If the archive + were to be mutated after ingestion, header and content could become inconsistent. + DRS readiness (non-normative): Designed for compatibility with GA4GH DRS 1.5+ (see https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.5.0/docs/). From 8cc315b8645e0954d5369089426cfed1e84bd659 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 14:50:43 +0100 Subject: [PATCH 050/530] fix(download-api): remove rigid response schema constraints to align with DRS extensibility Removes additionalProperties: false from key response schemas; resolves mismatch with documented DRS-readiness/extensibility claim. --- sda/cmd/download/swagger_v2.0.0.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index b6e33df845..1e696225c2 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -663,7 +663,6 @@ components: DatasetInfo: type: object - additionalProperties: false properties: datasetId: $ref: "#/components/schemas/DatasetId" @@ -683,7 +682,6 @@ components: DatasetListResponse: type: object - additionalProperties: false properties: datasets: type: array @@ -699,7 +697,6 @@ components: FileListResponse: type: object - additionalProperties: false properties: files: type: array @@ -715,7 +712,6 @@ components: FileInfo: type: object - additionalProperties: false properties: fileId: type: string From c9938a2f6a3e87b41000a56134b66ebca78a9d22 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 14:58:13 +0100 Subject: [PATCH 051/530] docs(download-api): recommend split endpoints over combined for production use --- sda/cmd/download/swagger_v2.0.0.yml | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 1e696225c2..c46596a866 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -266,8 +266,10 @@ paths: requests and If-Range will fall back to a full 200 (correct per RFC 9110 but not a true resume). For guaranteed stable ETag, use HEAD /files/{fileId}/content. - For robust resume and CDN caching, prefer the split sub-resources: - HEAD /files/{fileId}/header and HEAD /files/{fileId}/content. + For production clients, prefer the split endpoints + (HEAD /files/{fileId}/header + HEAD /files/{fileId}/content) for stable ETags, + robust If-Range resume, and better caching behavior. This combined endpoint + is a convenience path for simple clients. parameters: - $ref: "#/components/parameters/FileIdPath" - $ref: "#/components/parameters/C4ghPublicKey" @@ -322,9 +324,10 @@ paths: not a true resume). For guaranteed robust resume, use GET /files/{fileId}/content which has a stable ETag independent of the recipient key. - For CDN caching and htsget-rs integration, prefer the split sub-resources: - GET /files/{fileId}/header (per-recipient header) and - GET /files/{fileId}/content (static data segments with stable ETag and Range). + For production clients, prefer the split endpoints + (GET /files/{fileId}/header + GET /files/{fileId}/content) for stable ETags, + robust If-Range resume, and better caching behavior. This combined endpoint + is a convenience path for simple clients. parameters: - $ref: "#/components/parameters/FileIdPath" - $ref: "#/components/parameters/C4ghPublicKey" From 59085f30d4b7804d4132375d56eb26e5ad5672ea Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 15:27:20 +0100 Subject: [PATCH 052/530] fix(download-api): add 500 error response to GET /service-info --- sda/cmd/download/swagger_v2.0.0.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index c46596a866..488e70da8d 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -94,6 +94,8 @@ paths: url: https://neic.no version: 2.0.0 environment: production + "500": + $ref: "#/components/responses/InternalServerError" /datasets: get: From 0eea56ecbad864ffd9ed13f911ab5a8b65ccc04e Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 15:32:07 +0100 Subject: [PATCH 053/530] fix(download-api): tighten examples, defaults, and parameter constraints - Add RFC 9457 default "about:blank" for ProblemDetails.type - Clarify downloadUrl is relative to the servers block base URL - Replace empty-string SHA-256 hash with realistic example - Add minLength: 1 to DatasetId schema and FileIdPath parameter --- sda/cmd/download/swagger_v2.0.0.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 488e70da8d..9998e5a3db 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -216,7 +216,7 @@ paths: decryptedSize: 56600000 checksums: - type: sha256 - checksum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + checksum: 7d2c8b4a9f3e1d6b5c0a8f7e2d4b6a9c3f1e5d8b7a0c4f2e6d9b3a7c1f5e8d0a downloadUrl: /files/aa-file-123456-asdfgh nextPageToken: null filePathNoMatch: @@ -234,7 +234,7 @@ paths: decryptedSize: 56600000 checksums: - type: sha256 - checksum: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + checksum: 7d2c8b4a9f3e1d6b5c0a8f7e2d4b6a9c3f1e5d8b7a0c4f2e6d9b3a7c1f5e8d0a downloadUrl: /files/aa-file-123456-asdfgh nextPageToken: "ptk_7f9K2mQxVb3N" "400": @@ -663,6 +663,7 @@ components: schemas: DatasetId: type: string + minLength: 1 description: Dataset identifier example: aa-dataset-123456-asdfgh @@ -748,7 +749,9 @@ components: $ref: "#/components/schemas/Checksum" downloadUrl: type: string - description: Relative download URL for this file + description: >- + Relative download URL for this file (relative to the API base URL + defined in the servers block). example: /files/aa-file-123456-asdfgh required: [fileId, filePath, size, decryptedSize, checksums, downloadUrl] @@ -763,7 +766,7 @@ components: checksum: type: string description: Hex-encoded digest value - example: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + example: 7d2c8b4a9f3e1d6b5c0a8f7e2d4b6a9c3f1e5d8b7a0c4f2e6d9b3a7c1f5e8d0a required: [type, checksum] ServiceInfo: @@ -824,6 +827,7 @@ components: type: type: string format: uri-reference + default: "about:blank" description: A URI reference that identifies the problem type. title: type: string @@ -877,6 +881,7 @@ components: required: true schema: type: string + minLength: 1 description: File identifier C4ghPublicKey: From 1d853f7db3dfb15c54ffa3a5485221e5d8905e98 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 15:47:33 +0100 Subject: [PATCH 054/530] fix(download-api): add Cache-Control headers on metadata endpoints and token-expiry note - Add Cache-Control to /service-info (public, max-age=300) and authenticated metadata endpoints (private, max-age=60, must-revalidate) to support FUSE performance (US7, #1680) - Document token-expiry behavior on bearerAuth security scheme --- sda/cmd/download/swagger_v2.0.0.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index 9998e5a3db..e8fe978be6 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -77,6 +77,9 @@ paths: responses: "200": description: Service information + headers: + Cache-Control: + $ref: "#/components/headers/CacheControlPublic" content: application/json: schema: @@ -111,6 +114,9 @@ paths: responses: "200": description: Successful operation (an empty datasets array is valid). + headers: + Cache-Control: + $ref: "#/components/headers/CacheControlPrivate" content: application/json: schema: @@ -137,6 +143,9 @@ paths: responses: "200": description: Successful operation + headers: + Cache-Control: + $ref: "#/components/headers/CacheControlPrivate" content: application/json: schema: @@ -201,6 +210,9 @@ paths: description: | Successful operation. When filePath is provided, the files array contains 0 or 1 elements (200 with empty array when no file matches, not 404). + headers: + Cache-Control: + $ref: "#/components/headers/CacheControlPrivate" content: application/json: schema: @@ -995,6 +1007,18 @@ components: type: string example: bytes + CacheControlPrivate: + description: Cache directive for authenticated metadata responses. + schema: + type: string + example: "private, max-age=60, must-revalidate" + + CacheControlPublic: + description: Cache directive for public (unauthenticated) responses. + schema: + type: string + example: "public, max-age=300" + ContentLength: description: Size of the response body in bytes. schema: @@ -1141,3 +1165,6 @@ components: type: http scheme: bearer bearerFormat: JWT + description: >- + If an access token expires during transfer, server SHOULD return + 401 Unauthorized; clients SHOULD re-authenticate and retry/resume. From 747f6b4db2de12a9dc54c191183259afd2073fe2 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Feb 2026 17:44:07 +0100 Subject: [PATCH 055/530] feat(download-api): add SDA-Content-ETag header to header sub-resource responses Let's clients verify header/content consistency without an extra HEAD request to the content endpoint. The value is the content endpoint's ETag at the time the header was generated. Co-Authored-By: @pontus --- sda/cmd/download/swagger_v2.0.0.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.0.0.yml index e8fe978be6..9eec484f89 100644 --- a/sda/cmd/download/swagger_v2.0.0.yml +++ b/sda/cmd/download/swagger_v2.0.0.yml @@ -458,6 +458,8 @@ paths: headers: Content-Length: $ref: "#/components/headers/ContentLength" + SDA-Content-ETag: + $ref: "#/components/headers/SDA-Content-ETag" "400": $ref: "#/components/responses/BadRequest" "401": @@ -495,6 +497,8 @@ paths: headers: Content-Length: $ref: "#/components/headers/ContentLength" + SDA-Content-ETag: + $ref: "#/components/headers/SDA-Content-ETag" content: application/octet-stream: schema: @@ -1053,6 +1057,16 @@ components: type: string example: "\"a1b2c3d4e5f6g7h8\"" + SDA-Content-ETag: + description: | + Entity tag of the content endpoint (GET /files/{fileId}/content) at the + time the header was generated. Returned on header sub-resource responses + so clients can verify header/content consistency without an extra HEAD + request. Use the value with If-Range when fetching content. + schema: + type: string + example: "\"a1b2c3d4e5f6g7h8\"" + LastModified: description: Last modification time of the resource (HTTP-date format per RFC 9110). schema: From ffcb36b85292f35d7523260802207a397d744f46 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 18 Feb 2026 09:30:11 +0100 Subject: [PATCH 056/530] refactor(download-api): rename swagger_v2.0.0.yml to swagger_v2.yml Full semver in the filename adds overhead on every patch/minor bump. The version lives in info.version inside the file; the filename only needs to distinguish major versions (v1 vs v2). --- sda/cmd/download/{swagger_v2.0.0.yml => swagger_v2.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename sda/cmd/download/{swagger_v2.0.0.yml => swagger_v2.yml} (100%) diff --git a/sda/cmd/download/swagger_v2.0.0.yml b/sda/cmd/download/swagger_v2.yml similarity index 100% rename from sda/cmd/download/swagger_v2.0.0.yml rename to sda/cmd/download/swagger_v2.yml From e8ed75d5d4496aa000cdc4a654a9ed10e60d3f3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 18 Feb 2026 14:37:16 +0100 Subject: [PATCH 057/530] fix: address linter issues since linter update, mainly explicitly mark as safe with some TODOs to verify if we should sanitize user inputted fields to avoid risk of SSRF etc --- sda-download/api/api.go | 2 +- sda-download/api/api_test.go | 3 +- sda-download/internal/config/config.go | 8 +- sda-download/internal/storage/storage.go | 2 +- sda-validator/orchestrator/api/api.go | 4 +- sda-validator/orchestrator/api/api_test.go | 8 +- .../commandexecutor/command_executor.go | 2 +- .../jobpreparationworker/worker.go | 4 +- .../jobpreparationworker/worker_test.go | 2 +- sda/cmd/api/api_test.go | 82 +++++++++++-------- sda/cmd/auth/cega.go | 2 +- sda/cmd/ingest/ingest_test.go | 8 +- sda/cmd/rotatekey/rotatekey_test.go | 2 +- sda/cmd/s3inbox/healthchecks.go | 2 +- sda/cmd/s3inbox/proxy.go | 4 +- sda/cmd/s3inbox/s3inbox_test.go | 6 +- sda/cmd/sync/sync.go | 2 +- sda/cmd/syncapi/syncapi_test.go | 6 +- sda/internal/broker/broker.go | 4 +- sda/internal/broker/broker_test.go | 2 +- sda/internal/config/config.go | 8 +- sda/internal/database/database.go | 4 +- sda/internal/jsonadapter/jsonadapter_test.go | 4 +- sda/internal/storage/storage.go | 2 +- sda/internal/storage/storage_test.go | 2 +- .../storage/v2/posix/writer/write_file.go | 2 + sda/internal/storage/v2/s3/reader/config.go | 2 +- .../storage/v2/s3/reader/reader_test.go | 12 +-- sda/internal/storage/v2/s3/writer/config.go | 2 +- .../storage/v2/s3/writer/writer_test.go | 8 +- sda/internal/userauth/userauth_test.go | 2 +- 31 files changed, 108 insertions(+), 95 deletions(-) diff --git a/sda-download/api/api.go b/sda-download/api/api.go index 1c8f1816f8..ae53b7a5dc 100644 --- a/sda-download/api/api.go +++ b/sda-download/api/api.go @@ -1,4 +1,4 @@ -package api +package api // nolint:revive import ( "crypto/tls" diff --git a/sda-download/api/api_test.go b/sda-download/api/api_test.go index eff6d19e3f..b7dbbc3c0d 100644 --- a/sda-download/api/api_test.go +++ b/sda-download/api/api_test.go @@ -1,5 +1,4 @@ -// nolint:revive -package api +package api // nolint:revive import ( "testing" diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 18517aec63..b53a233c86 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -71,7 +71,7 @@ type AppConfig struct { // Stores the Crypt4GH private key used internally type transientKeyConf struct { - PrivateKey [32]byte + PrivateKey [32]byte // #nosec G117 -- Export needed to access configuration atm PublicKeyB64 string } @@ -128,7 +128,7 @@ type DatabaseConfig struct { // Database password for username // Optional. Default value lega_out - Password string + Password string // #nosec G117 -- Export needed to access configuration atm // Database name // Optional. Default value lega @@ -148,7 +148,7 @@ type DatabaseConfig struct { // TLS Key for database connection // Optional. - ClientKey string + ClientKey string // #nosec G117 -- Export needed to access configuration atm } type ReencryptConfig struct { @@ -156,7 +156,7 @@ type ReencryptConfig struct { Port int CACert string ClientCert string - ClientKey string + ClientKey string // #nosec G117 -- Export needed to access configuration atm Timeout int } diff --git a/sda-download/internal/storage/storage.go b/sda-download/internal/storage/storage.go index 9c499b81e0..7fc9099332 100644 --- a/sda-download/internal/storage/storage.go +++ b/sda-download/internal/storage/storage.go @@ -131,7 +131,7 @@ type s3Backend struct { type S3Conf struct { URL string Port int - AccessKey string + AccessKey string // #nosec G117 -- Export needed to access configuration atm SecretKey string Bucket string Region string diff --git a/sda-validator/orchestrator/api/api.go b/sda-validator/orchestrator/api/api.go index 7147df26e8..96cdbcb73e 100644 --- a/sda-validator/orchestrator/api/api.go +++ b/sda-validator/orchestrator/api/api.go @@ -1,4 +1,4 @@ -package api +package api // nolint:revive import ( "encoding/json" @@ -297,7 +297,7 @@ func (api *validatorAPIImpl) getUserFiles(userID string, requestedFilePaths []st // Send the request client := &http.Client{} - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- host originates from configuration, TODO verify if to sanitize userID if err != nil { return nil, fmt.Errorf("failed to get response, reason: %v", err) } diff --git a/sda-validator/orchestrator/api/api_test.go b/sda-validator/orchestrator/api/api_test.go index e3746ef6be..a4bde8ed39 100644 --- a/sda-validator/orchestrator/api/api_test.go +++ b/sda-validator/orchestrator/api/api_test.go @@ -83,7 +83,7 @@ func (ts *ValidatorAPITestSuite) SetupSuite() { openapi.ApiHandleFunctions{ ValidatorOrchestratorAPI: &validatorAPIImpl{ sdaAPIURL: ts.httpTestServer.URL, - sdaAPIToken: "mock-sdaAPIToken", + sdaAPIToken: "mock-sdaAPIToken", // #nosec G101 -- hardcoded credentials for unit test validationFileSizeLimit: 1024 * 4, validationJobPreparationQueue: "job-preparation-queue", broker: ts.mockBroker, @@ -324,7 +324,7 @@ func (ts *ValidatorAPITestSuite) TestAdminValidatePost_NoTokenInContext() { ginEngine := openapi.NewRouter(openapi.ApiHandleFunctions{ ValidatorOrchestratorAPI: &validatorAPIImpl{ sdaAPIURL: ts.httpTestServer.URL, - sdaAPIToken: "mock-sdaAPIToken", + sdaAPIToken: "mock-sdaAPIToken", // #nosec G101 -- hardcoded credentials for unit test validationFileSizeLimit: 1024 * 4, validationJobPreparationQueue: "job-preparation-queue", broker: ts.mockBroker, @@ -345,7 +345,7 @@ func (ts *ValidatorAPITestSuite) TestValidatePost_NoTokenInContext() { ginEngine := openapi.NewRouter(openapi.ApiHandleFunctions{ ValidatorOrchestratorAPI: &validatorAPIImpl{ sdaAPIURL: ts.httpTestServer.URL, - sdaAPIToken: "mock-sdaAPIToken", + sdaAPIToken: "mock-sdaAPIToken", // #nosec G101 -- hardcoded credentials for unit test validationFileSizeLimit: 1024 * 4, validationJobPreparationQueue: "job-preparation-queue", broker: ts.mockBroker, @@ -510,7 +510,7 @@ func (ts *ValidatorAPITestSuite) TestResultGet_NoTokenInContext() { ginEngine := openapi.NewRouter(openapi.ApiHandleFunctions{ ValidatorOrchestratorAPI: &validatorAPIImpl{ sdaAPIURL: ts.httpTestServer.URL, - sdaAPIToken: "mock-sdaAPIToken", + sdaAPIToken: "mock-sdaAPIToken", // #nosec G101 -- hardcoded credentials for unit test validationFileSizeLimit: 1024 * 4, validationJobPreparationQueue: "job-preparation-queue", broker: ts.mockBroker, diff --git a/sda-validator/orchestrator/internal/commandexecutor/command_executor.go b/sda-validator/orchestrator/internal/commandexecutor/command_executor.go index a971436df5..18cf50d1c0 100644 --- a/sda-validator/orchestrator/internal/commandexecutor/command_executor.go +++ b/sda-validator/orchestrator/internal/commandexecutor/command_executor.go @@ -11,5 +11,5 @@ type OsCommandExecutor struct { } func (OsCommandExecutor) Execute(name string, args ...string) ([]byte, error) { - return exec.Command(name, args...).Output() + return exec.Command(name, args...).Output() // #nosec G204 launched subprocess hardcoded to be apptainer at callers } diff --git a/sda-validator/orchestrator/jobpreparationworker/worker.go b/sda-validator/orchestrator/jobpreparationworker/worker.go index f121f74296..80b1bc59e7 100644 --- a/sda-validator/orchestrator/jobpreparationworker/worker.go +++ b/sda-validator/orchestrator/jobpreparationworker/worker.go @@ -261,7 +261,7 @@ func (w *worker) downloadFiles(ctx context.Context, validationFilesDir string, v } func (w *worker) downloadFile(_ context.Context, userID, fileID string, file *os.File, pubKeyBase64 string, privateKeyData [32]byte) error { - req, err := http.NewRequest("GET", fmt.Sprintf("%s/users/%s/file/%s", w.conf.sdaAPIURL, userID, fileID), nil) + req, err := http.NewRequest("GET", fmt.Sprintf("%s/users/%s/file/%s", w.conf.sdaAPIURL, userID, fileID), nil) // #nosec G704 -- host controlled by configuration, TODO verify if to sanitize userID and fileID if err != nil { return fmt.Errorf("failed to create the request, reason: %v", err) } @@ -273,7 +273,7 @@ func (w *worker) downloadFile(_ context.Context, userID, fileID string, file *os // Send the request client := &http.Client{} - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- host controlled by configuration, TODO verify if to sanitize userID and fileID if err != nil { return fmt.Errorf("failed to get response, reason: %v", err) } diff --git a/sda-validator/orchestrator/jobpreparationworker/worker_test.go b/sda-validator/orchestrator/jobpreparationworker/worker_test.go index 42d30d3402..07335808e3 100644 --- a/sda-validator/orchestrator/jobpreparationworker/worker_test.go +++ b/sda-validator/orchestrator/jobpreparationworker/worker_test.go @@ -68,7 +68,7 @@ func (ts *JobPreparationWorkerTestSuite) SetupSuite() { } file := bytes.Buffer{} - _, _ = file.Write([]byte(fmt.Sprintf("this is file: %s", filepath.Base(req.URL.Path)))) + _, _ = file.Write([]byte("this is file: " + filepath.Base(req.URL.Path))) _, _ = io.Copy(encryptedFileWriter, &file) _ = encryptedFileWriter.Close() diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 9b07706e0b..1f2e754133 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -164,7 +164,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } @@ -225,7 +225,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } @@ -267,7 +267,7 @@ func TestMain(m *testing.M) { type UserKey struct { PublicKey [32]byte - PrivateKey [32]byte + privateKey [32]byte PrivateKeyPath string PubKeyBase64 string } @@ -489,7 +489,7 @@ func (s *TestSuite) SetupSuite() { s.FailNow("failed to create temp folder") } - s.UserKey.PublicKey, s.UserKey.PrivateKey, err = keys.GenerateKeyPair() + s.UserKey.PublicKey, s.UserKey.privateKey, err = keys.GenerateKeyPair() if err != nil { s.T().FailNow() } @@ -499,7 +499,7 @@ func (s *TestSuite) SetupSuite() { if err != nil { s.T().FailNow() } - if err := keys.WriteCrypt4GHX25519PrivateKey(key, s.UserKey.PrivateKey, []byte("password")); err != nil { + if err := keys.WriteCrypt4GHX25519PrivateKey(key, s.UserKey.privateKey, []byte("password")); err != nil { s.T().FailNow() } @@ -612,7 +612,7 @@ func (s *TestSuite) SetupTest() { req, err := http.NewRequest(http.MethodDelete, "http://"+BrokerAPI+"/api/queues/sda/"+queue+"/contents", http.NoBody) assert.NoError(s.T(), err, "failed to generate query") req.SetBasicAuth("guest", "guest") - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") _ = res.Body.Close() } @@ -656,7 +656,7 @@ func (s *TestSuite) TestAPIGetFiles() { req.Header.Add("Authorization", "Bearer "+s.Token) // Test query when no files is in db - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, resp.StatusCode) @@ -676,7 +676,7 @@ func (s *TestSuite) TestAPIGetFiles() { err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, s.User, "{}", "{}") assert.NoError(s.T(), err, "got (%v) when trying to update file status") - resp, err = client.Do(req) + resp, err = client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, resp.StatusCode) @@ -696,7 +696,7 @@ func (s *TestSuite) TestAPIGetFiles() { err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, s.User, "{}", "{}") assert.NoError(s.T(), err, "got (%v) when trying to update file status") - resp, err = client.Do(req) + resp, err = client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, resp.StatusCode) @@ -716,7 +716,7 @@ func (s *TestSuite) TestAPIGetFiles() { _, err = Conf.API.DB.RegisterFile(nil, file2, s.User) assert.NoError(s.T(), err, "failed to register file in database") - resp, err = client.Do(req) + resp, err = client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, resp.StatusCode) @@ -891,11 +891,14 @@ func (s *TestSuite) TestGinLogLevel_Debug() { } r.Header.Add("Authorization", "Bearer "+s.Token) - _, err = client.Do(r) // nolint: bodyclose + rsp, err := client.Do(r) // #nosec G704 -- request controlled by unit test if err != nil { s.T().Logf("failure: %v", err) s.FailNow("failed to execute HTTP request") } + if rsp != nil && rsp.Body != nil { + _ = rsp.Body.Close() + } // Allow logs to flush time.Sleep(500 * time.Millisecond) @@ -916,11 +919,14 @@ func (s *TestSuite) TestGinLogLevel_Debug() { s.FailNow("failed to create new request instance for /ready") } - _, err = client.Do(r) // nolint: bodyclose + rsp, err = client.Do(r) // #nosec G704 -- request controlled by unit test if err != nil { s.T().Logf("failure: %v", err) s.FailNow("failed to execute HTTP request to /ready") } + if rsp != nil && rsp.Body != nil { + _ = rsp.Body.Close() + } // Allow logs to flush time.Sleep(500 * time.Millisecond) @@ -968,11 +974,14 @@ func (s *TestSuite) TestGinLogLevel_Info() { } r.Header.Add("Authorization", "Bearer "+s.Token) - _, err = client.Do(r) // nolint: bodyclose + rsp, err := client.Do(r) // #nosec G704 -- request controlled by unit test if err != nil { s.T().Logf("failure: %v", err) s.FailNow("failed to execute HTTP request") } + if rsp != nil && rsp.Body != nil { + _ = rsp.Body.Close() + } // Allow logs to flush time.Sleep(500 * time.Millisecond) @@ -993,11 +1002,14 @@ func (s *TestSuite) TestGinLogLevel_Info() { s.FailNow("failed to create new request instance for /ready") } - _, err = client.Do(r) // nolint: bodyclose + rsp, err = client.Do(r) // #nosec G704 -- request controlled by unit test if err != nil { s.T().Logf("failure: %v", err) s.FailNow("failed to execute HTTP request to /ready") } + if rsp != nil && rsp.Body != nil { + _ = rsp.Body.Close() + } // Allow logs to flush time.Sleep(500 * time.Millisecond) @@ -1175,7 +1187,7 @@ func (s *TestSuite) TestIngestFile_WithPayload() { client := http.Client{Timeout: 5 * time.Second} req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/ingest", http.NoBody) req.SetBasicAuth("guest", "guest") - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -1332,7 +1344,7 @@ func (s *TestSuite) TestIngestFile_WithFileID() { client := http.Client{Timeout: 5 * time.Second} req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/ingest", http.NoBody) req.SetBasicAuth("guest", "guest") - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -1457,7 +1469,7 @@ func (s *TestSuite) TestSetAccession_WithPayload() { client := http.Client{Timeout: 5 * time.Second} req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody) req.SetBasicAuth("guest", "guest") - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -1570,7 +1582,7 @@ func (s *TestSuite) TestSetAccession_WithParams() { client := http.Client{Timeout: 5 * time.Second} req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody) req.SetBasicAuth("guest", "guest") - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -1777,7 +1789,7 @@ func (s *TestSuite) TestCreateDataset() { client := http.Client{Timeout: 5 * time.Second} req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/mappings", http.NoBody) req.SetBasicAuth("guest", "guest") - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -2029,7 +2041,7 @@ func (s *TestSuite) TestReleaseDataset() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/mappings", http.NoBody) req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -2370,13 +2382,13 @@ func (s *TestSuite) TestAddC4ghHash() { req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() // Isert pubkey again and expect error - resp2, err := client.Do(req) + resp2, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusConflict, resp2.StatusCode) defer resp2.Body.Close() @@ -2412,7 +2424,7 @@ func (s *TestSuite) TestAddC4ghHash_emptyJson() { req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() @@ -2447,7 +2459,7 @@ func (s *TestSuite) TestAddC4ghHash_notBase64() { req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() @@ -2478,7 +2490,7 @@ func (s *TestSuite) TestListC4ghHashes() { assert.NoError(s.T(), err) req.Header.Add("Authorization", "Bearer "+s.Token) - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() @@ -2513,13 +2525,13 @@ func (s *TestSuite) TestDeprecateC4ghHash() { assert.NoError(s.T(), err) req.Header.Add("Authorization", "Bearer "+s.Token) - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() // a second time gives an error since the key is alreadu deprecated - resp2, err := client.Do(req) + resp2, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusBadRequest, resp2.StatusCode) defer resp2.Body.Close() @@ -2543,7 +2555,7 @@ func (s *TestSuite) TestDeprecateC4ghHash_wrongHash() { assert.NoError(s.T(), err) req.Header.Add("Authorization", "Bearer "+s.Token) - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() @@ -2791,7 +2803,7 @@ func (s *TestSuite) TestReVerifyFile() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/archived", http.NoBody) req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -2898,7 +2910,7 @@ func (s *TestSuite) TestReVerifyDataset() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/archived", http.NoBody) req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` @@ -2959,7 +2971,7 @@ func (s *TestSuite) TestDownloadFile() { req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) defer resp.Body.Close() @@ -2987,7 +2999,7 @@ func (s *TestSuite) TestDownloadFile_badPublicKey() { req.Header.Set("C4GH-Public-Key", "invalid_key") client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) defer resp.Body.Close() @@ -3013,7 +3025,7 @@ func (s *TestSuite) TestDownloadFile_fileIDNotExist() { req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) defer resp.Body.Close() @@ -3046,7 +3058,7 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) defer resp.Body.Close() @@ -3078,7 +3090,7 @@ func (s *TestSuite) TestDownloadFile_badC4ghFile() { req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) client := &http.Client{} - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) defer resp.Body.Close() diff --git a/sda/cmd/auth/cega.go b/sda/cmd/auth/cega.go index 2cd199bdbd..ba41400580 100644 --- a/sda/cmd/auth/cega.go +++ b/sda/cmd/auth/cega.go @@ -49,7 +49,7 @@ func authenticateWithCEGA(conf config.CegaConfig, username string) (*http.Respon req.Header.Add("Authorization", "Basic "+getb64Credentials(conf.ID, conf.Secret)) req.Header.Add("Content-Type", "application/json") - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- auth url controlled by configuration, username originates from login form TODO, verify if username needs sanitizing return res, err } diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 1ac59eacb5..23f77b7f2b 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -129,7 +129,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil || res.StatusCode != 200 { return err } @@ -278,7 +278,7 @@ func (ts *TestSuite) SetupTest() { } outFileName := f.Name() + ".c4gh" - outFile, err := os.Create(outFileName) + outFile, err := os.Create(outFileName) // #nosec G703 -- file controlled by unit test if err != nil { ts.FailNow("failed to create encrypted test file") } @@ -344,7 +344,7 @@ func (ts *TestSuite) TestTryDecrypt() { unencryptedFile, err := os.CreateTemp(tempDir, "unencryptedFile-") assert.NoError(ts.T(), err) - err = os.WriteFile(unencryptedFile.Name(), []byte("content"), 0600) + err = os.WriteFile(unencryptedFile.Name(), []byte("content"), 0600) // #nosec G703 -- file controlled by unit test assert.NoError(ts.T(), err) encryptedFile, err := os.CreateTemp(tempDir, "encryptedFile-") @@ -357,7 +357,7 @@ func (ts *TestSuite) TestTryDecrypt() { assert.NoError(ts.T(), err) crypt4GHWriter.Close() - file, err := os.Open(encryptedFile.Name()) + file, err := os.Open(encryptedFile.Name()) // #nosec G703 -- file controlled by unit test assert.NoError(ts.T(), err) defer file.Close() buf, err := io.ReadAll(file) diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go index 74df8fe2dd..b862195f7b 100644 --- a/sda/cmd/rotatekey/rotatekey_test.go +++ b/sda/cmd/rotatekey/rotatekey_test.go @@ -124,7 +124,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil || res.StatusCode != 200 { return err } diff --git a/sda/cmd/s3inbox/healthchecks.go b/sda/cmd/s3inbox/healthchecks.go index 9cba134a4d..3c33b4e5a0 100644 --- a/sda/cmd/s3inbox/healthchecks.go +++ b/sda/cmd/s3inbox/healthchecks.go @@ -74,7 +74,7 @@ func (p *Proxy) CheckHealth(w http.ResponseWriter, _ *http.Request) { // httpsGetCheck sends a request to the S3 backend and makes sure it is healthy func (p *Proxy) httpsGetCheck(uri string) error { - resp, e := p.client.Get(uri) + resp, e := p.client.Get(uri) // #nosec G704 uri originates from configuration if e != nil { return e } diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index ce0d086b58..94697f6c30 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -342,7 +342,7 @@ func (p *Proxy) forwardToBackend(r *http.Request) (*http.Response, error) { p.resignHeader(r, p.s3.AccessKey, p.s3.SecretKey, fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)) // Redirect request - nr, err := http.NewRequest(r.Method, fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)+r.URL.String(), r.Body) + nr, err := http.NewRequest(r.Method, fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)+r.URL.String(), r.Body) // #nosec G704 -- endpoint and port controlled by configuration, TODO verify if r.URL needs to be sanitized if err != nil { log.Debug("error when redirecting the request") log.Debug(err) @@ -353,7 +353,7 @@ func (p *Proxy) forwardToBackend(r *http.Request) (*http.Response, error) { contentLength, _ := strconv.ParseInt(r.Header.Get("content-length"), 10, 64) nr.ContentLength = contentLength - return p.client.Do(nr) + return p.client.Do(nr) // #nosec G704 -- endpoint and port controlled by configuration, TODO verify if r.URL needs to be sanitized } // Add bucket to host path diff --git a/sda/cmd/s3inbox/s3inbox_test.go b/sda/cmd/s3inbox/s3inbox_test.go index c0dcfae2ba..d8f426d325 100644 --- a/sda/cmd/s3inbox/s3inbox_test.go +++ b/sda/cmd/s3inbox/s3inbox_test.go @@ -74,7 +74,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } @@ -157,7 +157,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } @@ -214,7 +214,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 67c174136c..18e2be26d7 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -262,7 +262,7 @@ func sendPOST(payload []byte) error { } req.Header.Set("Content-Type", "application/json") req.SetBasicAuth(conf.Sync.RemoteUser, conf.Sync.RemotePassword) - resp, err := client.Do(req) + resp, err := client.Do(req) // #nosec G704 host originates from configuration if err != nil { return err } diff --git a/sda/cmd/syncapi/syncapi_test.go b/sda/cmd/syncapi/syncapi_test.go index 9eeeed92be..dc74c3ba5c 100644 --- a/sda/cmd/syncapi/syncapi_test.go +++ b/sda/cmd/syncapi/syncapi_test.go @@ -76,7 +76,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } @@ -267,13 +267,13 @@ func (s *SyncAPITest) TestBasicAuth() { req, err := http.NewRequest("POST", ts.URL+"/metadata", bytes.NewBuffer(goodJSON)) assert.NoError(s.T(), err) req.SetBasicAuth(Conf.SyncAPI.APIUser, Conf.SyncAPI.APIPassword) - good, err := ts.Client().Do(req) + good, err := ts.Client().Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusOK, good.StatusCode) defer good.Body.Close() req.SetBasicAuth(Conf.SyncAPI.APIUser, "wrongpass") - bad, err := ts.Client().Do(req) + bad, err := ts.Client().Do(req) // #nosec G704 -- request controlled by unit test assert.NoError(s.T(), err) assert.Equal(s.T(), http.StatusUnauthorized, bad.StatusCode) defer bad.Body.Close() diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index c3807a7f11..7bf2e67bb6 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -25,7 +25,7 @@ type MQConf struct { Host string Port int User string - Password string + Password string // #nosec G117 -- Export needed to access configuration atm Vhost string Queue string Exchange string @@ -34,7 +34,7 @@ type MQConf struct { VerifyPeer bool CACert string ClientCert string - ClientKey string + ClientKey string // #nosec G117 -- Export needed to access configuration atm ServerName string SchemasPath string PrefetchCount int diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index 843e74b9d5..9418aba5bc 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -82,7 +82,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 58a08e2e9c..4f2da2a18f 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -56,7 +56,7 @@ type Config struct { type Grpc struct { CACert string ClientCert string - ClientKey string + ClientKey string // #nosec G117 -- Export needed to access configuration atm ClientCreds credentials.TransportCredentials ServerCert string ServerKey string @@ -118,7 +118,7 @@ type SessionConfig struct { } type SMTPConf struct { - Password string + Password string // #nosec G117 -- Export needed to access configuration atm FromAddr string Host string Port int @@ -157,14 +157,14 @@ type OIDCConfig struct { Provider string RedirectURL string RevocationURL string - Secret string + Secret string // #nosec G117 -- Export needed to access configuration atm JwkURL string } type CegaConfig struct { AuthURL string ID string - Secret string + Secret string // #nosec G117 -- Export needed to access configuration atm } type CORSConfig struct { diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 7e72a3486c..5d12f041f9 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -16,12 +16,12 @@ type DBConf struct { Host string Port int User string - Password string + Password string // #nosec G117 -- Export needed to access configuration atm Database string CACert string SslMode string ClientCert string - ClientKey string + ClientKey string // #nosec G117 -- Export needed to access configuration atm } // SDAdb struct that acts as a receiver for the DB update methods diff --git a/sda/internal/jsonadapter/jsonadapter_test.go b/sda/internal/jsonadapter/jsonadapter_test.go index d501c932fe..5bcfe6802e 100644 --- a/sda/internal/jsonadapter/jsonadapter_test.go +++ b/sda/internal/jsonadapter/jsonadapter_test.go @@ -52,7 +52,7 @@ func (ts *AdapterTestSuite) SetupSuite() { } func (ts *AdapterTestSuite) TearDownSuite() { - _ = os.RemoveAll(ts.File.Name()) + _ = os.RemoveAll(ts.File.Name()) // #nosec G703 -- file controlled by unit test } func (ts *AdapterTestSuite) TestAdapter_empty() { @@ -79,7 +79,7 @@ func (ts *AdapterTestSuite) TestAdapter() { } func (ts *AdapterTestSuite) TestAdapter_fromFile() { - b, err := os.ReadFile(ts.File.Name()) + b, err := os.ReadFile(ts.File.Name()) // #nosec G703 -- file controlled by unit test assert.NoError(ts.T(), err, "failed to read json file") e, err := casbin.NewEnforcer(ts.Model, NewAdapter(&b)) assert.NoError(ts.T(), err, "New enforcer failed with policy from file: %s", ts.File.Name()) diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index 41f5aa8d20..cddaee0135 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -156,7 +156,7 @@ type s3Backend struct { type S3Conf struct { URL string Port int - AccessKey string + AccessKey string // #nosec G117 -- Export needed to access configuration atm SecretKey string Bucket string Region string diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index 67b626dc21..c87c5d50d8 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -107,7 +107,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } diff --git a/sda/internal/storage/v2/posix/writer/write_file.go b/sda/internal/storage/v2/posix/writer/write_file.go index c9d1be2e81..2bbec37d02 100644 --- a/sda/internal/storage/v2/posix/writer/write_file.go +++ b/sda/internal/storage/v2/posix/writer/write_file.go @@ -54,6 +54,7 @@ func (writer *Writer) WriteFile(ctx context.Context, filePath string, fileConten if noEmptyParentsErr := writer.ensureNoEmptyParentDirectories(location, filePath); noEmptyParentsErr != nil { log.Errorf("failed to ensure no empty parent directories exist due to: %v", noEmptyParentsErr) } + // #nosec G703 -- TODO file path to be validated in upcoming PR if osRemoveErr := os.Remove(tempFile.Name()); err != nil && !errors.Is(osRemoveErr, os.ErrNotExist) { log.Errorf("failed to remove temp file due to: %v after write failed", osRemoveErr) } @@ -75,6 +76,7 @@ func (writer *Writer) WriteFile(ctx context.Context, filePath string, fileConten return "", fmt.Errorf("failed to ensure parent directories: %s parentDirectories exists at location: %s, due to: %v", parentDirectories, location, err) } + // #nosec G703 -- TODO file path to be validated in upcoming PR if err = os.Rename(tempFile.Name(), filepath.Join(location, filePath)); err != nil { return "", fmt.Errorf("failed to rename temporary file: %s to %s at location: %s, due to: %v", tempFile.Name(), filePath, location, err) } diff --git a/sda/internal/storage/v2/s3/reader/config.go b/sda/internal/storage/v2/s3/reader/config.go index bf82ecf160..533d858d44 100644 --- a/sda/internal/storage/v2/s3/reader/config.go +++ b/sda/internal/storage/v2/s3/reader/config.go @@ -21,7 +21,7 @@ import ( ) type endpointConfig struct { - AccessKey string `mapstructure:"access_key"` + AccessKey string `mapstructure:"access_key"` // #nosec G117 -- needs to be exported for unmarshalling CACert string `mapstructure:"ca_cert"` ChunkSize string `mapstructure:"chunk_size"` chunkSizeBytes uint64 diff --git a/sda/internal/storage/v2/s3/reader/reader_test.go b/sda/internal/storage/v2/s3/reader/reader_test.go index cfbe731c31..c74c7db3ee 100644 --- a/sda/internal/storage/v2/s3/reader/reader_test.go +++ b/sda/internal/storage/v2/s3/reader/reader_test.go @@ -94,7 +94,7 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test case strings.HasPrefix(req.RequestURI, "/mock_s3_1_bucket_2"): if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file2.txt") { @@ -141,10 +141,10 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test default: w.WriteHeader(http.StatusInternalServerError) - _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test } })) @@ -214,7 +214,7 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test case strings.HasPrefix(req.RequestURI, "/mock_s3_2_bucket_2"): if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file4.txt") { w.WriteHeader(http.StatusOK) @@ -231,10 +231,10 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test default: w.WriteHeader(http.StatusInternalServerError) - _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test } })) diff --git a/sda/internal/storage/v2/s3/writer/config.go b/sda/internal/storage/v2/s3/writer/config.go index b3d96487e4..d0ef2cef12 100644 --- a/sda/internal/storage/v2/s3/writer/config.go +++ b/sda/internal/storage/v2/s3/writer/config.go @@ -25,7 +25,7 @@ import ( ) type endpointConfig struct { - AccessKey string `mapstructure:"access_key"` + AccessKey string `mapstructure:"access_key"` // #nosec G117 -- needs to be exported for unmarshalling BucketPrefix string `mapstructure:"bucket_prefix"` CACert string `mapstructure:"ca_cert"` ChunkSize string `mapstructure:"chunk_size"` diff --git a/sda/internal/storage/v2/s3/writer/writer_test.go b/sda/internal/storage/v2/s3/writer/writer_test.go index 782ef6e90a..1078616307 100644 --- a/sda/internal/storage/v2/s3/writer/writer_test.go +++ b/sda/internal/storage/v2/s3/writer/writer_test.go @@ -60,12 +60,12 @@ func (m *mockS3) ListBuckets(w http.ResponseWriter) { `) for bucket := range m.buckets { - _, _ = b.WriteString(fmt.Sprintf(` + _, _ = b.WriteString(` - %s + ` + bucket + ` us-east-1 - %s - `, bucket, bucket)) + ` + bucket + ` + `) } _, _ = b.WriteString(` diff --git a/sda/internal/userauth/userauth_test.go b/sda/internal/userauth/userauth_test.go index 4de8aa0c0d..d669afc9d4 100644 --- a/sda/internal/userauth/userauth_test.go +++ b/sda/internal/userauth/userauth_test.go @@ -90,7 +90,7 @@ func TestMain(m *testing.M) { // exponential backoff-retry, because the application in the container might not be ready to accept connections yet if err := pool.Retry(func() error { - res, err := client.Do(req) + res, err := client.Do(req) // #nosec G704 -- request controlled by unit test if err != nil { return err } From bd5c5e4d5850906dab2a0dc1a136c0597d34b44c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 18 Feb 2026 14:51:51 +0100 Subject: [PATCH 058/530] fix: add // nolint:revive in all api package files --- sda-validator/orchestrator/api/api_test.go | 2 +- sda-validator/orchestrator/api/options.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-validator/orchestrator/api/api_test.go b/sda-validator/orchestrator/api/api_test.go index a4bde8ed39..06d226b8fc 100644 --- a/sda-validator/orchestrator/api/api_test.go +++ b/sda-validator/orchestrator/api/api_test.go @@ -1,4 +1,4 @@ -package api +package api // nolint:revive import ( "bytes" diff --git a/sda-validator/orchestrator/api/options.go b/sda-validator/orchestrator/api/options.go index f085e5c085..690757c227 100644 --- a/sda-validator/orchestrator/api/options.go +++ b/sda-validator/orchestrator/api/options.go @@ -1,4 +1,4 @@ -package api +package api // nolint:revive import "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/broker" From 15864b7574136b8db32d0742df27c2a66aeb5b04 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 12 Feb 2026 10:49:54 +0100 Subject: [PATCH 059/530] docs(adr): bootstrap ADR framework with MADR 4.0.0 Add ADR-0000 establishing the use of Markdown Architectural Decision Records, the MADR 4.0.0 template, a README guide for the docs/adr/ directory, and an ADR consideration checkbox in the PR template. --- .github/PULL_REQUEST_TEMPLATE.md | 5 + ...markdown-architectural-decision-records.md | 109 ++++++++++++++++++ docs/decisions/README.md | 94 +++++++++++++++ docs/decisions/template.md | 75 ++++++++++++ 4 files changed, 283 insertions(+) create mode 100644 docs/decisions/0000-use-markdown-architectural-decision-records.md create mode 100644 docs/decisions/README.md create mode 100644 docs/decisions/template.md diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9a3333b6bb..3ff06bd75a 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,4 +5,9 @@ This PR closes [issue number]. ## Description +## ADR + +- [ ] This PR includes an architecturally significant decision (see `docs/decisions/README.md`) + - [ ] A decision record has been added or updated in `docs/decisions/` + ## How to test diff --git a/docs/decisions/0000-use-markdown-architectural-decision-records.md b/docs/decisions/0000-use-markdown-architectural-decision-records.md new file mode 100644 index 0000000000..829af77ab3 --- /dev/null +++ b/docs/decisions/0000-use-markdown-architectural-decision-records.md @@ -0,0 +1,109 @@ +--- +status: accepted +date: 2026-02-12 +decision-makers: + - "@neicnordic/sensitive-data-development-collaboration" +--- + +# Use Markdown Architectural Decision Records + +## Context and Problem Statement + +The SDA project is developed by a distributed team across multiple organisations. +Architectural decisions and their rationale are currently scattered across pull +request discussions, GitHub issues, Slack threads, and meeting notes. This makes it +difficult for contributors — both existing and new — to discover *why* the system is +designed the way it is. + +We need a lightweight, discoverable way to record architecturally significant +decisions so that the reasoning behind them is preserved alongside the code. + +## Decision Drivers + +* **Async-first communication** — team members work across time zones; decisions + must be readable without attending the meeting where they were made. +* **Contributor onboarding** — new contributors should be able to understand past + design choices without archaeology through old PRs and Slack logs. +* **Markdown-native** — records should live in the repository, render on GitHub, and + be reviewable in pull requests like any other change. +* **Low ceremony** — the process must be lightweight enough that people actually use + it; a heavy process will be ignored. + +## Considered Options + +1. **MADR 4.0.0** — Markdown Any Decision Records with YAML front matter +2. **Nygard template** — Michael Nygard's original ADR format (Title, Status, + Context, Decision, Consequences) +3. **Y-Statements** — single-sentence structured rationale +4. **Formless** — free-form Markdown files with no template + +## Decision Outcome + +Chosen option: **MADR 4.0.0**, because it provides enough structure to enable +consistent records and future tooling (via YAML front matter) while remaining +lightweight enough for a small distributed team. The optional-section design means +simple decisions stay simple, while complex decisions can use the full template. + +### Consequences + +* **Good**: Consistent format across all ADRs makes them easier to read, write, and + review. +* **Good**: YAML front matter enables future automation (status dashboards, linting, + index generation). +* **Good**: Living in the repo means decisions are versioned, searchable, and + co-located with the code they describe. +* **Neutral**: Team members must remember to consider whether a change warrants an + ADR; the PR template checkbox serves as a nudge. +* **Bad**: Small per-ADR overhead (creating a file, filling in sections, reviewing); + mitigated by the template and optional sections. + +### Confirmation + +The existence of this file (ADR-0000) in the repository confirms the decision has +been implemented. + +## Pros and Cons of the Options + +### MADR 4.0.0 + +* Good, because YAML front matter enables tooling and metadata queries. +* Good, because optional sections keep simple ADRs short. +* Good, because it is actively maintained with clear versioning. +* Neutral, because the template is slightly longer than Nygard's. + +### Nygard template + +* Good, because it is the most widely known ADR format. +* Good, because it is very concise (five sections). +* Bad, because it lacks structured metadata (no front matter). +* Bad, because it has no concept of decision drivers or options comparison. + +### Y-Statements + +* Good, because they are extremely concise (one sentence). +* Bad, because they lack room for context, alternatives, or consequences. +* Bad, because they are too terse for decisions that need explanation. + +### Formless + +* Good, because there is zero ceremony. +* Bad, because inconsistent structure makes records harder to read and compare. +* Bad, because no metadata means no tooling support. + +## More Information + +### Conventions Established by This ADR + +| Convention | Detail | +|---|---| +| Directory | `docs/decisions/` | +| File naming | `NNNN-title-with-dashes.md` — zero-padded, lowercase, dashes between words | +| Numbering | Starts at 0000; numbers are never reused, even for rejected or superseded decisions | +| Status lifecycle | `proposed` → `accepted` \| `rejected` \| `deprecated` \| `superseded` | +| Template | `docs/decisions/template.md` (MADR 4.0.0) | +| Backfilling | Incremental — no mandate to retroactively document every past decision | +| PR nudge | A checkbox in the pull request template reminds authors to consider adding a decision record | + +For guidance on when and how to write a decision record, see [`docs/decisions/README.md`](README.md). + +The template used here is [MADR 4.0.0](https://github.com/adr/madr/blob/4.0.0/template/adr-template.md). diff --git a/docs/decisions/README.md b/docs/decisions/README.md new file mode 100644 index 0000000000..30a23e988f --- /dev/null +++ b/docs/decisions/README.md @@ -0,0 +1,94 @@ +# Decisions + +This directory captures the decisions made in the SDA project. For the full +rationale behind adopting this practice, see +[ADR-0000](0000-use-markdown-architectural-decision-records.md). + +## Philosophy + +Decision records are a way for the team to think smarter and communicate +better — they are not after-the-fact paperwork. Focus on the *"why"*, not the +*"what"*. A good decision record helps a future contributor (or your future self) +understand the reasoning behind a choice. + +Decisions are not limited to software architecture. Technology choices, vendor +decisions, conventions, and process changes all belong here if they are worth +explaining to someone who wasn't in the room. + +Anyone can propose a decision record. If you think future contributors will +wonder *"why did we do it this way?"*, that is a good signal to write one. + +## When to write a decision record + +Consider writing one when a change involves: + +* Structural changes to the codebase (new services, module boundaries) +* Technology or library choices +* Conventions and standards (coding style, API design, naming) +* Non-obvious trade-offs that future contributors will wonder about + +Each decision record should cover a single decision — not a bundle of +unrelated choices. + +Skip a decision record when the choice is small-scoped, easily reversible, and +already obvious from the code or commit message. + +Not every change needs a record — use your judgement. The pull request template +includes a checkbox as a reminder. + +## How to create a decision record + +1. Copy the template: + ```sh + cp docs/decisions/template.md docs/decisions/NNNN-title-with-dashes.md + ``` + Use the next available number, zero-padded to four digits. + +2. Fill in the sections. Remove any optional sections (marked by HTML comments) + you do not need. + +3. Set `status: proposed` in the front matter. + +4. Open a pull request. The decision record is accepted when the PR is merged + (update the status to `accepted`). + +## Tips for good decision records + +* **Focus on rationale** — explain *why* the decision was made, not just *what* + was decided. The "what" is visible in the code; the "why" is not. +* **Be specific** — each record should address one decision with enough context + to stand on its own. +* **Keep them living documents** — if circumstances change, update the record + with a date-stamped note rather than letting it silently go stale. For major + reversals, create a new record and mark the old one as `superseded`. + +## Status lifecycle + +| Status | Meaning | +|---|---| +| `proposed` | Under discussion in a PR | +| `accepted` | Merged and in effect | +| `rejected` | Considered but not adopted | +| `deprecated` | No longer applies (but kept for history) | +| `superseded` | Replaced by a newer decision record (link to it in the front matter) | + +Numbers are never reused. Superseded and rejected records stay in the directory +for historical context. + +**To supersede a decision**: create the new record, then update the old record's +front matter to `status: superseded` and set `superseded-by` to the new file name: + +```yaml +status: superseded +superseded-by: "0005-use-new-approach.md" +``` + +## Index + +| # | Decision | Status | +|---|----------|--------| +| [0000](0000-use-markdown-architectural-decision-records.md) | Use Markdown Architectural Decision Records | accepted | + +## More information + +The template follows [MADR 4.0.0](https://github.com/adr/madr/blob/4.0.0/template/adr-template.md). diff --git a/docs/decisions/template.md b/docs/decisions/template.md new file mode 100644 index 0000000000..ba343b58d5 --- /dev/null +++ b/docs/decisions/template.md @@ -0,0 +1,75 @@ +--- +# These are optional metadata elements. Feel free to remove any of them. +status: proposed # proposed | accepted | rejected | deprecated | superseded +superseded-by: # e.g. "0005-new-approach.md" (only when status is superseded) +date: {YYYY-MM-DD when the decision was last updated} +decision-makers: {list everyone involved in the decision} +consulted: {list everyone whose opinions are sought (typically subject-matter experts); and with whom there is a two-way communication} +informed: {list everyone who is kept up-to-date on progress; and with whom there is a one-way communication} +--- + +# {short title, representative of solved problem and found solution} + +## Context and Problem Statement + +{Describe the context and problem statement, e.g., in free form using two to three sentences or in the form of an illustrative story. You may want to articulate the problem in form of a question and add links to collaboration boards or issue management systems.} + + +## Decision Drivers + +* {decision driver 1, e.g., a force, facing concern, …} +* {decision driver 2, e.g., a force, facing concern, …} +* … + +## Considered Options + +* {title of option 1} +* {title of option 2} +* {title of option 3} +* … + +## Decision Outcome + +Chosen option: "{title of option 1}", because {justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force {force} | … | comes out best (see below)}. + + +### Consequences + +* Good, because {positive consequence, e.g., improvement of one or more desired qualities, …} +* Bad, because {negative consequence, e.g., compromising one or more desired qualities, …} +* … + + +### Confirmation + +{Describe how the implementation / compliance of the ADR can/will be confirmed. Is there any automated or manual fitness function? If so, list it and explain how it is applied. Is the chosen design and its implementation in line with the decision? E.g., a design/code review or a test with a library such as ArchUnit can help validate this. Note that although we classify this element as optional, it is included in many ADRs.} + + +## Pros and Cons of the Options + +### {title of option 1} + + +{example | description | pointer to more information | …} + +* Good, because {argument a} +* Good, because {argument b} + +* Neutral, because {argument c} +* Bad, because {argument d} +* … + +### {title of other option} + +{example | description | pointer to more information | …} + +* Good, because {argument a} +* Good, because {argument b} +* Neutral, because {argument c} +* Bad, because {argument d} +* … + + +## More Information + +{You might want to provide additional evidence/confidence for the decision outcome here and/or document the team agreement on the decision and/or define when/how this decision the decision should be realized and if/when it should be re-visited. Links to other decisions and resources might appear here as well.} From fb7eb4f5754597ea7861e2d685693116168b9e8f Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Sat, 14 Feb 2026 07:21:21 +0100 Subject: [PATCH 060/530] docs(adr): document acceptance process via bi-weekly meeting Add step to the README clarifying that proposed decision records are discussed at the NeIC SDA-Devs bi-weekly meet-up before being accepted and merged. Addresses review feedback from Karl. --- docs/decisions/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 30a23e988f..6c90f28718 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -49,8 +49,11 @@ includes a checkbox as a reminder. 3. Set `status: proposed` in the front matter. -4. Open a pull request. The decision record is accepted when the PR is merged - (update the status to `accepted`). +4. Open a pull request and label it `adr`. + +5. The PR is brought up at the next **NeIC SDA-Devs bi-weekly meet-up** for + discussion. The decision record is accepted when the team agrees and the PR + is merged (update the status to `accepted`). ## Tips for good decision records From dd766601dfdb3cd5ab1516cbf00ec403c8fdccbd Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 12 Feb 2026 14:36:37 +0100 Subject: [PATCH 061/530] feat(ci): add markdownlint for decision records Add markdownlint-cli2 workflow to lint docs/decisions/**/*.md with MADR-appropriate rules (disabled MD013 line length and MD024 duplicate headings). Fix existing table formatting and code block spacing to pass linting. Workflow includes merge_group trigger and path filters to avoid unnecessary CI runs. Config placed at repo root for global applicability. --- .github/workflows/lint-markdown.yml | 29 +++++++++++++++++++ .markdownlint.yml | 19 ++++++++++++ ...markdown-architectural-decision-records.md | 2 +- docs/decisions/README.md | 6 ++-- 4 files changed, 53 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/lint-markdown.yml create mode 100644 .markdownlint.yml diff --git a/.github/workflows/lint-markdown.yml b/.github/workflows/lint-markdown.yml new file mode 100644 index 0000000000..769f721c4d --- /dev/null +++ b/.github/workflows/lint-markdown.yml @@ -0,0 +1,29 @@ +name: Lint Markdown + +on: + merge_group: + pull_request: + paths: + - "docs/decisions/**" + - ".github/workflows/lint-markdown.yml" + push: + branches: + - main + paths: + - "docs/decisions/**" + - ".github/workflows/lint-markdown.yml" + workflow_dispatch: + +jobs: + markdownlint: + name: Decision records + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Lint decision records + uses: DavidAnson/markdownlint-cli2-action@v18 + with: + globs: | + docs/decisions/**/*.md diff --git a/.markdownlint.yml b/.markdownlint.yml new file mode 100644 index 0000000000..0c5882d8d1 --- /dev/null +++ b/.markdownlint.yml @@ -0,0 +1,19 @@ +default: true + +# Allow arbitrary line length +# +# Reason: We apply the one-sentence-per-line rule. A sentence may get longer than 80 characters, especially if links are contained. +# +# Details: https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md013---line-length +MD013: false + +# Allow duplicate headings +# +# Reasons: +# +# - The chosen option is considerably often used as title of the decision record (e.g., "Use MADR 4.0.0" appears in both the title and the decision outcome). +# - We use "Examples" or similar headings multiple times in some records. +# - Markdown lint should support the user and not annoy them. +# +# Details: https://github.com/DavidAnson/markdownlint/blob/main/doc/Rules.md#md024---multiple-headings-with-the-same-content +MD024: false diff --git a/docs/decisions/0000-use-markdown-architectural-decision-records.md b/docs/decisions/0000-use-markdown-architectural-decision-records.md index 829af77ab3..7e13da8b8e 100644 --- a/docs/decisions/0000-use-markdown-architectural-decision-records.md +++ b/docs/decisions/0000-use-markdown-architectural-decision-records.md @@ -95,7 +95,7 @@ been implemented. ### Conventions Established by This ADR | Convention | Detail | -|---|---| +| --- | --- | | Directory | `docs/decisions/` | | File naming | `NNNN-title-with-dashes.md` — zero-padded, lowercase, dashes between words | | Numbering | Starts at 0000; numbers are never reused, even for rejected or superseded decisions | diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 6c90f28718..f3353c14c6 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -39,9 +39,11 @@ includes a checkbox as a reminder. ## How to create a decision record 1. Copy the template: + ```sh cp docs/decisions/template.md docs/decisions/NNNN-title-with-dashes.md ``` + Use the next available number, zero-padded to four digits. 2. Fill in the sections. Remove any optional sections (marked by HTML comments) @@ -68,7 +70,7 @@ includes a checkbox as a reminder. ## Status lifecycle | Status | Meaning | -|---|---| +| --- | --- | | `proposed` | Under discussion in a PR | | `accepted` | Merged and in effect | | `rejected` | Considered but not adopted | @@ -89,7 +91,7 @@ superseded-by: "0005-use-new-approach.md" ## Index | # | Decision | Status | -|---|----------|--------| +| --- | --- | --- | | [0000](0000-use-markdown-architectural-decision-records.md) | Use Markdown Architectural Decision Records | accepted | ## More information From f316310740610d89dc575db3cdd4ca97273343fd Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 17 Feb 2026 14:26:30 +0100 Subject: [PATCH 062/530] fix(adr): address review feedback on template and CI workflow - Use valid YAML in template front matter placeholders - Fix duplicated "decision the decision" wording in template - Add .markdownlint.yml to CI path filters so config changes trigger lint --- .github/workflows/lint-markdown.yml | 2 ++ docs/decisions/template.md | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint-markdown.yml b/.github/workflows/lint-markdown.yml index 769f721c4d..32572efa1c 100644 --- a/.github/workflows/lint-markdown.yml +++ b/.github/workflows/lint-markdown.yml @@ -6,12 +6,14 @@ on: paths: - "docs/decisions/**" - ".github/workflows/lint-markdown.yml" + - ".markdownlint.yml" push: branches: - main paths: - "docs/decisions/**" - ".github/workflows/lint-markdown.yml" + - ".markdownlint.yml" workflow_dispatch: jobs: diff --git a/docs/decisions/template.md b/docs/decisions/template.md index ba343b58d5..bdd53d17e2 100644 --- a/docs/decisions/template.md +++ b/docs/decisions/template.md @@ -2,10 +2,10 @@ # These are optional metadata elements. Feel free to remove any of them. status: proposed # proposed | accepted | rejected | deprecated | superseded superseded-by: # e.g. "0005-new-approach.md" (only when status is superseded) -date: {YYYY-MM-DD when the decision was last updated} -decision-makers: {list everyone involved in the decision} -consulted: {list everyone whose opinions are sought (typically subject-matter experts); and with whom there is a two-way communication} -informed: {list everyone who is kept up-to-date on progress; and with whom there is a one-way communication} +date: "YYYY-MM-DD" # when the decision was last updated +decision-makers: [] # list everyone involved in the decision +consulted: [] # list everyone whose opinions are sought (typically subject-matter experts); and with whom there is a two-way communication +informed: [] # list everyone who is kept up-to-date on progress; and with whom there is a one-way communication --- # {short title, representative of solved problem and found solution} @@ -72,4 +72,4 @@ Chosen option: "{title of option 1}", because {justification. e.g., only option, ## More Information -{You might want to provide additional evidence/confidence for the decision outcome here and/or document the team agreement on the decision and/or define when/how this decision the decision should be realized and if/when it should be re-visited. Links to other decisions and resources might appear here as well.} +{You might want to provide additional evidence/confidence for the decision outcome here and/or document the team agreement on the decision and/or define when/how the decision should be realized and if/when it should be re-visited. Links to other decisions and resources might appear here as well.} From 2af7331b8737092004e7b72e5ae8bc1cdf321f52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 20:23:04 +0000 Subject: [PATCH 063/530] build(deps): bump the go_modules group across 3 directories with 1 update Bumps the go_modules group with 1 update in the /.github/integration/tests/validator-orchestrator directory: [filippo.io/edwards25519](https://github.com/FiloSottile/edwards25519). Bumps the go_modules group with 1 update in the /sda-admin directory: [filippo.io/edwards25519](https://github.com/FiloSottile/edwards25519). Bumps the go_modules group with 1 update in the /sda-validator/orchestrator directory: [filippo.io/edwards25519](https://github.com/FiloSottile/edwards25519). Updates `filippo.io/edwards25519` from 1.1.0 to 1.1.1 - [Commits](https://github.com/FiloSottile/edwards25519/compare/v1.1.0...v1.1.1) Updates `filippo.io/edwards25519` from 1.1.0 to 1.1.1 - [Commits](https://github.com/FiloSottile/edwards25519/compare/v1.1.0...v1.1.1) Updates `filippo.io/edwards25519` from 1.1.0 to 1.1.1 - [Commits](https://github.com/FiloSottile/edwards25519/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: filippo.io/edwards25519 dependency-version: 1.1.1 dependency-type: indirect dependency-group: go_modules - dependency-name: filippo.io/edwards25519 dependency-version: 1.1.1 dependency-type: indirect dependency-group: go_modules - dependency-name: filippo.io/edwards25519 dependency-version: 1.1.1 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- .github/integration/tests/validator-orchestrator/go.mod | 2 +- .github/integration/tests/validator-orchestrator/go.sum | 4 ++-- sda-admin/go.mod | 2 +- sda-admin/go.sum | 4 ++-- sda-validator/orchestrator/go.mod | 2 +- sda-validator/orchestrator/go.sum | 4 ++-- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/integration/tests/validator-orchestrator/go.mod b/.github/integration/tests/validator-orchestrator/go.mod index cd4302ddbc..e684f09d91 100644 --- a/.github/integration/tests/validator-orchestrator/go.mod +++ b/.github/integration/tests/validator-orchestrator/go.mod @@ -15,7 +15,7 @@ require ( ) require ( - filippo.io/edwards25519 v1.1.0 // indirect + filippo.io/edwards25519 v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect diff --git a/.github/integration/tests/validator-orchestrator/go.sum b/.github/integration/tests/validator-orchestrator/go.sum index fdfa907be7..d8fe3dc9fe 100644 --- a/.github/integration/tests/validator-orchestrator/go.sum +++ b/.github/integration/tests/validator-orchestrator/go.sum @@ -1,5 +1,5 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= +filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk= github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= diff --git a/sda-admin/go.mod b/sda-admin/go.mod index 6a388feef5..f1b18a1e02 100644 --- a/sda-admin/go.mod +++ b/sda-admin/go.mod @@ -9,7 +9,7 @@ require ( ) require ( - filippo.io/edwards25519 v1.1.0 // indirect + filippo.io/edwards25519 v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/sda-admin/go.sum b/sda-admin/go.sum index b9308fa25f..c8a47e14c0 100644 --- a/sda-admin/go.sum +++ b/sda-admin/go.sum @@ -1,5 +1,5 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= +filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index cc3a3a84ce..175fdae368 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -20,7 +20,7 @@ require ( ) require ( - filippo.io/edwards25519 v1.1.0 // indirect + filippo.io/edwards25519 v1.1.1 // indirect github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.14.2 // indirect diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index 6b1c880a92..dc069db626 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -1,5 +1,5 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= +filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= From 7aecfb178ae15e756f45a4377a6ba3ff8cf24b8b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Feb 2026 19:14:19 +0000 Subject: [PATCH 064/530] build(deps): bump filippo.io/edwards25519 in /sda-download Bumps [filippo.io/edwards25519](https://github.com/FiloSottile/edwards25519) from 1.1.0 to 1.1.1. - [Commits](https://github.com/FiloSottile/edwards25519/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: filippo.io/edwards25519 dependency-version: 1.1.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 2af6780cce..cd1729e1d9 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -23,7 +23,7 @@ require ( ) require ( - filippo.io/edwards25519 v1.1.0 // indirect + filippo.io/edwards25519 v1.1.1 // indirect github.com/bytedance/sonic v1.14.0 // indirect github.com/bytedance/sonic/loader v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 8878e0753e..26548e8277 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -1,5 +1,5 @@ -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= +filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= From e4c7e0bed6851bc863eb4c99dda14f51361e4abc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 20 Feb 2026 08:01:37 +0000 Subject: [PATCH 065/530] build(deps): bump filippo.io/edwards25519 from 1.1.0 to 1.1.1 in /sda Bumps [filippo.io/edwards25519](https://github.com/FiloSottile/edwards25519) from 1.1.0 to 1.1.1. - [Commits](https://github.com/FiloSottile/edwards25519/compare/v1.1.0...v1.1.1) --- updated-dependencies: - dependency-name: filippo.io/edwards25519 dependency-version: 1.1.1 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index de695512b3..fa0b5a9d9c 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -41,7 +41,7 @@ require ( require ( dario.cat/mergo v1.0.1 // indirect - filippo.io/edwards25519 v1.1.0 // indirect + filippo.io/edwards25519 v1.1.1 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect github.com/BurntSushi/toml v1.5.0 // indirect github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect diff --git a/sda/go.sum b/sda/go.sum index ce505d03a9..1fdab56d82 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -1,7 +1,7 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= -filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= +filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= From 11facb5fe0725378d1b98d23832fb3790bc8aef7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 20:44:58 +0000 Subject: [PATCH 066/530] build(deps): bump DavidAnson/markdownlint-cli2-action from 18 to 22 Bumps [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action) from 18 to 22. - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v18...v22) --- updated-dependencies: - dependency-name: DavidAnson/markdownlint-cli2-action dependency-version: '22' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/lint-markdown.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint-markdown.yml b/.github/workflows/lint-markdown.yml index 32572efa1c..a2f1ba7420 100644 --- a/.github/workflows/lint-markdown.yml +++ b/.github/workflows/lint-markdown.yml @@ -25,7 +25,7 @@ jobs: uses: actions/checkout@v6 - name: Lint decision records - uses: DavidAnson/markdownlint-cli2-action@v18 + uses: DavidAnson/markdownlint-cli2-action@v22 with: globs: | docs/decisions/**/*.md From 3be9f33919a4967ba03f17f21d14645f14f627f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 20:45:05 +0000 Subject: [PATCH 067/530] build(deps): bump aquasecurity/trivy-action from 0.34.0 to 0.34.1 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.34.0 to 0.34.1. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.34.0...0.34.1) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.34.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- .github/workflows/scan-images.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 31c0d5c559..9b2090723c 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -117,7 +117,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.34.0 + uses: aquasecurity/trivy-action@0.34.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -135,7 +135,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.34.0 + uses: aquasecurity/trivy-action@0.34.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -184,7 +184,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.34.0 + uses: aquasecurity/trivy-action@0.34.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -215,7 +215,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.34.0 + uses: aquasecurity/trivy-action@0.34.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index 4334f51fd4..d4f9d3f897 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -27,7 +27,7 @@ jobs: image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.34.0 + uses: aquasecurity/trivy-action@0.34.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -53,7 +53,7 @@ jobs: security-events: write steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.34.0 + uses: aquasecurity/trivy-action@0.34.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From fef76e95be8d6d9a3e6a6437507fb9c32c42e46a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 20:52:10 +0000 Subject: [PATCH 068/530] build(deps): bump the all-modules group in /sda with 6 updates Bumps the all-modules group in /sda with 6 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.1` | `1.41.2` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.7` | `1.32.10` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.19.7` | `1.19.10` | | [github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager](https://github.com/aws/aws-sdk-go-v2) | `0.1.2` | `0.1.5` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.96.0` | `1.96.1` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.24.0` | `1.24.1` | Updates `github.com/aws/aws-sdk-go-v2` from 1.41.1 to 1.41.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.1...v1.41.2) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.7 to 1.32.10 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.32.7...config/v1.32.10) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.7 to 1.19.10 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/m2/v1.19.7...service/iam/v1.19.10) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager` from 0.1.2 to 0.1.5 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/ec2imds/v0.1.2...ec2imds/v0.1.5) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.96.0 to 1.96.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.96.0...service/s3/v1.96.1) Updates `github.com/aws/smithy-go` from 1.24.0 to 1.24.1 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.24.0...v1.24.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager dependency-version: 0.1.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.96.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/smithy-go dependency-version: 1.24.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 38 +++++++++++++-------------- sda/go.sum | 76 +++++++++++++++++++++++++++--------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index fa0b5a9d9c..b82f5a2cd1 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,12 +3,12 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.41.1 - github.com/aws/aws-sdk-go-v2/config v1.32.7 - github.com/aws/aws-sdk-go-v2/credentials v1.19.7 - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 - github.com/aws/smithy-go v1.24.0 + github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2/config v1.32.10 + github.com/aws/aws-sdk-go-v2/credentials v1.19.10 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.5 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.1 + github.com/aws/smithy-go v1.24.1 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 github.com/coreos/go-oidc/v3 v3.17.0 @@ -51,20 +51,20 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.14.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 1fdab56d82..f3531c8048 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,46 +24,46 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= -github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= -github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY= -github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY= -github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8= -github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2 h1:1q8/WwEqZnM/vO4q1gx2g7lHYmyN+o4P7G6EW4zKbRQ= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.2/go.mod h1:owKRexW+Ir5ACD2UTesmjkQ+w7mcmknLNfwOiKfVLTg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= +github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= +github.com/aws/aws-sdk-go-v2/config v1.32.10 h1:9DMthfO6XWZYLfzZglAgW5Fyou2nRI5CuV44sTedKBI= +github.com/aws/aws-sdk-go-v2/config v1.32.10/go.mod h1:2rUIOnA2JaiqYmSKYmRJlcMWy6qTj1vuRFscppSBMcw= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.5 h1:ArBXPSauOifB8IvX1R77qw9YbGzvZJBFJMdpKBAO5YM= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.5/go.mod h1:BHwO94hM6u8TGk7pxs0GCh7t/inrq300kYRksFyB2LY= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= -github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= -github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.9 h1:IJRzQTvdpjHRPItx9gzNcz7Y1F+xqAR+xiy9rr5ZYl8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.9/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.1 h1:giB30dEeoar5bgDnkE0q+z7cFjcHaCjulpmPVmuKR84= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.1/go.mod h1:071TH4M3botFLWDbzQLfBR7tXYi7Fs2RsXSiH7nlUlY= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= +github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0= +github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= From 408bf5c84f9ab4a0a052c0a69432f9995e96012f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 11:11:19 +0100 Subject: [PATCH 069/530] [sda-admin] update to GO 1.25 --- sda-admin/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-admin/go.mod b/sda-admin/go.mod index f1b18a1e02..33d046d3f2 100644 --- a/sda-admin/go.mod +++ b/sda-admin/go.mod @@ -1,6 +1,6 @@ module github.com/neicnordic/sensitive-data-archive/sda-admin -go 1.24.0 +go 1.25.7 require ( github.com/neicnordic/crypt4gh v1.12.0 From bae53387c67ed8fe9ea285865ed575284a32c434 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 11:12:29 +0100 Subject: [PATCH 070/530] [sda-download] update to GO 1.25 --- sda-download/Dockerfile | 2 +- sda-download/go.mod | 45 +++++++++---------- sda-download/go.sum | 96 +++++++++++++++++++++-------------------- 3 files changed, 73 insertions(+), 70 deletions(-) diff --git a/sda-download/Dockerfile b/sda-download/Dockerfile index 30f9453531..f050ab04c6 100644 --- a/sda-download/Dockerfile +++ b/sda-download/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24-alpine AS builder +FROM golang:1.25-alpine AS builder ENV GOPATH=$PWD ENV CGO_ENABLED=0 diff --git a/sda-download/go.mod b/sda-download/go.mod index cd1729e1d9..fd2a7711fb 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -1,51 +1,52 @@ module github.com/neicnordic/sda-download -go 1.24.1 +go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/Masterminds/semver/v3 v3.4.0 github.com/aws/aws-sdk-go v1.55.8 github.com/dgraph-io/ristretto v1.0.0 - github.com/gin-gonic/gin v1.11.0 + github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.11.2 - github.com/neicnordic/crypt4gh v1.14.1 + github.com/neicnordic/crypt4gh v1.14.2 github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.48.0 - golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 + golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa google.golang.org/grpc v1.79.1 google.golang.org/protobuf v1.36.11 ) require ( - filippo.io/edwards25519 v1.1.1 // indirect - github.com/bytedance/sonic v1.14.0 // indirect - github.com/bytedance/sonic/loader v0.3.0 // indirect + filippo.io/edwards25519 v1.2.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.27.0 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/goccy/go-yaml v1.18.0 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lestrrat-go/blackmagic v1.0.3 // indirect + github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect @@ -57,25 +58,25 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.57.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/sagikazarmark/locafero v0.11.0 // indirect - github.com/segmentio/asm v1.2.0 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect + github.com/segmentio/asm v1.2.1 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect - github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.3.0 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.20.0 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/arch v0.24.0 // indirect + golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect - golang.org/x/tools v0.41.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + golang.org/x/tools v0.42.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 26548e8277..7f28a686fd 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -1,5 +1,5 @@ -filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= -filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= @@ -7,10 +7,12 @@ github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lpr github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= -github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= -github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= -github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= -github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= @@ -23,8 +25,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84= github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= @@ -35,12 +37,12 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= -github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= -github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -51,14 +53,14 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= -github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= -github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= @@ -83,8 +85,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/blackmagic v1.0.3 h1:94HXkVLxkZO9vJI/w2u1T0DAoprShFd13xtnSINtDWs= -github.com/lestrrat-go/blackmagic v1.0.3/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= +github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= +github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= @@ -104,8 +106,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.14.1 h1:Pkfd3GfNz6meca6cypfpXiR4EkgcTE2Epsx4VM2/kK8= -github.com/neicnordic/crypt4gh v1.14.1/go.mod h1:293fdfGR0Gs7Ef5A+b/kQyIwpMuLLOdXel0p5M7Yj4s= +github.com/neicnordic/crypt4gh v1.14.2 h1:iUYbA6djJWql5WUet7J9Mw+iQ9DYRGJWvNWC8WfuZBc= +github.com/neicnordic/crypt4gh v1.14.2/go.mod h1:+y61sviowARFg7hMiggWvXe0LQXPDcUDOdflddsU1l4= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -115,23 +117,21 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE= -github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= -github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= -github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= -github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= @@ -144,22 +144,26 @@ github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjb github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= -github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= @@ -172,18 +176,18 @@ go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2W go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= -go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= -go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= -golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= +golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= -golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= -golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -193,8 +197,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -225,21 +229,19 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.41.0 h1:a9b8iMweWG+S0OBnlU36rzLp20z1Rp10w+IY2czHTQc= -golang.org/x/tools v0.41.0/go.mod h1:XSY6eDqxVNiYgezAVqqCeihT4j1U2CCsqvH3WhQpnlg= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= From 3442d0fe6e79fd67e8032f5534768658ec37078a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 11:14:21 +0100 Subject: [PATCH 071/530] [SDA] update Go to 1.25 --- sda/Dockerfile | 9 +-- sda/go.mod | 81 +++++++++++------------ sda/go.sum | 172 +++++++++++++++++++++++++------------------------ 3 files changed, 132 insertions(+), 130 deletions(-) diff --git a/sda/Dockerfile b/sda/Dockerfile index c3415e736e..2a0f437fad 100644 --- a/sda/Dockerfile +++ b/sda/Dockerfile @@ -1,14 +1,11 @@ -FROM "golang:${GOLANG_VERSION:-1.24}-bullseye" AS Build -ENV GO111MODULE=on -ENV GOPATH=$PWD +FROM "golang:${GOLANG_VERSION:-1.25}-trixie" AS Build ENV CGO_ENABLED=0 -ENV GOOS=linux COPY . . SHELL ["bash", "-c"] RUN set -ex; for p in cmd/*; do go build -buildvcs=false -o "${p/cmd\//sda-}" "./$p"; done -FROM debian:bullseye-slim AS Debug +FROM debian:trixie-slim AS Debug ARG SOURCE_COMMIT @@ -24,7 +21,7 @@ COPY --from=Build /go/schemas /schemas USER 65534 -FROM gcr.io/distroless/static-debian11 +FROM gcr.io/distroless/static-debian13 ARG SOURCE_COMMIT diff --git a/sda/go.mod b/sda/go.mod index b82f5a2cd1..8d15c9f77b 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -1,28 +1,28 @@ module github.com/neicnordic/sensitive-data-archive -go 1.24.1 +go 1.25.7 require ( github.com/aws/aws-sdk-go-v2 v1.41.2 github.com/aws/aws-sdk-go-v2/config v1.32.10 github.com/aws/aws-sdk-go-v2/credentials v1.19.10 - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.5 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.1 - github.com/aws/smithy-go v1.24.1 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.6 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 + github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 github.com/coreos/go-oidc/v3 v3.17.0 - github.com/gin-gonic/gin v1.11.0 + github.com/gin-gonic/gin v1.12.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 - github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c + github.com/iris-contrib/middleware/cors v0.0.0-20251225090426-92c6f28facda github.com/kataras/iris/v12 v12.2.11 github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.11.2 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 - github.com/neicnordic/crypt4gh v1.14.1 + github.com/neicnordic/crypt4gh v1.14.2 github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 github.com/ory/dockertest v3.3.5+incompatible github.com/ory/dockertest/v3 v3.12.0 @@ -41,16 +41,16 @@ require ( require ( dario.cat/mergo v1.0.1 // indirect - filippo.io/edwards25519 v1.1.1 // indirect + filippo.io/edwards25519 v1.2.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/BurntSushi/toml v1.5.0 // indirect - github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect + github.com/BurntSushi/toml v1.6.0 // indirect + github.com/CloudyKit/fastprinter v0.0.0-20251202014920-1725d2651bd4 // indirect github.com/CloudyKit/jet/v6 v6.3.1 // indirect github.com/Joker/jade v1.1.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect - github.com/andybalholm/brotli v1.1.1 // indirect + github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 // indirect + github.com/andybalholm/brotli v1.2.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect @@ -58,7 +58,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect @@ -66,17 +66,18 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect - github.com/bytedance/sonic v1.14.0 // indirect - github.com/bytedance/sonic/loader v0.3.0 // indirect - github.com/casbin/govaluate v1.6.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect + github.com/casbin/govaluate v1.10.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/containerd/continuity v0.4.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/docker/cli v27.4.1+incompatible // indirect github.com/docker/docker v28.0.0+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect @@ -84,41 +85,41 @@ require ( github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.4 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.27.0 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/goccy/go-yaml v1.18.0 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/snappy v1.0.0 // indirect - github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b // indirect + github.com/gomarkdown/markdown v0.0.0-20260217112301-37c66b85d6ab // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kataras/blocks v0.0.11 // indirect - github.com/kataras/golog v0.1.13 // indirect + github.com/kataras/blocks v0.0.12 // indirect + github.com/kataras/golog v0.1.12 // indirect github.com/kataras/pio v0.0.14 // indirect github.com/kataras/sitemap v0.0.6 // indirect github.com/kataras/tunnel v0.0.4 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.4 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lestrrat-go/blackmagic v1.0.3 // indirect + github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/mailgun/raymond/v2 v2.0.48 // indirect - github.com/mailru/easyjson v0.9.0 // indirect + github.com/mailru/easyjson v0.9.1 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -133,22 +134,21 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.57.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.11.0 // indirect + github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect - github.com/segmentio/asm v1.2.0 // indirect - github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/segmentio/asm v1.2.1 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tdewolff/minify/v2 v2.23.6 // indirect - github.com/tdewolff/parse/v2 v2.8.1 // indirect + github.com/tdewolff/minify/v2 v2.24.10 // indirect + github.com/tdewolff/parse/v2 v2.8.10 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect - github.com/ugorji/go/codec v1.3.0 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/vmihailenco/msgpack/v5 v5.4.1 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect @@ -156,15 +156,16 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yosssi/ace v0.0.5 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.20.0 // indirect - golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect - golang.org/x/net v0.49.0 // indirect + golang.org/x/arch v0.24.0 // indirect + golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect + golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect - golang.org/x/time v0.12.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect + gopkg.in/ini.v1 v1.67.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools v2.2.0+incompatible // indirect diff --git a/sda/go.sum b/sda/go.sum index f3531c8048..bbc2ef47b8 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -1,13 +1,14 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= -filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= -filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= -github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= -github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c= +github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= +github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= +github.com/CloudyKit/fastprinter v0.0.0-20251202014920-1725d2651bd4 h1:DQ1+lDdBve+u+aovjh4wV6sYnvZKH0Hx8GaQOi4vYl8= +github.com/CloudyKit/fastprinter v0.0.0-20251202014920-1725d2651bd4/go.mod h1:eauGmjfZG874MOAEPVeqg21mZCbTOLW+tFe8F7NpfnY= github.com/CloudyKit/jet/v6 v6.3.1 h1:6IAo5Cx21xrHVaR8zzXN5gJatKV/wO7Nf6bfCnCSbUw= github.com/CloudyKit/jet/v6 v6.3.1/go.mod h1:lf8ksdNsxZt7/yH/3n4vJQWA9RUq4wpaHtArHhGVMOw= github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= @@ -18,12 +19,12 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b h1:3QuwXO2TnDssBmmNKfRlO3ur9zOupJRdM7evq61Oi84= -github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= +github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 h1:dG7/gLroJGht/jSQtHiLvT48Hxn+crbmvyItZC8cWOs= +github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= -github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= @@ -34,8 +35,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iK github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.5 h1:ArBXPSauOifB8IvX1R77qw9YbGzvZJBFJMdpKBAO5YM= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.5/go.mod h1:BHwO94hM6u8TGk7pxs0GCh7t/inrq300kYRksFyB2LY= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.6 h1:AaBtONvA5gHSZHo2EgjXMQCvYdQWclo3MznTkQnQ8w0= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.6/go.mod h1:+gC6IbVU+BWeCB0W9MWNgOpKuTmggJ/ES54eyKA8zQM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= @@ -46,14 +47,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.9 h1:IJRzQTvdpjHRPItx9gzNcz7Y1F+xqAR+xiy9rr5ZYl8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.9/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.1 h1:giB30dEeoar5bgDnkE0q+z7cFjcHaCjulpmPVmuKR84= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.1/go.mod h1:071TH4M3botFLWDbzQLfBR7tXYi7Fs2RsXSiH7nlUlY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0= github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= @@ -62,24 +63,26 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWA github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= -github.com/aws/smithy-go v1.24.1 h1:VbyeNfmYkWoxMVpGUAbQumkODcYmfMRfZ8yQiH30SK0= -github.com/aws/smithy-go v1.24.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= -github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= -github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= -github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= -github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4= github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/casbin/casbin/v2 v2.135.0 h1:6BLkMQiGotYyS5yYeWgW19vxqugUlvHFkFiLnLR/bxk= github.com/casbin/casbin/v2 v2.135.0/go.mod h1:FmcfntdXLTcYXv/hxgNntcRPqAbwOG9xsism0yXT+18= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= -github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= -github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/casbin/govaluate v1.10.0 h1:ffGw51/hYH3w3rZcxO/KcaUIDOLP84w7nsidMVgaDG0= +github.com/casbin/govaluate v1.10.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= @@ -101,8 +104,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= @@ -112,8 +115,8 @@ github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= -github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw= @@ -122,12 +125,12 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= -github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= -github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= @@ -142,8 +145,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= -github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= @@ -152,8 +155,8 @@ github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= -github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= @@ -164,8 +167,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60qA17CptGXhwfZ+k1sFNJIUNR8DdbcuUk= -github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/gomarkdown/markdown v0.0.0-20260217112301-37c66b85d6ab h1:VYNivV7P8IRHUam2swVUNkhIdp0LRRFKe4hXNnoZKTc= +github.com/gomarkdown/markdown v0.0.0-20260217112301-37c66b85d6ab/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= @@ -189,8 +192,8 @@ github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE= -github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c h1:7gyBIX75xKttBJo35lyVLUb/OQh7jY2wcEIVUygmsL8= -github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c/go.mod h1:SvRi2dE8S9rgUvgDrGg7hpb/1gt9zVnvLnWjMQJUONo= +github.com/iris-contrib/middleware/cors v0.0.0-20251225090426-92c6f28facda h1:wtN/fj3B3xmjLLuDmcykBwx/IlUpwypYBZe+HG1tN5o= +github.com/iris-contrib/middleware/cors v0.0.0-20251225090426-92c6f28facda/go.mod h1:BGIjcv4ouBVUE0JzrrT1gEG6z8NmG911Jnm3ofNYM74= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -199,10 +202,10 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kataras/blocks v0.0.11 h1:JJdYW0AUaJKLx5kEWs/oRVCvKVXo+6CAAeaVAiJf7wE= -github.com/kataras/blocks v0.0.11/go.mod h1:b4UySrJySEOq6drKH9U3bOpMI+dRH148mayYfS3RFb8= -github.com/kataras/golog v0.1.13 h1:bGbPglTdCutekqwOUf8L1jq3tZ5ADG9gfPBd5p5SzKA= -github.com/kataras/golog v0.1.13/go.mod h1:oQmzBTCv/35TetBosjJl/k+LPdlJEblaTupkNwJlwj8= +github.com/kataras/blocks v0.0.12 h1:2OnEYFcLtYPjyEMhyDk1pdHm+b75hay5uobuPTacnIc= +github.com/kataras/blocks v0.0.12/go.mod h1:CtCOQ+YDdd0NJTMW019YPV9D+q6dWO2b9d2cSRgifpk= +github.com/kataras/golog v0.1.12 h1:Bu7I/G4ilJlbfzjmU39O9N+2uO1pBcMK045fzZ4ytNg= +github.com/kataras/golog v0.1.12/go.mod h1:wrGSbOiBqbQSQznleVNX4epWM8rl9SJ/rmEacl0yqy4= github.com/kataras/iris/v12 v12.2.11 h1:sGgo43rMPfzDft8rjVhPs6L3qDJy3TbBrMD/zGL1pzk= github.com/kataras/iris/v12 v12.2.11/go.mod h1:uMAeX8OqG9vqdhyrIPv8Lajo/wXTtAF43wchP9WHt2w= github.com/kataras/pio v0.0.14 h1:VGBHOmhwrMMrZeuRqoSfOrFwG+v1JxQge8N50DhmRYQ= @@ -213,8 +216,8 @@ github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA= github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= +github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= @@ -229,8 +232,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/blackmagic v1.0.3 h1:94HXkVLxkZO9vJI/w2u1T0DAoprShFd13xtnSINtDWs= -github.com/lestrrat-go/blackmagic v1.0.3/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= +github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA= +github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= @@ -245,8 +248,8 @@ github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= -github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= -github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= +github.com/mailru/easyjson v0.9.1/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -276,8 +279,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.14.1 h1:Pkfd3GfNz6meca6cypfpXiR4EkgcTE2Epsx4VM2/kK8= -github.com/neicnordic/crypt4gh v1.14.1/go.mod h1:293fdfGR0Gs7Ef5A+b/kQyIwpMuLLOdXel0p5M7Yj4s= +github.com/neicnordic/crypt4gh v1.14.2 h1:iUYbA6djJWql5WUet7J9Mw+iQ9DYRGJWvNWC8WfuZBc= +github.com/neicnordic/crypt4gh v1.14.2/go.mod h1:+y61sviowARFg7hMiggWvXe0LQXPDcUDOdflddsU1l4= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 h1:9bCMuD3TcnjeqjPT2gSlha4asp8NvgcFRYExCaikCxk= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25/go.mod h1:eDjgYHYDJbPLBLsyZ6qRaugP0mX8vePOhZ5id1fdzJw= @@ -302,34 +305,32 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE= -github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= -github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= +github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= +github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY= github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= -github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= -github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= +github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= -github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -349,21 +350,22 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tdewolff/minify/v2 v2.23.6 h1:s0ktOnrlqKfhignkv9M/rh9wCfRlIPiIpjPOq3xUJ84= -github.com/tdewolff/minify/v2 v2.23.6/go.mod h1:VW3ISUd3gDOZuQ/jwZr4sCzsuX+Qvsx87FDMjk6Rvno= -github.com/tdewolff/parse/v2 v2.8.1 h1:J5GSHru6o3jF1uLlEKVXkDxxcVx6yzOlIVIotK4w2po= -github.com/tdewolff/parse/v2 v2.8.1/go.mod h1:Hwlni2tiVNKyzR1o6nUs4FOF07URA+JLBLd6dlIXYqo= +github.com/tdewolff/minify/v2 v2.24.10 h1:SjOOY2Y3Uv34WY4wtyUzJA2T1Xd1v1zQVSZvPP0A/h4= +github.com/tdewolff/minify/v2 v2.24.10/go.mod h1:fXkGpJ4gel+z1nmeIjVtKmxGZ4ZXd7g1gA3dfTz5/j8= +github.com/tdewolff/parse/v2 v2.8.10 h1:5a8o388UmuiU3zlOBJ56PN0rxVi67LRNED/zzuHAfC0= +github.com/tdewolff/parse/v2 v2.8.10/go.mod h1:Hwlni2tiVNKyzR1o6nUs4FOF07URA+JLBLd6dlIXYqo= github.com/tdewolff/test v1.0.11 h1:FdLbwQVHxqG16SlkGveC0JVyrJN62COWTRyUFzfbtBE= github.com/tdewolff/test v1.0.11/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= -github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= -github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8= @@ -391,6 +393,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= @@ -405,12 +409,12 @@ go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6 go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko= -go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= -golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= +golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -419,8 +423,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= -golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= -golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= +golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= @@ -438,8 +442,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= -golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -482,8 +486,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -499,8 +503,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= @@ -510,8 +514,8 @@ gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= +gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 0cadd1ce081ac6b4e9d9bcb1a8e29c002331d3a9 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 11:17:27 +0100 Subject: [PATCH 072/530] [sda-validator] update GO modules --- sda-validator/orchestrator/go.mod | 52 ++++++------ sda-validator/orchestrator/go.sum | 132 +++++++++++++++--------------- 2 files changed, 92 insertions(+), 92 deletions(-) diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index 175fdae368..bd016408f3 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -1,44 +1,44 @@ module github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator -go 1.25.0 +go 1.25.7 require ( - github.com/casbin/casbin/v2 v2.131.0 + github.com/casbin/casbin/v2 v2.135.0 github.com/docker/go-units v0.5.0 - github.com/gin-gonic/gin v1.11.0 + github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.10.9 - github.com/neicnordic/crypt4gh v1.14.0 + github.com/lib/pq v1.11.2 + github.com/neicnordic/crypt4gh v1.14.2 github.com/rabbitmq/amqp091-go v1.10.0 - github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.10.1 + github.com/sirupsen/logrus v1.9.4 + github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - google.golang.org/grpc v1.76.0 + google.golang.org/grpc v1.79.1 ) require ( - filippo.io/edwards25519 v1.1.1 // indirect - github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect + filippo.io/edwards25519 v1.2.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect - github.com/bytedance/sonic v1.14.2 // indirect - github.com/bytedance/sonic/loader v0.4.0 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/casbin/govaluate v1.10.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.11 // indirect + github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.28.0 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.5 // indirect - github.com/goccy/go-yaml v1.18.0 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect @@ -54,7 +54,7 @@ require ( github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.6.0 // indirect - github.com/quic-go/quic-go v0.57.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/segmentio/asm v1.2.1 // indirect github.com/spf13/afero v1.15.0 // indirect @@ -63,14 +63,14 @@ require ( github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.3.1 // indirect - go.uber.org/mock v0.6.0 // indirect + go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/arch v0.22.0 // indirect - golang.org/x/crypto v0.45.0 // indirect - golang.org/x/net v0.47.0 // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.31.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect - google.golang.org/protobuf v1.36.10 // indirect + golang.org/x/arch v0.24.0 // indirect + golang.org/x/crypto v0.48.0 // indirect + golang.org/x/net v0.51.0 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect + google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index dc069db626..04a728461d 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -1,19 +1,21 @@ -filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= -filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= +filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE= -github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bmatcuk/doublestar/v4 v4.10.0 h1:zU9WiOla1YA122oLM6i4EXvGW62DvKZVxIe6TYWexEs= +github.com/bmatcuk/doublestar/v4 v4.10.0/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= -github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE= -github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980= -github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o= -github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= -github.com/casbin/casbin/v2 v2.131.0 h1:JWkedeOpvxkBE7iIrWqf2RSMOU/xmxsxgbQ8QzsmGiQ= -github.com/casbin/casbin/v2 v2.131.0/go.mod h1:FmcfntdXLTcYXv/hxgNntcRPqAbwOG9xsism0yXT+18= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/casbin/casbin/v2 v2.135.0 h1:6BLkMQiGotYyS5yYeWgW19vxqugUlvHFkFiLnLR/bxk= +github.com/casbin/casbin/v2 v2.135.0/go.mod h1:FmcfntdXLTcYXv/hxgNntcRPqAbwOG9xsism0yXT+18= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.10.0 h1:ffGw51/hYH3w3rZcxO/KcaUIDOLP84w7nsidMVgaDG0= github.com/casbin/govaluate v1.10.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= @@ -23,20 +25,20 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik= -github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/gabriel-vasile/mimetype v1.4.13 h1:46nXokslUBsAJE/wMsp5gtO500a4F3Nkz9Ufpk2AcUM= +github.com/gabriel-vasile/mimetype v1.4.13/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= -github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= +github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= +github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -47,14 +49,14 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688= -github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= -github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= -github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -88,8 +90,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= -github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= +github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -97,8 +99,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U= -github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI= +github.com/neicnordic/crypt4gh v1.14.2 h1:iUYbA6djJWql5WUet7J9Mw+iQ9DYRGJWvNWC8WfuZBc= +github.com/neicnordic/crypt4gh v1.14.2/go.mod h1:+y61sviowARFg7hMiggWvXe0LQXPDcUDOdflddsU1l4= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -106,8 +108,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= -github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE= -github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= @@ -117,14 +119,14 @@ github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88ee github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= -github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s= -github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -137,7 +139,6 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= @@ -150,54 +151,53 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= -go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= -go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= -go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= -go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= -go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= -go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= -go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= -go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= -go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= -go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= +go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI= -golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= +golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= +golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q= -golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4= +golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= +golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= -golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= +golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= -golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= +golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= -golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= -golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE= -golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= -google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= -google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= -google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= +google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From cfa476b93e7acff90a92c7823df0b455ac76df42 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 11:19:48 +0100 Subject: [PATCH 073/530] [GH actions] update to GO 1.25 --- .github/workflows/code-linter.yaml | 2 +- .github/workflows/test.yml | 2 +- sda-download/.github/workflows/test.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 755363c783..4a9258e300 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -8,7 +8,7 @@ on: - '**/go.*' env: - go-version: '1.24' + go-version: '1.25' go-version-validator-orchestrator: '1.25' jobs: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6d55b7a797..f75be56669 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: branches-ignore: [main] env: - go-version: '1.24' + go-version: '1.25' go-version-validator-orchestrator: '1.25' jobs: diff --git a/sda-download/.github/workflows/test.yml b/sda-download/.github/workflows/test.yml index 047f9154c0..8e05d33eff 100644 --- a/sda-download/.github/workflows/test.yml +++ b/sda-download/.github/workflows/test.yml @@ -8,7 +8,7 @@ on: - "schemas/**" env: - go-version: '1.24' + go-version: '1.25 ' jobs: build: From 285cf8d467bc205e9a89b12a5252632b2b1757d9 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 11:21:25 +0100 Subject: [PATCH 074/530] [GO test] remove deprecated flag --- .github/workflows/test.yml | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f75be56669..d619625b7b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,11 +50,7 @@ jobs: - name: Get dependencies run: | cd sda-download - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi + go get -v -t ./... - name: Test run: | cd sda-download @@ -85,11 +81,7 @@ jobs: - name: Get dependencies run: | cd sda - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi + go get -v -t ./... - name: Test run: | cd sda @@ -108,10 +100,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Set up Go ${{ matrix.go-version }} + - name: Set up Go ${{ env.go-version }} uses: actions/setup-go@v6 with: - go-version: ${{ matrix.go-version }} + go-version: ${{ env.go-version }} id: go - name: Check out code into the Go module directory @@ -120,11 +112,7 @@ jobs: - name: Get dependencies run: | cd sda-admin - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi + go get -v -t ./... - name: Test run: | cd sda-admin @@ -155,11 +143,7 @@ jobs: - name: Get dependencies run: | cd ./sda-validator/orchestrator - go get -v -t -d ./... - if [ -f Gopkg.toml ]; then - curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh - dep ensure - fi + go get -v -t ./... - name: Test run: | cd ./sda-validator/orchestrator From 8a8d1d6942936b2c01693ae69cd2f67dc9062a9f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 11:53:41 +0100 Subject: [PATCH 075/530] [linter] disable false positive --- .golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.golangci.yml b/.golangci.yml index 9edab1a144..55debf5957 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -23,6 +23,7 @@ linters: - G304 - G401 - G501 + - G704 revive: enable-all-rules: true rules: From 784414f1f601c05a593b41823a5da65bd19420e1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 12:59:32 +0100 Subject: [PATCH 076/530] [sda-admin] ignore linter issue about package name --- sda-admin/user/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-admin/user/user.go b/sda-admin/user/user.go index 315dddfdb8..3e365fff4b 100644 --- a/sda-admin/user/user.go +++ b/sda-admin/user/user.go @@ -1,4 +1,4 @@ -package user +package user // nolint:revive import ( "fmt" From 46caab75d2f2597de98c9f214d3379318bc5c5fa Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 13:28:38 +0100 Subject: [PATCH 077/530] [integration-tests] update to debian trixie --- .github/integration/sda-s3-integration.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index e18b91666b..0ac7df74bd 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -11,7 +11,7 @@ services: condition: service_healthy environment: - PGPASSWORD=rootpasswd - image: python:3.11-slim-bookworm + image: python:3.13-slim-trixie volumes: - ./scripts:/scripts - shared:/shared @@ -414,7 +414,7 @@ services: environment: - PGPASSWORD=rootpasswd - STORAGETYPE=s3 - image: python:3.11-slim-bullseye + image: python:3.13-slim-trixie profiles: - tests volumes: From ee6ddcd20b1f3f82bd763329952018ebc5da11b9 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 2 Mar 2026 15:00:35 +0100 Subject: [PATCH 078/530] [integrations-tests] fix bug in 70_rotate_key_test --- .github/integration/tests/sda/70_rotate_key_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index 4a8536be1a..32bd916ec0 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -22,7 +22,7 @@ checkErrors() { checkConsumers() { RETRY_TIMES=0 - until [ "$(curl -su guest:guest http://localhost:15672/api/consumers | jq '.[].queue.name' | grep -c "$1")" -eq "$2" ]; do + until [ "$(curl -su guest:guest http://rabbitmq:15672/api/consumers | jq '.[].queue.name' | grep -c "$1")" -eq "$2" ]; do echo "waiting for $1 consumer status" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then From a21f05924e32fcc19fa92e9936d4df638248121b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2026 20:52:14 +0000 Subject: [PATCH 079/530] build(deps): bump aquasecurity/trivy-action from 0.34.1 to 0.34.2 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.34.1 to 0.34.2. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.34.1...0.34.2) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.34.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- .github/workflows/scan-images.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 9b2090723c..3358a07fe0 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -117,7 +117,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.34.1 + uses: aquasecurity/trivy-action@0.34.2 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -135,7 +135,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.34.1 + uses: aquasecurity/trivy-action@0.34.2 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -184,7 +184,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.34.1 + uses: aquasecurity/trivy-action@0.34.2 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -215,7 +215,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.34.1 + uses: aquasecurity/trivy-action@0.34.2 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index d4f9d3f897..80c2fd6a84 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -27,7 +27,7 @@ jobs: image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.34.1 + uses: aquasecurity/trivy-action@0.34.2 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -53,7 +53,7 @@ jobs: security-events: write steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.34.1 + uses: aquasecurity/trivy-action@0.34.2 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From e16bda03692dcaa14ba3f8773fcae21d0279509c Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 2 Mar 2026 19:12:29 +0100 Subject: [PATCH 080/530] docs: add rotatekey to overview and fix doc gaps for reencrypt/rotatekey - Add rotatekey to sda.md support services list - Fix copy-paste error in rotatekey.md (sync -> rotatekey) - Add BROKER_PREFETCHCOUNT to rotatekey.md - Add GRPC_HOST and GRPC_PORT to reencrypt docs Ref: #2237 --- sda/cmd/reencrypt/Reencrypt.md | 5 +++++ sda/cmd/rotatekey/rotatekey.md | 3 ++- sda/sda.md | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sda/cmd/reencrypt/Reencrypt.md b/sda/cmd/reencrypt/Reencrypt.md index 1aeaa13dbd..cf2f301178 100644 --- a/sda/cmd/reencrypt/Reencrypt.md +++ b/sda/cmd/reencrypt/Reencrypt.md @@ -54,6 +54,11 @@ These settings control which crypt4gh keyfile is loaded. - `fatal` - `panic` +### GRPC server settings + +- `GRPC_HOST`: hostname or IP the gRPC server will listen on (default: `0.0.0.0`) +- `GRPC_PORT`: port the gRPC server will listen on (default: `50051`, changes to `50443` when TLS is enabled) + ### TLS settings - `GRPC_CACERT`: Certificate Authority (CA) certificate for validating incoming request diff --git a/sda/cmd/rotatekey/rotatekey.md b/sda/cmd/rotatekey/rotatekey.md index 007d6a4aa5..42072895e5 100644 --- a/sda/cmd/rotatekey/rotatekey.md +++ b/sda/cmd/rotatekey/rotatekey.md @@ -55,7 +55,7 @@ This setting controls which crypt4gh keyfile is loaded. ### RabbitMQ broker settings -These settings control how sync connects to the RabbitMQ message broker. +These settings control how `rotatekey` connects to the RabbitMQ message broker. - `BROKER_HOST`: hostname of the rabbitmq server - `BROKER_PORT`: rabbitmq broker port (commonly `5671` with TLS and `5672` without) @@ -63,6 +63,7 @@ These settings control how sync connects to the RabbitMQ message broker. - `BROKER_USER`: username to connect to rabbitmq - `BROKER_PASSWORD`: password to connect to rabbitmq - `BROKER_ROUTINGKEY`: routing from a rabbitmq exchange to the rotatekey queue +- `BROKER_PREFETCHCOUNT`: Number of messages to pull from the message server at the time (default to `2`) ### PostgreSQL Database settings diff --git a/sda/sda.md b/sda/sda.md index a9d1dec53a..a240a1ceb6 100644 --- a/sda/sda.md +++ b/sda/sda.md @@ -23,3 +23,4 @@ There are also additional support services: 4. [s3inbox](cmd/s3inbox/s3inbox.md) proxies uploads to the an S3 compatible storage backend. 5. [sync](cmd/sync/sync.md) mirrors ingested data between sites in the [Bigpicture](https://bigpicture.eu/) project. 6. [syncapi](cmd/syncapi/syncapi.md) is used in the [Bigpicture](https://bigpicture.eu/) project for mirroring data between two installations of SDA. +7. [RotateKey](cmd/rotatekey/rotatekey.md) re-encrypts file headers with a configured target key. From 4dc7395c99bae575791a5461d9a9e065993db2f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 16:16:45 +0100 Subject: [PATCH 081/530] feat(sda-download): add storage/v2 (and remove v1 and its config, etc), update to use storage/v2, update READMD.md to specify expected storage config --- sda-download/README.md | 9 + sda-download/api/s3/s3.go | 2 + sda-download/api/sda/sda.go | 9 +- sda-download/api/sda/sda_test.go | 104 ++- sda-download/cmd/main.go | 8 +- sda-download/go.mod | 35 +- sda-download/go.sum | 118 ++- sda-download/internal/config/config.go | 50 -- sda-download/internal/config/config_test.go | 9 - sda-download/internal/database/database.go | 4 +- .../internal/database/database_test.go | 6 +- .../internal/reencrypt/reencrypt.proto | 2 +- .../internal/storage/seekable_test.go | 339 -------- sda-download/internal/storage/storage.go | 312 -------- sda-download/internal/storage/storage_test.go | 356 --------- sda-download/internal/storage/v2/README.md | 80 ++ .../storage/v2/posix/reader/config.go | 36 + .../storage/v2/posix/reader/find_file.go | 29 + .../storage/v2/posix/reader/get_file_size.go | 37 + .../v2/posix/reader/new_file_read_seeker.go | 21 + .../v2/posix/reader/new_file_reader.go | 40 + .../storage/v2/posix/reader/reader.go | 39 + .../storage/v2/posix/reader/reader_test.go | 174 ++++ sda-download/internal/storage/v2/reader.go | 113 +++ .../internal/storage/v2/s3/reader/config.go | 158 ++++ .../storage/v2/s3/reader/find_file.go | 50 ++ .../storage/v2/s3/reader/get_file_size.go | 50 ++ .../s3/reader/new_file_read_seeker.go} | 217 ++--- .../storage/v2/s3/reader/new_file_reader.go | 41 + .../internal/storage/v2/s3/reader/reader.go | 80 ++ .../storage/v2/s3/reader/reader_test.go | 742 ++++++++++++++++++ .../storage/v2/seekable_multi_reader.go | 105 +++ .../storage/v2/seekable_multi_reader_test.go | 59 ++ .../storage/v2/storageerrors/errors.go | 9 + 34 files changed, 2091 insertions(+), 1352 deletions(-) delete mode 100644 sda-download/internal/storage/seekable_test.go delete mode 100644 sda-download/internal/storage/storage.go delete mode 100644 sda-download/internal/storage/storage_test.go create mode 100644 sda-download/internal/storage/v2/README.md create mode 100644 sda-download/internal/storage/v2/posix/reader/config.go create mode 100644 sda-download/internal/storage/v2/posix/reader/find_file.go create mode 100644 sda-download/internal/storage/v2/posix/reader/get_file_size.go create mode 100644 sda-download/internal/storage/v2/posix/reader/new_file_read_seeker.go create mode 100644 sda-download/internal/storage/v2/posix/reader/new_file_reader.go create mode 100644 sda-download/internal/storage/v2/posix/reader/reader.go create mode 100644 sda-download/internal/storage/v2/posix/reader/reader_test.go create mode 100644 sda-download/internal/storage/v2/reader.go create mode 100644 sda-download/internal/storage/v2/s3/reader/config.go create mode 100644 sda-download/internal/storage/v2/s3/reader/find_file.go create mode 100644 sda-download/internal/storage/v2/s3/reader/get_file_size.go rename sda-download/internal/storage/{seekable.go => v2/s3/reader/new_file_read_seeker.go} (62%) create mode 100644 sda-download/internal/storage/v2/s3/reader/new_file_reader.go create mode 100644 sda-download/internal/storage/v2/s3/reader/reader.go create mode 100644 sda-download/internal/storage/v2/s3/reader/reader_test.go create mode 100644 sda-download/internal/storage/v2/seekable_multi_reader.go create mode 100644 sda-download/internal/storage/v2/seekable_multi_reader_test.go create mode 100644 sda-download/internal/storage/v2/storageerrors/errors.go diff --git a/sda-download/README.md b/sda-download/README.md index 12a8928c8b..46b3fa9848 100644 --- a/sda-download/README.md +++ b/sda-download/README.md @@ -44,3 +44,12 @@ There is an README file in the [dev_utils](/dev_utils) folder with sections for |---------------|------| | auth | Auth pkg is used by the middleware to parse OIDC Details and extract GA4GH Visas from a [GA4GH Passport](https://github.com/ga4gh-duri/ga4gh-duri.github.io/blob/master/researcher_ids/ga4gh_passport_v1.md) | | request | This pkg Stores a HTTP client, so that it doesn't need to be initialised on every request. | + +## Storage settings +The download service requires access to the "archive" storage, to configure such the following config is needed: +```yaml +storage: + archive: + ${STORAGE_IMPLEMENTATION}: +``` +For more details on available configuration see [storage/v2 README.md](internal/storage/v2/README.md) \ No newline at end of file diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go index 87418d4a84..6b7dc61913 100644 --- a/sda-download/api/s3/s3.go +++ b/sda-download/api/s3/s3.go @@ -99,6 +99,8 @@ func ListBuckets(c *gin.Context) { if err != nil { log.Errorf("Failed to get dataset information: %v", err) c.AbortWithStatus(http.StatusInternalServerError) + + return } // TODO: Add real creation date buckets = append(buckets, Bucket{ diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index 94b4a33116..4c087b360b 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -23,14 +23,14 @@ import ( "github.com/neicnordic/sda-download/internal/config" "github.com/neicnordic/sda-download/internal/database" "github.com/neicnordic/sda-download/internal/reencrypt" - "github.com/neicnordic/sda-download/internal/storage" + "github.com/neicnordic/sda-download/internal/storage/v2" log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" ) -var Backend storage.Backend +var ArchiveReader storage.Reader func sanitizeString(str string) string { var pattern = regexp.MustCompile(`(https?://[^\s/$.?#].[^\s]+|[A-Za-z0-9-_:.]+)`) @@ -320,11 +320,10 @@ func Download(c *gin.Context) { // Get archive file handle var file io.Reader - if wholeFile { - file, err = Backend.NewFileReader(fileDetails.ArchivePath) + file, err = ArchiveReader.NewFileReader(context.Background(), fileDetails.ArchiveLocation, fileDetails.ArchivePath) } else { - file, err = Backend.NewFileReadSeeker(fileDetails.ArchivePath) + file, err = ArchiveReader.NewFileReadSeeker(context.Background(), fileDetails.ArchiveLocation, fileDetails.ArchivePath) } if err != nil { diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go index ede5d403c1..3db7fdfaf9 100644 --- a/sda-download/api/sda/sda_test.go +++ b/sda-download/api/sda/sda_test.go @@ -2,6 +2,7 @@ package sda import ( "bytes" + "context" "crypto/x509" "encoding/base64" "encoding/binary" @@ -13,6 +14,7 @@ import ( "net/http/httptest" "net/url" "os" + "path/filepath" "strconv" "strings" "testing" @@ -31,7 +33,7 @@ import ( "github.com/neicnordic/sda-download/internal/database" "github.com/neicnordic/sda-download/internal/reencrypt" "github.com/neicnordic/sda-download/internal/session" - "github.com/neicnordic/sda-download/internal/storage" + "github.com/neicnordic/sda-download/internal/storage/v2" ) func TestDatasets(t *testing.T) { @@ -62,7 +64,7 @@ func TestDatasets(t *testing.T) { if response.StatusCode != expectedStatusCode { t.Errorf("TestDatasets failed, got %d expected %d", response.StatusCode, expectedStatusCode) } - if !bytes.Equal(body, []byte(expectedBody)) { + if !bytes.Equal(body, expectedBody) { // visual byte comparison in terminal (easier to find string differences) t.Error(body) t.Error([]byte(expectedBody)) @@ -554,7 +556,27 @@ func TestDownload_Fail_OpenFile(t *testing.T) { originalGetFile := database.GetFile originalServeUnencryptedDataTrigger := config.Config.C4GH.PublicKeyB64 originalC4ghPrivateKeyFilepath := config.Config.C4GH.PrivateKey - Backend, _ = storage.NewBackend(config.Config.Archive) + + tempDir := t.TempDir() + + if err := os.WriteFile(filepath.Join(tempDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + archive: + posix: + - path: %s +`, tempDir)), 0600); err != nil { + assert.FailNow(t, err.Error()) + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + viper.SetConfigFile(filepath.Join(tempDir, "config.yaml")) + + if err := viper.ReadInConfig(); err != nil { + assert.FailNow(t, err.Error()) + } + + ArchiveReader, _ = storage.NewReader(context.Background(), "archive") // Substitute mock functions database.CheckFilePermission = func(_ string) (string, error) { @@ -567,9 +589,10 @@ func TestDownload_Fail_OpenFile(t *testing.T) { } database.GetFile = func(_ string) (*database.FileDownload, error) { fileDetails := &database.FileDownload{ - ArchivePath: "non-existant-file.txt", - ArchiveSize: 0, - Header: []byte{}, + ArchivePath: "non-existant-file.txt", + ArchiveSize: 0, + ArchiveLocation: tempDir, + Header: []byte{}, } return fileDetails, nil @@ -722,9 +745,27 @@ func TestDownload_Whole_Range_Encrypted(t *testing.T) { originalGetFile := database.GetFile originalServeUnencryptedDataTrigger := config.Config.C4GH.PublicKeyB64 originalC4ghPrivateKeyFilepath := config.Config.C4GH.PrivateKey - archive := config.Config.Archive - archive.Posix.Location = "." - Backend, _ = storage.NewBackend(archive) + + tempDir := t.TempDir() + + if err := os.WriteFile(filepath.Join(tempDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + archive: + posix: + - path: %s +`, tempDir)), 0600); err != nil { + assert.FailNow(t, err.Error()) + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + viper.SetConfigFile(filepath.Join(tempDir, "config.yaml")) + + if err := viper.ReadInConfig(); err != nil { + assert.FailNow(t, err.Error()) + } + + ArchiveReader, _ = storage.NewReader(context.TODO(), "archive") // Fix to run fake GRPC server faker := fakeGRPC{t: t} @@ -771,7 +812,7 @@ func TestDownload_Whole_Range_Encrypted(t *testing.T) { assert.NoError(t, err, "Could not load c4gh keys") // Make a file to hold the archive file - datafile, err := os.CreateTemp(".", "datafile.") + datafile, err := os.Create(filepath.Join(tempDir, "datafile.txt")) assert.NoError(t, err, "Could not create datafile for test") datafileName := datafile.Name() defer os.Remove(datafileName) @@ -821,9 +862,10 @@ func TestDownload_Whole_Range_Encrypted(t *testing.T) { } database.GetFile = func(_ string) (*database.FileDownload, error) { fileDetails := &database.FileDownload{ - ArchivePath: datafileName, - ArchiveSize: 0, - Header: headerBytes, + ArchivePath: strings.TrimPrefix(datafileName, tempDir+"/"), + ArchiveSize: 0, + ArchiveLocation: tempDir, + Header: headerBytes, } return fileDetails, nil @@ -1113,7 +1155,7 @@ func TestDownload_Range_FromStartOfFile(t *testing.T) { return fileDetails, nil } - Backend = &mockBackend{ + ArchiveReader = &mockBackend{ file: []byte("this is a mock file"), } @@ -1158,7 +1200,7 @@ func TestDownload_Range_FromStartOfFile(t *testing.T) { privateKeyFilePath, err := GenerateTestC4ghKey(t) if err != nil { - t.FailNow() + assert.FailNow(t, err.Error()) } viper.Set("c4gh.transientKeyPath", privateKeyFilePath) viper.Set("c4gh.transientPassphrase", "password") @@ -1193,7 +1235,7 @@ func TestDownload_Range_FromStartOfFile(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - Backend = nil + ArchiveReader = nil } func TestDownload_Range_FromMiddleOfFile(t *testing.T) { @@ -1222,7 +1264,7 @@ func TestDownload_Range_FromMiddleOfFile(t *testing.T) { return fileDetails, nil } - Backend = &mockBackend{ + ArchiveReader = &mockBackend{ file: []byte("this is a mock file"), } @@ -1267,7 +1309,7 @@ func TestDownload_Range_FromMiddleOfFile(t *testing.T) { privateKeyFilePath, err := GenerateTestC4ghKey(t) if err != nil { - t.FailNow() + assert.FailNow(t, err.Error()) } viper.Set("c4gh.transientKeyPath", privateKeyFilePath) viper.Set("c4gh.transientPassphrase", "password") @@ -1302,32 +1344,32 @@ func TestDownload_Range_FromMiddleOfFile(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - Backend = nil + ArchiveReader = nil } type mockBackend struct { file []byte } -func (m *mockBackend) GetFileSize(_ string) (int64, error) { - return int64(len(m.file)), nil -} - -func (m *mockBackend) NewFileReader(_ string) (io.ReadCloser, error) { +func (m *mockBackend) NewFileReader(_ context.Context, _, _ string) (io.ReadCloser, error) { return io.NopCloser(bytes.NewReader(m.file)), nil } -type mockReedSeekCloser struct { - io.ReadSeeker - io.Closer -} - -func (m *mockBackend) NewFileReadSeeker(_ string) (io.ReadSeekCloser, error) { +func (m *mockBackend) NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) { return mockReedSeekCloser{ ReadSeeker: io.ReadSeeker(bytes.NewReader(m.file)), }, nil } -func (m *mockBackend) NewFileWriter(_ string) (io.WriteCloser, error) { +func (m *mockBackend) FindFile(ctx context.Context, filePath string) (string, error) { panic("not expected to be used by unit tests") } + +func (m *mockBackend) GetFileSize(_ context.Context, _, _ string) (int64, error) { + return int64(len(m.file)), nil +} + +type mockReedSeekCloser struct { + io.ReadSeeker + io.Closer +} diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index 2c55930b67..5ad97a3ab4 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -1,12 +1,14 @@ package main import ( + "context" + "github.com/neicnordic/sda-download/api" "github.com/neicnordic/sda-download/api/sda" "github.com/neicnordic/sda-download/internal/config" "github.com/neicnordic/sda-download/internal/database" "github.com/neicnordic/sda-download/internal/session" - "github.com/neicnordic/sda-download/internal/storage" + "github.com/neicnordic/sda-download/internal/storage/v2" "github.com/neicnordic/sda-download/pkg/auth" "github.com/neicnordic/sda-download/pkg/request" log "github.com/sirupsen/logrus" @@ -55,11 +57,11 @@ func init() { } session.SessionCache = sessionCache - backend, err := storage.NewBackend(conf.Archive) + archiveReader, err := storage.NewReader(context.Background(), "archive") if err != nil { log.Panicf("Error initiating storage backend, reason: %v", err) } - sda.Backend = backend + sda.ArchiveReader = archiveReader } // main starts the web server diff --git a/sda-download/go.mod b/sda-download/go.mod index fd2a7711fb..6af46fbcba 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -5,14 +5,19 @@ go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/Masterminds/semver/v3 v3.4.0 - github.com/aws/aws-sdk-go v1.55.8 + github.com/aws/aws-sdk-go-v2 v1.41.2 + github.com/aws/aws-sdk-go-v2/config v1.32.10 + github.com/aws/aws-sdk-go-v2/credentials v1.19.10 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 + github.com/aws/smithy-go v1.24.2 + github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.12.0 + github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/uuid v1.6.0 - github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.11.2 - github.com/neicnordic/crypt4gh v1.14.2 + github.com/neicnordic/crypt4gh v1.15.0 github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 @@ -24,12 +29,26 @@ require ( require ( filippo.io/edwards25519 v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect @@ -39,10 +58,8 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.30.1 // indirect - github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -56,13 +73,11 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect - github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/segmentio/asm v1.2.1 // indirect - github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect github.com/spf13/pflag v1.0.10 // indirect @@ -75,8 +90,6 @@ require ( golang.org/x/net v0.51.0 // indirect golang.org/x/sys v0.41.0 // indirect golang.org/x/text v0.34.0 // indirect - golang.org/x/tools v0.42.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 7f28a686fd..21b66a63ab 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -4,25 +4,59 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= -github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= +github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= +github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= +github.com/aws/aws-sdk-go-v2/config v1.32.10 h1:9DMthfO6XWZYLfzZglAgW5Fyou2nRI5CuV44sTedKBI= +github.com/aws/aws-sdk-go-v2/config v1.32.10/go.mod h1:2rUIOnA2JaiqYmSKYmRJlcMWy6qTj1vuRFscppSBMcw= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= +github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 h1:6lhrsTEnloDPXyeZBvSYvQf8u86jbKehZPVDDlkgDl4= +github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= -github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= @@ -68,12 +102,6 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 h1:dUIw4/lPu6PA52q+jwYhaJn40NJ6Mr3DN9PwW5+XRNg= -github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250/go.mod h1:mVu8PpZKcm+CsTQvUCABkP3+5a8eGWhPrZKrx+cWP9s= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= @@ -106,33 +134,26 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.14.2 h1:iUYbA6djJWql5WUet7J9Mw+iQ9DYRGJWvNWC8WfuZBc= -github.com/neicnordic/crypt4gh v1.14.2/go.mod h1:+y61sviowARFg7hMiggWvXe0LQXPDcUDOdflddsU1l4= +github.com/neicnordic/crypt4gh v1.15.0 h1:as+O2Y2IwXAKJzSJX5RPE7PLTCtWkygdjIMk3N34YCo= +github.com/neicnordic/crypt4gh v1.15.0/go.mod h1:4FYcQUA0mtdEosuktXaWw8IID2VYL8lKKHbSKxoI4g4= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= -github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI= github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0= github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= -github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIGMa7oSwp8SQPp7Hb4ZiI38K3RNBKD2LLeEM= -github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= -github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -146,7 +167,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -160,8 +180,6 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= @@ -182,62 +200,19 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= -golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= @@ -249,11 +224,6 @@ google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index b53a233c86..2268bdcd76 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -14,7 +14,6 @@ import ( "github.com/Masterminds/semver/v3" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/neicnordic/crypt4gh/keys" - "github.com/neicnordic/sda-download/internal/storage" log "github.com/sirupsen/logrus" "github.com/spf13/viper" "golang.org/x/exp/slices" @@ -36,7 +35,6 @@ type Map struct { Session SessionConfig DB DatabaseConfig OIDC OIDCConfig - Archive storage.Conf Reencrypt ReencryptConfig C4GH transientKeyConf } @@ -220,7 +218,6 @@ func NewConfig() (*Map, error) { c := &Map{} c.applyDefaults() c.sessionConfig() - c.configArchive() err := c.configReencrypt() if err != nil { return nil, err @@ -257,41 +254,6 @@ func (c *Map) applyDefaults() { viper.SetDefault("session.name", "sda_session_key") } -// configS3Storage populates and returns a S3Conf from the -// configuration -func configS3Storage(prefix string) storage.S3Conf { - s3 := storage.S3Conf{} - // All these are required - s3.URL = viper.GetString(prefix + ".url") - s3.AccessKey = viper.GetString(prefix + ".accesskey") - s3.SecretKey = viper.GetString(prefix + ".secretkey") - s3.Bucket = viper.GetString(prefix + ".bucket") - - // Defaults (move to viper?) - - s3.Port = 443 - s3.Region = "us-east-1" - s3.NonExistRetryTime = 2 * time.Minute - - if viper.IsSet(prefix + ".port") { - s3.Port = viper.GetInt(prefix + ".port") - } - - if viper.IsSet(prefix + ".region") { - s3.Region = viper.GetString(prefix + ".region") - } - - if viper.IsSet(prefix + ".chunksize") { - s3.Chunksize = viper.GetInt(prefix+".chunksize") * 1024 * 1024 - } - - if viper.IsSet(prefix + ".cacert") { - s3.Cacert = viper.GetString(prefix + ".cacert") - } - - return s3 -} - func (c *Map) configureOIDC() error { c.OIDC.ConfigurationURL = viper.GetString("oidc.configuration.url") c.OIDC.Whitelist = nil @@ -311,18 +273,6 @@ func (c *Map) configureOIDC() error { return nil } -// configArchive provides configuration for the archive storage -// we default to POSIX unless S3 specified -func (c *Map) configArchive() { - if viper.GetString("archive.type") == S3 { - c.Archive.Type = S3 - c.Archive.S3 = configS3Storage("archive") - } else { - c.Archive.Type = POSIX - c.Archive.Posix.Location = viper.GetString("archive.location") - } -} - func (c *Map) configReencrypt() error { c.Reencrypt.Host = viper.GetString("grpc.host") viper.SetDefault("grpc.port", 50051) diff --git a/sda-download/internal/config/config_test.go b/sda-download/internal/config/config_test.go index e06308ec03..83cf003885 100644 --- a/sda-download/internal/config/config_test.go +++ b/sda-download/internal/config/config_test.go @@ -142,15 +142,6 @@ func (ts *TestSuite) TestAppConfig() { assert.ErrorContains(ts.T(), err, "chacha20poly1305: message authentication failed") } -func (ts *TestSuite) TestArchiveConfig() { - viper.Set("archive.type", POSIX) - viper.Set("archive.location", "/test") - - c := &Map{} - c.configArchive() - assert.Equal(ts.T(), "/test", c.Archive.Posix.Location) -} - func (ts *TestSuite) TestSessionConfig() { viper.Set("session.expiration", 3600) viper.Set("session.domain", "test") diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index 8e39999928..12c10c8f94 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -390,6 +390,7 @@ func (dbs *SQLdb) checkFilePermission(fileID string) (string, error) { type FileDownload struct { ArchivePath string ArchiveSize int + ArchiveLocation string DecryptedSize int DecryptedChecksum string LastModified string @@ -427,6 +428,7 @@ func (dbs *SQLdb) getFile(fileID string) (*FileDownload, error) { const query = ` SELECT f.archive_file_path, f.archive_file_size, + f.archive_location, f.decrypted_file_size, dc.checksum AS decrypted_checksum, f.last_modified, @@ -440,7 +442,7 @@ func (dbs *SQLdb) getFile(fileID string) (*FileDownload, error) { fd := &FileDownload{} var hexString string - err := db.QueryRow(query, fileID).Scan(&fd.ArchivePath, &fd.ArchiveSize, + err := db.QueryRow(query, fileID).Scan(&fd.ArchivePath, &fd.ArchiveSize, &fd.ArchiveLocation, &fd.DecryptedSize, &fd.DecryptedChecksum, &fd.LastModified, &hexString) if err != nil { log.Errorf("could not retrieve details for file %s, reason %s", sanitizeString(fileID), err) diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index 501a878fce..a92bbd441f 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -278,6 +278,7 @@ func TestGetFile(t *testing.T) { expected := &FileDownload{ ArchivePath: "file.txt", ArchiveSize: 32, + ArchiveLocation: "/archive", DecryptedSize: 1024, DecryptedChecksum: "sha256checksum", LastModified: "now", @@ -286,6 +287,7 @@ func TestGetFile(t *testing.T) { query := ` SELECT f.archive_file_path, f.archive_file_size, + f.archive_location, f.decrypted_file_size, dc.checksum AS decrypted_checksum, f.last_modified, @@ -299,9 +301,9 @@ func TestGetFile(t *testing.T) { mock.ExpectQuery(query). WithArgs("file1"). - WillReturnRows(sqlmock.NewRows([]string{"file_path", "archive_file_size", + WillReturnRows(sqlmock.NewRows([]string{"file_path", "archive_file_size", "archive_location", "decrypted_file_size", "decrypted_checksum", "last_modified", "header"}).AddRow( - expected.ArchivePath, expected.ArchiveSize, expected.DecryptedSize, + expected.ArchivePath, expected.ArchiveSize, expected.ArchiveLocation, expected.DecryptedSize, expected.DecryptedChecksum, expected.LastModified, "abc123")) x, err := testDb.getFile("file1") diff --git a/sda-download/internal/reencrypt/reencrypt.proto b/sda-download/internal/reencrypt/reencrypt.proto index 7ff5eb525d..f8940d2fd8 100644 --- a/sda-download/internal/reencrypt/reencrypt.proto +++ b/sda-download/internal/reencrypt/reencrypt.proto @@ -4,7 +4,7 @@ syntax = "proto3"; // // protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative PATH_TO/reencrypt.proto -option go_package = "github.com/neicnordic/sensitive-data-archive/internal/reencrypt"; +option go_package = "github.com/neicnordic/sda-download/internal/reencrypt"; package reencrypt; diff --git a/sda-download/internal/storage/seekable_test.go b/sda-download/internal/storage/seekable_test.go deleted file mode 100644 index 83598a53e5..0000000000 --- a/sda-download/internal/storage/seekable_test.go +++ /dev/null @@ -1,339 +0,0 @@ -package storage - -import ( - "bytes" - "fmt" - "io" - "os" - "slices" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - log "github.com/sirupsen/logrus" -) - -func TestSeekableBackend(t *testing.T) { - for _, backendType := range []string{posixType, s3Type} { - testConf.Type = backendType - - backend, err := NewBackend(testConf) - assert.Nil(t, err, "Backend failed") - - var buf bytes.Buffer - - path := fmt.Sprintf("%v.%v", s3Creatable, time.Now().UnixNano()) - - if testConf.Type == s3Type { - s3back := backend.(*s3Backend) - assert.IsType(t, s3back, &s3Backend{}, "Wrong type from NewBackend with seekable s3") - } - if testConf.Type == posixType { - path, err = writeName() - posix := backend.(*posixBackend) - assert.Nil(t, err, "File creation for backend failed") - assert.IsType(t, posix, &posixBackend{}, "Wrong type from NewBackend with seekable posix") - } - - writer, err := backend.NewFileWriter(path) - - assert.NotNil(t, writer, "Got a nil reader for writer from s3") - assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") - - for i := 0; i < 1000; i++ { - written, err := writer.Write(writeData) - assert.Nil(t, err, "Failure when writing to s3 writer") - assert.Equal(t, len(writeData), written, "Did not write all writeData") - } - - _ = writer.Close() - - reader, err := backend.NewFileReadSeeker(path) - assert.Nil(t, err, "s3 NewFileReadSeeker failed when it should work") - assert.NotNil(t, reader, "Got a nil reader for s3") - - size, err := backend.GetFileSize(path) - assert.Nil(t, err, "s3 GetFileSize failed when it should work") - assert.Equal(t, int64(len(writeData))*1000, size, "Got an incorrect file size") - - if reader == nil { - t.Error("reader that should be usable is not, bailing out") - - return - } - - var readBackBuffer [4096]byte - seeker := reader - - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData, readBackBuffer[:14], "did not read back data as expected") - assert.Nil(t, err, "read returned unexpected error") - - if testConf.Type == s3Type { - // POSIX is more allowing - _, err := seeker.Seek(95000, io.SeekStart) - assert.NotNil(t, err, "Seek didn't fail when it should") - - _, err = seeker.Seek(-95000, io.SeekStart) - assert.NotNil(t, err, "Seek didn't fail when it should") - - _, err = seeker.Seek(-95000, io.SeekCurrent) - assert.NotNil(t, err, "Seek didn't fail when it should") - - _, err = seeker.Seek(95000, io.SeekCurrent) - assert.NotNil(t, err, "Seek didn't fail when it should") - - _, err = seeker.Seek(95000, io.SeekEnd) - assert.NotNil(t, err, "Seek didn't fail when it should") - - _, err = seeker.Seek(-95000, io.SeekEnd) - assert.NotNil(t, err, "Seek didn't fail when it should") - - _, err = seeker.Seek(0, 4) - assert.NotNil(t, err, "Seek didn't fail when it should") - } - - offset, err := seeker.Seek(15, io.SeekStart) - assert.Nil(t, err, "Seek failed when it shouldn't") - assert.Equal(t, int64(15), offset, "Seek did not return expected offset") - - offset, err = seeker.Seek(5, io.SeekCurrent) - assert.Nil(t, err, "Seek failed when it shouldn't") - assert.Equal(t, int64(20), offset, "Seek did not return expected offset") - - offset, err = seeker.Seek(-5, io.SeekEnd) - assert.Nil(t, err, "Seek failed when it shouldn't") - assert.Equal(t, int64(13995), offset, "Seek did not return expected offset") - - n, err := seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, 5, n, "Unexpected amount of read bytes") - assert.Nil(t, err, "Read failed when it shouldn't") - - n, err = seeker.Read(readBackBuffer[0:4096]) - - assert.Equal(t, io.EOF, err, "Expected EOF") - assert.Equal(t, 0, n, "Unexpected amount of read bytes") - - offset, err = seeker.Seek(0, io.SeekEnd) - assert.Nil(t, err, "Seek failed when it shouldn't") - assert.Equal(t, int64(14000), offset, "Seek did not return expected offset") - - n, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, 0, n, "Unexpected amount of read bytes") - assert.Equal(t, io.EOF, err, "Read returned unexpected error when EOF") - - offset, err = seeker.Seek(6302, io.SeekStart) - assert.Nil(t, err, "Seek failed") - assert.Equal(t, int64(6302), offset, "Seek did not return expected offset") - - n = 0 - for i := 0; i < 500000 && n == 0 && err == nil; i++ { - // Allow 0 sizes while waiting for prefetch - n, err = seeker.Read(readBackBuffer[0:4096]) - } - - assert.Equal(t, 4096, n, "Read did not return expected amounts of bytes for %v", seeker) - assert.Equal(t, writeData[2:], readBackBuffer[:12], "did not read back data as expected") - assert.Nil(t, err, "unexpected error when reading back data") - - offset, err = seeker.Seek(6302, io.SeekStart) - assert.Nil(t, err, "unexpected error when seeking to read back data") - assert.Equal(t, int64(6302), offset, "returned offset wasn't expected") - - largeBuf := make([]byte, 65536) - readLen, err := seeker.Read(largeBuf) - assert.Equal(t, 7698, readLen, "did not read back expected amount of data") - assert.Nil(t, err, "unexpected error when reading back data") - - buf.Reset() - - log.SetOutput(&buf) - - if !testing.Short() { - _, err = backend.GetFileSize(s3DoesNotExist) - assert.NotNil(t, err, "s3 GetFileSize worked when it should not") - assert.NotZero(t, buf.Len(), "Expected warning missing") - - buf.Reset() - - reader, err = backend.NewFileReadSeeker(s3DoesNotExist) - assert.NotNil(t, err, "s3 NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a non-nil reader for s3") - assert.NotZero(t, buf.Len(), "Expected warning missing") - } - - log.SetOutput(os.Stdout) - } -} - -func TestS3SeekablePrefetchSize(t *testing.T) { - testConf.Type = s3Type - chunkSize := testConf.S3.Chunksize - testConf.S3.Chunksize = 5 * 1024 * 1024 - backend, err := NewBackend(testConf) - s3back := backend.(*s3Backend) - assert.IsType(t, s3back, &s3Backend{}, "Wrong type from NewBackend with seekable s3") - assert.Nil(t, err, "S3 backend failed") - path := fmt.Sprintf("%v.%v", s3Creatable, time.Now().UnixNano()) - - writer, err := backend.NewFileWriter(path) - - assert.NotNil(t, writer, "Got a nil reader for writer from s3") - assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") - - _ = writer.Close() - - reader, err := backend.NewFileReadSeeker(path) - assert.Nil(t, err, "s3 NewFileReadSeeker failed when it should work") - assert.NotNil(t, reader, "Got a nil reader for s3") - - s := reader.(*s3Reader) - - assert.Equal(t, int64(5*1024*1024), s.prefetchSize(), "Prefetch size not as expected with chunksize 5MB") - s.Conf.Chunksize = 0 - assert.Equal(t, int64(50*1024*1024), s.prefetchSize(), "Prefetch size not as expected") - - s.Conf.Chunksize = 1024 * 1024 - assert.Equal(t, int64(50*1024*1024), s.prefetchSize(), "Prefetch size not as expected") - - testConf.S3.Chunksize = chunkSize -} - -func TestS3SeekableSpecial(t *testing.T) { - // Some special tests here, messing with internals to expose behaviour - - testConf.Type = s3Type - - backend, err := NewBackend(testConf) - assert.Nil(t, err, "Backend failed") - - path := fmt.Sprintf("%v.%v", s3Creatable, time.Now().UnixNano()) - - s3back := backend.(*s3Backend) - assert.IsType(t, s3back, &s3Backend{}, "Wrong type from NewBackend with seekable s3") - - writer, err := backend.NewFileWriter(path) - - assert.NotNil(t, writer, "Got a nil reader for writer from s3") - assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") - - for i := 0; i < 1000; i++ { - written, err := writer.Write(writeData) - assert.Nil(t, err, "Failure when writing to s3 writer") - assert.Equal(t, len(writeData), written, "Did not write all writeData") - } - - _ = writer.Close() - - reader, err := backend.NewFileReadSeeker(path) - reader.(*s3Reader).seeked = true - - assert.Nil(t, err, "s3 NewFileReader failed when it should work") - assert.NotNil(t, reader, "Got a nil reader for s3") - size, err := backend.GetFileSize(path) - assert.Nil(t, err, "s3 GetFileSize failed when it should work") - assert.Equal(t, int64(len(writeData))*1000, size, "Got an incorrect file size") - - if reader == nil { - t.Error("reader that should be usable is not, bailing out") - - return - } - - var readBackBuffer [4096]byte - seeker := reader - - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData, readBackBuffer[:14], "did not read back data as expected") - assert.Nil(t, err, "read returned unexpected error") - - err = seeker.Close() - assert.Nil(t, err, "unexpected error when closing") - - reader, err = backend.NewFileReadSeeker(path) - assert.Nil(t, err, "unexpected error when creating reader") - - s := reader.(*s3Reader) - s.seeked = true - s.prefetchAt(0) - assert.Equal(t, 1, len(s.local), "nothing cached after prefetch") - // Clear cache - s.local = s.local[:0] - - s.outstandingPrefetches = []int64{0} - t.Logf("Cache %v, outstanding %v", s.local, s.outstandingPrefetches) - - n, err := s.Read(readBackBuffer[0:4096]) - assert.Nil(t, err, "read returned unexpected error") - assert.Equal(t, 0, n, "got data when we should get 0 because of prefetch") - - for i := 0; i < 30; i++ { - s.local = append(s.local, s3CacheBlock{90000000, int64(0), nil}) - } - s.prefetchAt(0) - assert.Equal(t, 8, len(s.local), "unexpected length of cache after prefetch") - - s.outstandingPrefetches = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} - s.removeFromOutstanding(9) - assert.Equal(t, s.outstandingPrefetches, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8}, "unexpected outstanding prefetches after remove") - s.removeFromOutstanding(19) - assert.Equal(t, s.outstandingPrefetches, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8}, "unexpected outstanding prefetches after remove") - s.removeFromOutstanding(5) - // We don't care about the internal order, sort for simplicity - slices.Sort(s.outstandingPrefetches) - assert.Equal(t, s.outstandingPrefetches, []int64{0, 1, 2, 3, 4, 6, 7, 8}, "unexpected outstanding prefetches after remove") - - s.objectReader = nil - s.Bucket = "" - s.filePath = "" - data := make([]byte, 100) - _, err = s.wholeReader(data) - assert.NotNil(t, err, "wholeReader object instantiation worked when it should have failed") -} - -func TestSeekableMultiReader(t *testing.T) { - readers := make([]io.Reader, 10) - for i := 0; i < 10; i++ { - readers[i] = bytes.NewReader(writeData) - } - - seeker, err := SeekableMultiReader(readers...) - assert.Nil(t, err, "unexpected error from creating SeekableMultiReader") - - var readBackBuffer [4096]byte - - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData, readBackBuffer[:14], "did not read back data as expected") - assert.Nil(t, err, "unexpected error from read") - - offset, err := seeker.Seek(60, io.SeekStart) - - assert.Nil(t, err, "Seek failed") - assert.Equal(t, int64(60), offset, "Seek did not return expected offset") - - // We don't know how many bytes this should return - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData[4:], readBackBuffer[:10], "did not read back data as expected") - assert.Nil(t, err, "Read failed when it should not") - - offset, err = seeker.Seek(0, io.SeekEnd) - assert.Equal(t, int64(140), offset, "Seek did not return expected offset") - assert.Nil(t, err, "Seek failed when it should not") - - n, err := seeker.Read(readBackBuffer[0:4096]) - - assert.Equal(t, 0, n, "Read did not return expected amounts of bytes") - assert.Equal(t, io.EOF, err, "did not get EOF as expected") - - offset, err = seeker.Seek(56, io.SeekStart) - assert.Equal(t, int64(56), offset, "Seek did not return expected offset") - assert.Nil(t, err, "Seek failed unexpectedly") - - largeBuf := make([]byte, 65536) - readLen, err := seeker.Read(largeBuf) - assert.Nil(t, err, "unexpected error when reading back data") - assert.Equal(t, 14, readLen, "did not read back expect amount of data") - - log.SetOutput(os.Stdout) -} diff --git a/sda-download/internal/storage/storage.go b/sda-download/internal/storage/storage.go deleted file mode 100644 index 7fc9099332..0000000000 --- a/sda-download/internal/storage/storage.go +++ /dev/null @@ -1,312 +0,0 @@ -// Package storage provides interface for storage areas, e.g. s3 or POSIX file system. -package storage - -import ( - "crypto/tls" - "crypto/x509" - "errors" - "fmt" - "io" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3manager" - - log "github.com/sirupsen/logrus" -) - -// Backend defines methods to be implemented by PosixBackend and S3Backend -type Backend interface { - GetFileSize(filePath string) (int64, error) - NewFileReader(filePath string) (io.ReadCloser, error) - NewFileReadSeeker(filePath string) (io.ReadSeekCloser, error) - NewFileWriter(filePath string) (io.WriteCloser, error) -} - -// Conf is a wrapper for the storage config -type Conf struct { - Type string - S3 S3Conf - Posix posixConf -} - -type posixBackend struct { - FileReader io.Reader - FileWriter io.Writer - Location string -} - -type posixConf struct { - Location string -} - -// NewBackend initiates a storage backend -func NewBackend(config Conf) (Backend, error) { - switch config.Type { - case "s3": - return newS3Backend(config.S3) - default: - return newPosixBackend(config.Posix) - } -} - -func newPosixBackend(config posixConf) (*posixBackend, error) { - fileInfo, err := os.Stat(config.Location) - - if err != nil { - return nil, err - } - - if !fileInfo.IsDir() { - return nil, fmt.Errorf("%s is not a directory", config.Location) - } - - return &posixBackend{Location: config.Location}, nil -} - -// NewFileReader returns an io.Reader instance -func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { - if pb == nil { - return nil, errors.New("invalid posixBackend") - } - - file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) - if err != nil { - log.Error(err) - - return nil, err - } - - return file, nil -} - -// NewFileWriter returns an io.Writer instance -func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { - if pb == nil { - return nil, errors.New("invalid posixBackend") - } - - file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) - if err != nil { - log.Error(err) - - return nil, err - } - - return file, nil -} - -// GetFileSize returns the size of the file -func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { - if pb == nil { - return 0, errors.New("invalid posixBackend") - } - - stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) - if err != nil { - log.Error(err) - - return 0, err - } - - return stat.Size(), nil -} - -type s3Backend struct { - Client *s3.S3 - Uploader *s3manager.Uploader - Bucket string - Conf *S3Conf -} - -// S3Conf stores information about the S3 storage backend -type S3Conf struct { - URL string - Port int - AccessKey string // #nosec G117 -- Export needed to access configuration atm - SecretKey string - Bucket string - Region string - UploadConcurrency int - Chunksize int - Cacert string - NonExistRetryTime time.Duration -} - -func newS3Backend(config S3Conf) (*s3Backend, error) { - s3Transport := transportConfigS3(config) - client := http.Client{Transport: s3Transport} - s3Session := session.Must(session.NewSession( - &aws.Config{ - Endpoint: aws.String(fmt.Sprintf("%s:%d", config.URL, config.Port)), - Region: aws.String(config.Region), - HTTPClient: &client, - S3ForcePathStyle: aws.Bool(true), - DisableSSL: aws.Bool(strings.HasPrefix(config.URL, "http:")), - Credentials: credentials.NewStaticCredentials(config.AccessKey, config.SecretKey, ""), - }, - )) - - // Attempt to create a bucket, but we really expect an error here - // (BucketAlreadyOwnedByYou) - _, err := s3.New(s3Session).CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(config.Bucket), - }) - - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && - aerr.Code() != s3.ErrCodeBucketAlreadyExists { - log.Error("unexpected issue while creating bucket", err) - } - } - } - - sb := &s3Backend{ - Bucket: config.Bucket, - Uploader: s3manager.NewUploader(s3Session, func(u *s3manager.Uploader) { - u.PartSize = int64(config.Chunksize) - u.Concurrency = config.UploadConcurrency - u.LeavePartsOnError = false - }), - Client: s3.New(s3Session), - Conf: &config} - - _, err = sb.Client.ListObjectsV2(&s3.ListObjectsV2Input{Bucket: &config.Bucket}) - - if err != nil { - return nil, err - } - - return sb, nil -} - -// NewFileReader returns an io.Reader instance -func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { - if sb == nil { - return nil, errors.New("invalid s3Backend") - } - - r, err := sb.Client.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(sb.Bucket), - Key: aws.String(filePath), - }) - - retryTime := 2 * time.Minute - if sb.Conf != nil { - retryTime = sb.Conf.NonExistRetryTime - } - - start := time.Now() - for err != nil && time.Since(start) < retryTime { - r, err = sb.Client.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(sb.Bucket), - Key: aws.String(filePath), - }) - time.Sleep(1 * time.Second) - } - - if err != nil { - log.Error(err) - - return nil, err - } - - return r.Body, nil -} - -// NewFileWriter uploads the contents of an io.Reader to a S3 bucket -func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { - if sb == nil { - return nil, errors.New("invalid s3Backend") - } - - reader, writer := io.Pipe() - go func() { - _, err := sb.Uploader.Upload(&s3manager.UploadInput{ - Body: reader, - Bucket: aws.String(sb.Bucket), - Key: aws.String(filePath), - ContentEncoding: aws.String("application/octet-stream"), - }) - - if err != nil { - _ = reader.CloseWithError(err) - } - }() - - return writer, nil -} - -// GetFileSize returns the size of a specific object -func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { - if sb == nil { - return 0, errors.New("invalid s3Backend") - } - - r, err := sb.Client.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(sb.Bucket), - Key: aws.String(filePath)}) - - start := time.Now() - - retryTime := 2 * time.Minute - if sb.Conf != nil { - retryTime = sb.Conf.NonExistRetryTime - } - - // Retry on error up to five minutes to allow for - // "slow writes' or s3 eventual consistency - for err != nil && time.Since(start) < retryTime { - r, err = sb.Client.HeadObject(&s3.HeadObjectInput{ - Bucket: aws.String(sb.Bucket), - Key: aws.String(filePath)}) - - time.Sleep(1 * time.Second) - } - - if err != nil { - log.Errorln(err) - - return 0, err - } - - return *r.ContentLength, nil -} - -// transportConfigS3 is a helper method to setup TLS for the S3 client. -func transportConfigS3(config S3Conf) http.RoundTripper { - cfg := new(tls.Config) - - // Read system CAs - systemCAs, err := x509.SystemCertPool() - if err != nil { - log.Errorf("failed to read system CAs: %v, using an empty pool as base", err) - systemCAs = x509.NewCertPool() - } - cfg.RootCAs = systemCAs - - if config.Cacert != "" { - cacert, e := os.ReadFile(config.Cacert) // #nosec this file comes from our config - if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main - } - if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { - log.Debug("no certs appended, using system certs only") - } - } - - var trConfig http.RoundTripper = &http.Transport{ - TLSClientConfig: cfg, - ForceAttemptHTTP2: true} - - return trConfig -} diff --git a/sda-download/internal/storage/storage_test.go b/sda-download/internal/storage/storage_test.go deleted file mode 100644 index 70352eb618..0000000000 --- a/sda-download/internal/storage/storage_test.go +++ /dev/null @@ -1,356 +0,0 @@ -package storage - -import ( - "bytes" - "io" - "net/http/httptest" - "os" - "strconv" - "strings" - "testing" - "time" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/johannesboyne/gofakes3/backend/s3mem" - - "github.com/johannesboyne/gofakes3" - "github.com/stretchr/testify/assert" - - log "github.com/sirupsen/logrus" -) - -// posixType is the configuration type used for posix backends -const posixType = "posix" - -// s3Type is the configuration type used for posix backends -const s3Type = "s3" - -var testS3Conf = S3Conf{ - "http://127.0.0.1", - 9000, - "accesskey", - "secretkey", - "bucket", - "region", - 10, - 5 * 1024 * 1024, - "../../README.md", - 2 * time.Second} - -var testConf = Conf{posixType, testS3Conf, testPosixConf} - -var posixDoesNotExist = "/this/does/not/exist" -var posixNotCreatable = posixDoesNotExist - -var ts *httptest.Server - -var s3DoesNotExist = "nothing such" -var s3Creatable = "somename" - -var writeData = []byte("this is a test") - -var cleanupFilesBack [1000]string -var cleanupFiles = cleanupFilesBack[0:0] - -var testPosixConf = posixConf{ - "/"} - -func writeName() (name string, err error) { - f, err := os.CreateTemp("", "writablefile") - - if err != nil { - return "", err - } - - name = f.Name() - - // Add to cleanup - cleanupFiles = append(cleanupFiles, name) - - return name, err -} - -func doCleanup() { - for _, name := range cleanupFiles { - _ = os.Remove(name) - } - - cleanupFiles = cleanupFilesBack[0:0] -} -func TestNewBackend(t *testing.T) { - testConf.Type = posixType - p, err := NewBackend(testConf) - assert.Nil(t, err, "Backend posix failed") - - testConf.Type = s3Type - s, err := NewBackend(testConf) - assert.Nil(t, err, "Backend s3 failed") - - assert.IsType(t, p, &posixBackend{}, "Wrong type from NewBackend with posix") - assert.IsType(t, s, &s3Backend{}, "Wrong type from NewBackend with S3") - - // test some extra ssl handling - testConf.S3.Cacert = "/dev/null" - s, err = NewBackend(testConf) - assert.Nil(t, err, "Backend s3 failed") - assert.IsType(t, s, &s3Backend{}, "Wrong type from NewBackend with S3") -} - -func TestMain(m *testing.M) { - err := setupFakeS3() - - if err != nil { - log.Error("Setup of fake s3 failed, bailing out") - os.Exit(1) - } - - ret := m.Run() - ts.Close() - os.Exit(ret) -} -func TestPosixBackend(t *testing.T) { - defer doCleanup() - testConf.Type = posixType - backend, err := NewBackend(testConf) - assert.Nil(t, err, "POSIX backend failed unexpectedly") - - var buf bytes.Buffer - - assert.IsType(t, backend, &posixBackend{}, "Wrong type from NewBackend with posix") - - log.SetOutput(os.Stdout) - - writable, err := writeName() - if err != nil { - t.Error("could not find a writable name, bailing out from test") - - return - } - - writer, err := backend.NewFileWriter(writable) - - assert.NotNil(t, writer, "Got a nil reader for writer from posix") - assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") - - written, err := writer.Write(writeData) - - assert.Nil(t, err, "Failure when writing to posix writer") - assert.Equal(t, len(writeData), written, "Did not write all writeData") - _ = writer.Close() - - log.SetOutput(&buf) - writer, err = backend.NewFileWriter(posixNotCreatable) - - assert.Nil(t, writer, "Got a non-nil reader for writer from posix") - assert.NotNil(t, err, "posix NewFileWriter worked when it shouldn't") - assert.NotZero(t, buf.Len(), "Expected warning missing") - - log.SetOutput(os.Stdout) - - reader, err := backend.NewFileReader(writable) - assert.Nil(t, err, "posix NewFileReader failed when it should work") - assert.NotNil(t, reader, "Got a nil reader for posix") - - if reader == nil { - t.Error("reader that should be usable is not, bailing out") - - return - } - - var readBackBuffer [4096]byte - readBack, err := reader.Read(readBackBuffer[0:4096]) - - assert.Equal(t, len(writeData), readBack, "did not read back data as expected") - assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") - assert.Nil(t, err, "unexpected error when reading back data") - - size, err := backend.GetFileSize(writable) - assert.Nil(t, err, "posix NewFileReader failed when it should work") - assert.NotNil(t, size, "Got a nil size for posix") - - log.SetOutput(&buf) - - reader, err = backend.NewFileReader(posixDoesNotExist) - assert.NotNil(t, err, "posix NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a non-nil reader for posix") - assert.NotZero(t, buf.Len(), "Expected warning missing") - - buf.Reset() - - _, err = backend.GetFileSize(posixDoesNotExist) // nolint - assert.NotNil(t, err, "posix GetFileSize worked when it should not") - assert.NotZero(t, buf.Len(), "Expected warning missing") - - buf.Reset() -} - -func setupFakeS3() (err error) { - // fake s3 - - if ts != nil { - // Setup done already? - return - } - - backend := s3mem.New() - faker := gofakes3.New(backend) - ts = httptest.NewServer(faker.Server()) - - portAt := strings.LastIndex(ts.URL, ":") - - testConf.S3.URL = ts.URL[:portAt] - testConf.S3.Port, err = strconv.Atoi(ts.URL[portAt+1:]) - testConf.Type = s3Type - - if err != nil { - log.Error("Unexpected error while setting up fake s3") - - return err - } - - backEnd, err := NewBackend(testConf) - if err != nil { - return err - } - - s3back := backEnd.(*s3Backend) - - _, err = s3back.Client.CreateBucket(&s3.CreateBucketInput{ - Bucket: aws.String(testConf.S3.Bucket)}) - - if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && - aerr.Code() != s3.ErrCodeBucketAlreadyExists { - log.Error("Unexpected issue while creating bucket: ", err) - } else { - // Do not flag an error for this - err = nil - } - } - } - - return err -} - -func TestS3Fail(t *testing.T) { - testConf.Type = s3Type - - tmp := testConf.S3.URL - - defer func() { testConf.S3.URL = tmp }() - testConf.S3.URL = "file://tmp/" - _, err := NewBackend(testConf) - assert.NotNil(t, err, "Backend worked when it should not") - - var dummyBackend *s3Backend - reader, err := dummyBackend.NewFileReader("/") - assert.NotNil(t, err, "NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a Reader when expected not to") - - writer, err := dummyBackend.NewFileWriter("/") - assert.NotNil(t, err, "NewFileWriter worked when it should not") - assert.Nil(t, writer, "Got a Writer when expected not to") - - _, err = dummyBackend.GetFileSize("/") - assert.NotNil(t, err, "GetFileSize worked when it should not") -} - -func TestPOSIXFail(t *testing.T) { - testConf.Type = posixType - - tmp := testConf.Posix.Location - - defer func() { testConf.Posix.Location = tmp }() - - testConf.Posix.Location = "/thisdoesnotexist" - backEnd, err := NewBackend(testConf) - assert.NotNil(t, err, "Backend worked when it should not") - assert.Nil(t, backEnd, "Got a backend when expected not to") - - testConf.Posix.Location = "/etc/passwd" - - backEnd, err = NewBackend(testConf) - assert.NotNil(t, err, "Backend worked when it should not") - assert.Nil(t, backEnd, "Got a backend when expected not to") - - var dummyBackend *posixBackend - reader, err := dummyBackend.NewFileReader("/") - assert.NotNil(t, err, "NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a Reader when expected not to") - - writer, err := dummyBackend.NewFileWriter("/") - assert.NotNil(t, err, "NewFileWriter worked when it should not") - assert.Nil(t, writer, "Got a Writer when expected not to") - - _, err = dummyBackend.GetFileSize("/") - assert.NotNil(t, err, "GetFileSize worked when it should not") -} - -func TestS3Backend(t *testing.T) { - testConf.Type = s3Type - backend, err := NewBackend(testConf) - assert.Nil(t, err, "Backend failed") - - s3back := backend.(*s3Backend) - - var buf bytes.Buffer - - assert.IsType(t, s3back, &s3Backend{}, "Wrong type from NewBackend with s3") - - writer, err := s3back.NewFileWriter(s3Creatable) - - assert.NotNil(t, writer, "Got a nil reader for writer from s3") - assert.Nil(t, err, "posix NewFileWriter failed when it shouldn't") - - written, err := writer.Write(writeData) - - assert.Nil(t, err, "Failure when writing to s3 writer") - assert.Equal(t, len(writeData), written, "Did not write all writeData") - _ = writer.Close() - - reader, err := s3back.NewFileReader(s3Creatable) - assert.Nil(t, err, "s3 NewFileReader failed when it should work") - assert.NotNil(t, reader, "Got a nil reader for s3") - - size, err := s3back.GetFileSize(s3Creatable) - assert.Nil(t, err, "s3 GetFileSize failed when it should work") - assert.Equal(t, int64(len(writeData)), size, "Got an incorrect file size") - - if reader == nil { - t.Error("reader that should be usable is not, bailing out") - - return - } - - var readBackBuffer [4096]byte - readBack, err := reader.Read(readBackBuffer[0:4096]) - - assert.Equal(t, len(writeData), readBack, "did not read back data as expected") - assert.Equal(t, writeData, readBackBuffer[:readBack], "did not read back data as expected") - - if err != nil && err != io.EOF { - assert.Nil(t, err, "unexpected error when reading back data") - } - - buf.Reset() - - log.SetOutput(&buf) - - if !testing.Short() { - _, err = backend.GetFileSize(s3DoesNotExist) - assert.NotNil(t, err, "s3 GetFileSize worked when it should not") - assert.NotZero(t, buf.Len(), "Expected warning missing") - - buf.Reset() - - reader, err = backend.NewFileReader(s3DoesNotExist) - assert.NotNil(t, err, "s3 NewFileReader worked when it should not") - assert.Nil(t, reader, "Got a non-nil reader for s3") - assert.NotZero(t, buf.Len(), "Expected warning missing") - } - - log.SetOutput(os.Stdout) -} diff --git a/sda-download/internal/storage/v2/README.md b/sda-download/internal/storage/v2/README.md new file mode 100644 index 0000000000..67b26c90b9 --- /dev/null +++ b/sda-download/internal/storage/v2/README.md @@ -0,0 +1,80 @@ +# Storage v2 + +> **_NOTE:_** The content of the storage/v2 package is copied from the "sda" implementation at [storage v2](../../../../sda/internal/storage/v2) just with deletion of the writer + +The storage v2 package is responsible for the interfacing to a storage implementation, the supported storage +implementations are posix, and s3. + +The [reader.go](reader.go) supports reading from multiple different storage implementations and +which is to be used is decided by the caller through the requested location. + +## Config + +The storage implementation is configured through the Config File loaded in by the [config.go](../../config/config.go) +with the following structure: + +```yaml +storage: + ${STORAGE_NAME}: + ${STORAGE_IMPLEMENTATION}: + - ${STORAGE_IMPLEMENTATION_DEPENDANT_CONFIG} +``` + +Where ${STORAGE_NAME} is the name of the storage, and this is decided when initializing the reader, eg: +`NewReader(..., "Inbox", ...).` +${STORAGE_IMPLEMENTATION} is which storage implementation is to be loaded, there can be multiple storage $ +{STORAGE_IMPLEMENTATION}, +supported values are "s3", and "posix", eg if an application is to be able to read from both s3 and posix the config would be: + +```yaml +storage: + ${STORAGE_NAME}: + s3: + - ${S3_READER_CONFIG} + posix: + - ${POSIX_READER_CONFIG} +``` + +${STORAGE_IMPLEMENTATION_DEPENDANT_CONFIG} is the required configuration for the different storage implementations +[s3 reader](#s3-reader-config), [posix reader](#posix-reader-config) + +There can be multiple ${STORAGE_IMPLEMENTATION_DEPENDANT_CONFIG} if we want to be able to read / write to multiple of +the same storage implementation. eg: + +```yaml +storage: + ${STORAGE_NAME}: + s3: + - ${S3_READER_CONFIG_1} + - ${S3_READER_CONFIG_2} +``` + +## S3 + +The s3 storage implementation uses the [AWS s3](https://docs.aws.amazon.com/s3/) to connect to a s3 storage location. + +### S3 Reader Config + +A s3 reader has the following configuration: + +| Name: | Type: | Default Value: | Description: | +|---------------|--------|----------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| access_key | string | | The access key used to authenticate when connecting to the s3 | +| secret_key | string | | The secret key used to authenticate when connecting to the s3 | +| ca_cert | string | | The ca certificate of the s3 to be appended to the certs of the system | +| chunk_size | string | 50MB | The chunk size (in bytes) when writing data S3, also when reading chunks with the Seekable Reader. The minimum allowed is 5MB, and max is 1gb | +| region | string | us-east-1 | The region of the s3 bucket | +| endpoint | string | | The address of the s3 buckets | +| disable_https | bool | false | If to disable https when connecting to the s3 bucket | +| bucket_prefix | string | | How the reader will identify which buckets to look through when looking for a file for which the location is not known by the caller | + +## Posix + +### Posix Reader Config + +A posix reader has the following configuration: + +| Name: | Type: | Default Value: | Description: | +|-------|--------|----------------|---------------------------------------------------------| +| path | string | | The path of the directory/volume where to store objects | + diff --git a/sda-download/internal/storage/v2/posix/reader/config.go b/sda-download/internal/storage/v2/posix/reader/config.go new file mode 100644 index 0000000000..d3c44f0b4a --- /dev/null +++ b/sda-download/internal/storage/v2/posix/reader/config.go @@ -0,0 +1,36 @@ +package reader + +import ( + "errors" + "strings" + + "github.com/go-viper/mapstructure/v2" + "github.com/spf13/viper" +) + +type endpointConfig struct { + Path string `mapstructure:"path"` +} + +func loadConfig(backendName string) ([]*endpointConfig, error) { + var endpointConf []*endpointConfig + + if err := viper.UnmarshalKey( + "storage."+backendName+".posix", + &endpointConf, + func(config *mapstructure.DecoderConfig) { + config.WeaklyTypedInput = true + config.ZeroFields = true + }, + ); err != nil { + return nil, err + } + + for _, e := range endpointConf { + if !strings.HasPrefix(e.Path, "/") { + return nil, errors.New("posix paths must be absolute") + } + } + + return endpointConf, nil +} diff --git a/sda-download/internal/storage/v2/posix/reader/find_file.go b/sda-download/internal/storage/v2/posix/reader/find_file.go new file mode 100644 index 0000000000..9ab6454c0a --- /dev/null +++ b/sda-download/internal/storage/v2/posix/reader/find_file.go @@ -0,0 +1,29 @@ +package reader + +import ( + "context" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +func (reader *Reader) FindFile(_ context.Context, filePath string) (string, error) { + for _, endpointConf := range reader.configuredEndpoints { + _, err := os.Stat(filepath.Join(endpointConf.Path, filePath)) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + continue + } + + return "", fmt.Errorf("failed to stat file: %s, at location: %s, due to: %v", filePath, endpointConf.Path, err) + } + + return endpointConf.Path, nil + } + + return "", storageerrors.ErrorFileNotFoundInLocation +} diff --git a/sda-download/internal/storage/v2/posix/reader/get_file_size.go b/sda-download/internal/storage/v2/posix/reader/get_file_size.go new file mode 100644 index 0000000000..4634c34530 --- /dev/null +++ b/sda-download/internal/storage/v2/posix/reader/get_file_size.go @@ -0,0 +1,37 @@ +package reader + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +// GetFileSize returns the size of a specific object +func (reader *Reader) GetFileSize(_ context.Context, location, filePath string) (int64, error) { + var locationConfigured bool + for _, endpoint := range reader.configuredEndpoints { + if endpoint.Path == location { + locationConfigured = true + + break + } + } + if !locationConfigured { + return 0, storageerrors.ErrorNoEndpointConfiguredForLocation + } + + stat, err := os.Stat(filepath.Join(location, filePath)) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return 0, storageerrors.ErrorFileNotFoundInLocation + } + + return 0, fmt.Errorf("failed to stat file: %s, at location: %s, due to: %v", filePath, location, err) + } + + return stat.Size(), nil +} diff --git a/sda-download/internal/storage/v2/posix/reader/new_file_read_seeker.go b/sda-download/internal/storage/v2/posix/reader/new_file_read_seeker.go new file mode 100644 index 0000000000..ffe94f9553 --- /dev/null +++ b/sda-download/internal/storage/v2/posix/reader/new_file_read_seeker.go @@ -0,0 +1,21 @@ +package reader + +import ( + "context" + "errors" + "io" +) + +func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) { + r, err := reader.NewFileReader(ctx, location, filePath) + if err != nil { + return nil, err + } + + seeker, ok := r.(io.ReadSeekCloser) + if !ok { + return nil, errors.New("unexpected error: could not cast io.ReadCloser to io.ReadSeekCloser") + } + + return seeker, nil +} diff --git a/sda-download/internal/storage/v2/posix/reader/new_file_reader.go b/sda-download/internal/storage/v2/posix/reader/new_file_reader.go new file mode 100644 index 0000000000..10bb49f71a --- /dev/null +++ b/sda-download/internal/storage/v2/posix/reader/new_file_reader.go @@ -0,0 +1,40 @@ +package reader + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +// NewFileReader returns an io.Reader instance +func (reader *Reader) NewFileReader(_ context.Context, location, filePath string) (io.ReadCloser, error) { + var locationConfigured bool + for _, endpoint := range reader.configuredEndpoints { + if endpoint.Path == location { + locationConfigured = true + + break + } + } + if !locationConfigured { + return nil, storageerrors.ErrorNoEndpointConfiguredForLocation + } + + fullFilePath := filepath.Join(location, filePath) + + if _, err := os.Stat(fullFilePath); errors.Is(err, os.ErrNotExist) { + return nil, storageerrors.ErrorFileNotFoundInLocation + } + + file, err := os.Open(fullFilePath) + if err != nil { + return nil, fmt.Errorf("failed to open file: %s, at location: %s, due to: %v", filePath, location, err) + } + + return file, nil +} diff --git a/sda-download/internal/storage/v2/posix/reader/reader.go b/sda-download/internal/storage/v2/posix/reader/reader.go new file mode 100644 index 0000000000..9be95f0783 --- /dev/null +++ b/sda-download/internal/storage/v2/posix/reader/reader.go @@ -0,0 +1,39 @@ +package reader + +import ( + "fmt" + "os" + + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +type Reader struct { + configuredEndpoints []*endpointConfig +} + +func NewReader(backendName string) (*Reader, error) { + endPointsConfigurations, err := loadConfig(backendName) + if err != nil { + return nil, err + } + + backend := &Reader{ + configuredEndpoints: endPointsConfigurations, + } + // Verify configuredEndpoints + for _, loc := range backend.configuredEndpoints { + fileInfo, err := os.Stat(loc.Path) + if err != nil { + return nil, fmt.Errorf("failed to stat location: %s, due to: %v", loc.Path, err) + } + + if !fileInfo.IsDir() { + return nil, fmt.Errorf("%s is not a directory", loc) + } + } + if len(backend.configuredEndpoints) == 0 { + return nil, storageerrors.ErrorNoValidLocations + } + + return backend, nil +} diff --git a/sda-download/internal/storage/v2/posix/reader/reader_test.go b/sda-download/internal/storage/v2/posix/reader/reader_test.go new file mode 100644 index 0000000000..9d9507171a --- /dev/null +++ b/sda-download/internal/storage/v2/posix/reader/reader_test.go @@ -0,0 +1,174 @@ +package reader + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" + "github.com/spf13/viper" + "github.com/stretchr/testify/suite" +) + +type ReaderTestSuite struct { + suite.Suite + reader *Reader + dir1, dir2 string +} + +func TestReaderTestSuite(t *testing.T) { + suite.Run(t, new(ReaderTestSuite)) +} + +func (ts *ReaderTestSuite) SetupSuite() { + ts.dir1 = ts.T().TempDir() + ts.dir2 = ts.T().TempDir() + + if err := os.WriteFile(filepath.Join(ts.dir1, "config.yaml"), []byte(fmt.Sprintf(` +storage: + test: + posix: + - path: %s + - path: %s +`, ts.dir1, ts.dir2)), 0600); err != nil { + ts.FailNow(err.Error()) + } + + // create some files + for i := 0; i < 10; i++ { + if err := os.WriteFile(filepath.Join(ts.dir1, fmt.Sprintf("dir1_file%d.txt", i)), []byte(fmt.Sprintf("file %d content in dir1", i)), 0600); err != nil { + ts.FailNow(err.Error()) + } + if err := os.WriteFile(filepath.Join(ts.dir2, fmt.Sprintf("dir2_file%d.txt", i)), []byte(fmt.Sprintf("file %d content in dir2", i)), 0600); err != nil { + ts.FailNow(err.Error()) + } + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + viper.SetConfigFile(filepath.Join(ts.dir1, "config.yaml")) + + if err := viper.ReadInConfig(); err != nil { + ts.FailNow(err.Error()) + } + + var err error + ts.reader, err = NewReader("test") + if err != nil { + ts.FailNow(err.Error()) + } +} + +func (ts *ReaderTestSuite) TestNewFileReader() { + for _, test := range []struct { + testName string + locationToReadFrom string + filePathToRead string + expectedError error + expectedFileContent string + }{ + { + testName: "FromDir1", + expectedError: nil, + expectedFileContent: "file 2 content in dir1", + locationToReadFrom: ts.dir1, + filePathToRead: "dir1_file2.txt", + }, + { + testName: "FromDir2", + expectedError: nil, + expectedFileContent: "file 9 content in dir2", + locationToReadFrom: ts.dir2, + filePathToRead: "dir2_file9.txt", + }, + { + testName: "FromDir1_FileNotExists", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedFileContent: "", + locationToReadFrom: ts.dir1, + filePathToRead: "not_exists.txt", + }, + } { + ts.T().Run(test.testName, func(t *testing.T) { + fileReader, err := ts.reader.NewFileReader(context.Background(), test.locationToReadFrom, test.filePathToRead) + ts.Equal(test.expectedError, err) + + if fileReader != nil { + content, err := io.ReadAll(fileReader) + ts.NoError(err) + + ts.Equal(test.expectedFileContent, string(content)) + _ = fileReader.Close() + } + }) + } +} + +func (ts *ReaderTestSuite) TestNewFileSeekReader_ReadFromDir() { + fileSeekReader, err := ts.reader.NewFileReadSeeker(context.Background(), ts.dir2, "dir2_file5.txt") + if err != nil { + ts.FailNow(err.Error()) + } + + start := int64(5) + lenToRead := int64(9) + + _, err = fileSeekReader.Seek(start, 0) + ts.NoError(err) + + content := make([]byte, lenToRead) + + read, err := fileSeekReader.Read(content) + ts.NoError(err) + ts.Equal(lenToRead, int64(read)) + + ts.Equal("5 content", string(content)) + _ = fileSeekReader.Close() +} + +func (ts *ReaderTestSuite) TestGetFileSize() { + fileSize, err := ts.reader.GetFileSize(context.Background(), ts.dir2, "dir2_file6.txt") + if err != nil { + ts.FailNow(err.Error()) + } + + ts.Equal(int64(len("file 6 content in dir2")), fileSize) +} + +func (ts *ReaderTestSuite) TestFindFile() { + for _, test := range []struct { + testName string + filePathToFind string + expectedError error + expectedLocation string + }{ + { + testName: "FoundInDir1", + expectedError: nil, + expectedLocation: ts.dir1, + filePathToFind: "dir1_file3.txt", + }, { + testName: "FoundInDir2", + expectedError: nil, + expectedLocation: ts.dir2, + filePathToFind: "dir2_file9.txt", + }, + { + testName: "NotFound", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedLocation: "", + filePathToFind: "not_exists.txt", + }, + } { + ts.T().Run(test.testName, func(t *testing.T) { + loc, err := ts.reader.FindFile(context.TODO(), test.filePathToFind) + ts.Equal(test.expectedError, err) + + ts.Equal(test.expectedLocation, loc) + }) + } +} diff --git a/sda-download/internal/storage/v2/reader.go b/sda-download/internal/storage/v2/reader.go new file mode 100644 index 0000000000..6646f7f809 --- /dev/null +++ b/sda-download/internal/storage/v2/reader.go @@ -0,0 +1,113 @@ +package storage + +import ( + "context" + "errors" + "io" + "strings" + + posixreader "github.com/neicnordic/sda-download/internal/storage/v2/posix/reader" + s3reader "github.com/neicnordic/sda-download/internal/storage/v2/s3/reader" + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +// Reader defines methods to read files from a backend +type Reader interface { + // NewFileReader will open a reader of the file at the file path in the specified location + NewFileReader(ctx context.Context, location, filePath string) (io.ReadCloser, error) + // NewFileReadSeeker will open a read seeker of the file at the file path in the specified location + NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) + // FindFile will look through all configured storages for the specified file path and return the first location in which it was found + FindFile(ctx context.Context, filePath string) (string, error) + // GetFileSize will return the size of the file specified by the file path and location + GetFileSize(ctx context.Context, location, filePath string) (int64, error) +} + +type reader struct { + posixReader Reader + s3Reader Reader +} + +func NewReader(ctx context.Context, backendName string) (Reader, error) { + r := &reader{} + + s3Reader, err := s3reader.NewReader(ctx, backendName) + if err != nil && !errors.Is(err, storageerrors.ErrorNoValidLocations) { + return nil, err + } + posixReader, err := posixreader.NewReader(backendName) + if err != nil && !errors.Is(err, storageerrors.ErrorNoValidLocations) { + return nil, err + } + + if s3Reader == nil && posixReader == nil { + return nil, storageerrors.ErrorNoValidReader + } + if posixReader != nil { + r.posixReader = posixReader + } + if s3Reader != nil { + r.s3Reader = s3Reader + } + + return r, nil +} + +func (r *reader) NewFileReader(ctx context.Context, location, filePath string) (io.ReadCloser, error) { + if strings.HasPrefix(location, "/") && r.posixReader != nil { + return r.posixReader.NewFileReader(ctx, location, filePath) + } + + if !strings.HasPrefix(location, "/") && r.s3Reader != nil { + return r.s3Reader.NewFileReader(ctx, location, filePath) + } + + return nil, storageerrors.ErrorNoValidReader +} + +func (r *reader) NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) { + if strings.HasPrefix(location, "/") && r.posixReader != nil { + return r.posixReader.NewFileReadSeeker(ctx, location, filePath) + } + + if !strings.HasPrefix(location, "/") && r.s3Reader != nil { + return r.s3Reader.NewFileReadSeeker(ctx, location, filePath) + } + + return nil, storageerrors.ErrorNoValidReader +} + +func (r *reader) GetFileSize(ctx context.Context, location, filePath string) (int64, error) { + if strings.HasPrefix(location, "/") && r.posixReader != nil { + return r.posixReader.GetFileSize(ctx, location, filePath) + } + + if !strings.HasPrefix(location, "/") && r.s3Reader != nil { + return r.s3Reader.GetFileSize(ctx, location, filePath) + } + + return 0, storageerrors.ErrorNoValidReader +} + +func (r *reader) FindFile(ctx context.Context, filePath string) (string, error) { + if r.s3Reader != nil { + loc, err := r.s3Reader.FindFile(ctx, filePath) + if err != nil && !errors.Is(err, storageerrors.ErrorFileNotFoundInLocation) { + return "", err + } + if loc != "" { + return loc, nil + } + } + if r.posixReader != nil { + loc, err := r.posixReader.FindFile(ctx, filePath) + if err != nil && !errors.Is(err, storageerrors.ErrorFileNotFoundInLocation) { + return "", err + } + if loc != "" { + return loc, nil + } + } + + return "", storageerrors.ErrorFileNotFoundInLocation +} diff --git a/sda-download/internal/storage/v2/s3/reader/config.go b/sda-download/internal/storage/v2/s3/reader/config.go new file mode 100644 index 0000000000..bf82ecf160 --- /dev/null +++ b/sda-download/internal/storage/v2/s3/reader/config.go @@ -0,0 +1,158 @@ +package reader + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "os" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/c2h5oh/datasize" + "github.com/go-viper/mapstructure/v2" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" +) + +type endpointConfig struct { + AccessKey string `mapstructure:"access_key"` + CACert string `mapstructure:"ca_cert"` + ChunkSize string `mapstructure:"chunk_size"` + chunkSizeBytes uint64 + Region string `mapstructure:"region"` + SecretKey string `mapstructure:"secret_key"` + Endpoint string `mapstructure:"endpoint"` + BucketPrefix string `mapstructure:"bucket_prefix"` + DisableHTTPS bool `mapstructure:"disable_https"` + + s3Client *s3.Client // cached s3 client for this endpoint, created by getS3Client +} + +func loadConfig(backendName string) ([]*endpointConfig, error) { + var endpointConf []*endpointConfig + + if err := viper.UnmarshalKey( + "storage."+backendName+".s3", + &endpointConf, + func(config *mapstructure.DecoderConfig) { + config.WeaklyTypedInput = true + config.ZeroFields = true + }, + ); err != nil { + return nil, err + } + + for _, e := range endpointConf { + switch { + case e.Endpoint == "": + return nil, errors.New("missing required parameter: endpoint") + case e.AccessKey == "": + return nil, errors.New("missing required parameter: access_key") + case e.SecretKey == "": + return nil, errors.New("missing required parameter: secret_key") + default: + switch { + case strings.HasPrefix(e.Endpoint, "http:") && !e.DisableHTTPS: + return nil, errors.New("http scheme in endpoint when using HTTPS") + case strings.HasPrefix(e.Endpoint, "https:") && e.DisableHTTPS: + return nil, errors.New("https scheme in endpoint when HTTPS is disabled") + default: + } + + e.chunkSizeBytes = 50 * 1024 * 1024 + if e.ChunkSize != "" { + s, err := datasize.ParseString(e.ChunkSize) + if err != nil { + return nil, errors.New("could not parse chunk_size as a valid data size") + } + if s < 5*datasize.MB { + return nil, errors.New("chunk_size can not be smaller than 5mb") + } + if s > 1*datasize.GB { + return nil, errors.New("chunk_size can not be bigger than 1gb") + } + } + if e.Region == "" { + e.Region = "us-east-1" + } + } + } + + return endpointConf, nil +} + +func (endpointConf *endpointConfig) getS3Client(ctx context.Context) (*s3.Client, error) { + if endpointConf.s3Client != nil { + return endpointConf.s3Client, nil + } + + transport, err := endpointConf.transportConfigS3() + if err != nil { + return nil, fmt.Errorf("failed to config s3 transport, due to: %v", err) + } + s3cfg, err := config.LoadDefaultConfig( + ctx, + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(endpointConf.AccessKey, endpointConf.SecretKey, "")), + config.WithHTTPClient(&http.Client{Transport: transport}), + ) + if err != nil { + return nil, fmt.Errorf("failed to load default S3 config, due to: %v", err) + } + + endpoint := endpointConf.Endpoint + switch { + case !strings.HasPrefix(endpoint, "http") && endpointConf.DisableHTTPS: + endpoint = "http://" + endpoint + case !strings.HasPrefix(endpoint, "https") && !endpointConf.DisableHTTPS: + endpoint = "https://" + endpoint + default: + } + + endpointConf.s3Client = s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(endpoint) + o.EndpointOptions.DisableHTTPS = endpointConf.DisableHTTPS + o.Region = endpointConf.Region + o.UsePathStyle = true + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }, + ) + + return endpointConf.s3Client, nil +} + +// transportConfigS3 is a helper method to setup TLS for the S3 client. +func (endpointConf *endpointConfig) transportConfigS3() (http.RoundTripper, error) { + cfg := new(tls.Config) + + // Read system CAs + systemCAs, err := x509.SystemCertPool() + if err != nil { + log.Warnf("failed to read system CAs: %v, using an empty pool as base", err) + systemCAs = x509.NewCertPool() + } + cfg.RootCAs = systemCAs + + if endpointConf.CACert != "" { + caCert, err := os.ReadFile(endpointConf.CACert) + if err != nil { + return nil, fmt.Errorf("failed to append %q to RootCAs, due to: %v", caCert, err) + } + if ok := cfg.RootCAs.AppendCertsFromPEM(caCert); !ok { + log.Debug("no certs appended, using system certs only") + } + } + + return &http.Transport{ + TLSClientConfig: cfg, + ForceAttemptHTTP2: true, + }, nil +} diff --git a/sda-download/internal/storage/v2/s3/reader/find_file.go b/sda-download/internal/storage/v2/s3/reader/find_file.go new file mode 100644 index 0000000000..f5c1f2e870 --- /dev/null +++ b/sda-download/internal/storage/v2/s3/reader/find_file.go @@ -0,0 +1,50 @@ +package reader + +import ( + "context" + "errors" + "slices" + "strings" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +func (reader *Reader) FindFile(ctx context.Context, filePath string) (string, error) { + for _, endpointConf := range reader.endpoints { + client, err := reader.getS3ClientForEndpoint(ctx, endpointConf.Endpoint) + if err != nil { + return "", err + } + + bucketsRsp, err := client.ListBuckets(ctx, &s3.ListBucketsInput{}) + if err != nil { + return "", err + } + + var bucketsWithPrefix []string + for _, bucket := range bucketsRsp.Buckets { + if strings.HasPrefix(aws.ToString(bucket.Name), endpointConf.BucketPrefix) { + bucketsWithPrefix = append(bucketsWithPrefix, aws.ToString(bucket.Name)) + } + } + slices.SortFunc(bucketsWithPrefix, strings.Compare) + + for _, bucket := range bucketsWithPrefix { + _, err := reader.getFileSize(ctx, client, bucket, filePath) + + if err != nil { + if errors.Is(err, storageerrors.ErrorFileNotFoundInLocation) { + continue + } + + return "", err + } + + return endpointConf.Endpoint + "/" + bucket, nil + } + } + + return "", storageerrors.ErrorFileNotFoundInLocation +} diff --git a/sda-download/internal/storage/v2/s3/reader/get_file_size.go b/sda-download/internal/storage/v2/s3/reader/get_file_size.go new file mode 100644 index 0000000000..e0ab0cf1dc --- /dev/null +++ b/sda-download/internal/storage/v2/s3/reader/get_file_size.go @@ -0,0 +1,50 @@ +// Package storage provides interface for storage areas, e.g. s3 or POSIX file system. +package reader + +import ( + "context" + "errors" + "fmt" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +// GetFileSize returns the size of a specific object +func (reader *Reader) GetFileSize(ctx context.Context, location, filePath string) (int64, error) { + endpoint, bucket, err := parseLocation(location) + if err != nil { + return 0, err + } + + client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + if err != nil { + return 0, err + } + + size, err := reader.getFileSize(ctx, client, bucket, filePath) + if err != nil { + return 0, err + } + + return size, nil +} +func (reader *Reader) getFileSize(ctx context.Context, client *s3.Client, bucket, filePath string) (int64, error) { + r, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(filePath), + }) + + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey") { + return 0, storageerrors.ErrorFileNotFoundInLocation + } + + return 0, fmt.Errorf("failed to head object: %s, bucket: %s, due to: %v", filePath, bucket, err) + } + + return *r.ContentLength, nil +} diff --git a/sda-download/internal/storage/seekable.go b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go similarity index 62% rename from sda-download/internal/storage/seekable.go rename to sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go index 5a6a36156c..4c35d7f3d0 100644 --- a/sda-download/internal/storage/seekable.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -1,33 +1,18 @@ -// Package storage provides interface for storage areas, e.g. s3 or POSIX file system. -package storage +package reader import ( "bytes" + "context" "errors" "fmt" "io" "strings" "sync" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" ) -// NewFileReadSeeker returns an io.ReadSeeker instance -func (pb *posixBackend) NewFileReadSeeker(filePath string) (io.ReadSeekCloser, error) { - reader, err := pb.NewFileReader(filePath) - if err != nil { - return nil, err - } - - seeker, ok := reader.(io.ReadSeekCloser) - if !ok { - return nil, errors.New("invalid posixBackend") - } - - return seeker, nil -} - // s3CacheBlock is used to keep track of cached data type s3CacheBlock struct { start int64 @@ -35,9 +20,10 @@ type s3CacheBlock struct { data []byte } -// s3Reader is the vehicle to keep track of needed state for the reader -type s3Reader struct { - s3Backend +// s3SeekableReader is the vehicle to keep track of needed state for the reader +type s3SeekableReader struct { + client *s3.Client + bucket string currentOffset int64 local []s3CacheBlock filePath string @@ -46,35 +32,54 @@ type s3Reader struct { outstandingPrefetches []int64 seeked bool objectReader io.Reader + chunkSize uint64 } -func (sb *s3Backend) NewFileReadSeeker(filePath string) (io.ReadSeekCloser, error) { - objectSize, err := sb.GetFileSize(filePath) +func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) { + endpoint, bucket, err := parseLocation(location) + if err != nil { + return nil, err + } + + client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + if err != nil { + return nil, err + } + objectSize, err := reader.getFileSize(ctx, client, bucket, filePath) if err != nil { return nil, err } - reader := &s3Reader{ - *sb, - 0, - make([]s3CacheBlock, 0, 32), - filePath, - objectSize, - sync.Mutex{}, - make([]int64, 0, 32), - false, - nil, + var chunkSizeBytes uint64 + for _, e := range reader.endpoints { + if e.Endpoint == endpoint { + chunkSizeBytes = e.chunkSizeBytes + + break + } } - return reader, nil + return &s3SeekableReader{ + client: client, + bucket: bucket, + currentOffset: 0, + local: make([]s3CacheBlock, 0, 32), + filePath: filePath, + objectSize: objectSize, + lock: sync.Mutex{}, + outstandingPrefetches: make([]int64, 0, 32), + seeked: false, + objectReader: nil, + chunkSize: chunkSizeBytes, + }, nil } -func (r *s3Reader) Close() (err error) { +func (r *s3SeekableReader) Close() (err error) { return nil } -func (r *s3Reader) pruneCache() { +func (r *s3SeekableReader) pruneCache() { r.lock.Lock() defer r.lock.Unlock() @@ -87,17 +92,13 @@ func (r *s3Reader) pruneCache() { r.local = r.local[keepfrom:] } -func (r *s3Reader) prefetchSize() int64 { - n := r.Conf.Chunksize - - if n >= 5*1024*1024 { - return int64(n) - } - - return 50 * 1024 * 1024 +func (r *s3SeekableReader) prefetchSize() int64 { + // Type conversation safe as chunkSizeBytes checked to be between 5mb and 1gb (in bytes) + //nolint:gosec // disable G115 + return int64(r.chunkSize) } -func (r *s3Reader) prefetchAt(offset int64) { +func (r *s3SeekableReader) prefetchAt(offset int64) { r.pruneCache() r.lock.Lock() @@ -117,8 +118,6 @@ func (r *s3Reader) prefetchAt(offset int64) { } // Not found in cache, we should fetch the data - bucket := aws.String(r.Bucket) - key := aws.String(r.filePath) prefetchSize := r.prefetchSize() r.outstandingPrefetches = append(r.outstandingPrefetches, offset) @@ -127,9 +126,9 @@ func (r *s3Reader) prefetchAt(offset int64) { wantedRange := aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+prefetchSize-1)) - object, err := r.Client.GetObject(&s3.GetObjectInput{ - Bucket: bucket, - Key: key, + object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(r.bucket), + Key: aws.String(r.filePath), Range: wantedRange, }) @@ -165,7 +164,7 @@ func (r *s3Reader) prefetchAt(offset int64) { r.local = append(r.local, s3CacheBlock{offset, int64(len(cacheBytes)), cacheBytes}) } -func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { +func (r *s3SeekableReader) Seek(offset int64, whence int) (int64, error) { r.lock.Lock() defer r.lock.Unlock() @@ -216,7 +215,7 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { } // removeFromOutstanding removes a prefetch from the list of outstanding prefetches once it's no longer active -func (r *s3Reader) removeFromOutstanding(toRemove int64) { +func (r *s3SeekableReader) removeFromOutstanding(toRemove int64) { switch len(r.outstandingPrefetches) { case 0: // Nothing to do @@ -243,7 +242,7 @@ func (r *s3Reader) removeFromOutstanding(toRemove int64) { } // isPrefetching checks if the data is already being fetched -func (r *s3Reader) isPrefetching(offset int64) bool { +func (r *s3SeekableReader) isPrefetching(offset int64) bool { // Walk through the outstanding prefetches for _, p := range r.outstandingPrefetches { if offset >= p && offset < p+r.prefetchSize() { @@ -257,11 +256,11 @@ func (r *s3Reader) isPrefetching(offset int64) bool { } // wholeReader is a helper for when we read the whole object -func (r *s3Reader) wholeReader(dst []byte) (int, error) { +func (r *s3SeekableReader) wholeReader(dst []byte) (int, error) { if r.objectReader == nil { // First call, setup a reader for the object - object, err := r.Client.GetObject(&s3.GetObjectInput{ - Bucket: aws.String(r.Bucket), + object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + Bucket: aws.String(r.bucket), Key: aws.String(r.filePath), }) @@ -277,7 +276,7 @@ func (r *s3Reader) wholeReader(dst []byte) (int, error) { return r.objectReader.Read(dst) } -func (r *s3Reader) Read(dst []byte) (n int, err error) { +func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { r.lock.Lock() defer r.lock.Unlock() @@ -325,7 +324,7 @@ func (r *s3Reader) Read(dst []byte) (n int, err error) { // Not found in cache, need to fetch data - bucket := aws.String(r.Bucket) + bucket := aws.String(r.bucket) key := aws.String(r.filePath) wantedRange := aws.String(fmt.Sprintf("bytes=%d-%d", r.currentOffset, r.currentOffset+r.prefetchSize()-1)) @@ -334,7 +333,7 @@ func (r *s3Reader) Read(dst []byte) (n int, err error) { r.lock.Unlock() - object, err := r.Client.GetObject(&s3.GetObjectInput{ + object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ Bucket: bucket, Key: key, Range: wantedRange, @@ -368,101 +367,3 @@ func (r *s3Reader) Read(dst []byte) (n int, err error) { return n, err } - -// seekableMultiReader is a helper struct to allow io.MultiReader to be used with a seekable reader -type seekableMultiReader struct { - readers []io.Reader - sizes []int64 - currentOffset int64 - totalSize int64 -} - -// SeekableMultiReader constructs a multireader that supports seeking. Requires -// all passed readers to be seekable -func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { - r := make([]io.Reader, len(readers)) - sizes := make([]int64, len(readers)) - - copy(r, readers) - - var totalSize int64 - for i, reader := range readers { - seeker, ok := reader.(io.ReadSeeker) - if !ok { - return nil, fmt.Errorf("reader %d to SeekableMultiReader is not seekable", i) - } - - size, err := seeker.Seek(0, io.SeekEnd) - if err != nil { - return nil, fmt.Errorf("size determination failed for reader %d to SeekableMultiReader: %v", i, err) - } - - sizes[i] = size - totalSize += size - } - - return &seekableMultiReader{r, sizes, 0, totalSize}, nil -} - -func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - r.currentOffset = offset - case io.SeekCurrent: - r.currentOffset += offset - case io.SeekEnd: - r.currentOffset = r.totalSize + offset - default: - return 0, errors.New("unsupported whence") - } - - return r.currentOffset, nil -} - -func (r *seekableMultiReader) Read(dst []byte) (int, error) { - var readerStartAt int64 - - for i, reader := range r.readers { - if r.currentOffset < readerStartAt { - // We want data from a previous reader (? HELP ?) - readerStartAt += r.sizes[i] - - continue - } - - if readerStartAt+r.sizes[i] < r.currentOffset { - // We want data from a later reader - readerStartAt += r.sizes[i] - - continue - } - - // At least part of the data is in this reader - - seekable, ok := reader.(io.ReadSeeker) - if !ok { - return 0, errors.New("expected seekable reader but changed") - } - - _, err := seekable.Seek(r.currentOffset-int64(readerStartAt), 0) - if err != nil { - return 0, fmt.Errorf("unexpected error while seeking: %v", err) - } - - n, err := seekable.Read(dst) - r.currentOffset += int64(n) - - if n > 0 || err != io.EOF { - if err == io.EOF && r.currentOffset < r.totalSize { - // More data left, hold that EOF - err = nil - } - - return n, err - } - - readerStartAt += r.sizes[i] - } - - return 0, io.EOF -} diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_reader.go b/sda-download/internal/storage/v2/s3/reader/new_file_reader.go new file mode 100644 index 0000000000..a8bbaa16dc --- /dev/null +++ b/sda-download/internal/storage/v2/s3/reader/new_file_reader.go @@ -0,0 +1,41 @@ +package reader + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +func (reader *Reader) NewFileReader(ctx context.Context, location, filePath string) (io.ReadCloser, error) { + endpoint, bucket, err := parseLocation(location) + if err != nil { + return nil, err + } + + client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + if err != nil { + return nil, err + } + + r, err := client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(filePath), + }) + + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) && (apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "NoSuchKey") { + return nil, storageerrors.ErrorFileNotFoundInLocation + } + + return nil, fmt.Errorf("failed to get object: %s, bucket: %s, endpoint: %s, due to: %v", filePath, bucket, endpoint, err) + } + + return r.Body, nil +} diff --git a/sda-download/internal/storage/v2/s3/reader/reader.go b/sda-download/internal/storage/v2/s3/reader/reader.go new file mode 100644 index 0000000000..99d394c69e --- /dev/null +++ b/sda-download/internal/storage/v2/s3/reader/reader.go @@ -0,0 +1,80 @@ +// Package storage provides interface for storage areas, e.g. s3 or POSIX file system. +package reader + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" +) + +type Reader struct { + endpoints []*endpointConfig +} + +func NewReader(ctx context.Context, backendName string) (*Reader, error) { + endPoints, err := loadConfig(backendName) + if err != nil { + return nil, err + } + + backend := &Reader{ + endpoints: endPoints, + } + // Verify endpoint connections + for _, e := range backend.endpoints { + client, err := e.getS3Client(ctx) + if err != nil { + return nil, err + } + // Use list buckets to verify if client valid + _, err = client.ListBuckets(ctx, &s3.ListBucketsInput{}) + if err != nil { + return nil, fmt.Errorf("failed to call S3 client at endpoint: %s, due to: %v", e.Endpoint, err) + } + } + if len(backend.endpoints) == 0 { + return nil, storageerrors.ErrorNoValidLocations + } + + return backend, nil +} + +func (reader *Reader) getS3ClientForEndpoint(ctx context.Context, endpoint string) (*s3.Client, error) { + for _, e := range reader.endpoints { + if e.Endpoint != endpoint { + continue + } + client, err := e.getS3Client(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create S3 client to endpoint: %s, due to %v", endpoint, err) + } + + return client, nil + } + + return nil, storageerrors.ErrorNoEndpointConfiguredForLocation +} + +// parseLocation attempts to parse a location to a s3 endpoint, and a bucket +// expected format of location is "${ENDPOINT}/${BUCKET} +func parseLocation(location string) (string, string, error) { + locAsURL, err := url.Parse(location) + if err != nil { + return "", "", storageerrors.ErrorInvalidLocation + } + + endpoint := strings.TrimSuffix(location, locAsURL.RequestURI()) + if endpoint == "" { + return "", "", storageerrors.ErrorInvalidLocation + } + bucketName := strings.TrimPrefix(locAsURL.RequestURI(), "/") + if bucketName == "" { + return "", "", storageerrors.ErrorInvalidLocation + } + + return endpoint, bucketName, nil +} diff --git a/sda-download/internal/storage/v2/s3/reader/reader_test.go b/sda-download/internal/storage/v2/s3/reader/reader_test.go new file mode 100644 index 0000000000..854d3e3055 --- /dev/null +++ b/sda-download/internal/storage/v2/s3/reader/reader_test.go @@ -0,0 +1,742 @@ +package reader + +import ( + "context" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" + + "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" +) + +type ReaderTestSuite struct { + suite.Suite + reader *Reader + + configDir string + + bigFilePath string + s3Mock1, s3Mock2 *httptest.Server +} + +func TestReaderTestSuite(t *testing.T) { + suite.Run(t, new(ReaderTestSuite)) +} + +func (ts *ReaderTestSuite) SetupSuite() { + ts.configDir = ts.T().TempDir() + + // create a big file + bigFile, err := os.CreateTemp(ts.configDir, "bigfile-") + if err != nil { + ts.FailNow("failed to create big test file", err) + } + + if _, err = bigFile.WriteString("This is a big file for testing seekable s3 reader"); err != nil { + ts.FailNow("failed to write big test file", err) + } + for range 6 * 1000 * 1000 { + if _, err = bigFile.WriteString("a"); err != nil { + ts.FailNow("failed to write big test file", err) + } + } + if _, err = bigFile.WriteString("file is ending now"); err != nil { + ts.FailNow("failed to write big test file", err) + } + _ = bigFile.Close() + ts.bigFilePath = bigFile.Name() + + ts.s3Mock1 = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch { + case strings.HasSuffix(req.RequestURI, "ListBuckets"): + w.WriteHeader(http.StatusOK) + + _, _ = fmt.Fprint(w, ` + + + + + mock_s3_1_bucket_1 + us-east-1 + mock_s3_1_bucket_1 + + + mock_s3_1_bucket_2 + us-east-1 + mock_s3_1_bucket_2 + + + +`) + case strings.HasPrefix(req.RequestURI, "/mock_s3_1_bucket_1"): + if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file1.txt") { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, "file 1 content in mock s3 1, bucket 1") + + return + } + + if req.Method == "HEAD" && strings.Contains(req.RequestURI, "file1.txt") { + w.Header().Set("Content-Length", "101") + w.WriteHeader(http.StatusOK) + + return + } + + w.WriteHeader(http.StatusNotFound) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + + case strings.HasPrefix(req.RequestURI, "/mock_s3_1_bucket_2"): + if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file2.txt") { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, "file 2 content in mock s3 1, bucket 2") + + return + } + + if req.Method == "HEAD" && strings.Contains(req.RequestURI, "file2.txt") { + w.Header().Set("Content-Length", "102") + w.WriteHeader(http.StatusOK) + + return + } + + if strings.Contains(req.RequestURI, "seekable_big_file.txt") { + content, err := os.ReadFile(ts.bigFilePath) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + _, _ = fmt.Fprintf(w, "could not read the big seekable file due to: %v", err) + } + + switch { + case req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject"): + byteRange := req.Header.Get("Range") + start, _ := strconv.Atoi(strings.Split(strings.Split(byteRange, "=")[1], "-")[0]) + end, _ := strconv.Atoi(strings.Split(strings.Split(byteRange, "=")[1], "-")[1]) + if len(content) < end { + end = len(content) + } + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, len(content), len(content))) + seekedContent := content[start:end] + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, string(seekedContent)) + + case req.Method == "HEAD": + w.Header().Set("Content-Length", strconv.Itoa(len(content))) + w.WriteHeader(http.StatusOK) + default: + } + + return + } + + w.WriteHeader(http.StatusNotFound) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + default: + w.WriteHeader(http.StatusInternalServerError) + _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) + } + })) + + ts.s3Mock2 = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + switch { + case strings.HasSuffix(req.RequestURI, "ListBuckets"): + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, ` + + + + + mock_s3_2_bucket_1 + us-east-1 + mock_s3_2_bucket_1 + + + mock_s3_2_bucket_2 + us-east-1 + mock_s3_2_bucket_2 + + + + mock + mock + + +`) + case strings.HasPrefix(req.RequestURI, "/mock_s3_2_bucket_1"): + if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file3.txt") { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, "file 3 content in mock s3 2, bucket 1") + + return + } + + if req.Method == "HEAD" && strings.Contains(req.RequestURI, "file3.txt") { + w.Header().Set("Content-Length", "103") + w.WriteHeader(http.StatusOK) + + return + } + + if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "seekable.txt") { + byteRange := req.Header.Get("Range") + start, _ := strconv.Atoi(strings.Split(strings.Split(byteRange, "=")[1], "-")[0]) + end, _ := strconv.Atoi(strings.Split(strings.Split(byteRange, "=")[1], "-")[1]) + content := "this file is mocked to be seekable following s3 logic in mock s3 2, bucket 1" + + if len(content) < end { + end = len(content) + } + seekedContent := content[start:end] + w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", start, end, len(content))) + w.WriteHeader(http.StatusOK) + + _, _ = fmt.Fprint(w, seekedContent) + + return + } + + if req.Method == "HEAD" && strings.Contains(req.RequestURI, "seekable.txt") { + w.Header().Set("Content-Length", "77") + w.WriteHeader(http.StatusOK) + + return + } + + w.WriteHeader(http.StatusNotFound) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + case strings.HasPrefix(req.RequestURI, "/mock_s3_2_bucket_2"): + if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file4.txt") { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, "file 4 content in mock s3 2, bucket 2") + + return + } + + if req.Method == "HEAD" && strings.Contains(req.RequestURI, "file4.txt") { + w.Header().Set("Content-Length", "104") + w.WriteHeader(http.StatusOK) + + return + } + + w.WriteHeader(http.StatusNotFound) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + default: + w.WriteHeader(http.StatusInternalServerError) + _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) + } + })) + + if err := os.WriteFile(filepath.Join(ts.configDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + test: + s3: + - endpoint: %s + access_key: access_key1 + secret_key: secret_key1 + disable_https: true + region: us-east-1 + - endpoint: %s + access_key: access_key2 + secret_key: secret_key2 + disable_https: true + region: us-east-1 +`, ts.s3Mock1.URL, ts.s3Mock2.URL)), 0600); err != nil { + ts.FailNow(err.Error()) + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + viper.SetConfigFile(filepath.Join(ts.configDir, "config.yaml")) + + if err := viper.ReadInConfig(); err != nil { + ts.FailNow(err.Error()) + } + + ts.reader, err = NewReader(context.TODO(), "test") + if err != nil { + ts.FailNow(err.Error()) + } +} + +func (ts *ReaderTestSuite) TestLoadConfig() { + for _, test := range []struct { + testName string + config string + expectedErrorMsg string + }{ + { + testName: "MissingEndpoint", + config: ` +storage: + config_test: + s3: + - access_key: access_key1 + secret_key: secret_key1 + disable_https: true + region: us-east-1 + chunk_size: 1mb +`, + expectedErrorMsg: "missing required parameter: endpoint", + }, { + testName: "MissingAccessKey", + config: ` +storage: + config_test: + s3: + - endpoint: 123 + secret_key: secret_key1 + disable_https: true + region: us-east-1 + chunk_size: 1mb +`, + expectedErrorMsg: "missing required parameter: access_key", + }, { + testName: "MissingSecretKey", + config: ` +storage: + config_test: + s3: + - endpoint: 123 + access_key: access_key1 + disable_https: true + region: us-east-1 + chunk_size: 1mb +`, + expectedErrorMsg: "missing required parameter: secret_key", + }, { + testName: "InvalidChunkSize", + config: ` +storage: + config_test: + s3: + - endpoint: 123 + access_key: access_key1 + secret_key: secret_key1 + disable_https: true + region: us-east-1 + chunk_size: -100 +`, + expectedErrorMsg: "could not parse chunk_size as a valid data size", + }, { + testName: "HTTPSEndpointWithDisableHttps", + config: ` +storage: + config_test: + s3: + - endpoint: https://123 + access_key: access_key1 + secret_key: secret_key1 + disable_https: true + region: us-east-1 + chunk_size: -100 +`, + expectedErrorMsg: "https scheme in endpoint when HTTPS is disabled", + }, { + testName: "HTTPEndpointWithEnabledHttps", + config: ` +storage: + config_test: + s3: + - endpoint: http://123 + access_key: access_key1 + secret_key: secret_key1 + disable_https: false + region: us-east-1 + chunk_size: -100 +`, + expectedErrorMsg: "http scheme in endpoint when using HTTPS", + }, + } { + ts.T().Run(test.testName, func(t *testing.T) { + if err := os.WriteFile(filepath.Join(ts.configDir, "config_.yaml"), []byte(test.config), 0600); err != nil { + assert.FailNow(t, err.Error()) + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + viper.SetConfigFile(filepath.Join(ts.configDir, "config_.yaml")) + + if err := viper.ReadInConfig(); err != nil { + assert.FailNow(t, err.Error()) + } + + _, err := NewReader(context.TODO(), "config_test") + assert.EqualError(t, err, test.expectedErrorMsg) + }) + } +} + +func (ts *ReaderTestSuite) TestNewFileReader() { + for _, test := range []struct { + testName string + + endpointToReadFrom string + bucketToReadFrom string + filePathToRead string + expectedError error + expectedFileContent string + }{ + { + testName: "InvalidLocation", + endpointToReadFrom: "", + bucketToReadFrom: "", + filePathToRead: "", + expectedError: storageerrors.ErrorInvalidLocation, + expectedFileContent: "", + }, { + testName: "ReadFrom1Bucket1", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_1", + filePathToRead: "file1.txt", + expectedError: nil, + expectedFileContent: "file 1 content in mock s3 1, bucket 1", + }, { + testName: "ReadFrom1Bucket2", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_2", + filePathToRead: "file2.txt", + expectedError: nil, + expectedFileContent: "file 2 content in mock s3 1, bucket 2", + }, { + testName: "ReadFrom2Bucket1", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_1", + filePathToRead: "file3.txt", + expectedError: nil, + expectedFileContent: "file 3 content in mock s3 2, bucket 1", + }, { + testName: "ReadFrom2Bucket2", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_2", + filePathToRead: "file4.txt", + expectedError: nil, + expectedFileContent: "file 4 content in mock s3 2, bucket 2", + }, { + testName: "ReadFrom1Bucket1_NotFound", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_1", + filePathToRead: "file2.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedFileContent: "", + }, { + testName: "ReadFrom1Bucket2_NotFound", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_2", + filePathToRead: "file1.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedFileContent: "", + }, { + testName: "ReadFrom2Bucket1_NotFound", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_1", + filePathToRead: "file1.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedFileContent: "", + }, { + testName: "ReadFrom2Bucket2_NotFound", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_2", + filePathToRead: "file1.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedFileContent: "", + }, + } { + ts.T().Run(test.testName, func(t *testing.T) { + fileReader, err := ts.reader.NewFileReader(context.TODO(), test.endpointToReadFrom+"/"+test.bucketToReadFrom, test.filePathToRead) + assert.Equal(t, test.expectedError, err) + + if fileReader != nil { + content, err := io.ReadAll(fileReader) + assert.NoError(t, err) + + assert.Equal(t, test.expectedFileContent, string(content)) + _ = fileReader.Close() + } + }) + } +} + +func (ts *ReaderTestSuite) TestGetFileSize() { + for _, test := range []struct { + testName string + + endpointToReadFrom string + bucketToReadFrom string + filePathToRead string + expectedError error + expectedSize int64 + }{ + { + testName: "InvalidLocation", + endpointToReadFrom: "", + bucketToReadFrom: "", + filePathToRead: "", + expectedError: storageerrors.ErrorInvalidLocation, + expectedSize: 0, + }, { + testName: "ReadFrom1Bucket1", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_1", + filePathToRead: "file1.txt", + expectedError: nil, + expectedSize: 101, + }, { + testName: "ReadFrom1Bucket2", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_2", + filePathToRead: "file2.txt", + expectedError: nil, + expectedSize: 102, + }, { + testName: "ReadFrom2Bucket1", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_1", + filePathToRead: "file3.txt", + expectedError: nil, + expectedSize: 103, + }, { + testName: "ReadFrom2Bucket2", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_2", + filePathToRead: "file4.txt", + expectedError: nil, + expectedSize: 104, + }, { + testName: "ReadFrom1Bucket1_NotFound", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_1", + filePathToRead: "file2.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedSize: 0, + }, { + testName: "ReadFrom1Bucket2_NotFound", + endpointToReadFrom: ts.s3Mock1.URL, + bucketToReadFrom: "mock_s3_1_bucket_2", + filePathToRead: "file1.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedSize: 0, + }, { + testName: "ReadFrom2Bucket2_NotFound", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_2", + filePathToRead: "file1.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedSize: 0, + }, { + testName: "ReadFrom2Bucket2_NotFound", + endpointToReadFrom: ts.s3Mock2.URL, + bucketToReadFrom: "mock_s3_2_bucket_2", + filePathToRead: "file1.txt", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + expectedSize: 0, + }, + } { + ts.T().Run(test.testName, func(t *testing.T) { + size, err := ts.reader.GetFileSize(context.TODO(), test.endpointToReadFrom+"/"+test.bucketToReadFrom, test.filePathToRead) + assert.Equal(t, test.expectedError, err) + assert.Equal(t, test.expectedSize, size) + }) + } +} + +func (ts *ReaderTestSuite) TestFindFile() { + for _, test := range []struct { + testName string + + fileToFind string + expectedLocation string + expectedError error + }{ + { + testName: "NotFound", + fileToFind: "not_found.txt", + expectedLocation: "", + expectedError: storageerrors.ErrorFileNotFoundInLocation, + }, { + testName: "FoundIn1Bucket1", + fileToFind: "file1.txt", + expectedLocation: ts.s3Mock1.URL + "/mock_s3_1_bucket_1", + expectedError: nil, + }, { + testName: "FoundIn1Bucket2", + fileToFind: "file2.txt", + expectedLocation: ts.s3Mock1.URL + "/mock_s3_1_bucket_2", + expectedError: nil, + }, { + testName: "FoundIn2Bucket1", + fileToFind: "file3.txt", + expectedLocation: ts.s3Mock2.URL + "/mock_s3_2_bucket_1", + expectedError: nil, + }, { + testName: "FoundIn2Bucket2", + fileToFind: "file4.txt", + expectedLocation: ts.s3Mock2.URL + "/mock_s3_2_bucket_2", + expectedError: nil, + }, + } { + ts.T().Run(test.testName, func(t *testing.T) { + location, err := ts.reader.FindFile(context.TODO(), test.fileToFind) + assert.Equal(t, test.expectedError, err) + assert.Equal(t, test.expectedLocation, location) + }) + } +} + +func (ts *ReaderTestSuite) TestNewFileSeekReader_ReadFrom2Bucket1() { + fileSeekReader, err := ts.reader.NewFileReadSeeker(context.TODO(), ts.s3Mock2.URL+"/mock_s3_2_bucket_1", "seekable.txt") + if err != nil { + ts.FailNow(err.Error()) + } + + start := int64(5) + lenToRead := int64(14) + + _, err = fileSeekReader.Seek(start, 0) + ts.NoError(err) + + content := make([]byte, lenToRead) + + read, err := fileSeekReader.Read(content) + ts.NoError(err) + ts.Equal(lenToRead, int64(read)) + + ts.Equal("file is mocked", string(content)) + _ = fileSeekReader.Close() +} + +func (ts *ReaderTestSuite) TestNewFileSeekReader_ReadFrom1Bucket2_BigFile() { + fileSeekReader, err := ts.reader.NewFileReadSeeker(context.TODO(), ts.s3Mock1.URL+"/mock_s3_1_bucket_2", "seekable_big_file.txt") + if err != nil { + ts.FailNow(err.Error()) + } + + start := int64(14) + end := int64(6000060) + + _, err = fileSeekReader.Seek(start, 0) + ts.NoError(err) + + togo := end - start + + buf := make([]byte, 4096) + + var content []byte + // Loop until we've read what we should (if no/faulty end given, that's EOF) + for end == 0 || togo > 0 { + rbuf := buf + + if end != 0 && togo < 4096 { + // If we don't want to read as much as 4096 bytes + rbuf = buf[:togo] + } + r, err := fileSeekReader.Read(rbuf) + togo -= int64(r) + + if err == io.EOF && r == 0 { + break + } + + if err != nil && err != io.EOF { + ts.FailNow(err.Error()) + } + + content = append(content, rbuf[:r]...) + } + + ts.NoError(err) + ts.Equal(end-start, int64(len(content))) + + ts.Equal("aaaaaaafile is end", string(content[len(content)-len("aaaaaaafile is end"):])) + ts.Equal("file for testing seekable s3 readeraaaaaaaaaa", string(content[:len("file for testing seekable s3 readeraaaaaaaaaa")])) + _ = fileSeekReader.Close() +} + +func (ts *ReaderTestSuite) TestNewFileSeekReader_ReadFrom2Bucket1FileNotFound() { + _, err := ts.reader.NewFileReadSeeker(context.TODO(), ts.s3Mock2.URL+"/mock_s3_2_bucket_1", "not_exist.txt") + ts.EqualError(err, storageerrors.ErrorFileNotFoundInLocation.Error()) +} + +// s3 seekable reader internal test here, messing with internals to expose behaviour +func (ts *ReaderTestSuite) TestFileReadSeeker_Internal_NoPrefetchFromSeek() { + fileSeekReader, err := ts.reader.NewFileReadSeeker(context.TODO(), ts.s3Mock1.URL+"/mock_s3_1_bucket_2", "seekable_big_file.txt") + if err != nil { + ts.FailNow(err.Error()) + } + s3SeekableReaderCast, ok := fileSeekReader.(*s3SeekableReader) + if !ok { + ts.FailNow("could not cast io.ReadSeekerCloser to *s3SeekableReader returned from NewFileReadSeeker") + } + s3SeekableReaderCast.seeked = true + + var readBackBuffer [4096]byte + + _, err = s3SeekableReaderCast.Read(readBackBuffer[0:4096]) + ts.Equal("This is a big file", string(readBackBuffer[:18]), "did not read back data as expected") + ts.NoError(err, "read returned unexpected error") + _ = fileSeekReader.Close() +} + +// s3 seekable reader internal test here, messing with internals to expose behaviour +func (ts *ReaderTestSuite) TestFileReadSeeker_Internal_Cache() { + fileSeekReader, err := ts.reader.NewFileReadSeeker(context.TODO(), ts.s3Mock1.URL+"/mock_s3_1_bucket_2", "seekable_big_file.txt") + if err != nil { + ts.FailNow(err.Error()) + } + s3SeekableReaderCast, ok := fileSeekReader.(*s3SeekableReader) + if !ok { + ts.FailNow("could not cast io.ReadSeekerCloser to *s3SeekableReader returned from NewFileReadSeeker") + } + s3SeekableReaderCast.seeked = true + + s3SeekableReaderCast.prefetchAt(0) + ts.Equal(1, len(s3SeekableReaderCast.local), "nothing cached after prefetch") + // Clear cache + s3SeekableReaderCast.local = s3SeekableReaderCast.local[:0] + + s3SeekableReaderCast.outstandingPrefetches = []int64{0} + + var readBackBuffer [4096]byte + n, err := s3SeekableReaderCast.Read(readBackBuffer[0:4096]) + ts.Nil(err, "read returned unexpected error") + ts.Equal(0, n, "got data when we should get 0 because of prefetch") + + for i := 0; i < 30; i++ { + s3SeekableReaderCast.local = append(s3SeekableReaderCast.local, s3CacheBlock{90000000, int64(0), nil}) + } + s3SeekableReaderCast.prefetchAt(0) + ts.Equal(8, len(s3SeekableReaderCast.local), "unexpected length of cache after prefetch") + + s3SeekableReaderCast.outstandingPrefetches = []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} + s3SeekableReaderCast.removeFromOutstanding(9) + ts.Equal(s3SeekableReaderCast.outstandingPrefetches, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8}, "unexpected outstanding prefetches after remove") + s3SeekableReaderCast.removeFromOutstanding(19) + ts.Equal(s3SeekableReaderCast.outstandingPrefetches, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8}, "unexpected outstanding prefetches after remove") + s3SeekableReaderCast.removeFromOutstanding(5) + // We don't care about the internal order, sort for simplicity + slices.Sort(s3SeekableReaderCast.outstandingPrefetches) + ts.Equal(s3SeekableReaderCast.outstandingPrefetches, []int64{0, 1, 2, 3, 4, 6, 7, 8}, "unexpected outstanding prefetches after remove") + + s3SeekableReaderCast.objectReader = nil + s3SeekableReaderCast.bucket = "" + s3SeekableReaderCast.filePath = "" + data := make([]byte, 100) + _, err = s3SeekableReaderCast.wholeReader(data) + ts.NotNil(err, "wholeReader object instantiation worked when it should have failed") +} + +func (ts *ReaderTestSuite) TestNewFileReaderSeeker_InvalidLocation() { + _, err := ts.reader.NewFileReader(context.TODO(), "", "") + ts.EqualError(err, storageerrors.ErrorInvalidLocation.Error()) +} diff --git a/sda-download/internal/storage/v2/seekable_multi_reader.go b/sda-download/internal/storage/v2/seekable_multi_reader.go new file mode 100644 index 0000000000..fa1e54ef9c --- /dev/null +++ b/sda-download/internal/storage/v2/seekable_multi_reader.go @@ -0,0 +1,105 @@ +package storage + +import ( + "errors" + "fmt" + "io" +) + +// seekableMultiReader is a helper struct to allow io.MultiReader to be used with a seekable reader +type seekableMultiReader struct { + readers []io.Reader + sizes []int64 + currentOffset int64 + totalSize int64 +} + +// SeekableMultiReader constructs a multireader that supports seeking. Requires +// all passed readers to be seekable +func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { + r := make([]io.Reader, len(readers)) + sizes := make([]int64, len(readers)) + + copy(r, readers) + + var totalSize int64 + for i, reader := range readers { + seeker, ok := reader.(io.ReadSeeker) + if !ok { + return nil, fmt.Errorf("reader %d to SeekableMultiReader is not seekable", i) + } + + size, err := seeker.Seek(0, io.SeekEnd) + if err != nil { + return nil, fmt.Errorf("size determination failed for reader %d to SeekableMultiReader: %v", i, err) + } + + sizes[i] = size + totalSize += size + } + + return &seekableMultiReader{r, sizes, 0, totalSize}, nil +} + +func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + r.currentOffset = offset + case io.SeekCurrent: + r.currentOffset += offset + case io.SeekEnd: + r.currentOffset = r.totalSize + offset + default: + return 0, errors.New("unsupported whence") + } + + return r.currentOffset, nil +} + +func (r *seekableMultiReader) Read(dst []byte) (int, error) { + var readerStartAt int64 + + for i, reader := range r.readers { + if r.currentOffset < readerStartAt { + // We want data from a previous reader (? HELP ?) + readerStartAt += r.sizes[i] + + continue + } + + if readerStartAt+r.sizes[i] < r.currentOffset { + // We want data from a later reader + readerStartAt += r.sizes[i] + + continue + } + + // At least part of the data is in this reader + + seekable, ok := reader.(io.ReadSeeker) + if !ok { + return 0, errors.New("expected seekable reader but changed") + } + + _, err := seekable.Seek(r.currentOffset-int64(readerStartAt), 0) + if err != nil { + return 0, fmt.Errorf("unexpected error while seeking: %v", err) + } + + n, err := seekable.Read(dst) + r.currentOffset += int64(n) + + if n > 0 || err != io.EOF { + if err == io.EOF && r.currentOffset < r.totalSize { + // More data left, hold that EOF + err = nil + } + + return n, err + } + + readerStartAt += r.sizes[i] + } + + return 0, io.EOF +} diff --git a/sda-download/internal/storage/v2/seekable_multi_reader_test.go b/sda-download/internal/storage/v2/seekable_multi_reader_test.go new file mode 100644 index 0000000000..1186f9a4ce --- /dev/null +++ b/sda-download/internal/storage/v2/seekable_multi_reader_test.go @@ -0,0 +1,59 @@ +package storage + +import ( + "bytes" + "io" + "os" + "testing" + + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" +) + +func TestSeekableMultiReader(t *testing.T) { + writeData := []byte("this is a test") + + readers := make([]io.Reader, 10) + for i := 0; i < 10; i++ { + readers[i] = bytes.NewReader(writeData) + } + + seeker, err := SeekableMultiReader(readers...) + assert.Nil(t, err, "unexpected error from creating SeekableMultiReader") + + var readBackBuffer [4096]byte + + _, err = seeker.Read(readBackBuffer[0:4096]) + assert.Equal(t, writeData, readBackBuffer[:14], "did not read back data as expected") + assert.Nil(t, err, "unexpected error from read") + + offset, err := seeker.Seek(60, io.SeekStart) + + assert.Nil(t, err, "Seek failed") + assert.Equal(t, int64(60), offset, "Seek did not return expected offset") + + // We don't know how many bytes this should return + _, err = seeker.Read(readBackBuffer[0:4096]) + assert.Equal(t, writeData[4:], readBackBuffer[:10], "did not read back data as expected") + assert.Nil(t, err, "Read failed when it should not") + + offset, err = seeker.Seek(0, io.SeekEnd) + assert.Equal(t, int64(140), offset, "Seek did not return expected offset") + assert.Nil(t, err, "Seek failed when it should not") + + n, err := seeker.Read(readBackBuffer[0:4096]) + + assert.Equal(t, 0, n, "Read did not return expected amounts of bytes") + assert.Equal(t, io.EOF, err, "did not get EOF as expected") + + offset, err = seeker.Seek(56, io.SeekStart) + assert.Equal(t, int64(56), offset, "Seek did not return expected offset") + assert.Nil(t, err, "Seek failed unexpectedly") + + largeBuf := make([]byte, 65536) + readLen, err := seeker.Read(largeBuf) + assert.Nil(t, err, "unexpected error when reading back data") + assert.Equal(t, 14, readLen, "did not read back expect amount of data") + + log.SetOutput(os.Stdout) +} diff --git a/sda-download/internal/storage/v2/storageerrors/errors.go b/sda-download/internal/storage/v2/storageerrors/errors.go new file mode 100644 index 0000000000..bec0e3da54 --- /dev/null +++ b/sda-download/internal/storage/v2/storageerrors/errors.go @@ -0,0 +1,9 @@ +package storageerrors + +import "errors" + +var ErrorFileNotFoundInLocation = errors.New("file not found in location") +var ErrorNoValidLocations = errors.New("no valid locations") +var ErrorInvalidLocation = errors.New("invalid location") +var ErrorNoEndpointConfiguredForLocation = errors.New("no endpoint configured for location") +var ErrorNoValidReader = errors.New("no valid reader configured") From 3b67020f739a9b1f53a6f4df78a633f1c7291717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 21 Jan 2026 09:20:49 +0100 Subject: [PATCH 082/530] feat(download): update db to get schema version on NewDB (and replace ping as it calls the database), add db version check at v23 --- sda-download/cmd/main.go | 5 ++++ sda-download/internal/database/database.go | 24 +++++++++++++++---- .../internal/database/database_test.go | 2 +- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index 5ad97a3ab4..e8ab74c9a7 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -33,6 +33,11 @@ func init() { log.Panicf("database connection failed, reason: %v", err) } defer db.Close() + if db.Version < 23 { + log.Error("database schema v23 is required") + + return + } database.DB = db // Initialise HTTP client for making requests diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index 12c10c8f94..007f970b7a 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -21,6 +21,7 @@ var DB *SQLdb // SQLdb struct that acts as a receiver for the DB update methods type SQLdb struct { DB *sql.DB + Version int ConnInfo string } @@ -72,15 +73,15 @@ func NewDB(conf config.DatabaseConfig) (*SQLdb, error) { return nil, err } - if err = db.Ping(); err != nil { - log.Errorf("could not get response from database, %s", err) - - return nil, err + pgdb := &SQLdb{DB: db, ConnInfo: connInfo} + pgdb.Version, err = pgdb.getVersion() + if err != nil { + return nil, fmt.Errorf("failed to fetch database schema version: %v", err) } log.Debug("database connection formed") - return &SQLdb{DB: db, ConnInfo: connInfo}, nil + return pgdb, nil } // buildConnInfo builds a connection string for the database @@ -125,6 +126,19 @@ func (dbs *SQLdb) checkAndReconnectIfNeeded() { } } +func (dbs *SQLdb) getVersion() (int, error) { + dbs.checkAndReconnectIfNeeded() + + log.Debug("Fetching database schema version") + + query := "SELECT MAX(version) FROM sda.dbschema_version;" + + var dbVersion = -1 + err := dbs.DB.QueryRow(query).Scan(&dbVersion) + + return dbVersion, err +} + // GetFiles retrieves the file details var GetFiles = func(datasetID string) ([]*FileInfo, error) { var ( diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index a92bbd441f..a881984e64 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -67,7 +67,7 @@ func TestCheckAndReconnect(t *testing.T) { mock.ExpectPing().WillReturnError(errors.New("ping fail for testing bad conn")) - err := CatchPanicCheckAndReconnect(SQLdb{db, ""}) + err := CatchPanicCheckAndReconnect(SQLdb{DB: db, ConnInfo: "", Version: 0}) assert.Error(t, err, "Should have received error from checkAndReconnectOnNeeded fataling") } From 479248e76b82fb108d4740cfe7d5fc7a7a299347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 22 Jan 2026 11:15:06 +0100 Subject: [PATCH 083/530] feat(database): update unit tests to expect version select statement when calling NewDB --- sda-download/internal/database/database_test.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index a881984e64..987785cc51 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "os" + "regexp" "strings" "testing" "time" @@ -149,8 +150,9 @@ func TestNewDB(t *testing.T) { if err = mock.ExpectationsWereMet(); err != nil { t.Errorf("there were unfulfilled expectations: %s", err) } - - mock.ExpectPing() + db, mock, _ = sqlmock.New(sqlmock.MonitorPingsOption(true)) + mock.ExpectPing().WillReturnError(nil) + mock.ExpectQuery(regexp.QuoteMeta(`SELECT MAX(version) FROM sda.dbschema_version`)).WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(1)) _, err = NewDB(testPgconf) assert.Nilf(t, err, "NewDB failed unexpectedly: %s", err) @@ -167,6 +169,7 @@ func sqlTesterHelper(t *testing.T, f func(sqlmock.Sqlmock, *SQLdb) error) error return db, err } + mock.ExpectQuery(regexp.QuoteMeta(`SELECT MAX(version) FROM sda.dbschema_version`)).WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(1)) testDb, err := NewDB(testPgconf) assert.Nil(t, err, "NewDB failed unexpectedly") From b017c8bacd5fc1b811add05b17cf133d338e585d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 22 Jan 2026 11:35:41 +0100 Subject: [PATCH 084/530] feat(download): remove unhelpful warning log at calculateCoords --- sda-download/api/sda/sda.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index 4c087b360b..d822a8c205 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -528,7 +528,6 @@ var sendStream = func(reader io.Reader, writer http.ResponseWriter, start, end i // it will be used as is. If not, the functions parameters will be used. // If in encrypted mode, the parameters will be adjusted to match the data block boundaries. var calculateCoords = func(start, end int64, htsget_range string, fileDetails *database.FileDownload, encryptedType string) (int64, int64, error) { - log.Warnf("calculate") if htsget_range != "" { startEnd := strings.Split(strings.TrimPrefix(htsget_range, "bytes="), "-") if len(startEnd) > 1 { From 074246e8d62531ba196ae23b8d49eb606fc06439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Feb 2026 09:50:13 +0100 Subject: [PATCH 085/530] feat(download): update config files used in integration tests --- sda-download/dev_utils/config-notls.yaml | 22 +++++++++--------- .../dev_utils/config-notls_local.yaml | 19 ++++++++------- sda-download/dev_utils/config.yaml | 23 +++++++++---------- 3 files changed, 31 insertions(+), 33 deletions(-) diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index 63f8e2e89d..c2fbf480c5 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -6,17 +6,17 @@ log: level: "debug" format: "json" -archive: - type: "" - # S3 backend - url: "http://localhost" - port: 9000 - accesskey: "access" - secretkey: "secretkey" - bucket: "archive" - chunksize: 32 - # posix backend - location: "/tmp" +storage: + archive: + s3: + endpoint: "http://localhost:9000" + access_key: "access" + secret_key: "secretkey" + bucket_prefix: "archive" + region: "us-east-1" + disable_https: true + posix: + path: "/tmp" grpc: host: reencrypt diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml index 57cd74d5d4..ac6d501f90 100644 --- a/sda-download/dev_utils/config-notls_local.yaml +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -6,16 +6,15 @@ log: level: "debug" format: "json" -archive: - type: "s3" - # S3 backend - url: "http://localhost" - port: 19000 - accesskey: "access" - secretkey: "secretKey" - bucket: "archive" - region: "us-east-1" - chunksize: 32 +storage: + archive: + s3: + endpoint: "http://localhost:19000" + access_key: "access" + secret_key: "secretkey" + bucket_prefix: "archive" + region: "us-east-1" + disable_https: true grpc: host: localhost diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index 4dea553fb2..d8ba904876 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -10,18 +10,17 @@ log: level: "debug" format: "json" -archive: - type: "" - # S3 backend - url: "https://localhost" - port: 9000 - accesskey: "access" - secretkey: "secretkey" - bucket: "archive" - chunksize: 32 - cacert: "./dev_utils/certs/ca.pem" - # posix backend - location: "/tmp" +storage: + archive: + s3: + endpoint: "https://localhost:9000" + access_key: "access" + secret_key: "secretkey" + bucket_prefix: "archive" + region: "us-east-1" + ca_cert: "./dev_utils/certs/ca.pem" + posix: + path: "/tmp" session: # session key expiration time in seconds From a7c103ff596b80394684a2cf23b3680871cc57c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Feb 2026 15:58:45 +0100 Subject: [PATCH 086/530] feat(download): use sql.NullString when scanning archive location to avoid error if column is NULL, and check if archive location == "" --- sda-download/api/sda/sda.go | 7 +++++++ sda-download/internal/database/database.go | 7 ++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index d822a8c205..e87842c3bc 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -268,6 +268,13 @@ func Download(c *gin.Context) { return } + if fileDetails.ArchiveLocation == "" { + log.Errorf("archive location for file: %s not known", fileID) + c.String(http.StatusInternalServerError, "archive location not known") + + return + } + // Get query params qStart := c.DefaultQuery("startCoordinate", "0") qEnd := c.DefaultQuery("endCoordinate", "0") diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index 007f970b7a..db348dc2f1 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -455,8 +455,9 @@ func (dbs *SQLdb) getFile(fileID string) (*FileDownload, error) { WHERE stable_id = $1` fd := &FileDownload{} + var archiveLocation sql.NullString var hexString string - err := db.QueryRow(query, fileID).Scan(&fd.ArchivePath, &fd.ArchiveSize, &fd.ArchiveLocation, + err := db.QueryRow(query, fileID).Scan(&fd.ArchivePath, &fd.ArchiveSize, &archiveLocation, &fd.DecryptedSize, &fd.DecryptedChecksum, &fd.LastModified, &hexString) if err != nil { log.Errorf("could not retrieve details for file %s, reason %s", sanitizeString(fileID), err) @@ -464,6 +465,10 @@ func (dbs *SQLdb) getFile(fileID string) (*FileDownload, error) { return nil, err } + if archiveLocation.Valid { + fd.ArchiveLocation = archiveLocation.String + } + fd.Header, err = hex.DecodeString(hexString) if err != nil { log.Errorf("could not decode file header for file %s, reason %s", sanitizeString(fileID), err) From cf18db704ecc1f2c7deff7a13d070205497511ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Feb 2026 15:59:24 +0100 Subject: [PATCH 087/530] feat(download): remove storage v1 required config --- sda-download/internal/config/config.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 2268bdcd76..156fad1c65 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -185,12 +185,6 @@ func NewConfig() (*Map, error) { "db.host", "db.user", "db.password", "db.database", "oidc.configuration.url", "grpc.host", } - if viper.GetString("archive.type") == S3 { - requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) - } else if viper.GetString("archive.type") == POSIX { - requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) - } - for _, s := range requiredConfVars { if !viper.IsSet(s) || viper.GetString(s) == "" { return nil, fmt.Errorf("%s not set", s) From 087eb013d141e8930608fbf38cc704d1b6bfd0f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Feb 2026 16:05:45 +0100 Subject: [PATCH 088/530] feat(download): update integration tests to work with sda-download using storage v2, use local built images in integration tests not hard coded versions, create db download user, use sda db instead of lega, etc --- .../integration/setup/common/10_services.sh | 9 +-- .../setup/common/23_create_db_entries.sh | 18 ++--- .../integration/setup/common/2_build_all.sh | 7 ++ .../setup/posix/1_posix_env_setup.sh | 1 - .../setup/posix/24_posix_update_db_entry.sh | 10 +++ .../setup/s3notls/99_s3_storage_setup.sh | 2 + .../integration/tests/common/30_check_db.sh | 6 +- sda-download/dev_utils/README.md | 6 +- sda-download/dev_utils/compose-no-tls.yml | 53 ++++++++++---- sda-download/dev_utils/compose-sda.yml | 44 +++++++++--- sda-download/dev_utils/compose.yml | 71 +++++++++++++------ sda-download/dev_utils/config-notls.yaml | 8 +-- .../dev_utils/config-notls_local.yaml | 6 +- sda-download/dev_utils/config.yaml | 10 +-- .../dev_utils/run_integration_test.sh | 18 ++--- .../dev_utils/run_integration_test_no_tls.sh | 18 ++--- sda-download/dev_utils/ssl.cnf | 6 -- 17 files changed, 191 insertions(+), 102 deletions(-) create mode 100644 sda-download/.github/integration/setup/common/2_build_all.sh create mode 100644 sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh diff --git a/sda-download/.github/integration/setup/common/10_services.sh b/sda-download/.github/integration/setup/common/10_services.sh index 1680ff9171..222a6947a6 100644 --- a/sda-download/.github/integration/setup/common/10_services.sh +++ b/sda-download/.github/integration/setup/common/10_services.sh @@ -1,15 +1,12 @@ #!/bin/bash # Build containers -docker build -t neicnordic/sda-download:latest . || exit 1 - cd dev_utils || exit 1 bash ./make_certs.sh if [ "$STORAGETYPE" = s3notls ]; then - - docker compose -f compose-no-tls.yml up -d + PR_NUMBER=$(date +%F) docker compose -f compose-no-tls.yml up -d RETRY_TIMES=0 for p in db s3 download; do @@ -34,7 +31,7 @@ else # We need to leave the $tostart variable unquoted here since we want it to split # shellcheck disable=SC2086 - docker compose -f compose.yml up -d $tostart + PR_NUMBER=$(date +%F) docker compose -f compose.yml up -d $tostart database_user_creation for p in $tostart; do RETRY_TIMES=0 @@ -54,7 +51,7 @@ else done done - docker compose -f compose.yml up -d + PR_NUMBER=$(date +%F) docker compose -f compose.yml up -d RETRY_TIMES=0 until docker ps -f name="download" --format "{{.Status}}" | grep "Up" diff --git a/sda-download/.github/integration/setup/common/23_create_db_entries.sh b/sda-download/.github/integration/setup/common/23_create_db_entries.sh index 14085487d3..8b9fe49452 100755 --- a/sda-download/.github/integration/setup/common/23_create_db_entries.sh +++ b/sda-download/.github/integration/setup/common/23_create_db_entries.sh @@ -12,12 +12,12 @@ file_id=$(docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.files (stable_id, submission_user, \ submission_file_path, submission_file_size, archive_file_path, \ - archive_file_size, decrypted_file_size, backup_path, header, \ + archive_file_size, archive_location, decrypted_file_size, backup_path, header, \ encryption_method) VALUES ('urn:neic:001-002', 'integration-test', 'dummy_data.c4gh', \ - 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 1048605, \ + 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 'https://s3:9000/archive', 1048605, \ '', '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'CRYPT4GH') RETURNING id;" | xargs) # insert "ready" database log event @@ -27,7 +27,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.file_event_log (file_id, event) \ VALUES ('$file_id', 'ready');" @@ -37,7 +37,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'UNENCRYPTED');" @@ -47,7 +47,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', 'UPLOADED');" @@ -57,7 +57,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 'ARCHIVED');" @@ -69,7 +69,7 @@ dataset_id=$(docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.datasets (stable_id) VALUES ('https://doi.example/ty009.sfrrss/600.45asasga') \ ON CONFLICT (stable_id) DO UPDATE \ SET stable_id=excluded.stable_id RETURNING id;") @@ -81,6 +81,6 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.file_dataset (file_id, dataset_id) \ VALUES ('$file_id', $dataset_id);" diff --git a/sda-download/.github/integration/setup/common/2_build_all.sh b/sda-download/.github/integration/setup/common/2_build_all.sh new file mode 100644 index 0000000000..cf0d492914 --- /dev/null +++ b/sda-download/.github/integration/setup/common/2_build_all.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Build containers +cd .. || exit 1 + +make build-all + diff --git a/sda-download/.github/integration/setup/posix/1_posix_env_setup.sh b/sda-download/.github/integration/setup/posix/1_posix_env_setup.sh index 913f1245f6..73bd1f509f 100644 --- a/sda-download/.github/integration/setup/posix/1_posix_env_setup.sh +++ b/sda-download/.github/integration/setup/posix/1_posix_env_setup.sh @@ -1,4 +1,3 @@ #!/bin/bash sed -i 's/=s3/=posix/g' dev_utils/env.download - diff --git a/sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh b/sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh new file mode 100644 index 0000000000..8a95672584 --- /dev/null +++ b/sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = '/tmp';" \ No newline at end of file diff --git a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh index 575cf96184..d6540764e2 100644 --- a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh +++ b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh @@ -1,6 +1,8 @@ #!/bin/bash set -e +sudo apt install -y jq s3cmd + cd dev_utils || exit 1 # Make buckets if they don't exist already diff --git a/sda-download/.github/integration/tests/common/30_check_db.sh b/sda-download/.github/integration/tests/common/30_check_db.sh index 4a25090938..f2a03350ca 100644 --- a/sda-download/.github/integration/tests/common/30_check_db.sh +++ b/sda-download/.github/integration/tests/common/30_check_db.sh @@ -7,15 +7,15 @@ chmod 600 certs/client-key.pem docker run --rm --name client --network dev_utils_default -v "$PWD/certs:/certs" \ -e PGSSLCERT=/certs/client.pem -e PGSSLKEY=/certs/client-key.pem -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/lega \ + neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/sda \ -t -c "SELECT * from sda.file_dataset" docker run --rm --name client --network dev_utils_default -v "$PWD/certs:/certs" \ -e PGSSLCERT=/certs/client.pem -e PGSSLKEY=/certs/client-key.pem -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/lega \ + neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/sda \ -t -c "SELECT * from sda.file_event_log" docker run --rm --name client --network dev_utils_default -v "$PWD/certs:/certs" \ -e PGSSLCERT=/certs/client.pem -e PGSSLKEY=/certs/client-key.pem -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/lega \ + neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/sda \ -t -c "SELECT id, stable_id, archive_file_path FROM sda.files ORDER BY id DESC" diff --git a/sda-download/dev_utils/README.md b/sda-download/dev_utils/README.md index a69e310df6..4048207463 100644 --- a/sda-download/dev_utils/README.md +++ b/sda-download/dev_utils/README.md @@ -19,7 +19,7 @@ sh make_certs.sh ## Getting up and running fast with docker compose ```command -docker compose -f compose-no-tls.yml up -d +PR_NUMBER=$(date +%F) docker compose -f compose-no-tls.yml up -d ``` For testing the API @@ -33,13 +33,13 @@ sh run_integration_test_no_tls.sh To start all the backend services using docker compose. ```command -docker compose up -d db s3 mockauth +PR_NUMBER=$(date +%F) docker compose up -d db s3 mockauth ``` To start all the sda services using docker compose. ```command -docker compose up -d +PR_NUMBER=$(date +%F) docker compose up -d ``` To see brief real-time logs at the terminal remove the -d flag. diff --git a/sda-download/dev_utils/compose-no-tls.yml b/sda-download/dev_utils/compose-no-tls.yml index af1aebb55f..efd6cbfb58 100644 --- a/sda-download/dev_utils/compose-no-tls.yml +++ b/sda-download/dev_utils/compose-no-tls.yml @@ -1,30 +1,48 @@ services: db: - command: server /data + user: postgres container_name: db environment: - - DB_LEGA_IN_PASSWORD=lega_in - - DB_LEGA_OUT_PASSWORD=lega_out - NOTLS=true - POSTGRES_PASSWORD=rootpassword healthcheck: - test: ["CMD", "pg_isready", "-h", "localhost", "-U", "lega_out"] + test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s timeout: 20s retries: 3 - image: ghcr.io/neicnordic/sda-db:v2.1.4 + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres ports: - "5432:5432" volumes: - /tmp/data:/data + database_user_creation: + container_name: database_user_creation + image: postgres:15.4-alpine + command: + - "/bin/sh" + - "-c" + - | + pg_isready + psql -U postgres -h db -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" + depends_on: + db: + condition: service_healthy + restart: on-failure + environment: + - PGHOST=postgres + - PGPORT=5432 + - PGUSER=postgres + - PGPASSWORD=rootpassword + s3: command: server /data container_name: s3 environment: - - MINIO_ACCESS_KEY=access - - MINIO_SECRET_KEY=secretkey + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretkey + - MINIO_SERVER_URL=http://127.0.0.1:9000 healthcheck: test: ["CMD", "curl", "-fq", "http://localhost:9000/minio/health/live"] interval: 5s @@ -52,17 +70,17 @@ services: command: sda-download container_name: download depends_on: - db: - condition: service_healthy + database_user_creation: + condition: service_completed_successfully s3: condition: service_healthy mockauth: - condition: service_started + condition: service_healthy environment: - ARCHIVE_URL=http://s3 - ARCHIVE_TYPE=s3 - DB_HOST=db - image: neicnordic/sda-download:latest + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download build: context: .. volumes: @@ -83,14 +101,14 @@ services: s3: condition: service_healthy mockauth: - condition: service_started + condition: service_healthy environment: - ARCHIVE_URL=http://s3 - ARCHIVE_TYPE=s3 - DB_HOST=db - C4GH_TRANSIENTKEYPATH=/dev_utils/c4gh.sec.pem - C4GH_TRANSIENTPASSPHRASE=oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm - image: neicnordic/sda-download:latest + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download build: context: .. volumes: @@ -107,11 +125,18 @@ services: - /bin/sh - -c - | + apt update + apt install curl -y pip install --upgrade pip pip install aiohttp Authlib python -u /mockoidc.py container_name: mockauth image: python:3.10-slim + healthcheck: + test: [ "CMD", "curl", "-fkq", "http://localhost:8000/.well-known/openid-configuration" ] + interval: 10s + timeout: 30s + retries: 10 volumes: - ./mockoidc/mockoidc.py:/mockoidc.py mem_limit: 256m @@ -120,7 +145,7 @@ services: restart: always reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:latest + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} build: ../../sda/ command: [ sda-reencrypt ] container_name: reencrypt diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 2d1a92b22f..c56482a983 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -17,37 +17,54 @@ services: - ./certs:/origcerts - certs:/certs db: - command: server /data + user: postgres container_name: db depends_on: - certfixer environment: - - DB_LEGA_IN_PASSWORD=lega_in - - DB_LEGA_OUT_PASSWORD=lega_out - PKI_VOLUME_PATH=/certs/ - PG_CA=/var/lib/postgresql/tls/ca.pem - PG_SERVER_CERT=/var/lib/postgresql/tls/db.pem - PG_SERVER_KEY=/var/lib/postgresql/tls/db-key.pem - POSTGRES_PASSWORD=rootpassword healthcheck: - test: ["CMD", "pg_isready", "-h", "localhost", "-U", "lega_out"] + test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s timeout: 20s retries: 3 - image: ghcr.io/neicnordic/sda-db:v2.1.4 + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres ports: - "5432:5432" volumes: - /tmp/data:/data - certs:/var/lib/postgresql/tls/ + database_user_creation: + container_name: database_user_creation + image: postgres:15.4-alpine + command: + - "/bin/sh" + - "-c" + - | + pg_isready + psql -U postgres -h db -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" + depends_on: + db: + condition: service_healthy + restart: on-failure + environment: + - PGHOST=postgres + - PGPORT=5432 + - PGUSER=postgres + - PGPASSWORD=rootpassword s3: command: server /data container_name: s3 environment: - - MINIO_ACCESS_KEY=access - - MINIO_SECRET_KEY=secretkey + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretkey + - MINIO_SERVER_URL=http://127.0.0.1:9000 healthcheck: - test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] + test: ["CMD", "curl", "-fkq", "https//localhost:9000/minio/health/live"] interval: 5s timeout: 20s retries: 3 @@ -78,10 +95,10 @@ services: container_name: download depends_on: - certfixer - - db + - database_user_creation - mockauth env_file: ./env.download - image: neicnordic/sda-download:latest + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download volumes: - ./config.yaml:/config.yaml - ./:/dev_utils/ @@ -97,10 +114,17 @@ services: - /bin/sh - -c - | + apt update + apt install curl -y pip install --upgrade pip pip install aiohttp Authlib python -u /mockoidc.py container_name: mockauth + healthcheck: + test: [ "CMD", "curl", "-fkq", "https://localhost:8000/.well-known/openid-configuration" ] + interval: 10s + timeout: 30s + retries: 10 image: python:3.11-slim volumes: - ./mockoidc/mockoidc.py:/mockoidc.py diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 0f84f7fc2e..2f5c44bb76 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -18,37 +18,61 @@ services: - certs:/certs db: - command: server /data + user: postgres container_name: db depends_on: certfixer: condition: service_completed_successfully environment: - - DB_LEGA_IN_PASSWORD=lega_in - - DB_LEGA_OUT_PASSWORD=lega_out - PKI_VOLUME_PATH=/certs/ - - PG_CA=/var/lib/postgresql/tls/ca.pem - - PG_SERVER_CERT=/var/lib/postgresql/tls/db.pem - - PG_SERVER_KEY=/var/lib/postgresql/tls/db-key.pem + - POSTGRES_SERVER_CACERT=/var/lib/postgresql/tls/ca.pem + - POSTGRES_SERVER_CERT=/var/lib/postgresql/tls/db.pem + - POSTGRES_SERVER_KEY=/var/lib/postgresql/tls/db-key.pem - POSTGRES_PASSWORD=rootpassword healthcheck: - test: ["CMD", "pg_isready", "-h", "localhost", "-U", "lega_out"] + test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s timeout: 20s retries: 3 - image: ghcr.io/neicnordic/sda-db:v2.1.4 + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres ports: - "5432:5432" volumes: - /tmp/data:/data - certs:/var/lib/postgresql/tls/ + database_user_creation: + container_name: database_user_creation + image: postgres:15.4-alpine + command: + - "/bin/sh" + - "-c" + - | + pg_isready + psql -U postgres -h db -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" + depends_on: + db: + condition: service_healthy + restart: on-failure + environment: + - PGHOST=postgres + - PGPORT=5432 + - PGUSER=postgres + - PGPASSWORD=rootpassword + - PGSSLMODE=verify-full + - PGSSLKEY=/var/lib/postgresql/tls/client-key.pem + - PGSSLCERT=/var/lib/postgresql/tls/client.pem + - PGSSLROOTCERT=/var/lib/postgresql/tls/ca.pem + volumes: + - certs:/var/lib/postgresql/tls/ + s3: - command: server /data + command: server /data --console-address ":9001" container_name: s3 environment: - - MINIO_ACCESS_KEY=access - - MINIO_SECRET_KEY=secretkey + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretkey + - MINIO_SERVER_URL=http://127.0.0.1:9000 healthcheck: test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] interval: 5s @@ -84,14 +108,14 @@ services: depends_on: certfixer: condition: service_completed_successfully - db: - condition: service_healthy + database_user_creation: + condition: service_completed_successfully s3: condition: service_healthy mockauth: - condition: service_started + condition: service_healthy env_file: ./env.download - image: neicnordic/sda-download:latest + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download build: context: .. volumes: @@ -111,19 +135,19 @@ services: depends_on: certfixer: condition: service_completed_successfully - db: - condition: service_healthy + database_user_creation: + condition: service_completed_successfully s3: condition: service_healthy mockauth: - condition: service_started + condition: service_healthy download: condition: service_started env_file: ./env.download environment: - C4GH_TRANSIENTKEYPATH=/dev_utils/c4gh.sec.pem - C4GH_TRANSIENTPASSPHRASE=oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm - image: neicnordic/sda-download:latest + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download volumes: - ./config.yaml:/config.yaml - ./:/dev_utils/ @@ -136,7 +160,7 @@ services: restart: always reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:latest + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} build: ../../sda command: [ sda-reencrypt ] container_name: reencrypt @@ -158,11 +182,18 @@ services: - /bin/sh - -c - | + apt update + apt install curl -y pip install --upgrade pip pip install aiohttp Authlib python -u /mockoidc.py container_name: mockauth image: python:3.8-slim + healthcheck: + test: [ "CMD", "curl", "-fkq", "https://localhost:8000/.well-known/openid-configuration" ] + interval: 10s + timeout: 30s + retries: 10 volumes: - ./mockoidc/mockoidc.py:/mockoidc.py - certs:/certs diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index c2fbf480c5..9f346c4703 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -9,7 +9,7 @@ log: storage: archive: s3: - endpoint: "http://localhost:9000" + endpoint: "http://s3:9000" access_key: "access" secret_key: "secretkey" bucket_prefix: "archive" @@ -32,9 +32,9 @@ c4gh: db: host: "localhost" port: 5432 - user: "lega_out" - password: "lega_out" - database: "lega" + user: "download" + password: "download" + database: "sda" sslmode: "disable" oidc: diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml index ac6d501f90..302f7e4ba8 100644 --- a/sda-download/dev_utils/config-notls_local.yaml +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -21,10 +21,10 @@ grpc: port: 50051 db: - host: "localhost" + host: "s3" port: 15432 - user: "postgres" - password: "rootpasswd" + user: "download" + password: "download" database: "sda" sslmode: "disable" diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index d8ba904876..10437b39d2 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -13,7 +13,7 @@ log: storage: archive: s3: - endpoint: "https://localhost:9000" + endpoint: "https://s3:9000" access_key: "access" secret_key: "secretkey" bucket_prefix: "archive" @@ -61,13 +61,13 @@ grpc: db: host: "localhost" port: 5432 - user: "lega_out" - password: "lega_out" - database: "lega" + user: "download" + password: "download" + database: "sda" cacert: "./dev_utils/certs/ca.pem" clientCert: "./dev_utils/certs/client.pem" clientKey: "./dev_utils/certs/client-key.pem" - sslmode: "verify-ca" + sslmode: "verify-full" oidc: cacert: "./dev_utils/certs/ca.pem" diff --git a/sda-download/dev_utils/run_integration_test.sh b/sda-download/dev_utils/run_integration_test.sh index f02d9526ea..091dfaadff 100644 --- a/sda-download/dev_utils/run_integration_test.sh +++ b/sda-download/dev_utils/run_integration_test.sh @@ -36,12 +36,12 @@ file_id=$(docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.files (stable_id, submission_user, \ submission_file_path, submission_file_size, archive_file_path, \ - archive_file_size, decrypted_file_size, backup_path, header, \ + archive_file_size, archive_location, decrypted_file_size, backup_path, header, \ encryption_method) VALUES ('urn:neic:001-002', 'integration-test', 'dummy_data.c4gh', \ - 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 1048605, \ + 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 'https://s3:9000/archive', 1048605, \ '', '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'CRYPT4GH') RETURNING id;" | xargs) # insert "ready" database log event @@ -51,7 +51,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.file_event_log (file_id, event) \ VALUES ('$file_id', 'ready');" @@ -61,7 +61,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'UNENCRYPTED');" @@ -71,7 +71,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', 'UPLOADED');" @@ -81,7 +81,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 'ARCHIVED');" @@ -93,7 +93,7 @@ dataset_id=$(docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.datasets (stable_id) VALUES ('https://doi.example/ty009.sfrrss/600.45asasga') \ ON CONFLICT (stable_id) DO UPDATE \ SET stable_id=excluded.stable_id RETURNING id;") @@ -105,7 +105,7 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLKEY=/certs/client-key.pem \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.file_dataset (file_id, dataset_id) \ VALUES ('$file_id', $dataset_id);" diff --git a/sda-download/dev_utils/run_integration_test_no_tls.sh b/sda-download/dev_utils/run_integration_test_no_tls.sh index b53c6ca88d..9c34000c7a 100644 --- a/sda-download/dev_utils/run_integration_test_no_tls.sh +++ b/sda-download/dev_utils/run_integration_test_no_tls.sh @@ -31,36 +31,36 @@ chmod 444 c4gh.sec.pem # insert file entry into database file_id=$(docker run --rm --name client --network dev_utils_default \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.files (stable_id, submission_user, \ submission_file_path, submission_file_size, archive_file_path, \ - archive_file_size, decrypted_file_size, backup_path, header, \ + archive_file_size, archive_location, decrypted_file_size, backup_path, header, \ encryption_method) VALUES ('urn:neic:001-002', 'integration-test', 'dummy_data.c4gh', \ - 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 1048605, \ + 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 'https://s3:9000/archive', 1048605, \ '', '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'CRYPT4GH') RETURNING id;" | xargs) # insert "ready" database log event docker run --rm --name client --network dev_utils_default \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.file_event_log (file_id, event) \ VALUES ('$file_id', 'ready');" docker run --rm --name client --network dev_utils_default \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'UNENCRYPTED');" docker run --rm --name client --network dev_utils_default \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', 'UPLOADED');" docker run --rm --name client --network dev_utils_default \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ VALUES ('$file_id', '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 'ARCHIVED');" @@ -68,7 +68,7 @@ docker run --rm --name client --network dev_utils_default \ # make sure that the dataset exists in the database dataset_id=$(docker run --rm --name client --network dev_utils_default \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.datasets (stable_id) VALUES ('https://doi.example/ty009.sfrrss/600.45asasga') \ ON CONFLICT (stable_id) DO UPDATE \ SET stable_id=excluded.stable_id RETURNING id;") @@ -76,7 +76,7 @@ dataset_id=$(docker run --rm --name client --network dev_utils_default \ # insert the file into the dataset docker run --rm --name client --network dev_utils_default \ neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/lega \ + postgresql://postgres:rootpassword@db:5432/sda \ -t -q -c "INSERT INTO sda.file_dataset (file_id, dataset_id) \ VALUES ('$file_id', $dataset_id);" diff --git a/sda-download/dev_utils/ssl.cnf b/sda-download/dev_utils/ssl.cnf index eee1863631..26c7e0a39f 100644 --- a/sda-download/dev_utils/ssl.cnf +++ b/sda-download/dev_utils/ssl.cnf @@ -59,7 +59,6 @@ commonName = SysDev root CA [ v3_ca ] # Extensions for a typical CA (`man x509v3_config`). subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid:always,issuer basicConstraints = critical, CA:true keyUsage = critical, digitalSignature, cRLSign, keyCertSign #nsCertType = sslCA @@ -71,7 +70,6 @@ basicConstraints = CA:FALSE nsCertType = server,client nsComment = "LocalEGA Server+Client Certificate" subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = critical, clientAuth, serverAuth @@ -79,7 +77,6 @@ extendedKeyUsage = critical, clientAuth, serverAuth # Extensions for client certificates (`man x509v3_config`). basicConstraints = critical,CA:FALSE subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth subjectAltName = DNS:localhost,DNS:client,IP:127.0.0.1 @@ -88,7 +85,6 @@ subjectAltName = DNS:localhost,DNS:client,IP:127.0.0.1 # Extensions for client certificates (`man x509v3_config`). basicConstraints = critical,CA:FALSE subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth subjectAltName = DNS:localhost,DNS:download,IP:127.0.0.1 @@ -97,11 +93,9 @@ subjectAltName = DNS:localhost,DNS:download,IP:127.0.0.1 # Extensions for server certificates (`man x509v3_config`). basicConstraints = critical,CA:FALSE subjectKeyIdentifier = hash -authorityKeyIdentifier = keyid,issuer:always keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth subjectAltName = DNS:localhost,DNS:db,DNS:s3,DNS:mockauth,DNS:reencrypt,IP:127.0.0.1 [ crl_ext ] # Extension for CRLs (`man x509v3_config`). -authorityKeyIdentifier=keyid:always From 738a0b40c268b587e70b88bcb3f0c274881695da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 08:37:01 +0100 Subject: [PATCH 089/530] feat(download): fix unit test after adding archive location validation, add unit test for archive location not set --- sda-download/api/sda/sda_test.go | 80 ++++++++++++++++++++++++++------ 1 file changed, 67 insertions(+), 13 deletions(-) diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go index 3db7fdfaf9..6a7f7bc2c3 100644 --- a/sda-download/api/sda/sda_test.go +++ b/sda-download/api/sda/sda_test.go @@ -998,9 +998,10 @@ func TestDownload_InvalidRange_NoEnd(t *testing.T) { } database.GetFile = func(_ string) (*database.FileDownload, error) { fileDetails := &database.FileDownload{ - ArchivePath: "/mocks3/somepath", - ArchiveSize: 0, - Header: []byte("mock header"), + ArchivePath: "/mocks3/somepath", + ArchiveSize: 0, + ArchiveLocation: "/archive", + Header: []byte("mock header"), } return fileDetails, nil @@ -1047,9 +1048,10 @@ func TestDownload_InvalidRange_NoStart(t *testing.T) { } database.GetFile = func(_ string) (*database.FileDownload, error) { fileDetails := &database.FileDownload{ - ArchivePath: "/mocks3/somepath", - ArchiveSize: 0, - Header: []byte("mock header"), + ArchivePath: "/mocks3/somepath", + ArchiveSize: 0, + ArchiveLocation: "/archive", + Header: []byte("mock header"), } return fileDetails, nil @@ -1080,7 +1082,8 @@ func TestDownload_InvalidRange_NoStart(t *testing.T) { middleware.GetCacheFromContext = originalGetCacheFromContext database.GetFile = originalGetFile } -func TestDownload_InvalidRange_NegativeStart(t *testing.T) { + +func TestDownload_ArchiveLocationUnset(t *testing.T) { originalCheckFilePermission := database.CheckFilePermission originalGetCacheFromContext := middleware.GetCacheFromContext originalGetFile := database.GetFile @@ -1104,6 +1107,55 @@ func TestDownload_InvalidRange_NegativeStart(t *testing.T) { return fileDetails, nil } + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = &http.Request{Method: "GET", URL: &url.URL{Path: "/mocks3/somepath", RawQuery: "filename=somepath"}} + c.Request.Header = http.Header{"Client-Public-Key": []string{config.Config.C4GH.PublicKeyB64}} + + c.Params = make(gin.Params, 1) + c.Params[0] = gin.Param{Key: "type", Value: "encrypted"} + + Download(c) + response := w.Result() + body, err := io.ReadAll(response.Body) + if err != nil { + assert.FailNow(t, err.Error()) + } + _ = response.Body.Close() + assert.Equal(t, http.StatusInternalServerError, response.StatusCode) + assert.Contains(t, string(body), "archive location not known") + + // Return mock functions to originals + database.CheckFilePermission = originalCheckFilePermission + middleware.GetCacheFromContext = originalGetCacheFromContext + database.GetFile = originalGetFile +} + +func TestDownload_InvalidRange_NegativeStart(t *testing.T) { + originalCheckFilePermission := database.CheckFilePermission + originalGetCacheFromContext := middleware.GetCacheFromContext + originalGetFile := database.GetFile + + // Substitute mock functions + database.CheckFilePermission = func(_ string) (string, error) { + return "dataset1", nil + } + middleware.GetCacheFromContext = func(_ *gin.Context) session.Cache { + return session.Cache{ + Datasets: []string{"dataset1"}, + } + } + database.GetFile = func(_ string) (*database.FileDownload, error) { + fileDetails := &database.FileDownload{ + ArchivePath: "/mocks3/somepath", + ArchiveSize: 0, + ArchiveLocation: "/archive", + Header: []byte("mock header"), + } + + return fileDetails, nil + } + w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = &http.Request{Method: "GET", URL: &url.URL{Path: "/mocks3/somepath", RawQuery: "filename=somepath"}} @@ -1147,9 +1199,10 @@ func TestDownload_Range_FromStartOfFile(t *testing.T) { } database.GetFile = func(_ string) (*database.FileDownload, error) { fileDetails := &database.FileDownload{ - ArchivePath: "/mocks3/somepath", - ArchiveSize: 0, - Header: []byte("mock header;"), + ArchivePath: "/mocks3/somepath", + ArchiveSize: 0, + ArchiveLocation: "/archive", + Header: []byte("mock header;"), } return fileDetails, nil @@ -1256,9 +1309,10 @@ func TestDownload_Range_FromMiddleOfFile(t *testing.T) { } database.GetFile = func(_ string) (*database.FileDownload, error) { fileDetails := &database.FileDownload{ - ArchivePath: "/mocks3/somepath", - ArchiveSize: 0, - Header: []byte("mock header;"), + ArchivePath: "/mocks3/somepath", + ArchiveSize: 0, + ArchiveLocation: "/archive", + Header: []byte("mock header;"), } return fileDetails, nil From 3cc03252260eb71af22e2b23097881b5134d3a20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 10:01:34 +0100 Subject: [PATCH 090/530] feat(download): fix notls integration test, update created file archive location to be http instead of https --- .../setup/s3notls/24_posix_update_db_entry.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh diff --git a/sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh b/sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh new file mode 100644 index 0000000000..7eff05fcfb --- /dev/null +++ b/sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'http://s3:9000/archive';" \ No newline at end of file From 30199a648f56c4bd748aa8321bc6caae26d55ab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 11:59:32 +0100 Subject: [PATCH 091/530] feat(download): revert change to 99_s3_storage_setup.sh --- .../.github/integration/setup/s3notls/99_s3_storage_setup.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh index d6540764e2..575cf96184 100644 --- a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh +++ b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh @@ -1,8 +1,6 @@ #!/bin/bash set -e -sudo apt install -y jq s3cmd - cd dev_utils || exit 1 # Make buckets if they don't exist already From 99bda64533bc7ba03c71199a5f099d2085d810f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 12:02:00 +0100 Subject: [PATCH 092/530] feat(download): fix typo in health check url --- sda-download/dev_utils/compose-sda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index c56482a983..b9b4f0fb2b 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -64,7 +64,7 @@ services: - MINIO_ROOT_PASSWORD=secretkey - MINIO_SERVER_URL=http://127.0.0.1:9000 healthcheck: - test: ["CMD", "curl", "-fkq", "https//localhost:9000/minio/health/live"] + test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] interval: 5s timeout: 20s retries: 3 From da2f214e597d9d8bcd8df62fbb65686e9c2d7c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 16:51:00 +0100 Subject: [PATCH 093/530] feat(download): fix proper panic if db version to old to start instead of partial init, move defer closing of db to main --- sda-download/cmd/main.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index e8ab74c9a7..14206326f8 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -32,11 +32,8 @@ func init() { if err != nil { log.Panicf("database connection failed, reason: %v", err) } - defer db.Close() if db.Version < 23 { - log.Error("database schema v23 is required") - - return + log.Panicf("database schema v23 is required") } database.DB = db @@ -72,6 +69,8 @@ func init() { // main starts the web server func main() { srv := api.Setup() + // Db is setup by the init() func + defer database.DB.Close() // Start the server log.Info("(5/5) Starting web server") From a92caefb05193d9c2e25844ca83eb3734f4af3a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 09:09:22 +0100 Subject: [PATCH 094/530] fix(storage-v2): fix acutally setting chunkSizeBytes for s3 reader --- sda-download/internal/storage/v2/s3/reader/config.go | 1 + sda/internal/storage/v2/s3/reader/config.go | 1 + 2 files changed, 2 insertions(+) diff --git a/sda-download/internal/storage/v2/s3/reader/config.go b/sda-download/internal/storage/v2/s3/reader/config.go index bf82ecf160..d80e0ed8c3 100644 --- a/sda-download/internal/storage/v2/s3/reader/config.go +++ b/sda-download/internal/storage/v2/s3/reader/config.go @@ -77,6 +77,7 @@ func loadConfig(backendName string) ([]*endpointConfig, error) { if s > 1*datasize.GB { return nil, errors.New("chunk_size can not be bigger than 1gb") } + e.chunkSizeBytes = s.Bytes() } if e.Region == "" { e.Region = "us-east-1" diff --git a/sda/internal/storage/v2/s3/reader/config.go b/sda/internal/storage/v2/s3/reader/config.go index 533d858d44..0bc206bd5a 100644 --- a/sda/internal/storage/v2/s3/reader/config.go +++ b/sda/internal/storage/v2/s3/reader/config.go @@ -77,6 +77,7 @@ func loadConfig(backendName string) ([]*endpointConfig, error) { if s > 1*datasize.GB { return nil, errors.New("chunk_size can not be bigger than 1gb") } + e.chunkSizeBytes = s.Bytes() } if e.Region == "" { e.Region = "us-east-1" From 62801d705d3f1603bc96a936627a09ab039ce751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 09:52:03 +0100 Subject: [PATCH 095/530] fix(storage-v2): s3 seekable reader add closing of GetObjectResponse bodies --- .../v2/s3/reader/new_file_read_seeker.go | 19 +++++++++++++++++-- .../v2/s3/reader/new_file_read_seeker.go | 19 +++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go index 4c35d7f3d0..dca31a3ddd 100644 --- a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -31,7 +31,7 @@ type s3SeekableReader struct { lock sync.Mutex outstandingPrefetches []int64 seeked bool - objectReader io.Reader + objectReader io.ReadCloser chunkSize uint64 } @@ -75,7 +75,11 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath }, nil } -func (r *s3SeekableReader) Close() (err error) { +func (r *s3SeekableReader) Close() error { + if r.objectReader != nil { + _ = r.objectReader.Close() + } + return nil } @@ -131,6 +135,11 @@ func (r *s3SeekableReader) prefetchAt(offset int64) { Key: aws.String(r.filePath), Range: wantedRange, }) + defer func() { + if object != nil && object.Body != nil { + _ = object.Body.Close() + } + }() r.lock.Lock() @@ -269,6 +278,7 @@ func (r *s3SeekableReader) wholeReader(dst []byte) (int, error) { } // Store for future use + // object.Body will be closed by r.Close() r.objectReader = object.Body } @@ -338,6 +348,11 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { Key: key, Range: wantedRange, }) + defer func() { + if object != nil && object.Body != nil { + _ = object.Body.Close() + } + }() r.lock.Lock() diff --git a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go index 4c35d7f3d0..dca31a3ddd 100644 --- a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -31,7 +31,7 @@ type s3SeekableReader struct { lock sync.Mutex outstandingPrefetches []int64 seeked bool - objectReader io.Reader + objectReader io.ReadCloser chunkSize uint64 } @@ -75,7 +75,11 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath }, nil } -func (r *s3SeekableReader) Close() (err error) { +func (r *s3SeekableReader) Close() error { + if r.objectReader != nil { + _ = r.objectReader.Close() + } + return nil } @@ -131,6 +135,11 @@ func (r *s3SeekableReader) prefetchAt(offset int64) { Key: aws.String(r.filePath), Range: wantedRange, }) + defer func() { + if object != nil && object.Body != nil { + _ = object.Body.Close() + } + }() r.lock.Lock() @@ -269,6 +278,7 @@ func (r *s3SeekableReader) wholeReader(dst []byte) (int, error) { } // Store for future use + // object.Body will be closed by r.Close() r.objectReader = object.Body } @@ -338,6 +348,11 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { Key: key, Range: wantedRange, }) + defer func() { + if object != nil && object.Body != nil { + _ = object.Body.Close() + } + }() r.lock.Lock() From 75dd98ebfb061628ff4f6736ef00449eeda211c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 10:01:54 +0100 Subject: [PATCH 096/530] fix(download): use request context when spawning resources in request handler such that those can reject to context cancanclled if request is aborted, etc --- sda-download/api/sda/sda.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index e87842c3bc..e59c814bef 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -38,7 +38,7 @@ func sanitizeString(str string) string { return pattern.ReplaceAllString(str, "[identifier]: $1") } -func reencryptHeader(oldHeader []byte, reencKey string) ([]byte, error) { +func reencryptHeader(ctx context.Context, oldHeader []byte, reencKey string) ([]byte, error) { var opts []grpc.DialOption switch { case config.Config.Reencrypt.ClientKey != "" && config.Config.Reencrypt.ClientCert != "": @@ -93,7 +93,7 @@ func reencryptHeader(oldHeader []byte, reencKey string) ([]byte, error) { defer conn.Close() timeoutDuration := time.Duration(config.Config.Reencrypt.Timeout) * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + ctx, cancel := context.WithTimeout(ctx, timeoutDuration) defer cancel() c := reencrypt.NewReencryptClient(conn) @@ -328,9 +328,9 @@ func Download(c *gin.Context) { // Get archive file handle var file io.Reader if wholeFile { - file, err = ArchiveReader.NewFileReader(context.Background(), fileDetails.ArchiveLocation, fileDetails.ArchivePath) + file, err = ArchiveReader.NewFileReader(c, fileDetails.ArchiveLocation, fileDetails.ArchivePath) } else { - file, err = ArchiveReader.NewFileReadSeeker(context.Background(), fileDetails.ArchiveLocation, fileDetails.ArchivePath) + file, err = ArchiveReader.NewFileReadSeeker(c, fileDetails.ArchiveLocation, fileDetails.ArchivePath) } if err != nil { @@ -361,7 +361,7 @@ func Download(c *gin.Context) { // Size of the header in the archive c.Header("Server-Additional-Bytes", fmt.Sprint(headerSize)) if requestPublicKey != "" { - newHeader, _ := reencryptHeader(fileDetails.Header, requestPublicKey) + newHeader, _ := reencryptHeader(c, fileDetails.Header, requestPublicKey) headerSize = bytes.NewReader(newHeader).Size() // Size of the header if the file is re-encrypted before downloading c.Header("Client-Additional-Bytes", fmt.Sprint(headerSize)) @@ -388,7 +388,7 @@ func Download(c *gin.Context) { } log.Debugf("Public key from the request header = %v", requestPublicKey) - newHeader, err := reencryptHeader(fileDetails.Header, requestPublicKey) + newHeader, err := reencryptHeader(c, fileDetails.Header, requestPublicKey) if err != nil { log.Errorf("Failed to reencrypt the file header, reason: %v", err) c.String(http.StatusInternalServerError, "file re-encryption error") @@ -420,7 +420,7 @@ func Download(c *gin.Context) { } default: // Reencrypt header for use with the loaded internal key - newHeader, err := reencryptHeader(fileDetails.Header, config.Config.C4GH.PublicKeyB64) + newHeader, err := reencryptHeader(c, fileDetails.Header, config.Config.C4GH.PublicKeyB64) if err != nil { log.Errorf("Failed to reencrypt the file header, reason: %v", err) c.String(http.StatusInternalServerError, "file re-encryption error") From ecb84f8d1892dead7671cc124368fc7b59a662a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 10:05:34 +0100 Subject: [PATCH 097/530] fix(storage-v2): update s3 file read seeker to propagate ctx with cancel from caller when calling s3.GetObject such that it could react if context is cancelled, and ensure context is cancelled on r.Close --- .../storage/v2/s3/reader/new_file_read_seeker.go | 16 +++++++++++++--- .../storage/v2/s3/reader/new_file_read_seeker.go | 16 +++++++++++++--- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go index dca31a3ddd..de6c06b55b 100644 --- a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -33,21 +33,28 @@ type s3SeekableReader struct { seeked bool objectReader io.ReadCloser chunkSize uint64 + ctx context.Context + cancel context.CancelFunc } func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) { + ctx, cancel := context.WithCancel(ctx) + endpoint, bucket, err := parseLocation(location) if err != nil { + cancel() return nil, err } client, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { + cancel() return nil, err } objectSize, err := reader.getFileSize(ctx, client, bucket, filePath) if err != nil { + cancel() return nil, err } @@ -72,6 +79,8 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath seeked: false, objectReader: nil, chunkSize: chunkSizeBytes, + ctx: ctx, + cancel: cancel, }, nil } @@ -80,6 +89,7 @@ func (r *s3SeekableReader) Close() error { _ = r.objectReader.Close() } + r.cancel() return nil } @@ -130,7 +140,7 @@ func (r *s3SeekableReader) prefetchAt(offset int64) { wantedRange := aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+prefetchSize-1)) - object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + object, err := r.client.GetObject(r.ctx, &s3.GetObjectInput{ Bucket: aws.String(r.bucket), Key: aws.String(r.filePath), Range: wantedRange, @@ -268,7 +278,7 @@ func (r *s3SeekableReader) isPrefetching(offset int64) bool { func (r *s3SeekableReader) wholeReader(dst []byte) (int, error) { if r.objectReader == nil { // First call, setup a reader for the object - object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + object, err := r.client.GetObject(r.ctx, &s3.GetObjectInput{ Bucket: aws.String(r.bucket), Key: aws.String(r.filePath), }) @@ -343,7 +353,7 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { r.lock.Unlock() - object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + object, err := r.client.GetObject(r.ctx, &s3.GetObjectInput{ Bucket: bucket, Key: key, Range: wantedRange, diff --git a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go index dca31a3ddd..de6c06b55b 100644 --- a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -33,21 +33,28 @@ type s3SeekableReader struct { seeked bool objectReader io.ReadCloser chunkSize uint64 + ctx context.Context + cancel context.CancelFunc } func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) { + ctx, cancel := context.WithCancel(ctx) + endpoint, bucket, err := parseLocation(location) if err != nil { + cancel() return nil, err } client, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { + cancel() return nil, err } objectSize, err := reader.getFileSize(ctx, client, bucket, filePath) if err != nil { + cancel() return nil, err } @@ -72,6 +79,8 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath seeked: false, objectReader: nil, chunkSize: chunkSizeBytes, + ctx: ctx, + cancel: cancel, }, nil } @@ -80,6 +89,7 @@ func (r *s3SeekableReader) Close() error { _ = r.objectReader.Close() } + r.cancel() return nil } @@ -130,7 +140,7 @@ func (r *s3SeekableReader) prefetchAt(offset int64) { wantedRange := aws.String(fmt.Sprintf("bytes=%d-%d", offset, offset+prefetchSize-1)) - object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + object, err := r.client.GetObject(r.ctx, &s3.GetObjectInput{ Bucket: aws.String(r.bucket), Key: aws.String(r.filePath), Range: wantedRange, @@ -268,7 +278,7 @@ func (r *s3SeekableReader) isPrefetching(offset int64) bool { func (r *s3SeekableReader) wholeReader(dst []byte) (int, error) { if r.objectReader == nil { // First call, setup a reader for the object - object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + object, err := r.client.GetObject(r.ctx, &s3.GetObjectInput{ Bucket: aws.String(r.bucket), Key: aws.String(r.filePath), }) @@ -343,7 +353,7 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { r.lock.Unlock() - object, err := r.client.GetObject(context.Background(), &s3.GetObjectInput{ + object, err := r.client.GetObject(r.ctx, &s3.GetObjectInput{ Bucket: bucket, Key: key, Range: wantedRange, From a1eb18dad0df0eaad6465ec3da177b67fdfdffb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 10:13:50 +0100 Subject: [PATCH 098/530] fix(storage-v2): update s3 reader, make getS3ClientForEndpoint return endpoint Config to remove need for 2nd loop if config is needed after client creation --- .../internal/storage/v2/s3/reader/find_file.go | 2 +- .../internal/storage/v2/s3/reader/get_file_size.go | 2 +- .../storage/v2/s3/reader/new_file_read_seeker.go | 13 ++----------- .../storage/v2/s3/reader/new_file_reader.go | 2 +- .../internal/storage/v2/s3/reader/reader.go | 8 ++++---- sda/internal/storage/v2/s3/reader/find_file.go | 2 +- sda/internal/storage/v2/s3/reader/get_file_size.go | 2 +- .../storage/v2/s3/reader/new_file_read_seeker.go | 13 ++----------- .../storage/v2/s3/reader/new_file_reader.go | 2 +- sda/internal/storage/v2/s3/reader/reader.go | 8 ++++---- 10 files changed, 18 insertions(+), 36 deletions(-) diff --git a/sda-download/internal/storage/v2/s3/reader/find_file.go b/sda-download/internal/storage/v2/s3/reader/find_file.go index f5c1f2e870..9bc42ad3f5 100644 --- a/sda-download/internal/storage/v2/s3/reader/find_file.go +++ b/sda-download/internal/storage/v2/s3/reader/find_file.go @@ -13,7 +13,7 @@ import ( func (reader *Reader) FindFile(ctx context.Context, filePath string) (string, error) { for _, endpointConf := range reader.endpoints { - client, err := reader.getS3ClientForEndpoint(ctx, endpointConf.Endpoint) + client, _, err := reader.getS3ClientForEndpoint(ctx, endpointConf.Endpoint) if err != nil { return "", err } diff --git a/sda-download/internal/storage/v2/s3/reader/get_file_size.go b/sda-download/internal/storage/v2/s3/reader/get_file_size.go index e0ab0cf1dc..0ce52b1a82 100644 --- a/sda-download/internal/storage/v2/s3/reader/get_file_size.go +++ b/sda-download/internal/storage/v2/s3/reader/get_file_size.go @@ -19,7 +19,7 @@ func (reader *Reader) GetFileSize(ctx context.Context, location, filePath string return 0, err } - client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + client, _, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { return 0, err } diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go index de6c06b55b..4abdf2de33 100644 --- a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -46,7 +46,7 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath return nil, err } - client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + client, endpointConf, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { cancel() return nil, err @@ -58,15 +58,6 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath return nil, err } - var chunkSizeBytes uint64 - for _, e := range reader.endpoints { - if e.Endpoint == endpoint { - chunkSizeBytes = e.chunkSizeBytes - - break - } - } - return &s3SeekableReader{ client: client, bucket: bucket, @@ -78,7 +69,7 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath outstandingPrefetches: make([]int64, 0, 32), seeked: false, objectReader: nil, - chunkSize: chunkSizeBytes, + chunkSize: endpointConf.chunkSizeBytes, ctx: ctx, cancel: cancel, }, nil diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_reader.go b/sda-download/internal/storage/v2/s3/reader/new_file_reader.go index a8bbaa16dc..5ff9ac1529 100644 --- a/sda-download/internal/storage/v2/s3/reader/new_file_reader.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_reader.go @@ -18,7 +18,7 @@ func (reader *Reader) NewFileReader(ctx context.Context, location, filePath stri return nil, err } - client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + client, _, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { return nil, err } diff --git a/sda-download/internal/storage/v2/s3/reader/reader.go b/sda-download/internal/storage/v2/s3/reader/reader.go index 99d394c69e..f5db9b7e87 100644 --- a/sda-download/internal/storage/v2/s3/reader/reader.go +++ b/sda-download/internal/storage/v2/s3/reader/reader.go @@ -43,20 +43,20 @@ func NewReader(ctx context.Context, backendName string) (*Reader, error) { return backend, nil } -func (reader *Reader) getS3ClientForEndpoint(ctx context.Context, endpoint string) (*s3.Client, error) { +func (reader *Reader) getS3ClientForEndpoint(ctx context.Context, endpoint string) (*s3.Client, *endpointConfig, error) { for _, e := range reader.endpoints { if e.Endpoint != endpoint { continue } client, err := e.getS3Client(ctx) if err != nil { - return nil, fmt.Errorf("failed to create S3 client to endpoint: %s, due to %v", endpoint, err) + return nil, nil, fmt.Errorf("failed to create S3 client to endpoint: %s, due to %v", endpoint, err) } - return client, nil + return client, e, nil } - return nil, storageerrors.ErrorNoEndpointConfiguredForLocation + return nil, nil, storageerrors.ErrorNoEndpointConfiguredForLocation } // parseLocation attempts to parse a location to a s3 endpoint, and a bucket diff --git a/sda/internal/storage/v2/s3/reader/find_file.go b/sda/internal/storage/v2/s3/reader/find_file.go index af8c29e57e..16146ec2c8 100644 --- a/sda/internal/storage/v2/s3/reader/find_file.go +++ b/sda/internal/storage/v2/s3/reader/find_file.go @@ -13,7 +13,7 @@ import ( func (reader *Reader) FindFile(ctx context.Context, filePath string) (string, error) { for _, endpointConf := range reader.endpoints { - client, err := reader.getS3ClientForEndpoint(ctx, endpointConf.Endpoint) + client, _, err := reader.getS3ClientForEndpoint(ctx, endpointConf.Endpoint) if err != nil { return "", err } diff --git a/sda/internal/storage/v2/s3/reader/get_file_size.go b/sda/internal/storage/v2/s3/reader/get_file_size.go index f740e13aed..1c8a41dbd2 100644 --- a/sda/internal/storage/v2/s3/reader/get_file_size.go +++ b/sda/internal/storage/v2/s3/reader/get_file_size.go @@ -19,7 +19,7 @@ func (reader *Reader) GetFileSize(ctx context.Context, location, filePath string return 0, err } - client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + client, _, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { return 0, err } diff --git a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go index de6c06b55b..4abdf2de33 100644 --- a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -46,7 +46,7 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath return nil, err } - client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + client, endpointConf, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { cancel() return nil, err @@ -58,15 +58,6 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath return nil, err } - var chunkSizeBytes uint64 - for _, e := range reader.endpoints { - if e.Endpoint == endpoint { - chunkSizeBytes = e.chunkSizeBytes - - break - } - } - return &s3SeekableReader{ client: client, bucket: bucket, @@ -78,7 +69,7 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath outstandingPrefetches: make([]int64, 0, 32), seeked: false, objectReader: nil, - chunkSize: chunkSizeBytes, + chunkSize: endpointConf.chunkSizeBytes, ctx: ctx, cancel: cancel, }, nil diff --git a/sda/internal/storage/v2/s3/reader/new_file_reader.go b/sda/internal/storage/v2/s3/reader/new_file_reader.go index 585ebc7f48..ce432571f3 100644 --- a/sda/internal/storage/v2/s3/reader/new_file_reader.go +++ b/sda/internal/storage/v2/s3/reader/new_file_reader.go @@ -18,7 +18,7 @@ func (reader *Reader) NewFileReader(ctx context.Context, location, filePath stri return nil, err } - client, err := reader.getS3ClientForEndpoint(ctx, endpoint) + client, _, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { return nil, err } diff --git a/sda/internal/storage/v2/s3/reader/reader.go b/sda/internal/storage/v2/s3/reader/reader.go index 550f5c4455..8d8e77e337 100644 --- a/sda/internal/storage/v2/s3/reader/reader.go +++ b/sda/internal/storage/v2/s3/reader/reader.go @@ -43,20 +43,20 @@ func NewReader(ctx context.Context, backendName string) (*Reader, error) { return backend, nil } -func (reader *Reader) getS3ClientForEndpoint(ctx context.Context, endpoint string) (*s3.Client, error) { +func (reader *Reader) getS3ClientForEndpoint(ctx context.Context, endpoint string) (*s3.Client, *endpointConfig, error) { for _, e := range reader.endpoints { if e.Endpoint != endpoint { continue } client, err := e.getS3Client(ctx) if err != nil { - return nil, fmt.Errorf("failed to create S3 client to endpoint: %s, due to %v", endpoint, err) + return nil, nil, fmt.Errorf("failed to create S3 client to endpoint: %s, due to %v", endpoint, err) } - return client, nil + return client, e, nil } - return nil, storageerrors.ErrorNoEndpointConfiguredForLocation + return nil, nil, storageerrors.ErrorNoEndpointConfiguredForLocation } // parseLocation attempts to parse a location to a s3 endpoint, and a bucket From 8458c0953d596f42454b000e17165a2fa5615198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 10:39:30 +0100 Subject: [PATCH 099/530] fix(download): update getVersion to scan to a NullInt64 and catch error case when there are now rows in the sda.dbschema_version table --- sda-download/internal/database/database.go | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index db348dc2f1..ebaefc6275 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -21,7 +21,7 @@ var DB *SQLdb // SQLdb struct that acts as a receiver for the DB update methods type SQLdb struct { DB *sql.DB - Version int + Version int64 ConnInfo string } @@ -126,17 +126,22 @@ func (dbs *SQLdb) checkAndReconnectIfNeeded() { } } -func (dbs *SQLdb) getVersion() (int, error) { +func (dbs *SQLdb) getVersion() (int64, error) { dbs.checkAndReconnectIfNeeded() log.Debug("Fetching database schema version") query := "SELECT MAX(version) FROM sda.dbschema_version;" - var dbVersion = -1 - err := dbs.DB.QueryRow(query).Scan(&dbVersion) + var version sql.NullInt64 + if err := dbs.DB.QueryRow(query).Scan(&version); err != nil { + return 0, err + } + if !version.Valid { + return 0, fmt.Errorf("database schema version not initialized") + } - return dbVersion, err + return version.Int64, nil } // GetFiles retrieves the file details From 8f5840060f4c26a2c68c67ec048dbafca3efb027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 11:47:22 +0100 Subject: [PATCH 100/530] fix(download): fix some linting issues with spacing, panic instead of fatal for defer func, errors without formatting, etc --- sda-download/cmd/main.go | 10 +++++++--- sda-download/internal/database/database.go | 3 ++- .../storage/v2/s3/reader/new_file_read_seeker.go | 5 ++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index 14206326f8..d1af8e275f 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -33,7 +33,7 @@ func init() { log.Panicf("database connection failed, reason: %v", err) } if db.Version < 23 { - log.Panicf("database schema v23 is required") + log.Panic("database schema v23 is required") } database.DB = db @@ -81,9 +81,13 @@ func main() { if config.Config.App.ServerCert != "" && config.Config.App.ServerKey != "" { log.Infof("Web server is ready to receive connections at https://%s:%d", config.Config.App.Host, config.Config.App.Port) - log.Fatal(srv.ListenAndServeTLS(config.Config.App.ServerCert, config.Config.App.ServerKey)) + if err := srv.ListenAndServeTLS(config.Config.App.ServerCert, config.Config.App.ServerKey); err != nil { + log.Panic(err) + } } log.Infof("Web server is ready to receive connections at http://%s:%d", config.Config.App.Host, config.Config.App.Port) - log.Fatal(srv.ListenAndServe()) + if err := srv.ListenAndServe(); err != nil { + log.Panic(err) + } } diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index ebaefc6275..c6b650c7aa 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "encoding/hex" + "errors" "fmt" "regexp" "strings" @@ -138,7 +139,7 @@ func (dbs *SQLdb) getVersion() (int64, error) { return 0, err } if !version.Valid { - return 0, fmt.Errorf("database schema version not initialized") + return 0, errors.New("database schema version not initialized") } return version.Int64, nil diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go index 4abdf2de33..e8efd638ed 100644 --- a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -43,18 +43,21 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath endpoint, bucket, err := parseLocation(location) if err != nil { cancel() + return nil, err } client, endpointConf, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { cancel() + return nil, err } objectSize, err := reader.getFileSize(ctx, client, bucket, filePath) if err != nil { cancel() + return nil, err } @@ -79,8 +82,8 @@ func (r *s3SeekableReader) Close() error { if r.objectReader != nil { _ = r.objectReader.Close() } - r.cancel() + return nil } From 4a0edc65d7ffbbfacea68f6b7c9c48707c43485f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 14:31:55 +0100 Subject: [PATCH 101/530] feat(download): update panic log when archive reader fails to initialize to explicitly mention archive --- sda-download/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index d1af8e275f..a41c7df1ab 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -61,7 +61,7 @@ func init() { archiveReader, err := storage.NewReader(context.Background(), "archive") if err != nil { - log.Panicf("Error initiating storage backend, reason: %v", err) + log.Panicf("Error initiating archive storage reader, reason: %v", err) } sda.ArchiveReader = archiveReader } From f1d363051b68b26e99110c0d4bd27d2015ce7810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 15:02:05 +0100 Subject: [PATCH 102/530] feat(storage-v2): update s3 reader unit test to more effectivly create a big file, update read file seeker to accept content into the cache if the currentOffset has moved since fetching content, but correntOffset is still within the content --- .../v2/s3/reader/new_file_read_seeker.go | 80 +++++++++++++++-- .../storage/v2/s3/reader/reader_test.go | 7 +- .../v2/s3/reader/new_file_read_seeker.go | 85 +++++++++++++++++-- .../storage/v2/s3/reader/reader_test.go | 7 +- 4 files changed, 158 insertions(+), 21 deletions(-) diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go index e8efd638ed..e6202c3659 100644 --- a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -6,11 +6,13 @@ import ( "errors" "fmt" "io" + "strconv" "strings" "sync" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + log "github.com/sirupsen/logrus" ) // s3CacheBlock is used to keep track of cached data @@ -153,10 +155,18 @@ func (r *s3SeekableReader) prefetchAt(offset int64) { return } - responseRange := fmt.Sprintf("bytes %d-", r.currentOffset) + if object.ContentRange == nil { + return + } + + contentRange, err := parseContentRange(*object.ContentRange) + if err != nil { + log.Errorf("failed to parse content range: %v", err) - if object.ContentRange == nil || !strings.HasPrefix(*object.ContentRange, responseRange) { - // Unexpected content range - ignore + return + } + + if r.currentOffset < contentRange.start || r.currentOffset > contentRange.end { return } @@ -366,10 +376,17 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { return 0, err } - responseRange := fmt.Sprintf("bytes %d-", r.currentOffset) + if object.ContentRange == nil { + return 0, errors.New("unexpected nil content range") + } - if object.ContentRange == nil || !strings.HasPrefix(*object.ContentRange, responseRange) { - return 0, fmt.Errorf("unexpected content range %v - expected prefix %v", object.ContentRange, responseRange) + contentRange, err := parseContentRange(*object.ContentRange) + if err != nil { + return 0, fmt.Errorf("failed to parse content range: %v", err) + } + + if r.currentOffset < contentRange.start || r.currentOffset > contentRange.end { + return 0, fmt.Errorf("unexpected content range %v - expected %v to be witin the content range", *object.ContentRange, r.currentOffset) } b := bytes.Buffer{} @@ -386,3 +403,54 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { return n, err } + +type contentRange struct { + start int64 + end int64 + size int64 // -1 if unknown (*) +} + +// parseContentRange parses a content range header, with the format bytes $start-$end/$size +func parseContentRange(v string) (*contentRange, error) { + if !strings.HasPrefix(v, "bytes ") { + return nil, errors.New("invalid Content-Range header") + } + + v = strings.TrimPrefix(v, "bytes ") + parts := strings.Split(v, "/") + if len(parts) != 2 { + return nil, errors.New("invalid Content-Range") + } + + rangePart := parts[0] + sizePart := parts[1] + + re := strings.Split(rangePart, "-") + if len(re) != 2 { + return nil, errors.New("invalid Content-Range header") + } + + start, err := strconv.ParseInt(re[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse start in content range header: %v", err) + } + + end, err := strconv.ParseInt(re[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse end in content range header: %v", err) + } + + size := int64(-1) + if sizePart != "*" { + size, err = strconv.ParseInt(sizePart, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse size in content range header: %v", err) + } + } + + return &contentRange{ + start: start, + end: end, + size: size, + }, nil +} diff --git a/sda-download/internal/storage/v2/s3/reader/reader_test.go b/sda-download/internal/storage/v2/s3/reader/reader_test.go index 854d3e3055..cf726923ed 100644 --- a/sda-download/internal/storage/v2/s3/reader/reader_test.go +++ b/sda-download/internal/storage/v2/s3/reader/reader_test.go @@ -1,6 +1,7 @@ package reader import ( + "bytes" "context" "fmt" "io" @@ -45,10 +46,8 @@ func (ts *ReaderTestSuite) SetupSuite() { if _, err = bigFile.WriteString("This is a big file for testing seekable s3 reader"); err != nil { ts.FailNow("failed to write big test file", err) } - for range 6 * 1000 * 1000 { - if _, err = bigFile.WriteString("a"); err != nil { - ts.FailNow("failed to write big test file", err) - } + if _, err := bigFile.Write(bytes.Repeat([]byte{'a'}, 6*1000*1000)); err != nil { + ts.FailNow("failed to write big test file", err) } if _, err = bigFile.WriteString("file is ending now"); err != nil { ts.FailNow("failed to write big test file", err) diff --git a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go index 4abdf2de33..e6202c3659 100644 --- a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -6,11 +6,13 @@ import ( "errors" "fmt" "io" + "strconv" "strings" "sync" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + log "github.com/sirupsen/logrus" ) // s3CacheBlock is used to keep track of cached data @@ -43,18 +45,21 @@ func (reader *Reader) NewFileReadSeeker(ctx context.Context, location, filePath endpoint, bucket, err := parseLocation(location) if err != nil { cancel() + return nil, err } client, endpointConf, err := reader.getS3ClientForEndpoint(ctx, endpoint) if err != nil { cancel() + return nil, err } objectSize, err := reader.getFileSize(ctx, client, bucket, filePath) if err != nil { cancel() + return nil, err } @@ -79,8 +84,8 @@ func (r *s3SeekableReader) Close() error { if r.objectReader != nil { _ = r.objectReader.Close() } - r.cancel() + return nil } @@ -150,10 +155,18 @@ func (r *s3SeekableReader) prefetchAt(offset int64) { return } - responseRange := fmt.Sprintf("bytes %d-", r.currentOffset) + if object.ContentRange == nil { + return + } + + contentRange, err := parseContentRange(*object.ContentRange) + if err != nil { + log.Errorf("failed to parse content range: %v", err) - if object.ContentRange == nil || !strings.HasPrefix(*object.ContentRange, responseRange) { - // Unexpected content range - ignore + return + } + + if r.currentOffset < contentRange.start || r.currentOffset > contentRange.end { return } @@ -363,10 +376,17 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { return 0, err } - responseRange := fmt.Sprintf("bytes %d-", r.currentOffset) + if object.ContentRange == nil { + return 0, errors.New("unexpected nil content range") + } - if object.ContentRange == nil || !strings.HasPrefix(*object.ContentRange, responseRange) { - return 0, fmt.Errorf("unexpected content range %v - expected prefix %v", object.ContentRange, responseRange) + contentRange, err := parseContentRange(*object.ContentRange) + if err != nil { + return 0, fmt.Errorf("failed to parse content range: %v", err) + } + + if r.currentOffset < contentRange.start || r.currentOffset > contentRange.end { + return 0, fmt.Errorf("unexpected content range %v - expected %v to be witin the content range", *object.ContentRange, r.currentOffset) } b := bytes.Buffer{} @@ -383,3 +403,54 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { return n, err } + +type contentRange struct { + start int64 + end int64 + size int64 // -1 if unknown (*) +} + +// parseContentRange parses a content range header, with the format bytes $start-$end/$size +func parseContentRange(v string) (*contentRange, error) { + if !strings.HasPrefix(v, "bytes ") { + return nil, errors.New("invalid Content-Range header") + } + + v = strings.TrimPrefix(v, "bytes ") + parts := strings.Split(v, "/") + if len(parts) != 2 { + return nil, errors.New("invalid Content-Range") + } + + rangePart := parts[0] + sizePart := parts[1] + + re := strings.Split(rangePart, "-") + if len(re) != 2 { + return nil, errors.New("invalid Content-Range header") + } + + start, err := strconv.ParseInt(re[0], 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse start in content range header: %v", err) + } + + end, err := strconv.ParseInt(re[1], 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse end in content range header: %v", err) + } + + size := int64(-1) + if sizePart != "*" { + size, err = strconv.ParseInt(sizePart, 10, 64) + if err != nil { + return nil, fmt.Errorf("failed to parse size in content range header: %v", err) + } + } + + return &contentRange{ + start: start, + end: end, + size: size, + }, nil +} diff --git a/sda/internal/storage/v2/s3/reader/reader_test.go b/sda/internal/storage/v2/s3/reader/reader_test.go index c74c7db3ee..d228b4e8ed 100644 --- a/sda/internal/storage/v2/s3/reader/reader_test.go +++ b/sda/internal/storage/v2/s3/reader/reader_test.go @@ -1,6 +1,7 @@ package reader import ( + "bytes" "context" "fmt" "io" @@ -45,10 +46,8 @@ func (ts *ReaderTestSuite) SetupSuite() { if _, err = bigFile.WriteString("This is a big file for testing seekable s3 reader"); err != nil { ts.FailNow("failed to write big test file", err) } - for range 6 * 1000 * 1000 { - if _, err = bigFile.WriteString("a"); err != nil { - ts.FailNow("failed to write big test file", err) - } + if _, err := bigFile.Write(bytes.Repeat([]byte{'a'}, 6*1000*1000)); err != nil { + ts.FailNow("failed to write big test file", err) } if _, err = bigFile.WriteString("file is ending now"); err != nil { ts.FailNow("failed to write big test file", err) From 627e220cc514d94429e7ea58af7ffea8a3251e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 15:17:48 +0100 Subject: [PATCH 103/530] feat(download): integration test small fixes with correct naming, tls, etc --- sda-download/dev_utils/compose-no-tls.yml | 4 ++-- sda-download/dev_utils/compose-sda.yml | 6 +++--- sda-download/dev_utils/compose.yml | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sda-download/dev_utils/compose-no-tls.yml b/sda-download/dev_utils/compose-no-tls.yml index efd6cbfb58..f0aefb19d7 100644 --- a/sda-download/dev_utils/compose-no-tls.yml +++ b/sda-download/dev_utils/compose-no-tls.yml @@ -24,14 +24,14 @@ services: - "/bin/sh" - "-c" - | - pg_isready + pg_isready -h db -U postgres psql -U postgres -h db -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" depends_on: db: condition: service_healthy restart: on-failure environment: - - PGHOST=postgres + - PGHOST=db - PGPORT=5432 - PGUSER=postgres - PGPASSWORD=rootpassword diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index b9b4f0fb2b..67af4b819a 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -45,14 +45,14 @@ services: - "/bin/sh" - "-c" - | - pg_isready + pg_isready -h db -U postgres psql -U postgres -h db -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" depends_on: db: condition: service_healthy restart: on-failure environment: - - PGHOST=postgres + - PGHOST=db - PGPORT=5432 - PGUSER=postgres - PGPASSWORD=rootpassword @@ -62,7 +62,7 @@ services: environment: - MINIO_ROOT_USER=access - MINIO_ROOT_PASSWORD=secretkey - - MINIO_SERVER_URL=http://127.0.0.1:9000 + - MINIO_SERVER_URL=https://127.0.0.1:9000 healthcheck: test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] interval: 5s diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 2f5c44bb76..688ba3af54 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -48,14 +48,14 @@ services: - "/bin/sh" - "-c" - | - pg_isready + pg_isready -h db -U postgres psql -U postgres -h db -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" depends_on: db: condition: service_healthy restart: on-failure environment: - - PGHOST=postgres + - PGHOST=db - PGPORT=5432 - PGUSER=postgres - PGPASSWORD=rootpassword From 51579d26b4fc702c9d4064c0e65e496f5abba7b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 15:23:24 +0100 Subject: [PATCH 104/530] feat(storage-v2): clarify chunk_size configuration description in README.md --- sda-download/internal/storage/v2/README.md | 30 +++++++------ sda/internal/storage/v2/README.md | 50 +++++++++++----------- 2 files changed, 41 insertions(+), 39 deletions(-) diff --git a/sda-download/internal/storage/v2/README.md b/sda-download/internal/storage/v2/README.md index 67b26c90b9..1f2efd7bb2 100644 --- a/sda-download/internal/storage/v2/README.md +++ b/sda-download/internal/storage/v2/README.md @@ -1,6 +1,7 @@ # Storage v2 -> **_NOTE:_** The content of the storage/v2 package is copied from the "sda" implementation at [storage v2](../../../../sda/internal/storage/v2) just with deletion of the writer +> **_NOTE:_** The content of the storage/v2 package is copied from the "sda" implementation +> at [storage v2](../../../../sda/internal/storage/v2) just with deletion of the writer The storage v2 package is responsible for the interfacing to a storage implementation, the supported storage implementations are posix, and s3. @@ -24,7 +25,8 @@ Where ${STORAGE_NAME} is the name of the storage, and this is decided when initi `NewReader(..., "Inbox", ...).` ${STORAGE_IMPLEMENTATION} is which storage implementation is to be loaded, there can be multiple storage $ {STORAGE_IMPLEMENTATION}, -supported values are "s3", and "posix", eg if an application is to be able to read from both s3 and posix the config would be: +supported values are "s3", and "posix", eg if an application is to be able to read from both s3 and posix the config +would be: ```yaml storage: @@ -38,14 +40,14 @@ storage: ${STORAGE_IMPLEMENTATION_DEPENDANT_CONFIG} is the required configuration for the different storage implementations [s3 reader](#s3-reader-config), [posix reader](#posix-reader-config) -There can be multiple ${STORAGE_IMPLEMENTATION_DEPENDANT_CONFIG} if we want to be able to read / write to multiple of +There can be multiple ${STORAGE_IMPLEMENTATION_DEPENDANT_CONFIG} if we want to be able to read from multiple of the same storage implementation. eg: ```yaml storage: ${STORAGE_NAME}: s3: - - ${S3_READER_CONFIG_1} + - ${S3_READER_CONFIG_1} - ${S3_READER_CONFIG_2} ``` @@ -57,16 +59,16 @@ The s3 storage implementation uses the [AWS s3](https://docs.aws.amazon.com/s3/) A s3 reader has the following configuration: -| Name: | Type: | Default Value: | Description: | -|---------------|--------|----------------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| access_key | string | | The access key used to authenticate when connecting to the s3 | -| secret_key | string | | The secret key used to authenticate when connecting to the s3 | -| ca_cert | string | | The ca certificate of the s3 to be appended to the certs of the system | -| chunk_size | string | 50MB | The chunk size (in bytes) when writing data S3, also when reading chunks with the Seekable Reader. The minimum allowed is 5MB, and max is 1gb | -| region | string | us-east-1 | The region of the s3 bucket | -| endpoint | string | | The address of the s3 buckets | -| disable_https | bool | false | If to disable https when connecting to the s3 bucket | -| bucket_prefix | string | | How the reader will identify which buckets to look through when looking for a file for which the location is not known by the caller | +| Name: | Type: | Default Value: | Description: | +|---------------|--------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| access_key | string | | The access key used to authenticate when connecting to the s3 | +| secret_key | string | | The secret key used to authenticate when connecting to the s3 | +| ca_cert | string | | The ca certificate of the s3 to be appended to the certs of the system | +| chunk_size | string | 50MB | The chunk size used when writing data to `S3` and when reading data with the `Seekable Reader`. Supports values like 50MB, 100MB. The minimum allowed value is 5MB, and the maximum is 1GB. | +| region | string | us-east-1 | The region of the s3 bucket | +| endpoint | string | | The address of the s3 buckets | +| disable_https | bool | false | If to disable https when connecting to the s3 bucket | +| bucket_prefix | string | | How the reader will identify which buckets to look through when looking for a file for which the location is not known by the caller | ## Posix diff --git a/sda/internal/storage/v2/README.md b/sda/internal/storage/v2/README.md index e45b887844..306205bf8c 100644 --- a/sda/internal/storage/v2/README.md +++ b/sda/internal/storage/v2/README.md @@ -22,7 +22,7 @@ storage: Where `${STORAGE_NAME}` is the name of the storage, and this is decided when initializing the writer / reader, eg: `NewWriter(..., "Inbox", ...).` -`${STORAGE_IMPLEMENTATION}` is which storage implementation is to be loaded, there can be multiple storage +`${STORAGE_IMPLEMENTATION}` is which storage implementation is to be loaded, there can be multiple storage `${STORAGE_IMPLEMENTATION}`, supported values are "s3", and "posix", eg if an application is to be able to read from both s3 and posix, but writer to s3 the config would be: @@ -61,35 +61,35 @@ The s3 storage implementation uses the [AWS s3](https://docs.aws.amazon.com/s3/) A s3 reader has the following configuration: -| Name: | Type: | Default Value: | Description: | -|---------------|--------|----------------|-----------------------------------------------------------------------------------------------------------------------------------------------| -| access_key | string | | The access key used to authenticate when connecting to the s3 | -| secret_key | string | | The secret key used to authenticate when connecting to the s3 | -| ca_cert | string | | The ca certificate of the s3 to be appended to the certs of the system | -| chunk_size | string | 50MB | The chunk size (in bytes) when writing data S3, also when reading chunks with the Seekable Reader. The minimum allowed is 5MB, and max is 1gb | -| region | string | us-east-1 | The region of the s3 bucket | -| endpoint | string | | The address of the s3 buckets | -| disable_https | bool | false | If to disable https when connecting to the s3 bucket | -| bucket_prefix | string | | How the reader will identify which buckets to look through when looking for a file for which the location is not known by the caller | +| Name: | Type: | Default Value: | Description: | +|---------------|--------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| access_key | string | | The access key used to authenticate when connecting to the s3 | +| secret_key | string | | The secret key used to authenticate when connecting to the s3 | +| ca_cert | string | | The ca certificate of the s3 to be appended to the certs of the system | +| chunk_size | string | 50MB | The chunk size used when writing data to `S3` and when reading data with the `Seekable Reader`. Supports values like 50MB, 100MB. The minimum allowed value is 5MB, and the maximum is 1GB. | +| region | string | us-east-1 | The region of the s3 bucket | +| endpoint | string | | The address of the s3 buckets | +| disable_https | bool | false | If to disable https when connecting to the s3 bucket | +| bucket_prefix | string | | How the reader will identify which buckets to look through when looking for a file for which the location is not known by the caller | ### S3 Writer Config A s3 writer has the following configuration: -| Name: | Type: | Default Value: | Description: | -|-----------------|--------------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------| -| access_key | string | | The access key used to authenticate when connecting to the s3 | -| secret_key | string | | The secret key used to authenticate when connecting to the s3 | -| ca_cert | string | | The ca certificate of the s3 to be appended to the certs of the system | -| chunk_size | string | 50MB | The chunk size (in bytes) when writing data S3, also when reading chunks with the Seekable Reader. The minimum allowed is 5MB, and max is 1gb | -| region | string | us-east-1 | The region of the s3 bucket | -| endpoint | string | | The address of the s3 buckets | -| disable_https | bool | false | If to disable https when connecting to the s3 bucket | -| bucket_prefix | string | | How the writer will identify which buckets to be used or named if created, the buckets will be named by the bucket_prefix with a following incremental number | -| max_buckets | unsigned int | 1 | How many buckets the writer will automatically create in the endpoint when previous ones have reached their quota | -| max_objects | unsigned int | 0 | How many objects the writer will write to a bucket before switching to the next one | -| max_size | string | 0 | How many bytes the writer will write to a bucket before switching to the next one | -| writer_disabled | bool | false | If the writer for this config should be disabled, i.e if this is just the config for a reader | +| Name: | Type: | Default Value: | Description: | +|-----------------|--------------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| access_key | string | | The access key used to authenticate when connecting to the s3 | +| secret_key | string | | The secret key used to authenticate when connecting to the s3 | +| ca_cert | string | | The ca certificate of the s3 to be appended to the certs of the system | +| chunk_size | string | 50MB | The chunk size used when writing data to `S3` and when reading data with the `Seekable Reader`. Supports values like 50MB, 100MB. The minimum allowed value is 5MB, and the maximum is 1GB. | +| region | string | us-east-1 | The region of the s3 bucket | +| endpoint | string | | The address of the s3 buckets | +| disable_https | bool | false | If to disable https when connecting to the s3 bucket | +| bucket_prefix | string | | How the writer will identify which buckets to be used or named if created, the buckets will be named by the bucket_prefix with a following incremental number | +| max_buckets | unsigned int | 1 | How many buckets the writer will automatically create in the endpoint when previous ones have reached their quota | +| max_objects | unsigned int | 0 | How many objects the writer will write to a bucket before switching to the next one | +| max_size | string | 0 | How many bytes the writer will write to a bucket before switching to the next one | +| writer_disabled | bool | false | If the writer for this config should be disabled, i.e if this is just the config for a reader | ## Posix From 283f94520cce51d05b501527ea79d008af5aec52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 15:25:35 +0100 Subject: [PATCH 105/530] feat(storage-v2): clarify posix reader path configuration --- sda-download/internal/storage/v2/README.md | 6 +++--- sda/internal/storage/v2/README.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda-download/internal/storage/v2/README.md b/sda-download/internal/storage/v2/README.md index 1f2efd7bb2..08e2c50e25 100644 --- a/sda-download/internal/storage/v2/README.md +++ b/sda-download/internal/storage/v2/README.md @@ -76,7 +76,7 @@ A s3 reader has the following configuration: A posix reader has the following configuration: -| Name: | Type: | Default Value: | Description: | -|-------|--------|----------------|---------------------------------------------------------| -| path | string | | The path of the directory/volume where to store objects | +| Name: | Type: | Default Value: | Description: | +|-------|--------|----------------|--------------------------------------------------------------| +| path | string | | The path of the directory/volume from which objects are read | diff --git a/sda/internal/storage/v2/README.md b/sda/internal/storage/v2/README.md index 306205bf8c..d1f6c958a7 100644 --- a/sda/internal/storage/v2/README.md +++ b/sda/internal/storage/v2/README.md @@ -97,9 +97,9 @@ A s3 writer has the following configuration: A posix reader has the following configuration: -| Name: | Type: | Default Value: | Description: | -|-------|--------|----------------|---------------------------------------------------------| -| path | string | | The path of the directory/volume where to store objects | +| Name: | Type: | Default Value: | Description: | +|-------|--------|----------------|--------------------------------------------------------------| +| path | string | | The path of the directory/volume from which objects are read | ### Posix Writer Config From acef30a8cd5fe5c44e601f39b6d922180bffcb82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 15:49:36 +0100 Subject: [PATCH 106/530] fix(storage-v2): fix error when posix reader path is not a directory --- sda-download/internal/storage/v2/posix/reader/reader.go | 2 +- sda/internal/storage/v2/posix/reader/reader.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-download/internal/storage/v2/posix/reader/reader.go b/sda-download/internal/storage/v2/posix/reader/reader.go index 9be95f0783..c2f2d1d0d6 100644 --- a/sda-download/internal/storage/v2/posix/reader/reader.go +++ b/sda-download/internal/storage/v2/posix/reader/reader.go @@ -28,7 +28,7 @@ func NewReader(backendName string) (*Reader, error) { } if !fileInfo.IsDir() { - return nil, fmt.Errorf("%s is not a directory", loc) + return nil, fmt.Errorf("%s is not a directory", loc.Path) } } if len(backend.configuredEndpoints) == 0 { diff --git a/sda/internal/storage/v2/posix/reader/reader.go b/sda/internal/storage/v2/posix/reader/reader.go index a5743cbe63..9f2dc80cee 100644 --- a/sda/internal/storage/v2/posix/reader/reader.go +++ b/sda/internal/storage/v2/posix/reader/reader.go @@ -28,7 +28,7 @@ func NewReader(backendName string) (*Reader, error) { } if !fileInfo.IsDir() { - return nil, fmt.Errorf("%s is not a directory", loc) + return nil, fmt.Errorf("%s is not a directory", loc.Path) } } if len(backend.configuredEndpoints) == 0 { From f8435d582141ccd1dfa7e2b6dd974d95676d530f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 15:52:06 +0100 Subject: [PATCH 107/530] fix(storage-v2): fix posix reader check if any endpoints configured before creating reader --- sda-download/internal/storage/v2/posix/reader/reader.go | 6 +++--- sda/internal/storage/v2/posix/reader/reader.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda-download/internal/storage/v2/posix/reader/reader.go b/sda-download/internal/storage/v2/posix/reader/reader.go index c2f2d1d0d6..beef052441 100644 --- a/sda-download/internal/storage/v2/posix/reader/reader.go +++ b/sda-download/internal/storage/v2/posix/reader/reader.go @@ -16,6 +16,9 @@ func NewReader(backendName string) (*Reader, error) { if err != nil { return nil, err } + if len(endPointsConfigurations) == 0 { + return nil, storageerrors.ErrorNoValidLocations + } backend := &Reader{ configuredEndpoints: endPointsConfigurations, @@ -31,9 +34,6 @@ func NewReader(backendName string) (*Reader, error) { return nil, fmt.Errorf("%s is not a directory", loc.Path) } } - if len(backend.configuredEndpoints) == 0 { - return nil, storageerrors.ErrorNoValidLocations - } return backend, nil } diff --git a/sda/internal/storage/v2/posix/reader/reader.go b/sda/internal/storage/v2/posix/reader/reader.go index 9f2dc80cee..a01853064f 100644 --- a/sda/internal/storage/v2/posix/reader/reader.go +++ b/sda/internal/storage/v2/posix/reader/reader.go @@ -16,6 +16,9 @@ func NewReader(backendName string) (*Reader, error) { if err != nil { return nil, err } + if len(endPointsConfigurations) == 0 { + return nil, storageerrors.ErrorNoValidLocations + } backend := &Reader{ configuredEndpoints: endPointsConfigurations, @@ -31,9 +34,6 @@ func NewReader(backendName string) (*Reader, error) { return nil, fmt.Errorf("%s is not a directory", loc.Path) } } - if len(backend.configuredEndpoints) == 0 { - return nil, storageerrors.ErrorNoValidLocations - } return backend, nil } From add454ad36810e9b67a55689c851f583b53fcffd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 09:06:37 +0100 Subject: [PATCH 108/530] feat(download): fix sentence in README.md --- sda-download/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-download/README.md b/sda-download/README.md index 46b3fa9848..f840a9c326 100644 --- a/sda-download/README.md +++ b/sda-download/README.md @@ -46,7 +46,7 @@ There is an README file in the [dev_utils](/dev_utils) folder with sections for | request | This pkg Stores a HTTP client, so that it doesn't need to be initialised on every request. | ## Storage settings -The download service requires access to the "archive" storage, to configure such the following config is needed: +The download service requires access to the "archive" storage. To configure it the following configuration is required: ```yaml storage: archive: From 200e12f97825701fe7da626f0a5ecab5d3ab819f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 09:08:27 +0100 Subject: [PATCH 109/530] feat(storage-v2): fix sentence in README.md --- sda-download/internal/storage/v2/README.md | 3 +-- sda/internal/storage/v2/README.md | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/sda-download/internal/storage/v2/README.md b/sda-download/internal/storage/v2/README.md index 08e2c50e25..97cd7e8d45 100644 --- a/sda-download/internal/storage/v2/README.md +++ b/sda-download/internal/storage/v2/README.md @@ -23,8 +23,7 @@ storage: Where ${STORAGE_NAME} is the name of the storage, and this is decided when initializing the reader, eg: `NewReader(..., "Inbox", ...).` -${STORAGE_IMPLEMENTATION} is which storage implementation is to be loaded, there can be multiple storage $ -{STORAGE_IMPLEMENTATION}, +${STORAGE_IMPLEMENTATION} is which storage implementation is to be loaded, there can be multiple storage implementations, supported values are "s3", and "posix", eg if an application is to be able to read from both s3 and posix the config would be: diff --git a/sda/internal/storage/v2/README.md b/sda/internal/storage/v2/README.md index d1f6c958a7..3285d6fef3 100644 --- a/sda/internal/storage/v2/README.md +++ b/sda/internal/storage/v2/README.md @@ -22,8 +22,7 @@ storage: Where `${STORAGE_NAME}` is the name of the storage, and this is decided when initializing the writer / reader, eg: `NewWriter(..., "Inbox", ...).` -`${STORAGE_IMPLEMENTATION}` is which storage implementation is to be loaded, there can be multiple storage -`${STORAGE_IMPLEMENTATION}`, +`${STORAGE_IMPLEMENTATION}` is which storage implementation is to be loaded, there can be multiple storage implementations, supported values are "s3", and "posix", eg if an application is to be able to read from both s3 and posix, but writer to s3 the config would be: From c020aa70e073307b7348c1536fa8b34355306a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 09:33:12 +0100 Subject: [PATCH 110/530] feat(storage-v2): update storage v2 seekable multi reader, add check for negative offset in seek, and error internal incase we want to read data from a previous --- .../internal/storage/v2/seekable_multi_reader.go | 12 +++++++----- sda/internal/storage/v2/seekable_multi_reader.go | 12 +++++++----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/sda-download/internal/storage/v2/seekable_multi_reader.go b/sda-download/internal/storage/v2/seekable_multi_reader.go index fa1e54ef9c..0ecbf17ada 100644 --- a/sda-download/internal/storage/v2/seekable_multi_reader.go +++ b/sda-download/internal/storage/v2/seekable_multi_reader.go @@ -53,6 +53,10 @@ func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { return 0, errors.New("unsupported whence") } + if r.currentOffset < 0 { + return 0, errors.New("negative seek offset") + } + return r.currentOffset, nil } @@ -61,10 +65,8 @@ func (r *seekableMultiReader) Read(dst []byte) (int, error) { for i, reader := range r.readers { if r.currentOffset < readerStartAt { - // We want data from a previous reader (? HELP ?) - readerStartAt += r.sizes[i] - - continue + // We want data from a previous reader + return 0, errors.New("unexpected error: could not find correct reader to read from") } if readerStartAt+r.sizes[i] < r.currentOffset { @@ -81,7 +83,7 @@ func (r *seekableMultiReader) Read(dst []byte) (int, error) { return 0, errors.New("expected seekable reader but changed") } - _, err := seekable.Seek(r.currentOffset-int64(readerStartAt), 0) + _, err := seekable.Seek(r.currentOffset-readerStartAt, 0) if err != nil { return 0, fmt.Errorf("unexpected error while seeking: %v", err) } diff --git a/sda/internal/storage/v2/seekable_multi_reader.go b/sda/internal/storage/v2/seekable_multi_reader.go index fa1e54ef9c..0ecbf17ada 100644 --- a/sda/internal/storage/v2/seekable_multi_reader.go +++ b/sda/internal/storage/v2/seekable_multi_reader.go @@ -53,6 +53,10 @@ func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { return 0, errors.New("unsupported whence") } + if r.currentOffset < 0 { + return 0, errors.New("negative seek offset") + } + return r.currentOffset, nil } @@ -61,10 +65,8 @@ func (r *seekableMultiReader) Read(dst []byte) (int, error) { for i, reader := range r.readers { if r.currentOffset < readerStartAt { - // We want data from a previous reader (? HELP ?) - readerStartAt += r.sizes[i] - - continue + // We want data from a previous reader + return 0, errors.New("unexpected error: could not find correct reader to read from") } if readerStartAt+r.sizes[i] < r.currentOffset { @@ -81,7 +83,7 @@ func (r *seekableMultiReader) Read(dst []byte) (int, error) { return 0, errors.New("expected seekable reader but changed") } - _, err := seekable.Seek(r.currentOffset-int64(readerStartAt), 0) + _, err := seekable.Seek(r.currentOffset-readerStartAt, 0) if err != nil { return 0, fmt.Errorf("unexpected error while seeking: %v", err) } From 4167108c25f09d7ad81cda40183caf2e6c99e91b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 14:08:48 +0100 Subject: [PATCH 111/530] feat(download): update integration tests to use 2 s3 with 2 separate buckets, remove deprecated stuff, update run_integration_test.sh to use same scripts as github actions etc --- .../setup/common/23_create_db_entries.sh | 139 ++++++----- .../setup/posix/24_posix_update_db_entry.sh | 29 ++- .../setup/s3/100_s3_storage_setup.sh | 17 +- .../setup/s3/24_s3_update_db_entry.sh | 37 +++ .../setup/s3notls/24_posix_update_db_entry.sh | 10 - .../s3notls/24_s3notls_update_db_entry.sh | 37 +++ .../setup/s3notls/99_s3_storage_setup.sh | 15 +- .../integration/tests/common/30_check_db.sh | 2 +- .../tests/common/50_check_endpoint.sh | 38 +-- .../tests/common/60_check_s3_endpoint.sh | 2 +- .../tests/common/80_check_reencrypt.sh | 4 +- .../tests/common/90_check_s3_errors.sh | 12 +- .../tests/s3notls/52_check_endpoint.sh | 14 +- sda-download/dev_utils/README.md | 2 +- ...-dc50-46db-b79a-27ddc0dad1c6 => test_file} | Bin sda-download/dev_utils/compose-no-tls.yml | 26 +- sda-download/dev_utils/compose-sda.yml | 139 ----------- sda-download/dev_utils/compose.yml | 32 ++- sda-download/dev_utils/config-notls.yaml | 11 +- .../dev_utils/config-notls_local.yaml | 34 --- sda-download/dev_utils/config.yaml | 11 +- .../dev_utils/run_integration_test.sh | 233 +----------------- .../dev_utils/run_integration_test_no_tls.sh | 191 +------------- sda-download/dev_utils/ssl.cnf | 2 +- 24 files changed, 333 insertions(+), 704 deletions(-) create mode 100644 sda-download/.github/integration/setup/s3/24_s3_update_db_entry.sh delete mode 100644 sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh create mode 100644 sda-download/.github/integration/setup/s3notls/24_s3notls_update_db_entry.sh rename sda-download/dev_utils/archive_data/{4293c9a7-dc50-46db-b79a-27ddc0dad1c6 => test_file} (100%) delete mode 100644 sda-download/dev_utils/compose-sda.yml delete mode 100644 sda-download/dev_utils/config-notls_local.yaml diff --git a/sda-download/.github/integration/setup/common/23_create_db_entries.sh b/sda-download/.github/integration/setup/common/23_create_db_entries.sh index 8b9fe49452..3ca23996ff 100755 --- a/sda-download/.github/integration/setup/common/23_create_db_entries.sh +++ b/sda-download/.github/integration/setup/common/23_create_db_entries.sh @@ -5,62 +5,6 @@ cd dev_utils || exit 1 chmod 600 certs/client-key.pem -# insert file entry into database -file_id=$(docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.files (stable_id, submission_user, \ - submission_file_path, submission_file_size, archive_file_path, \ - archive_file_size, archive_location, decrypted_file_size, backup_path, header, \ - encryption_method) VALUES ('urn:neic:001-002', 'integration-test', 'dummy_data.c4gh', \ - 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 'https://s3:9000/archive', 1048605, \ - '', '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'CRYPT4GH') RETURNING id;" | xargs) - -# insert "ready" database log event -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.file_event_log (file_id, event) \ - VALUES ('$file_id', 'ready');" - -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'UNENCRYPTED');" - -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', 'UPLOADED');" - -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 'ARCHIVED');" - # make sure that the dataset exists in the database dataset_id=$(docker run --rm --name client --network dev_utils_default \ @@ -74,13 +18,76 @@ dataset_id=$(docker run --rm --name client --network dev_utils_default \ ON CONFLICT (stable_id) DO UPDATE \ SET stable_id=excluded.stable_id RETURNING id;") -# insert the file into the dataset -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.file_dataset (file_id, dataset_id) \ - VALUES ('$file_id', $dataset_id);" +for file_id in 00000000-0000-0000-0000-000000000001 00000000-0000-0000-0000-000000000002 00000000-0000-0000-0000-000000000003 00000000-0000-0000-0000-000000000004; do + + docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "INSERT INTO sda.files (id, stable_id, submission_user, \ + submission_file_path, submission_file_size, archive_file_path, \ + archive_file_size, archive_location, decrypted_file_size, backup_path, header, \ + encryption_method) VALUES ('$file_id', 'urn:neic:001-$file_id', 'integration-test', 'file_$file_id.c4gh', \ + 1048729, '$file_id', 1049081, '', 1048605, \ + '', '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'CRYPT4GH') RETURNING id;" + + + # insert "ready" database log event + docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "INSERT INTO sda.file_event_log (file_id, event) \ + VALUES ('$file_id', 'ready');" + + docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ + VALUES ('$file_id', '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'UNENCRYPTED');" + + + docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ + VALUES ('$file_id', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', 'UPLOADED');" + + docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ + VALUES ('$file_id', '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 'ARCHIVED');" + + + + # insert the file into the dataset + docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "INSERT INTO sda.file_dataset (file_id, dataset_id) \ + VALUES ('$file_id', $dataset_id);" + +done diff --git a/sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh b/sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh index 8a95672584..31f46d5e16 100644 --- a/sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh +++ b/sda-download/.github/integration/setup/posix/24_posix_update_db_entry.sh @@ -7,4 +7,31 @@ docker run --rm --name client --network dev_utils_default \ -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest \ postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "UPDATE sda.files SET archive_location = '/tmp';" \ No newline at end of file + -t -q -c "UPDATE sda.files SET archive_location = '/tmp/dir1' WHERE id = '00000000-0000-0000-0000-000000000001';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = '/tmp/dir1' WHERE id = '00000000-0000-0000-0000-000000000002';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = '/tmp/dir2' WHERE id = '00000000-0000-0000-0000-000000000003';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = '/tmp/dir2' WHERE id = '00000000-0000-0000-0000-000000000004';" diff --git a/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh b/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh index ac2d6b7a7d..b61455ac98 100644 --- a/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh +++ b/sda-download/.github/integration/setup/s3/100_s3_storage_setup.sh @@ -4,7 +4,18 @@ set -e cd dev_utils || exit 1 # Make buckets if they don't exist already -s3cmd -c s3cmd.conf mb s3://archive || true +s3cmd -c s3cmd.conf mb s3://archive-1 || true +s3cmd -c s3cmd.conf mb s3://archive-2 || true + +# Make buckets if they don't exist already +s3cmd -c s3cmd.conf --host localhost:9001 --host-bucket localhost:9001 mb s3://archive-1 || true +s3cmd -c s3cmd.conf --host localhost:9001 --host-bucket localhost:9001 mb s3://archive-2 || true + +# Upload test file +s3cmd -c s3cmd.conf put archive_data/test_file s3://archive-1/00000000-0000-0000-0000-000000000001 +s3cmd -c s3cmd.conf put archive_data/test_file s3://archive-2/00000000-0000-0000-0000-000000000002 + +# Upload test file +s3cmd -c s3cmd.conf --host localhost:9001 --host-bucket localhost:9001 put archive_data/test_file s3://archive-1/00000000-0000-0000-0000-000000000003 +s3cmd -c s3cmd.conf --host localhost:9001 --host-bucket localhost:9001 put archive_data/test_file s3://archive-2/00000000-0000-0000-0000-000000000004 -# # Upload test file -s3cmd -c s3cmd.conf put archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 s3://archive/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 diff --git a/sda-download/.github/integration/setup/s3/24_s3_update_db_entry.sh b/sda-download/.github/integration/setup/s3/24_s3_update_db_entry.sh new file mode 100644 index 0000000000..33155d0f84 --- /dev/null +++ b/sda-download/.github/integration/setup/s3/24_s3_update_db_entry.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'https://s3:9000/archive-1' WHERE id = '00000000-0000-0000-0000-000000000001';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'https://s3:9000/archive-2' WHERE id = '00000000-0000-0000-0000-000000000002';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'https://s3-2nd:9000/archive-1' WHERE id = '00000000-0000-0000-0000-000000000003';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'https://s3-2nd:9000/archive-2' WHERE id = '00000000-0000-0000-0000-000000000004';" \ No newline at end of file diff --git a/sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh b/sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh deleted file mode 100644 index 7eff05fcfb..0000000000 --- a/sda-download/.github/integration/setup/s3notls/24_posix_update_db_entry.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "UPDATE sda.files SET archive_location = 'http://s3:9000/archive';" \ No newline at end of file diff --git a/sda-download/.github/integration/setup/s3notls/24_s3notls_update_db_entry.sh b/sda-download/.github/integration/setup/s3notls/24_s3notls_update_db_entry.sh new file mode 100644 index 0000000000..1167b6bacf --- /dev/null +++ b/sda-download/.github/integration/setup/s3notls/24_s3notls_update_db_entry.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'http://s3:9000/archive-1' WHERE id = '00000000-0000-0000-0000-000000000001';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'http://s3:9000/archive-2' WHERE id = '00000000-0000-0000-0000-000000000002';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'http://s3-2nd:9000/archive-1' WHERE id = '00000000-0000-0000-0000-000000000003';" + +docker run --rm --name client --network dev_utils_default \ + -v "dev_utils_certs:/certs" \ + -e PGSSLCERT=/certs/client.pem \ + -e PGSSLKEY=/certs/client-key.pem \ + -e PGSSLROOTCERT=/certs/ca.pem \ + neicnordic/pg-client:latest \ + postgresql://postgres:rootpassword@db:5432/sda \ + -t -q -c "UPDATE sda.files SET archive_location = 'http://s3-2nd:9000/archive-2' WHERE id = '00000000-0000-0000-0000-000000000004';" \ No newline at end of file diff --git a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh index 575cf96184..0d9e935890 100644 --- a/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh +++ b/sda-download/.github/integration/setup/s3notls/99_s3_storage_setup.sh @@ -4,7 +4,18 @@ set -e cd dev_utils || exit 1 # Make buckets if they don't exist already -s3cmd -c s3cmd-notls.conf mb s3://archive || true +s3cmd -c s3cmd-notls.conf mb s3://archive-1 || true +s3cmd -c s3cmd-notls.conf mb s3://archive-2 || true + +# Make buckets if they don't exist already +s3cmd -c s3cmd-notls.conf --host localhost:9001 --host-bucket localhost:9001 mb s3://archive-1 || true +s3cmd -c s3cmd-notls.conf --host localhost:9001 --host-bucket localhost:9001 mb s3://archive-2 || true + +# Upload test file +s3cmd -c s3cmd-notls.conf put archive_data/test_file s3://archive-1/00000000-0000-0000-0000-000000000001 +s3cmd -c s3cmd-notls.conf put archive_data/test_file s3://archive-2/00000000-0000-0000-0000-000000000002 # Upload test file -s3cmd -c s3cmd-notls.conf put archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 s3://archive/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 +s3cmd -c s3cmd-notls.conf --host localhost:9001 --host-bucket localhost:9001 put archive_data/test_file s3://archive-1/00000000-0000-0000-0000-000000000003 +s3cmd -c s3cmd-notls.conf --host localhost:9001 --host-bucket localhost:9001 put archive_data/test_file s3://archive-2/00000000-0000-0000-0000-000000000004 + diff --git a/sda-download/.github/integration/tests/common/30_check_db.sh b/sda-download/.github/integration/tests/common/30_check_db.sh index f2a03350ca..b5e0812b6c 100644 --- a/sda-download/.github/integration/tests/common/30_check_db.sh +++ b/sda-download/.github/integration/tests/common/30_check_db.sh @@ -18,4 +18,4 @@ docker run --rm --name client --network dev_utils_default -v "$PWD/certs:/certs" docker run --rm --name client --network dev_utils_default -v "$PWD/certs:/certs" \ -e PGSSLCERT=/certs/client.pem -e PGSSLKEY=/certs/client-key.pem -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/sda \ - -t -c "SELECT id, stable_id, archive_file_path FROM sda.files ORDER BY id DESC" + -t -c "SELECT id, stable_id, archive_file_path, archive_location FROM sda.files ORDER BY id DESC" diff --git a/sda-download/.github/integration/tests/common/50_check_endpoint.sh b/sda-download/.github/integration/tests/common/50_check_endpoint.sh index eb10a9ac34..61e5ce4545 100755 --- a/sda-download/.github/integration/tests/common/50_check_endpoint.sh +++ b/sda-download/.github/integration/tests/common/50_check_endpoint.sh @@ -9,7 +9,7 @@ fi # ------------------ # Test Health Endpoint -check_health=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem https://localhost:8443/health) +check_health=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem https://localhost:9443/health) if [ "$check_health" != "200" ]; then echo "Health endpoint does not respond properly" @@ -19,7 +19,7 @@ fi echo "Health endpoint is ok" -check_health_header=$(curl -o /dev/null -s -w "%{http_code}\n" -LI --cacert certs/ca.pem https://localhost:8443/) +check_health_header=$(curl -o /dev/null -s -w "%{http_code}\n" -LI --cacert certs/ca.pem https://localhost:9443/) if [ "$check_health_header" != "200" ]; then echo "Head request to health endpoint does not respond properly" @@ -32,7 +32,7 @@ echo "Head method health endpoint is ok" # ------------------ # Test empty token -check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) +check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:9443/metadata/datasets) if [ "$check_401" != "401" ]; then echo "no token provided should give 401" @@ -42,7 +42,7 @@ fi echo "got correct response when no token provided" -check_405=$(curl -o /dev/null -s -w "%{http_code}\n" -X POST --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) +check_405=$(curl -o /dev/null -s -w "%{http_code}\n" -X POST --cacert certs/ca.pem -H "SDA-Client-Version: v0.3.0" https://localhost:9443/metadata/datasets) if [ "$check_405" != "405" ]; then echo "POST should not be allowed" @@ -61,7 +61,7 @@ token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r ' # We assume the app is configured to require a minimum version (v0.2.0). # Fail - missing header (expected 412 Precondition Failed) -check_missing_header=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets") +check_missing_header=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:9443/metadata/datasets") if [ "$check_missing_header" != "412" ]; then echo "Client Version Test FAIL: missing header should return 412" @@ -71,7 +71,7 @@ fi echo "Client Version Test OK: Missing header correctly returns 412" # Fail - insufficient version (e.g., v0.1.0, Expected 412) -check_insufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.1.0" "https://localhost:8443/metadata/datasets") +check_insufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.1.0" "https://localhost:9443/metadata/datasets") if [ "$check_insufficient_version" != "412" ]; then echo "Client Version Test FAIL: insufficient version (v0.1.0) should return 412" @@ -81,7 +81,7 @@ fi echo "Client Version Test OK: Insufficient version (v0.1.0) correctly returns 412" # Success - sufficient version (e.g., v0.2.0, Expected 200) -check_sufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.2.0" "https://localhost:8443/metadata/datasets") +check_sufficient_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.2.0" "https://localhost:9443/metadata/datasets") if [ "$check_sufficient_version" != "200" ]; then echo "Client Version Test FAIL: sufficient version (v0.2.0) should pass version check and return 200" @@ -91,7 +91,7 @@ fi echo "Client Version Test OK: sufficient version (v0.2.0) correctly returns 200" # Success - newer version (e.g., v1.0.0, Expected 200) -check_newer_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v1.0.0" "https://localhost:8443/metadata/datasets") +check_newer_version=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v1.0.0" "https://localhost:9443/metadata/datasets") if [ "$check_newer_version" != "200" ]; then echo "Client Version Test FAIL: Newer version (v1.0.0) should pass version check and return 200" @@ -102,7 +102,7 @@ echo "Client Version Test OK: Newer version (v1.0.0) correctly returns 200" ## Test datasets endpoint -check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets | jq -r '.[0]') +check_dataset=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:9443/metadata/datasets | jq -r '.[0]') if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found" @@ -114,10 +114,10 @@ echo "expected dataset found" ## Test datasets/files endpoint -check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') +check_files=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[1].fileId') -if [ "$check_files" != "urn:neic:001-002" ]; then - echo "file with id urn:neic:001-002 not found" +if [ "$check_files" != "urn:neic:001-00000000-0000-0000-0000-000000000002" ]; then + echo "file with id urn:neic:001-00000000-0000-0000-0000-000000000002 not found" echo "got: ${check_files}" exit 1 fi @@ -132,7 +132,7 @@ export C4GH_PASSPHRASE crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002" --output test-download.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-00000000-0000-0000-0000-000000000002" --output test-download.txt cmp --silent old-file.txt test-download.txt status=$? @@ -143,7 +143,7 @@ else exit 1 fi -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-00000000-0000-0000-0000-000000000002?startCoordinate=0&endCoordinate=2" --output test-part.txt dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt @@ -156,7 +156,7 @@ else exit 1 fi -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-00000000-0000-0000-0000-000000000002?startCoordinate=7&endCoordinate=14" --output test-part2.txt dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt @@ -169,7 +169,7 @@ else exit 1 fi -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/files/urn:neic:001-00000000-0000-0000-0000-000000000002?startCoordinate=70000&endCoordinate=140000" --output test-part3.txt dd if=old-file.txt ibs=1 skip=70000 count=70000 > old-part3.txt @@ -184,7 +184,7 @@ fi # test that downloads of decrypted files from a download instance that # serves only encrypted files (here running at port 8443) should fail -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/files/urn:neic:001-002" --output test-download-fail.txt +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/files/urn:neic:001-00000000-0000-0000-0000-000000000002" --output test-download-fail.txt if ! grep -q "downloading unencrypted data is not supported" test-download-fail.txt; then echo "got unexpected response when trying to download unencrypted data from encrypted endpoint" @@ -198,7 +198,7 @@ token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r ' ## Test datasets endpoint -check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) +check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:9443/metadata/datasets) if [ "$check_empty_token" != "200" ]; then echo "response for empty token is not 200" @@ -216,7 +216,7 @@ token=$(curl -s --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r ' ## Test datasets endpoint -check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:8443/metadata/datasets) +check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" https://localhost:9443/metadata/datasets) if [ "$check_empty_token" != "200" ]; then echo "response for token with untrusted sources is not 200" diff --git a/sda-download/.github/integration/tests/common/60_check_s3_endpoint.sh b/sda-download/.github/integration/tests/common/60_check_s3_endpoint.sh index 4477f3813e..a9a4b96704 100644 --- a/sda-download/.github/integration/tests/common/60_check_s3_endpoint.sh +++ b/sda-download/.github/integration/tests/common/60_check_s3_endpoint.sh @@ -75,7 +75,7 @@ auth_token() { # # Create s3cmd configs # -# ports 9443/9080 refer to the download instance serving unencrypted files +# ports 9443/8080 refer to the download instance serving unencrypted files port="9443" [[ "$TLS" == "False" ]] && port="9080" diff --git a/sda-download/.github/integration/tests/common/80_check_reencrypt.sh b/sda-download/.github/integration/tests/common/80_check_reencrypt.sh index 2f90949e38..0b252f3d26 100644 --- a/sda-download/.github/integration/tests/common/80_check_reencrypt.sh +++ b/sda-download/.github/integration/tests/common/80_check_reencrypt.sh @@ -30,7 +30,7 @@ fi # Test reencrypt the file header with the client public key clientkey=$(base64 -w0 client.pub.pem) reencryptedFile=reencrypted.bam.c4gh -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" --output $reencryptedFile +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file" --output $reencryptedFile expected_encrypted_size=1049205 file_size=$(stat -c %s $reencryptedFile) @@ -55,7 +55,7 @@ fi # download reencrypted partial file, check file size partReencryptedFile=part1.bam.c4gh -curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile +curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file?startCoordinate=0&endCoordinate=1000" --output $partReencryptedFile file_size=$(stat -c %s $partReencryptedFile) # Get the size of the file part_expected_size=65688 diff --git a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh index 0c5cefe13b..830573b3eb 100644 --- a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh +++ b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh @@ -22,26 +22,26 @@ bad_token=BADeyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJyZXF1ZXN0ZXJAZGVtby # Test error codes and error messages returned to the user # try to download encrypted file without sending a public key -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file") if ! echo "$resp" | grep -q "c4gh public key is missing from the header"; then echo "Incorrect response, expected 'c4gh public key is missing from the header' got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 400 ]; then echo "Incorrect response with missing public key, expected 400 got $resp" exit 1 fi # try to download encrypted file with a bad public key -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file") if ! echo "$resp" | grep -q "file re-encryption error"; then echo "Incorrect response, expected 'file re-encryption error' got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: YmFkIGtleQ==" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 500 ]; then echo "Incorrect response with missing public key, expected 500 got $resp" fi @@ -63,13 +63,13 @@ fi # try to download a file the user doesn't have access to -resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file") +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file") if ! echo "$resp" | grep -q "get visas failed"; then echo "Incorrect response, expected 'get visas failed' got $resp" exit 1 fi -resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:8443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $bad_token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/$dataset/$file" -s -o /dev/null -w "%{http_code}") if [ "$resp" -ne 401 ]; then echo "Incorrect response, expected 401 got $resp" exit 1 diff --git a/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh b/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh index e70c08f63e..fbd15317bf 100644 --- a/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh +++ b/sda-download/.github/integration/tests/s3notls/52_check_endpoint.sh @@ -57,10 +57,10 @@ echo "expected dataset found" ## Test datasets/files endpoint -check_files=$(curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:8080/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') +check_files=$(curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[1].fileId') -if [ "$check_files" != "urn:neic:001-002" ]; then - echo "file with id urn:neic:001-002 not found" +if [ "$check_files" != "urn:neic:001-00000000-0000-0000-0000-000000000002" ]; then + echo "file with id urn:neic:001-00000000-0000-0000-0000-000000000002 not found" echo "got: ${check_files}" exit 1 fi @@ -76,7 +76,7 @@ export C4GH_PASSPHRASE crypt4gh decrypt -s c4gh.sec.pem -f dummy_data.c4gh && mv dummy_data old-file.txt # first try downloading from download instance serving encrypted data, should fail -curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:8080/files/urn:neic:001-002" --output test-download.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:8080/files/urn:neic:001-00000000-0000-0000-0000-000000000002" --output test-download.txt if ! grep -q "downloading unencrypted data is not supported" "test-download.txt"; then echo "wrong response when trying to download unencrypted data from encrypted endpoint" @@ -84,7 +84,7 @@ if ! grep -q "downloading unencrypted data is not supported" "test-download.txt" fi # now try downloading from download instance serving unencrypted data -curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-002" --output test-download.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-00000000-0000-0000-0000-000000000002" --output test-download.txt cmp --silent old-file.txt test-download.txt @@ -96,7 +96,7 @@ else fi # downloading from download instance serving unencrypted data -curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-002?startCoordinate=0&endCoordinate=2" --output test-part.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-00000000-0000-0000-0000-000000000002?startCoordinate=0&endCoordinate=2" --output test-part.txt dd if=old-file.txt ibs=1 skip=0 count=2 > old-part.txt @@ -110,7 +110,7 @@ else fi # downloading from download instance serving unencrypted data -curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-002?startCoordinate=7&endCoordinate=14" --output test-part2.txt +curl -s -H "Authorization: Bearer $token" -H "SDA-Client-Version: v0.3.0" "http://localhost:9080/files/urn:neic:001-00000000-0000-0000-0000-000000000002?startCoordinate=7&endCoordinate=14" --output test-part2.txt dd if=old-file.txt ibs=1 skip=7 count=7 > old-part2.txt diff --git a/sda-download/dev_utils/README.md b/sda-download/dev_utils/README.md index 4048207463..3f73dee7d7 100644 --- a/sda-download/dev_utils/README.md +++ b/sda-download/dev_utils/README.md @@ -66,7 +66,7 @@ See more information on how these are generated in [sda-pipeline](https://github docker run --rm --name client --network dev_utils_default -v "$PWD/certs:/certs" \ -e PGSSLCERT=/certs/client.pem -e PGSSLKEY=/certs/client-key.pem -e PGSSLROOTCERT=/certs/ca.pem \ neicnordic/pg-client:latest postgresql://postgres:rootpassword@db:5432/lega \ - -t -c "INSERT INTO local_ega.main (id, stable_id, status, submission_file_path, submission_file_extension, submission_file_calculated_checksum, submission_file_calculated_checksum_type, submission_file_size, submission_user, archive_file_reference, archive_file_type, archive_file_size, archive_file_checksum, archive_file_checksum_type, decrypted_file_size, decrypted_file_checksum, decrypted_file_checksum_type, encryption_method, version, header, created_by, last_modified_by, created_at, last_modified) VALUES (1, 'urn:neic:001-002', 'READY', 'dummy_data.c4gh', 'c4gh', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', NULL, 'test', '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', NULL, 1049081, '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 1048605, '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'CRYPT4GH', NULL, '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'lega_in', 'lega_in', '2021-12-13 16:18:06.512169+00', '2021-12-13 16:18:06.512169+00');" + -t -c "INSERT INTO local_ega.main (id, stable_id, status, submission_file_path, submission_file_extension, submission_file_calculated_checksum, submission_file_calculated_checksum_type, submission_file_size, submission_user, archive_file_reference, archive_file_type, archive_file_size, archive_file_checksum, archive_file_checksum_type, decrypted_file_size, decrypted_file_checksum, decrypted_file_checksum_type, encryption_method, version, header, created_by, last_modified_by, created_at, last_modified) VALUES (1, 'urn:neic:001-00000000-0000-0000-0000-000000000002', 'READY', 'dummy_data.c4gh', 'c4gh', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', NULL, 'test', '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', NULL, 1049081, '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 1048605, '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'CRYPT4GH', NULL, '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'lega_in', 'lega_in', '2021-12-13 16:18:06.512169+00', '2021-12-13 16:18:06.512169+00');" docker run --rm --name client --network dev_utils_default -v "$PWD/certs:/certs" \ -e PGSSLCERT=/certs/client.pem -e PGSSLKEY=/certs/client-key.pem -e PGSSLROOTCERT=/certs/ca.pem \ diff --git a/sda-download/dev_utils/archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 b/sda-download/dev_utils/archive_data/test_file similarity index 100% rename from sda-download/dev_utils/archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 rename to sda-download/dev_utils/archive_data/test_file diff --git a/sda-download/dev_utils/compose-no-tls.yml b/sda-download/dev_utils/compose-no-tls.yml index f0aefb19d7..55e6585e15 100644 --- a/sda-download/dev_utils/compose-no-tls.yml +++ b/sda-download/dev_utils/compose-no-tls.yml @@ -52,6 +52,22 @@ services: ports: - "9000:9000" + s3-2nd: + command: server /data + container_name: s3-2nd + environment: + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretkey + - MINIO_SERVER_URL=http://127.0.0.1:9000 + healthcheck: + test: ["CMD", "curl", "-fq", "http://localhost:9000/minio/health/live"] + interval: 5s + timeout: 20s + retries: 3 + image: minio/minio + ports: + - "9001:9000" + createbucket: container_name: buckets image: minio/mc @@ -86,7 +102,10 @@ services: volumes: - ./config-notls.yaml:/config.yaml - ./:/dev_utils/ - - ./archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6:/tmp/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000001 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000002 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000003 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000004 mem_limit: 256m ports: - "8080:8080" @@ -114,7 +133,10 @@ services: volumes: - ./config-notls.yaml:/config.yaml - ./:/dev_utils/ - - ./archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6:/tmp/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000001 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000002 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000003 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000004 mem_limit: 256m ports: - "9080:8080" diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml deleted file mode 100644 index 67af4b819a..0000000000 --- a/sda-download/dev_utils/compose-sda.yml +++ /dev/null @@ -1,139 +0,0 @@ -version: "2.4" -services: - certfixer: - command: - - /bin/sh - - -c - - | - cp /origcerts/* /certs - chown -R nobody:nobody /certs/* - chmod -R og-rw /certs/*-key.pem - chown -R 70:70 /certs/db* - ls -la /certs/ - - container_name: certfixer - image: alpine:latest - volumes: - - ./certs:/origcerts - - certs:/certs - db: - user: postgres - container_name: db - depends_on: - - certfixer - environment: - - PKI_VOLUME_PATH=/certs/ - - PG_CA=/var/lib/postgresql/tls/ca.pem - - PG_SERVER_CERT=/var/lib/postgresql/tls/db.pem - - PG_SERVER_KEY=/var/lib/postgresql/tls/db-key.pem - - POSTGRES_PASSWORD=rootpassword - healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres"] - interval: 5s - timeout: 20s - retries: 3 - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres - ports: - - "5432:5432" - volumes: - - /tmp/data:/data - - certs:/var/lib/postgresql/tls/ - database_user_creation: - container_name: database_user_creation - image: postgres:15.4-alpine - command: - - "/bin/sh" - - "-c" - - | - pg_isready -h db -U postgres - psql -U postgres -h db -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" - depends_on: - db: - condition: service_healthy - restart: on-failure - environment: - - PGHOST=db - - PGPORT=5432 - - PGUSER=postgres - - PGPASSWORD=rootpassword - s3: - command: server /data - container_name: s3 - environment: - - MINIO_ROOT_USER=access - - MINIO_ROOT_PASSWORD=secretkey - - MINIO_SERVER_URL=https://127.0.0.1:9000 - healthcheck: - test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] - interval: 5s - timeout: 20s - retries: 3 - image: minio/minio - ports: - - "9000:9000" - volumes: - - ./certs/ca.pem:/root/.minio/certs/CAs/public.crt - - ./certs/s3.pem:/root/.minio/certs/public.crt - - ./certs/s3-key.pem:/root/.minio/certs/private.key - createbucket: - container_name: buckets - image: minio/mc - depends_on: - - s3 - entrypoint: > - /bin/sh -c " - sleep 10; - /usr/bin/mc alias set s3 https://s3:9000 access secretkey; - /usr/bin/mc mb s3/archive; - exit 0; - " - volumes: - - ./certs/ca.pem:/etc/ssl/certs/public.crt - restart: always - download: - command: sda-download - container_name: download - depends_on: - - certfixer - - database_user_creation - - mockauth - env_file: ./env.download - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download - volumes: - - ./config.yaml:/config.yaml - - ./:/dev_utils/ - - ./iss.json:/iss.json - - certs:/dev_utils/certs - - ./archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6:/tmp/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 - mem_limit: 256m - ports: - - "8443:8443" - restart: always - mockauth: - command: - - /bin/sh - - -c - - | - apt update - apt install curl -y - pip install --upgrade pip - pip install aiohttp Authlib - python -u /mockoidc.py - container_name: mockauth - healthcheck: - test: [ "CMD", "curl", "-fkq", "https://localhost:8000/.well-known/openid-configuration" ] - interval: 10s - timeout: 30s - retries: 10 - image: python:3.11-slim - volumes: - - ./mockoidc/mockoidc.py:/mockoidc.py - - certs:/certs - mem_limit: 256m - ports: - - "8000:8000" - restart: always - -volumes: - archive: - certs: diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 688ba3af54..4d1bbd855b 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -72,7 +72,7 @@ services: environment: - MINIO_ROOT_USER=access - MINIO_ROOT_PASSWORD=secretkey - - MINIO_SERVER_URL=http://127.0.0.1:9000 + - MINIO_SERVER_URL=https://127.0.0.1:9000 healthcheck: test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] interval: 5s @@ -86,6 +86,26 @@ services: - ./certs/s3.pem:/root/.minio/certs/public.crt - ./certs/s3-key.pem:/root/.minio/certs/private.key + s3-2nd: + command: server /data --console-address ":9001" + container_name: s3-2nd + environment: + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretkey + - MINIO_SERVER_URL=https://127.0.0.1:9000 + healthcheck: + test: ["CMD", "curl", "-fkq", "https://localhost:9000/minio/health/live"] + interval: 5s + timeout: 20s + retries: 3 + image: minio/minio + ports: + - "9001:9000" + volumes: + - ./certs/ca.pem:/root/.minio/certs/CAs/public.crt + - ./certs/s3.pem:/root/.minio/certs/public.crt + - ./certs/s3-key.pem:/root/.minio/certs/private.key + createbucket: container_name: buckets image: minio/mc @@ -123,7 +143,10 @@ services: - ./:/dev_utils/ - ./iss.json:/iss.json - certs:/dev_utils/certs - - ./archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6:/tmp/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000001 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000002 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000003 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000004 mem_limit: 256m ports: - "8443:8443" @@ -153,7 +176,10 @@ services: - ./:/dev_utils/ - ./iss.json:/iss.json - certs:/dev_utils/certs - - ./archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6:/tmp/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000001 + - ./archive_data/test_file:/tmp/dir1/00000000-0000-0000-0000-000000000002 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000003 + - ./archive_data/test_file:/tmp/dir2/00000000-0000-0000-0000-000000000004 mem_limit: 256m ports: - "9443:8443" diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index 9f346c4703..f13194b47a 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -9,14 +9,21 @@ log: storage: archive: s3: - endpoint: "http://s3:9000" + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretkey" + bucket_prefix: "archive" + region: "us-east-1" + disable_https: true + - endpoint: "http://s3-2nd:9000" access_key: "access" secret_key: "secretkey" bucket_prefix: "archive" region: "us-east-1" disable_https: true posix: - path: "/tmp" + - path: "/tmp/dir1" + - path: "/tmp/dir2" grpc: host: reencrypt diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml deleted file mode 100644 index 302f7e4ba8..0000000000 --- a/sda-download/dev_utils/config-notls_local.yaml +++ /dev/null @@ -1,34 +0,0 @@ -app: - serveUnencryptedData: true - port: 18080 - -log: - level: "debug" - format: "json" - -storage: - archive: - s3: - endpoint: "http://localhost:19000" - access_key: "access" - secret_key: "secretkey" - bucket_prefix: "archive" - region: "us-east-1" - disable_https: true - -grpc: - host: localhost - port: 50051 - -db: - host: "s3" - port: 15432 - user: "download" - password: "download" - database: "sda" - sslmode: "disable" - -oidc: - # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri" - configuration: - url: "http://localhost:8800/oidc/.well-known/openid-configuration" diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index 10437b39d2..b4dfcc18fa 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -13,14 +13,21 @@ log: storage: archive: s3: - endpoint: "https://s3:9000" + - endpoint: "https://s3:9000" + access_key: "access" + secret_key: "secretkey" + bucket_prefix: "archive" + region: "us-east-1" + ca_cert: "./dev_utils/certs/ca.pem" + - endpoint: "https://s3-2nd:9000" access_key: "access" secret_key: "secretkey" bucket_prefix: "archive" region: "us-east-1" ca_cert: "./dev_utils/certs/ca.pem" posix: - path: "/tmp" + - path: "/tmp/dir1" + - path: "/tmp/dir2" session: # session key expiration time in seconds diff --git a/sda-download/dev_utils/run_integration_test.sh b/sda-download/dev_utils/run_integration_test.sh index 091dfaadff..22f63999e5 100644 --- a/sda-download/dev_utils/run_integration_test.sh +++ b/sda-download/dev_utils/run_integration_test.sh @@ -1,226 +1,15 @@ -#!/bin/sh +#!/bin/bash -for c in s3cmd jq yq -do - if ! command -v $c - then - echo "$c could not be found" - exit 1 - fi -done - -chmod 600 certs/client-key.pem - -cat << EOF > c4gh.pub.pem ------BEGIN CRYPT4GH PUBLIC KEY----- -avFAerx0ZWuJE6fTI8S/0wv3yMo1n3SuNTV6zvKdxQc= ------END CRYPT4GH PUBLIC KEY----- -EOF - -chmod 444 c4gh.pub.pem - -cat << EOF > c4gh.sec.pem ------BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- -YzRnaC12MQAGc2NyeXB0ABQAAAAAwAs5mVkXda50vqeYv6tbkQARY2hhY2hhMjBf -cG9seTEzMDUAPAd46aTuoVWAe+fMGl3VocCKCCWmgFUsFIHejJoWxNwy62c1L/Vc -R9haQsAPfJMLJSvUXStJ04cyZnDHSw== ------END CRYPT4GH ENCRYPTED PRIVATE KEY----- -EOF - -chmod 444 c4gh.sec.pem - -# insert file entry into database -file_id=$(docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.files (stable_id, submission_user, \ - submission_file_path, submission_file_size, archive_file_path, \ - archive_file_size, archive_location, decrypted_file_size, backup_path, header, \ - encryption_method) VALUES ('urn:neic:001-002', 'integration-test', 'dummy_data.c4gh', \ - 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 'https://s3:9000/archive', 1048605, \ - '', '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'CRYPT4GH') RETURNING id;" | xargs) - -# insert "ready" database log event -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.file_event_log (file_id, event) \ - VALUES ('$file_id', 'ready');" - -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'UNENCRYPTED');" - -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', 'UPLOADED');" - -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 'ARCHIVED');" - - -# make sure that the dataset exists in the database -dataset_id=$(docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.datasets (stable_id) VALUES ('https://doi.example/ty009.sfrrss/600.45asasga') \ - ON CONFLICT (stable_id) DO UPDATE \ - SET stable_id=excluded.stable_id RETURNING id;") - -# insert the file into the dataset -docker run --rm --name client --network dev_utils_default \ - -v "dev_utils_certs:/certs" \ - -e PGSSLCERT=/certs/client.pem \ - -e PGSSLKEY=/certs/client-key.pem \ - -e PGSSLROOTCERT=/certs/ca.pem \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.file_dataset (file_id, dataset_id) \ - VALUES ('$file_id', $dataset_id);" - -# Make buckets if they don't exist already -s3cmd -c s3cmd.conf mb s3://archive || true - -# Upload test file -s3cmd -c s3cmd.conf put archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 s3://archive/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 - -# Test Health Endpoint -check_health=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem https://localhost:8443/health) - -if [ "$check_health" != "200" ]; then - echo "Health endpoint does not respond properly" - echo "got: ${check_health}" - exit 1 -fi - -echo "Health endpoint is ok" - -# Test empty token - -check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET --cacert certs/ca.pem https://localhost:8443/metadata/datasets) - -if [ "$check_401" != "401" ]; then - echo "no token provided should give 401" - echo "got: ${check_401}" - exit 1 -fi - -echo "got correct response when no token provided" - - -check_405=$(curl -o /dev/null -s -w "%{http_code}\n" -X POST --cacert certs/ca.pem https://localhost:8443/metadata/datasets) - -if [ "$check_405" != "405" ]; then - echo "POST should not be allowed" - echo "got: ${check_405}" - exit 1 -fi +cd .. -echo "got correct response when POST method used" +export STORAGETYPE=s3 -# Test good token - -token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[0]') - -## Test datasets endpoint - -check_dataset=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets | jq -r '.[0]') - -if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then - echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found" - echo "got: ${check_dataset}" - exit 1 -fi - -echo "expected dataset found" - -## Test datasets/files endpoint, no scheme -## dataset=https://doi.example/ty009.sfrrss/600.45asasga - -check_files=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') - -if [ "$check_files" != "urn:neic:001-002" ]; then - echo "file with id urn:neic:001-002 not found" - echo "got: ${check_files}" - exit 1 -fi - -echo "expected file found" - -## Test datasets/files endpoint, with scheme param (scheme separated from dataset) -## dataset=doi.example/ty009.sfrrss/600.45asasga, scheme=https - -check_files=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/metadata/datasets/doi.example/ty009.sfrrss/600.45asasga/files?scheme=https" | jq -r '.[0].fileId') - -if [ "$check_files" != "urn:neic:001-002" ]; then - echo "file with id urn:neic:001-002 not found" - echo "got: ${check_files}" - exit 1 -fi - -echo "expected file found" - -# Test file can be decrypted -## test also the files endpoint - -C4GH_PASSPHRASE=$(yq .c4gh.passphrase config.yaml) -export C4GH_PASSPHRASE - -crypt4gh decrypt --sk c4gh.sec.pem < dummy_data.c4gh > old-file.txt - -curl --cacert certs/ca.pem -H "Authorization: Bearer $token" "https://localhost:8443/files/urn:neic:001-002" --output test-download.txt - -cmp --silent old-file.txt test-download.txt -status=$? -if [ $status = 0 ]; then - echo "Files are the same" -else - echo "Files are different" -fi - -# Test get visas failed - -token=$(curl --cacert certs/ca.pem "https://localhost:8000/tokens" | jq -r '.[1]') - -## Test datasets endpoint - -check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET -I --cacert certs/ca.pem -H "Authorization: Bearer $token" https://localhost:8443/metadata/datasets) - -if [ "$check_empty_token" != "200" ]; then - echo "response for empty token is not 200" - echo "got: ${check_empty_token}" - exit 1 -fi +find .github/integration/setup/{common,s3}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do + echo "Executing setup script $runscript"; + bash -x "$runscript"; +done -echo "got correct response when token has no permissions" +find .github/integration/tests/{common,s3}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do + echo "Executing test script $runscript"; + bash -x "$runscript"; +done diff --git a/sda-download/dev_utils/run_integration_test_no_tls.sh b/sda-download/dev_utils/run_integration_test_no_tls.sh index 9c34000c7a..03bef5e14e 100644 --- a/sda-download/dev_utils/run_integration_test_no_tls.sh +++ b/sda-download/dev_utils/run_integration_test_no_tls.sh @@ -1,184 +1,15 @@ -#!/bin/sh +#!/bin/bash -for c in s3cmd jq yq -do - if ! command -v $c - then - echo "$c could not be found" - exit 1 - fi -done - -cat << EOF > c4gh.pub.pem ------BEGIN CRYPT4GH PUBLIC KEY----- -avFAerx0ZWuJE6fTI8S/0wv3yMo1n3SuNTV6zvKdxQc= ------END CRYPT4GH PUBLIC KEY----- -EOF - -chmod 444 c4gh.pub.pem - -cat << EOF > c4gh.sec.pem ------BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- -YzRnaC12MQAGc2NyeXB0ABQAAAAAwAs5mVkXda50vqeYv6tbkQARY2hhY2hhMjBf -cG9seTEzMDUAPAd46aTuoVWAe+fMGl3VocCKCCWmgFUsFIHejJoWxNwy62c1L/Vc -R9haQsAPfJMLJSvUXStJ04cyZnDHSw== ------END CRYPT4GH ENCRYPTED PRIVATE KEY----- -EOF - -chmod 444 c4gh.sec.pem - - -# insert file entry into database -file_id=$(docker run --rm --name client --network dev_utils_default \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.files (stable_id, submission_user, \ - submission_file_path, submission_file_size, archive_file_path, \ - archive_file_size, archive_location, decrypted_file_size, backup_path, header, \ - encryption_method) VALUES ('urn:neic:001-002', 'integration-test', 'dummy_data.c4gh', \ - 1048729, '4293c9a7-dc50-46db-b79a-27ddc0dad1c6', 1049081, 'https://s3:9000/archive', 1048605, \ - '', '637279707434676801000000010000006c000000000000006af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc5073799e207ec5d022b2601340585ff082565e55fbff5b6cdbbbe6b12a0d0a19ef325a219f8b62344325e22c8d26a8e82e45f053f4dcee10c0ec4bb9e466d5253f139dcd4be', 'CRYPT4GH') RETURNING id;" | xargs) - -# insert "ready" database log event -docker run --rm --name client --network dev_utils_default \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.file_event_log (file_id, event) \ - VALUES ('$file_id', 'ready');" - -docker run --rm --name client --network dev_utils_default \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '06bb0a514b26497b4b41b30c547ad51d059d57fb7523eb3763cfc82fdb4d8fb7', 'SHA256', 'UNENCRYPTED');" - -docker run --rm --name client --network dev_utils_default \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '5e9c767958cc3f6e8d16512b8b8dcab855ad1e04e05798b86f50ef600e137578', 'SHA256', 'UPLOADED');" - -docker run --rm --name client --network dev_utils_default \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.checksums (file_id, checksum, type, source) \ - VALUES ('$file_id', '74820dbcf9d30f8ccd1ea59c17d5ec8a714aabc065ae04e46ad82fcf300a731e', 'SHA256', 'ARCHIVED');" - - -# make sure that the dataset exists in the database -dataset_id=$(docker run --rm --name client --network dev_utils_default \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.datasets (stable_id) VALUES ('https://doi.example/ty009.sfrrss/600.45asasga') \ - ON CONFLICT (stable_id) DO UPDATE \ - SET stable_id=excluded.stable_id RETURNING id;") - -# insert the file into the dataset -docker run --rm --name client --network dev_utils_default \ - neicnordic/pg-client:latest \ - postgresql://postgres:rootpassword@db:5432/sda \ - -t -q -c "INSERT INTO sda.file_dataset (file_id, dataset_id) \ - VALUES ('$file_id', $dataset_id);" - -# Make buckets if they don't exist already -s3cmd -c s3cmd-notls.conf mb s3://archive || true - -# Upload test file -s3cmd -c s3cmd-notls.conf put archive_data/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 s3://archive/4293c9a7-dc50-46db-b79a-27ddc0dad1c6 - - - -# Test Health Endpoint -check_health=$(curl -o /dev/null -s -w "%{http_code}\n" http://localhost:8080/health) - -if [ "$check_health" != "200" ]; then - echo "Health endpoint does not respond properly" - echo "got: ${check_health}" - exit 1 -fi - -echo "Health endpoint is ok" - -# Test empty token - -check_401=$(curl -o /dev/null -s -w "%{http_code}\n" -X GET http://localhost:8080/metadata/datasets) - -if [ "$check_401" != "401" ]; then - echo "no token provided should give 401" - echo "got: ${check_401}" - exit 1 -fi +cd .. -echo "got correct response when no token provided" +export STORAGETYPE=s3notls -check_405=$(curl -X POST -o /dev/null -s -w "%{http_code}\n" http://localhost:8080/metadata/datasets ) - -if [ "$check_405" != "405" ]; then - echo "POST should not be allowed" - echo "got: ${check_405}" - exit 1 -fi - -echo "got correct response when POST method used" - -# Test good token - -token=$(curl "http://localhost:8000/tokens" | jq -r '.[0]') - -## Test datasets endpoint - -check_dataset=$(curl -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets | jq -r '.[0]') - -if [ "$check_dataset" != "https://doi.example/ty009.sfrrss/600.45asasga" ]; then - echo "dataset https://doi.example/ty009.sfrrss/600.45asasga not found" - echo "got: ${check_dataset}" - exit 1 -fi - -echo "expected dataset found" - -## Test datasets/files endpoint - -check_files=$(curl -H "Authorization: Bearer $token" "http://localhost:8080/metadata/datasets/https://doi.example/ty009.sfrrss/600.45asasga/files" | jq -r '.[0].fileId') - -if [ "$check_files" != "urn:neic:001-002" ]; then - echo "file with id urn:neic:001-002 not found" - echo "got: ${check_files}" - exit 1 -fi - -echo "expected file found" - -# Test file can be decrypted -## test also the files endpoint - -C4GH_PASSPHRASE=$(yq .c4gh.passphrase config.yaml) -export C4GH_PASSPHRASE - -crypt4gh decrypt --sk c4gh.sec.pem < dummy_data.c4gh > old-file.txt - -curl -H "Authorization: Bearer $token" "http://localhost:8080/files/urn:neic:001-002" --output test-download.txt - -cmp --silent old-file.txt test-download.txt -status=$? -if [ $status = 0 ]; then - echo "Files are the same" -else - echo "Files are different" -fi - -# Test get visas failed - -token=$(curl "http://localhost:8000/tokens" | jq -r '.[1]') - -## Test datasets endpoint - -check_empty_token=$(curl -o /dev/null -s -w "%{http_code}\n" -H "Authorization: Bearer $token" http://localhost:8080/metadata/datasets) - -if [ "$check_empty_token" != "200" ]; then - echo "response for empty token is not 200" - echo "got: ${check_empty_token}" - exit 1 -fi +find .github/integration/setup/{common,s3notls}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do + echo "Executing setup script $runscript"; + bash -x "$runscript"; +done -echo "got correct response when token has no permissions" +find .github/integration/tests/{common,s3notls}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do + echo "Executing test script $runscript"; + bash -x "$runscript"; +done diff --git a/sda-download/dev_utils/ssl.cnf b/sda-download/dev_utils/ssl.cnf index 26c7e0a39f..472ac89cde 100644 --- a/sda-download/dev_utils/ssl.cnf +++ b/sda-download/dev_utils/ssl.cnf @@ -95,7 +95,7 @@ basicConstraints = critical,CA:FALSE subjectKeyIdentifier = hash keyUsage = critical, digitalSignature, keyEncipherment extendedKeyUsage = clientAuth, serverAuth -subjectAltName = DNS:localhost,DNS:db,DNS:s3,DNS:mockauth,DNS:reencrypt,IP:127.0.0.1 +subjectAltName = DNS:localhost,DNS:db,DNS:s3,DNS:s3-2nd,DNS:mockauth,DNS:reencrypt,IP:127.0.0.1 [ crl_ext ] # Extension for CRLs (`man x509v3_config`). From 3b913d91118d6be2cc32862e362e44141ac236ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 12:36:45 +0100 Subject: [PATCH 112/530] feat(api): update to use storage/v2 instead of v1 --- sda/cmd/api/api.go | 29 +++++-- sda/cmd/api/api_test.go | 94 ++++++++++++--------- sda/internal/database/database_interface.go | 6 +- sda/internal/database/db_functions.go | 50 +++++------ sda/internal/database/db_functions_test.go | 4 +- 5 files changed, 105 insertions(+), 78 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 2612f1204a..94c7965f10 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -32,7 +32,8 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" "google.golang.org/grpc" @@ -401,7 +402,14 @@ func ingestFile(c *gin.Context) { // The deleteFile function deletes files from the inbox and marks them as // discarded in the db. Files are identified by their ids and the user id. func deleteFile(c *gin.Context) { - inbox, err := storage.NewBackend(Conf.Inbox) + lb, err := locationbroker.NewLocationBroker(Conf.API.DB) + if err != nil { + log.Error(err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + inboxWriter, err := storage.NewWriter(c, "inbox", lb) if err != nil { log.Error(err) c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) @@ -422,7 +430,7 @@ func deleteFile(c *gin.Context) { } // Get the file path from the fileID and submission user - filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) + filePath, location, err := Conf.API.DB.GetSubmissionPathAndLocation(c, submissionUser, fileID) if err != nil { log.Errorf("getting file from fileID failed, reason: (%v)", err) c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") @@ -432,7 +440,7 @@ func deleteFile(c *gin.Context) { filePath = helper.UnanonymizeFilepath(filePath, submissionUser) for count := 1; count <= 5; count++ { - err = inbox.RemoveFile(filePath) + err = inboxWriter.RemoveFile(c, location, filePath) if err == nil { break } @@ -504,9 +512,9 @@ func downloadFile(c *gin.Context) { return } - inbox, err := storage.NewBackend(Conf.Inbox) + inboxReader, err := storage.NewReader(c, "inbox") if err != nil { - log.Errorf("failed to initialize inbox backend, reason: %v", err) + log.Errorf("failed to initialize inbox reader, reason: %v", err) c.AbortWithStatusJSON(http.StatusInternalServerError, "storage backend error") return @@ -514,7 +522,7 @@ func downloadFile(c *gin.Context) { // Retrieve the actual file path for the user's file. fileID := strings.TrimPrefix(c.Param("fileid"), "/") - filePath, err := Conf.API.DB.GetInboxFilePathFromID( + filePath, location, err := Conf.API.DB.GetSubmissionPathAndLocation(c, strings.TrimPrefix(c.Param("username"), "/"), fileID, ) @@ -524,9 +532,14 @@ func downloadFile(c *gin.Context) { return } + if location == "" { + log.Errorf("fileID: %s has no known submission location", fileID) + + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to find file location") + } // Get inbox file handle #noqa - file, err := inbox.NewFileReader( + file, err := inboxReader.NewFileReader(c, location, helper.UnanonymizeFilepath( filePath, strings.TrimPrefix(c.Param("username"), "/"), diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 1f2e754133..4260873364 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -39,6 +39,7 @@ import ( "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" log "github.com/sirupsen/logrus" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "google.golang.org/grpc" @@ -280,7 +281,6 @@ type GrpcListener struct { type TestSuite struct { suite.Suite - Path string PublicPath string PrivatePath string KeyName string @@ -295,10 +295,12 @@ type TestSuite struct { GoodC4ghFile string BadC4ghFile string GrpcListener GrpcListener + + inboxDir string } func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, hash.Hash) { - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update status of file in database") @@ -418,11 +420,13 @@ func (s *TestSuite) TestReadinessResponse() { // Initialise configuration and create jwt keys func (s *TestSuite) SetupSuite() { log.SetLevel(log.DebugLevel) - s.Path = "/tmp/keys/" + + s.inboxDir = s.T().TempDir() + s.KeyName = "example.demo" log.Print("Creating JWT keys for testing") - privpath, pubpath, err := helper.MakeFolder(s.Path) + privpath, pubpath, err := helper.MakeFolder(s.inboxDir) assert.NoError(s.T(), err) s.PrivatePath = privpath s.PublicPath = pubpath @@ -484,9 +488,21 @@ func (s *TestSuite) SetupSuite() { "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"admin"}]}`) - Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") - if err != nil { - s.FailNow("failed to create temp folder") + if err := os.WriteFile(filepath.Join(s.inboxDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + inbox: + posix: + - path: %s +`, s.inboxDir)), 0600); err != nil { + s.FailNow(err.Error()) + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + viper.SetConfigFile(filepath.Join(s.inboxDir, "config.yaml")) + + if err := viper.ReadInConfig(); err != nil { + s.FailNow(err.Error()) } s.UserKey.PublicKey, s.UserKey.privateKey, err = keys.GenerateKeyPair() @@ -494,7 +510,7 @@ func (s *TestSuite) SetupSuite() { s.T().FailNow() } - s.UserKey.PrivateKeyPath = s.Path + "/user.key" + s.UserKey.PrivateKeyPath = s.inboxDir + "/user.key" key, err := os.Create(s.UserKey.PrivateKeyPath) if err != nil { s.T().FailNow() @@ -518,14 +534,14 @@ func (s *TestSuite) SetupSuite() { s.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") s.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") - err = os.MkdirAll(path.Join(Conf.Inbox.Posix.Location, s.User), 0750) + err = os.MkdirAll(path.Join(s.inboxDir, s.User), 0750) assert.NoError(s.T(), err, "failed to create inbox directory") // Create test files in the inbox fileContent := []byte("This is the content of the test file.") contentReader := bytes.NewReader(fileContent) - s.GoodC4ghFile = path.Join(Conf.Inbox.Posix.Location, s.User, "test_download.c4gh") - s.BadC4ghFile = path.Join(Conf.Inbox.Posix.Location, s.User, "badc4ghfile.c4gh") + s.GoodC4ghFile = path.Join(s.inboxDir, s.User, "test_download.c4gh") + s.BadC4ghFile = path.Join(s.inboxDir, s.User, "badc4ghfile.c4gh") outFile, err := os.Create(s.GoodC4ghFile) if err != nil { @@ -570,8 +586,6 @@ func (s *TestSuite) SetupSuite() { }() } func (s *TestSuite) TearDownSuite() { - assert.NoError(s.T(), os.RemoveAll(s.Path)) - assert.NoError(s.T(), os.RemoveAll(Conf.Inbox.Posix.Location)) if s.GrpcListener.gs != nil { s.GrpcListener.gs.GracefulStop() } @@ -669,7 +683,7 @@ func (s *TestSuite) TestAPIGetFiles() { // Insert a file and make sure it is listed file1 := fmt.Sprintf("/%v/TestAPIGetFiles.c4gh", s.User) - fileID, err := Conf.API.DB.RegisterFile(nil, file1, s.User) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, file1, s.User) assert.NoError(s.T(), err, "failed to register file in database") latestStatus := "uploaded" @@ -713,7 +727,7 @@ func (s *TestSuite) TestAPIGetFiles() { // Insert a second file and make sure it is listed file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", s.User) - _, err = Conf.API.DB.RegisterFile(nil, file2, s.User) + _, err = Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, file2, s.User) assert.NoError(s.T(), err, "failed to register file in database") resp, err = client.Do(req) // #nosec G704 -- request controlled by unit test @@ -739,7 +753,7 @@ func (s *TestSuite) TestAPIGetFiles() { func (s *TestSuite) TestAPIGetFiles_SubmissionFileSize() { for i := 1; i <= 2; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", "submission_b", i), "dummy") + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", "submission_b", i), "dummy") if err != nil { s.FailNow("failed to register file in database") } @@ -807,7 +821,7 @@ func (s *TestSuite) TestAPIGetFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -1155,7 +1169,7 @@ func (s *TestSuite) TestIngestFile_WithPayload() { s.FailNow("failed to setup RBAC enforcer") } - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1214,7 +1228,7 @@ func (s *TestSuite) TestIngestFile_WithPayload_NoUser() { s.FailNow("failed to setup RBAC enforcer") } - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1248,7 +1262,7 @@ func (s *TestSuite) TestIngestFile_WithPayload_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1281,7 +1295,7 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1315,7 +1329,7 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { func (s *TestSuite) TestIngestFile_WithFileID() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) @@ -1360,7 +1374,7 @@ func (s *TestSuite) TestIngestFile_WithFileID() { func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) @@ -1386,7 +1400,7 @@ func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { func (s *TestSuite) TestIngestFile_BothFileIDAndPayloadProvided() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) @@ -1723,7 +1737,7 @@ func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update status of file in database") @@ -1805,7 +1819,7 @@ func (s *TestSuite) TestCreateDataset_BadFormat() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1927,7 +1941,7 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1985,7 +1999,7 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { func (s *TestSuite) TestReleaseDataset() { user := "TestReleaseDataset" for i := 0; i < 3; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2118,7 +2132,7 @@ func (s *TestSuite) TestReleaseDataset_DeprecatedDataset() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2177,7 +2191,7 @@ func (s *TestSuite) TestListActiveUsers() { testUsers := []string{"User-A", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 3; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) if err != nil { s.FailNow("failed to register file in database") } @@ -2236,7 +2250,7 @@ func (s *TestSuite) TestListUserFiles() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2302,7 +2316,7 @@ func (s *TestSuite) TestListUserFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2563,7 +2577,7 @@ func (s *TestSuite) TestDeprecateC4ghHash_wrongHash() { func (s *TestSuite) TestListDatasets() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") if err != nil { s.FailNow("failed to register file in database") } @@ -2621,7 +2635,7 @@ func (s *TestSuite) TestListDatasets() { func (s *TestSuite) TestListUserDatasets() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2678,7 +2692,7 @@ func (s *TestSuite) TestListUserDatasets() { func (s *TestSuite) TestListDatasetsAsUser() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) if err != nil { s.FailNow("failed to register file in database") } @@ -2737,7 +2751,7 @@ func (s *TestSuite) TestReVerifyFile() { user := "TestReVerify" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerify-00%d.c4gh", user, i) - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) if err != nil { s.FailNow("failed to register file in database") } @@ -2839,7 +2853,7 @@ func (s *TestSuite) TestReVerifyDataset() { user := "TestReVerifyDataset" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerifyDataset-00%d.c4gh", user, i) - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, user) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) if err != nil { s.FailNow("failed to register file in database") } @@ -2959,7 +2973,7 @@ func (s *TestSuite) TestDownloadFile() { defer ts.Close() // Register the file in the database - fileID, err := Conf.API.DB.RegisterFile(nil, filepath.Base(s.GoodC4ghFile), s.User) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filepath.Base(s.GoodC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -3046,7 +3060,7 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { // Register a file in the database (but don't create the actual file) filePath := fmt.Sprintf("/%v/nonexistent.c4gh", s.User) - fileID, err := Conf.API.DB.RegisterFile(nil, filePath, s.User) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -3078,7 +3092,7 @@ func (s *TestSuite) TestDownloadFile_badC4ghFile() { defer ts.Close() // Register a file in the database (but don't create the actual file) - fileID, err := Conf.API.DB.RegisterFile(nil, filepath.Base(s.BadC4ghFile), s.User) + fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filepath.Base(s.BadC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index a7a81eb844..e52c7e2fb2 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -1,7 +1,11 @@ package database -import "context" +import ( + "context" +) type Database interface { GetSizeAndObjectCountOfLocation(ctx context.Context, location string) (uint64, uint64, error) + + GetSubmissionPathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 114ee1c5d2..0b5cf0f527 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -65,40 +65,36 @@ func (dbs *SDAdb) RegisterFileWithLocation(fileID *string, inboxLocation, upload return createdFileID, err } -// GetInboxFilePathFromID checks if a file exists in the database for a given user and fileID +// GetSubmissionPathAndLocation checks if a file exists in the database for a given user and fileID // and that is not yet archived -func (dbs *SDAdb) GetInboxFilePathFromID(submissionUser, fileID string) (string, error) { - var ( - err error - count int - filePath string - ) - - for count == 0 || (err != nil && count < RetryTimes) { - filePath, err = dbs.getInboxFilePathFromID(submissionUser, fileID) - count++ - } - - return filePath, err -} - -func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, error) { +func (dbs *SDAdb) GetSubmissionPathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFilePath = `SELECT submission_file_path from sda.files where -submission_user= $1 and id = $2 -AND EXISTS (SELECT 1 FROM -(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) -as subquery WHERE event = 'uploaded');` - - var filePath string - err := db.QueryRow(getFilePath, submissionUser, fileID).Scan(&filePath) + const getFilePath = ` +SELECT submission_file_path, submission_location +FROM sda.files +WHERE + submission_user = $1 + AND id = $2 + AND EXISTS ( + SELECT 1 + FROM ( + SELECT event + FROM sda.file_event_log + WHERE file_id = $2 + ORDER BY started_at DESC limit 1 + ) AS subquery + WHERE event = 'uploaded' + );` + + var filePath, location string + err := db.QueryRowContext(ctx, getFilePath, submissionUser, fileID).Scan(&filePath, &location) if err != nil { - return "", err + return "", "", err } - return filePath, nil + return filePath, location, nil } // GetFileIDByUserPathAndStatus checks if a file exists in the database for a given user and submission filepath diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index a4744a22e9..71e9dc8631 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1572,13 +1572,13 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { if err != nil { suite.FailNow("Failed to update file event log") } - path, err := db.getInboxFilePathFromID(user, fileID) + path, err := db.getSubmissionPathAndLocation(user, fileID) assert.NoError(suite.T(), err) assert.Equal(suite.T(), path, filePath) err = db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}") assert.NoError(suite.T(), err) - _, err = db.getInboxFilePathFromID(user, fileID) + _, err = db.getSubmissionPathAndLocation(user, fileID) assert.Error(suite.T(), err) db.Close() } From 3edf77fc04001addd175d1823aa300fbc1c366be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:43:13 +0100 Subject: [PATCH 113/530] feat(api): add required db version check at v23 with new location columns --- sda/cmd/api/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 94c7965f10..f12a1070a4 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -66,6 +66,9 @@ func main() { if err != nil { log.Fatal(err) } + if Conf.API.DB.Version < 23 { + log.Fatalf("database schema v23 is required") + } if err := setupJwtAuth(); err != nil { log.Fatalf("error when setting up JWT auth, reason %s", err.Error()) From ba7913189778596830063e95151da9455faeeda9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 14:33:11 +0100 Subject: [PATCH 114/530] feat(api): fix database unit test after func rename, remove unnecessary format in fatal when db version to low --- sda/cmd/api/api.go | 2 +- sda/internal/database/db_functions_test.go | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index f12a1070a4..7e728bf7b6 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -67,7 +67,7 @@ func main() { log.Fatal(err) } if Conf.API.DB.Version < 23 { - log.Fatalf("database schema v23 is required") + log.Fatal("database schema v23 is required") } if err := setupJwtAuth(); err != nil { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 71e9dc8631..6dda57a925 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1558,13 +1558,13 @@ func (suite *DatabaseTests) TestGetDatasetFileIDs() { db.Close() } -func (suite *DatabaseTests) TestGetInboxFilePathFromID() { +func (suite *DatabaseTests) TestGetSubmissionPathAndLocation() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) user := "UserX" filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) - fileID, err := db.RegisterFile(nil, filePath, user) + fileID, err := db.RegisterFileWithLocation(nil, "/inbox", filePath, user) if err != nil { suite.FailNow("Failed to register file") } @@ -1572,13 +1572,14 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { if err != nil { suite.FailNow("Failed to update file event log") } - path, err := db.getSubmissionPathAndLocation(user, fileID) + path, location, err := db.GetSubmissionPathAndLocation(context.Background(), user, fileID) assert.NoError(suite.T(), err) assert.Equal(suite.T(), path, filePath) + assert.Equal(suite.T(), "/inbox", location) err = db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}") assert.NoError(suite.T(), err) - _, err = db.getSubmissionPathAndLocation(user, fileID) + _, _, err = db.GetSubmissionPathAndLocation(context.Background(), user, fileID) assert.Error(suite.T(), err) db.Close() } From 54cfa20660273a8cfa93fabd66ffd391f7cc4f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 14:46:52 +0100 Subject: [PATCH 115/530] feat(api): update READMD.md to now mention expected storage config --- sda/cmd/api/api.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 02d67ad2db..71f4692f85 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -385,3 +385,13 @@ The `roles` section defines the available roles ] } ``` + + +## Storage settings +The api service requires access to the "inbox" storage, to configure such the following config is needed: +```yaml +storage: + inbox: + ${STORAGE_IMPLEMENTATION}: +``` +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) From 2fed8c749386c1cdc09bbaaa2dff73c47c57c634 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 12:27:01 +0100 Subject: [PATCH 116/530] feat(api): add validation of fileid param on /file/ingest and /file/accession such that we do not get stuck in db retry logic until timeout due to invalid pgsql arg, and update unit tests to assert new invalid fileid message --- sda/cmd/api/api.go | 11 +++++++++++ sda/cmd/api/api_test.go | 4 ++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 7e728bf7b6..1060733ceb 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -22,6 +22,7 @@ import ( "github.com/casbin/casbin/v2" "github.com/casbin/casbin/v2/model" "github.com/gin-gonic/gin" + "github.com/google/uuid" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/crypt4gh/model/headers" @@ -342,6 +343,11 @@ func ingestFile(c *gin.Context) { return case c.Query("fileid") != "": + if _, err := uuid.Parse(c.Query("fileid")); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "fileid param is invalid, not a uuid") + + return + } // Get the user and the inbox filepath fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid"), "uploaded") if err != nil { @@ -614,6 +620,11 @@ func setAccession(c *gin.Context) { return case c.Query("fileid") != "" && c.Query("accessionid") != "": + if _, err := uuid.Parse(c.Query("fileid")); err != nil { + c.AbortWithStatusJSON(http.StatusBadRequest, "fileid param is invalid, not a uuid") + + return + } // Get the user and the inbox filepath fileDetails, err := Conf.API.DB.GetFileDetailsFromUUID(c.Query("fileid"), "verified") if err != nil { diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 4260873364..35d9ce2350 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -1394,7 +1394,7 @@ func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { defer resp.Body.Close() b, _ := io.ReadAll(resp.Body) assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) - assert.Contains(s.T(), string(b), "file information not found") + assert.Contains(s.T(), string(b), "fileid param is invalid, not a uuid") } func (s *TestSuite) TestIngestFile_BothFileIDAndPayloadProvided() { @@ -1634,7 +1634,7 @@ func (s *TestSuite) TestSetAccession_WithParams_WrongID() { defer resp.Body.Close() b, _ := io.ReadAll(resp.Body) assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) - assert.Contains(s.T(), string(b), "file details not found") + assert.Contains(s.T(), string(b), "fileid param is invalid, not a uuid") } func (s *TestSuite) TestSetAccession_WithParams_MissingAccession() { From 04c04ab4ba53e9c24b53654554d729c32a1694c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 13:32:25 +0100 Subject: [PATCH 117/530] feat(api): update config to not require and load in storage v1 for api service --- sda/internal/config/config.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 4f2da2a18f..21892c4e12 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -244,14 +244,6 @@ func NewConfig(app string) (*Config, error) { "db.database", "grpc.host", } - switch viper.GetString("inbox.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) - default: - return nil, errors.New("inbox.type not set") - } case "auth": requiredConfVars = []string{ "auth.s3Inbox", @@ -521,8 +513,6 @@ func NewConfig(app string) (*Config, error) { return nil, err } - c.configInbox() - err = c.configAPI() if err != nil { return nil, err From b81057ce07fd6f00cdfca1edefc41ba86f5cca0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 20 Jan 2026 15:39:03 +0100 Subject: [PATCH 118/530] feat(storage-v2): update location broker unit test mock database to implement new func GetSubmissionPathAndLocation with panic as not expected to be used in those unit tests --- .../storage/v2/locationbroker/location_broker_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index 87f0c714d1..d76ecf397e 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -23,6 +23,10 @@ func (m *mockDatabase) GetSizeAndObjectCountOfLocation(_ context.Context, locati return args.Get(0).(uint64), args.Get(1).(uint64), args.Error(2) } +func (m *mockDatabase) GetSubmissionPathAndLocation(_ context.Context, _, _ string) (string, string, error) { + panic("function not expected to be called in unit tests") +} + func TestLocationBrokerTestSuite(t *testing.T) { suite.Run(t, new(LocationBrokerTestSuite)) } From 1077a585fa37ae5f7115cf17302b59d839716e1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 08:42:48 +0100 Subject: [PATCH 119/530] feat(api): move initializing of inbox reader and writer to start of application not per request --- sda/cmd/api/api.go | 48 ++++++++++++++++------------------------- sda/cmd/api/api_test.go | 9 ++++++++ 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 1060733ceb..a139303aea 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -51,25 +51,38 @@ var ( Conf *config.Config err error auth *userauth.ValidateFromToken - auditLogger *log.Logger + inboxReader storage.Reader + inboxWriter storage.Writer ) func main() { Conf, err = config.NewConfig("api") if err != nil { - log.Fatal(err) + log.Fatalf("failed to load config, due to: %v", err) } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) if err != nil { - log.Fatal(err) + log.Fatalf("failed to initialize mq broker, due to: %v", err) } Conf.API.DB, err = database.NewSDAdb(Conf.Database) if err != nil { - log.Fatal(err) + log.Fatalf("failed to initialize sda db, due to: %v", err) } if Conf.API.DB.Version < 23 { log.Fatal("database schema v23 is required") } + lb, err := locationbroker.NewLocationBroker(Conf.API.DB) + if err != nil { + log.Fatalf("failed to initialize new location broker, due to: %v", err) + } + inboxWriter, err = storage.NewWriter(context.Background(), "inbox", lb) + if err != nil { + log.Fatalf("failed to initialize inbox writer, due to: %v", err) + } + inboxReader, err = storage.NewReader(context.Background(), "inbox") + if err != nil { + log.Fatalf("failed to initialize inbox reader, reason: %v", err) + } if err := setupJwtAuth(); err != nil { log.Fatalf("error when setting up JWT auth, reason %s", err.Error()) @@ -88,13 +101,13 @@ func main() { log.Infof("Starting web server at https://%s:%d", Conf.API.Host, Conf.API.Port) if err := srv.ListenAndServeTLS(Conf.API.ServerCert, Conf.API.ServerKey); err != nil { shutdown() - log.Fatalln(err) + log.Fatalf("failed to start https server, due to: %v", err) } } else { log.Infof("Starting web server at http://%s:%d", Conf.API.Host, Conf.API.Port) if err := srv.ListenAndServe(); err != nil { shutdown() - log.Fatalln(err) + log.Fatalf("failed to start http server, due to: %v", err) } } } @@ -411,21 +424,6 @@ func ingestFile(c *gin.Context) { // The deleteFile function deletes files from the inbox and marks them as // discarded in the db. Files are identified by their ids and the user id. func deleteFile(c *gin.Context) { - lb, err := locationbroker.NewLocationBroker(Conf.API.DB) - if err != nil { - log.Error(err) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - - return - } - inboxWriter, err := storage.NewWriter(c, "inbox", lb) - if err != nil { - log.Error(err) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - - return - } - submissionUser := c.Param("username") log.Debug("submission user:", submissionUser) @@ -521,14 +519,6 @@ func downloadFile(c *gin.Context) { return } - inboxReader, err := storage.NewReader(c, "inbox") - if err != nil { - log.Errorf("failed to initialize inbox reader, reason: %v", err) - c.AbortWithStatusJSON(http.StatusInternalServerError, "storage backend error") - - return - } - // Retrieve the actual file path for the user's file. fileID := strings.TrimPrefix(c.Param("fileid"), "/") filePath, location, err := Conf.API.DB.GetSubmissionPathAndLocation(c, diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 35d9ce2350..3cf6986014 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -36,6 +36,8 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" log "github.com/sirupsen/logrus" @@ -505,6 +507,13 @@ storage: s.FailNow(err.Error()) } + lb, err := locationbroker.NewLocationBroker(Conf.API.DB) + assert.NoError(s.T(), err) + inboxWriter, err = storage.NewWriter(context.Background(), "inbox", lb) + assert.NoError(s.T(), err) + inboxReader, err = storage.NewReader(context.Background(), "inbox") + assert.NoError(s.T(), err) + s.UserKey.PublicKey, s.UserKey.privateKey, err = keys.GenerateKeyPair() if err != nil { s.T().FailNow() From d1b08a0871f64f1fc48fbcf225aedbf3f0fa9351 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 13:49:17 +0100 Subject: [PATCH 120/530] feat(api): update main shutdown and error logic --- sda/cmd/api/api.go | 123 +++++++++++++++++++++++++--------------- sda/cmd/api/api_test.go | 49 +++------------- 2 files changed, 84 insertions(+), 88 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index a139303aea..7e8cefbc9e 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -36,6 +36,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/neicnordic/sensitive-data-archive/internal/userauth" + amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -56,68 +57,102 @@ var ( ) func main() { - Conf, err = config.NewConfig("api") - if err != nil { - log.Fatalf("failed to load config, due to: %v", err) + if err := run(); err != nil { + log.Fatal(err) } - Conf.API.MQ, err = broker.NewMQ(Conf.Broker) +} +func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + Conf, err = config.NewConfig("api") if err != nil { - log.Fatalf("failed to initialize mq broker, due to: %v", err) + return fmt.Errorf("failed to load config, due to: %v", err) } + Conf.API.DB, err = database.NewSDAdb(Conf.Database) if err != nil { - log.Fatalf("failed to initialize sda db, due to: %v", err) + return fmt.Errorf("failed to initialize sda db, due to: %v", err) } + defer Conf.API.DB.Close() if Conf.API.DB.Version < 23 { - log.Fatal("database schema v23 is required") + return errors.New("database schema v23 is required") } + + Conf.API.MQ, err = broker.NewMQ(Conf.Broker) + if err != nil { + return fmt.Errorf("failed to initialize mq broker, due to: %v", err) + } + defer func() { + if err := Conf.API.MQ.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + if err := Conf.API.MQ.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } + }() + lb, err := locationbroker.NewLocationBroker(Conf.API.DB) if err != nil { - log.Fatalf("failed to initialize new location broker, due to: %v", err) + return fmt.Errorf("failed to initialize new location broker, due to: %v", err) } - inboxWriter, err = storage.NewWriter(context.Background(), "inbox", lb) + inboxWriter, err = storage.NewWriter(ctx, "inbox", lb) if err != nil { - log.Fatalf("failed to initialize inbox writer, due to: %v", err) + return fmt.Errorf("failed to initialize inbox writer, due to: %v", err) } - inboxReader, err = storage.NewReader(context.Background(), "inbox") + inboxReader, err = storage.NewReader(ctx, "inbox") if err != nil { - log.Fatalf("failed to initialize inbox reader, reason: %v", err) + return fmt.Errorf("failed to initialize inbox reader, reason: %v", err) } if err := setupJwtAuth(); err != nil { - log.Fatalf("error when setting up JWT auth, reason %s", err.Error()) + return fmt.Errorf("error when setting up JWT auth, reason %s", err.Error()) } - sigc := make(chan os.Signal, 5) - signal.Notify(sigc, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + serverErr := make(chan error, 1) + srv, err := setup(Conf) + if err != nil { + return fmt.Errorf("failed to setup http/https server, due to: %v", err) + } go func() { - <-sigc - shutdown() - os.Exit(0) + if Conf.API.ServerCert != "" && Conf.API.ServerKey != "" { + log.Infof("Starting web server at https://%s:%d", Conf.API.Host, Conf.API.Port) + if err := srv.ListenAndServeTLS(Conf.API.ServerCert, Conf.API.ServerKey); err != nil { + serverErr <- fmt.Errorf("failed to start https server, due to: %v", err) + } + } else { + log.Infof("Starting web server at http://%s:%d", Conf.API.Host, Conf.API.Port) + if err := srv.ListenAndServe(); err != nil { + serverErr <- fmt.Errorf("failed to start http server, due to: %v", err) + } + } }() - srv := setup(Conf) - if Conf.API.ServerCert != "" && Conf.API.ServerKey != "" { - log.Infof("Starting web server at https://%s:%d", Conf.API.Host, Conf.API.Port) - if err := srv.ListenAndServeTLS(Conf.API.ServerCert, Conf.API.ServerKey); err != nil { - shutdown() - log.Fatalf("failed to start https server, due to: %v", err) - } - } else { - log.Infof("Starting web server at http://%s:%d", Conf.API.Host, Conf.API.Port) - if err := srv.ListenAndServe(); err != nil { - shutdown() - log.Fatalf("failed to start http server, due to: %v", err) - } + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + select { + case <-sigc: + case err := <-Conf.API.MQ.Connection.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-Conf.API.MQ.Channel.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-serverErr: + return err + } + + if err := srv.Shutdown(context.Background()); err != nil { + log.Errorf("failed to close http/https server due to: %v", err) } + + return nil } -func setup(conf *config.Config) *http.Server { +func setup(conf *config.Config) (*http.Server, error) { m, _ := model.NewModelFromString(jsonadapter.Model) e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) if err != nil { - shutdown() - log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) // nolint # FIXME Fatal should only be called from main + return nil, err } r := gin.New() @@ -188,7 +223,7 @@ func setup(conf *config.Config) *http.Server { WriteTimeout: 2 * time.Minute, } - return srv + return srv, nil } func setupJwtAuth() error { @@ -207,12 +242,6 @@ func setupJwtAuth() error { return nil } -func shutdown() { - defer Conf.API.MQ.Channel.Close() - defer Conf.API.MQ.Connection.Close() - defer Conf.API.DB.Close() -} - func readinessResponse(c *gin.Context) { statusCode := http.StatusOK @@ -237,7 +266,7 @@ func readinessResponse(c *gin.Context) { } } - if dbRes := checkDB(Conf.API.DB, 5*time.Millisecond); dbRes != nil { + if dbRes := checkDB(c, Conf.API.DB, 5*time.Millisecond); dbRes != nil { log.Debugf("DB connection error :%v", dbRes) Conf.API.DB.Reconnect() statusCode = http.StatusServiceUnavailable @@ -246,8 +275,8 @@ func readinessResponse(c *gin.Context) { c.JSON(statusCode, "") } -func checkDB(db *database.SDAdb, timeout time.Duration) error { - ctx, cancel := context.WithTimeout(context.Background(), timeout) +func checkDB(ctx context.Context, db *database.SDAdb, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() if db.DB == nil { return errors.New("database is nil") @@ -475,7 +504,7 @@ func deleteFile(c *gin.Context) { // gRPC to communicate with the re-encrypt service and handles TLS configuration // if needed. The function also handles the case where the CA certificate is // provided for secure communication. -func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { +func reencryptHeader(ctx context.Context, oldHeader []byte, c4ghPubKey string) ([]byte, error) { var opts []grpc.DialOption switch { case Conf.API.Grpc.ClientCreds != nil: @@ -492,7 +521,7 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { } defer conn.Close() - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(Conf.API.Grpc.Timeout)*time.Second) + ctx, cancel := context.WithTimeout(ctx, time.Duration(Conf.API.Grpc.Timeout)*time.Second) defer cancel() c := reencrypt.NewReencryptClient(conn) @@ -560,7 +589,7 @@ func downloadFile(c *gin.Context) { return } - newHeader, err := reencryptHeader(header, c4ghPubKey) + newHeader, err := reencryptHeader(c, header, c4ghPubKey) if err != nil { log.Errorf("failed to reencrypt header, reason: %v", err) c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to reencrypt header") diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 3cf6986014..182a88b403 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -332,41 +332,6 @@ func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, return fileID, decSha } -func (s *TestSuite) TestShutdown() { - Conf = &config.Config{} - Conf.Broker = broker.MQConf{ - Host: "localhost", - Port: mqPort, - User: "guest", - Password: "guest", - Exchange: "sda", - Vhost: "/sda", - } - Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(s.T(), err) - - Conf.Database = database.DBConf{ - Host: "localhost", - Port: dbPort, - User: "postgres", - Password: "rootpasswd", - Database: "sda", - SslMode: "disable", - } - Conf.API.DB, err = database.NewSDAdb(Conf.Database) - assert.NoError(s.T(), err) - - // make sure all conections are alive - assert.Equal(s.T(), false, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(s.T(), false, Conf.API.MQ.Connection.IsClosed()) - assert.Equal(s.T(), nil, Conf.API.DB.DB.Ping()) - - shutdown() - assert.Equal(s.T(), true, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(s.T(), true, Conf.API.MQ.Connection.IsClosed()) - assert.Equal(s.T(), "sql: database is closed", Conf.API.DB.DB.Ping().Error()) -} - func (s *TestSuite) TestReadinessResponse() { gin.SetMode(gin.ReleaseMode) r := gin.Default() @@ -643,11 +608,11 @@ func (s *TestSuite) SetupTest() { func (s *TestSuite) TestDatabasePingCheck() { emptyDB := database.SDAdb{} - assert.Error(s.T(), checkDB(&emptyDB, 1*time.Second), "nil DB should fail") + assert.Error(s.T(), checkDB(context.TODO(), &emptyDB, 1*time.Second), "nil DB should fail") db, err := database.NewSDAdb(Conf.Database) assert.NoError(s.T(), err) - assert.NoError(s.T(), checkDB(db, 1*time.Second), "ping should succeed") + assert.NoError(s.T(), checkDB(context.TODO(), db, 1*time.Second), "ping should succeed") } func (s *TestSuite) TestAPIGetFiles() { @@ -892,7 +857,8 @@ func (s *TestSuite) TestGinLogLevel_Debug() { // A specific port is enforced here so we don't have a conflict when running the LogLevel_Info test Conf.API.Port = 8081 - srv := setup(Conf) + srv, err := setup(Conf) + s.NoError(err) go func() { if err := srv.ListenAndServe(); err != nil { s.T().Logf("failure: %v", err) @@ -975,7 +941,8 @@ func (s *TestSuite) TestGinLogLevel_Info() { // A specific port is enforced here so we don't have a conflict when running the LogLevel_Debug test Conf.API.Port = 8082 - srv := setup(Conf) + srv, err := setup(Conf) + s.NoError(err) go func() { if err := srv.ListenAndServe(); err != nil { s.T().Logf("failure: %v", err) @@ -3133,7 +3100,7 @@ func (s *TestSuite) TestReencryptHeader() { } // Call the function under test - newHeader, err := reencryptHeader(s.FileHeader, s.UserKey.PubKeyBase64) + newHeader, err := reencryptHeader(context.TODO(), s.FileHeader, s.UserKey.PubKeyBase64) if err != nil { s.T().Fatal("reencryptHeader failed:", err) } @@ -3147,7 +3114,7 @@ func (s *TestSuite) TestReencryptHeader_failedToConnect() { // Mock the server address to an invalid one Conf.API.Grpc.Host, Conf.API.Grpc.Port, _ = splitHostPort("localhost:9999") - newHeader, err := reencryptHeader(s.FileHeader, s.UserKey.PubKeyBase64) + newHeader, err := reencryptHeader(context.TODO(), s.FileHeader, s.UserKey.PubKeyBase64) if err == nil { s.T().Fatal("Expected an error due to failed connection, but got nil") } From ca4eecefb2438717a9676e6bed580a25688e9860 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 27 Jan 2026 13:01:54 +0100 Subject: [PATCH 121/530] feat(api): add missing return if file has no submission location in downloadFile, add submission location != check when deleting file from inbox --- sda/cmd/api/api.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 7e8cefbc9e..123e6886d7 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -474,6 +474,13 @@ func deleteFile(c *gin.Context) { return } + if location == "" { + log.Errorf("fileID: %s has no known submission location", fileID) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to find file location") + + return + } + filePath = helper.UnanonymizeFilepath(filePath, submissionUser) for count := 1; count <= 5; count++ { err = inboxWriter.RemoveFile(c, location, filePath) @@ -562,8 +569,9 @@ func downloadFile(c *gin.Context) { } if location == "" { log.Errorf("fileID: %s has no known submission location", fileID) - c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to find file location") + + return } // Get inbox file handle #noqa From db961b9dc207d41b27625d30ae27c79d0039ee49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Feb 2026 08:55:27 +0100 Subject: [PATCH 122/530] feat(api): do not fatal incase rabbitmq broker connection or channel notifies of error, as readiness func will reconnect, defer the shutdown of the http/https server to ensure its closed incase of error --- sda/cmd/api/api.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 123e6886d7..88d28e2242 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -36,7 +36,6 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/neicnordic/sensitive-data-archive/internal/userauth" - amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" @@ -127,25 +126,21 @@ func run() error { } } }() + defer func() { + if err := srv.Shutdown(context.Background()); err != nil { + log.Errorf("failed to close http/https server due to: %v", err) + } + }() sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { case <-sigc: - case err := <-Conf.API.MQ.Connection.NotifyClose(make(chan *amqp.Error)): - return err - case err := <-Conf.API.MQ.Channel.NotifyClose(make(chan *amqp.Error)): - return err + return nil case err := <-serverErr: return err } - - if err := srv.Shutdown(context.Background()); err != nil { - log.Errorf("failed to close http/https server due to: %v", err) - } - - return nil } func setup(conf *config.Config) (*http.Server, error) { From b1be2cd1db1c677c847da6b1464dee549c75ad57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 10:32:02 +0100 Subject: [PATCH 123/530] feat(api): rename GetSubmissionPathAndLocation func -> GetUploadedSubmissionFilePathAndLocation to be more descriptive, as it only returns the file path and submission of files were ingestion, etc has not been triggered, so for that sense also include files with last event disabled --- sda/cmd/api/api.go | 4 ++-- sda/internal/database/database_interface.go | 3 +-- sda/internal/database/db_functions.go | 8 ++++---- sda/internal/database/db_functions_test.go | 4 ++-- .../storage/v2/locationbroker/location_broker_test.go | 2 +- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 88d28e2242..5bc900f93a 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -461,7 +461,7 @@ func deleteFile(c *gin.Context) { } // Get the file path from the fileID and submission user - filePath, location, err := Conf.API.DB.GetSubmissionPathAndLocation(c, submissionUser, fileID) + filePath, location, err := Conf.API.DB.GetUploadedSubmissionFilePathAndLocation(c, submissionUser, fileID) if err != nil { log.Errorf("getting file from fileID failed, reason: (%v)", err) c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") @@ -552,7 +552,7 @@ func downloadFile(c *gin.Context) { // Retrieve the actual file path for the user's file. fileID := strings.TrimPrefix(c.Param("fileid"), "/") - filePath, location, err := Conf.API.DB.GetSubmissionPathAndLocation(c, + filePath, location, err := Conf.API.DB.GetUploadedSubmissionFilePathAndLocation(c, strings.TrimPrefix(c.Param("username"), "/"), fileID, ) diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index e52c7e2fb2..b35163d509 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -6,6 +6,5 @@ import ( type Database interface { GetSizeAndObjectCountOfLocation(ctx context.Context, location string) (uint64, uint64, error) - - GetSubmissionPathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) + GetUploadedSubmissionFilePathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 0b5cf0f527..682e48b58c 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -65,9 +65,9 @@ func (dbs *SDAdb) RegisterFileWithLocation(fileID *string, inboxLocation, upload return createdFileID, err } -// GetSubmissionPathAndLocation checks if a file exists in the database for a given user and fileID -// and that is not yet archived -func (dbs *SDAdb) GetSubmissionPathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) { +// GetUploadedFilePathAndLocation returns the submission file path and location for a given user and fileID +// for a file which last event was 'uploaded' or +func (dbs *SDAdb) GetUploadedSubmissionFilePathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB @@ -85,7 +85,7 @@ WHERE WHERE file_id = $2 ORDER BY started_at DESC limit 1 ) AS subquery - WHERE event = 'uploaded' + WHERE event = 'uploaded' OR event = 'disabled' );` var filePath, location string diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 6dda57a925..e764a9c755 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1572,14 +1572,14 @@ func (suite *DatabaseTests) TestGetSubmissionPathAndLocation() { if err != nil { suite.FailNow("Failed to update file event log") } - path, location, err := db.GetSubmissionPathAndLocation(context.Background(), user, fileID) + path, location, err := db.GetUploadedSubmissionFilePathAndLocation(context.Background(), user, fileID) assert.NoError(suite.T(), err) assert.Equal(suite.T(), path, filePath) assert.Equal(suite.T(), "/inbox", location) err = db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}") assert.NoError(suite.T(), err) - _, _, err = db.GetSubmissionPathAndLocation(context.Background(), user, fileID) + _, _, err = db.GetUploadedSubmissionFilePathAndLocation(context.Background(), user, fileID) assert.Error(suite.T(), err) db.Close() } diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index d76ecf397e..f530923c3b 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -23,7 +23,7 @@ func (m *mockDatabase) GetSizeAndObjectCountOfLocation(_ context.Context, locati return args.Get(0).(uint64), args.Get(1).(uint64), args.Error(2) } -func (m *mockDatabase) GetSubmissionPathAndLocation(_ context.Context, _, _ string) (string, string, error) { +func (m *mockDatabase) GetUploadedSubmissionFilePathAndLocation(_ context.Context, _, _ string) (string, string, error) { panic("function not expected to be called in unit tests") } From 1ba3299e60a5ec5952f25409dc3eef202d9b3859 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 10:34:49 +0100 Subject: [PATCH 124/530] feat(database): use sql.Nullstring when scanning submission location in GetUploadedSubmissionFilePathAndLocation --- sda/internal/database/db_functions.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 682e48b58c..f7c83f66a9 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -65,7 +65,7 @@ func (dbs *SDAdb) RegisterFileWithLocation(fileID *string, inboxLocation, upload return createdFileID, err } -// GetUploadedFilePathAndLocation returns the submission file path and location for a given user and fileID +// GetUploadedSubmissionFilePathAndLocation returns the submission file path and location for a given user and fileID // for a file which last event was 'uploaded' or func (dbs *SDAdb) GetUploadedSubmissionFilePathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) { dbs.checkAndReconnectIfNeeded() @@ -88,13 +88,14 @@ WHERE WHERE event = 'uploaded' OR event = 'disabled' );` - var filePath, location string + var filePath string + var location sql.NullString err := db.QueryRowContext(ctx, getFilePath, submissionUser, fileID).Scan(&filePath, &location) if err != nil { return "", "", err } - - return filePath, location, nil + // dont really care if location is valid, we just want null -> "" + return filePath, location.String, nil } // GetFileIDByUserPathAndStatus checks if a file exists in the database for a given user and submission filepath From ec7fd645e8270911d1e261284c50d956ee8afd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:16:25 +0100 Subject: [PATCH 125/530] fix(api): fix typo in api.md --- sda/cmd/api/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 71f4692f85..b117454e6f 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -394,4 +394,4 @@ storage: inbox: ${STORAGE_IMPLEMENTATION}: ``` -For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) From 92f6298c7794c1cc30d2f303dbdddedb372487dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:33:00 +0100 Subject: [PATCH 126/530] feat(api): add closing defer func of file reader --- sda/cmd/api/api.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 5bc900f93a..5b9682dee0 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -582,6 +582,9 @@ func downloadFile(c *gin.Context) { return } + defer func() { + _ = file.Close() + }() // get the header of the crypt4gh file header, err := headers.ReadHeader(file) From 142c83fc9554816ba1d6b7db4c7dc28eb64c5f2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 11:45:02 +0100 Subject: [PATCH 127/530] feat(api): add timeout on shutting down http/https server at 10 sec --- sda/cmd/api/api.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 5b9682dee0..90520b27dc 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -127,9 +127,11 @@ func run() error { } }() defer func() { - if err := srv.Shutdown(context.Background()); err != nil { + serverShutdownCtx, serverShutdownCancel := context.WithTimeout(ctx, 10*time.Second) + if err := srv.Shutdown(serverShutdownCtx); err != nil { log.Errorf("failed to close http/https server due to: %v", err) } + serverShutdownCancel() }() sigc := make(chan os.Signal, 5) From a6f08aea1ba5f917829f5c575f2229fb1636f2a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:16:40 +0100 Subject: [PATCH 128/530] feat(api): do not interpret http.ErrServerClosed as a server error --- sda/cmd/api/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 90520b27dc..4fc99dc38d 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -116,12 +116,12 @@ func run() error { go func() { if Conf.API.ServerCert != "" && Conf.API.ServerKey != "" { log.Infof("Starting web server at https://%s:%d", Conf.API.Host, Conf.API.Port) - if err := srv.ListenAndServeTLS(Conf.API.ServerCert, Conf.API.ServerKey); err != nil { + if err := srv.ListenAndServeTLS(Conf.API.ServerCert, Conf.API.ServerKey); err != nil && !errors.Is(err, http.ErrServerClosed) { serverErr <- fmt.Errorf("failed to start https server, due to: %v", err) } } else { log.Infof("Starting web server at http://%s:%d", Conf.API.Host, Conf.API.Port) - if err := srv.ListenAndServe(); err != nil { + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { serverErr <- fmt.Errorf("failed to start http server, due to: %v", err) } } From 0ce2c0a06c678664554219ea2037b12ccf13d482 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:22:39 +0100 Subject: [PATCH 129/530] feat(api): update sigc chan buffer size to 1 instead of 5 as we just care that any of the signals triggered --- sda/cmd/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 4fc99dc38d..e15805a15a 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -134,7 +134,7 @@ func run() error { serverShutdownCancel() }() - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { From 573c2ce4a6255b77334f379949d3460d44cd53f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 14:58:02 +0100 Subject: [PATCH 130/530] feat(api): fix some sentance in api.md --- sda/cmd/api/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index b117454e6f..0968ff5242 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -388,7 +388,7 @@ The `roles` section defines the available roles ## Storage settings -The api service requires access to the "inbox" storage, to configure such the following config is needed: +The API service requires access to the "inbox" storage. To configure that, the following configuration is required: ```yaml storage: inbox: From c6558cdbe8ca0f85ef1291ddef5471b2094521e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 14:15:06 +0100 Subject: [PATCH 131/530] feat(finalize): update to use storage/v2 instead of v1, update GetArchived to return archive location, update db version check to v22 --- sda/cmd/finalize/finalize.go | 80 ++++++++++++++-------- sda/internal/database/db_functions.go | 28 ++++---- sda/internal/database/db_functions_test.go | 5 +- 3 files changed, 67 insertions(+), 46 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 4e9b9f8ff4..d4bb10f9e1 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -3,7 +3,9 @@ package main import ( + "context" "encoding/json" + "errors" "fmt" "io" @@ -11,19 +13,26 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" ) var db *database.SDAdb -var archive, backup storage.Backend +var archiveReader storage.Reader +var backupWriter storage.Writer var conf *config.Config var err error var message schema.IngestionAccession +var backupInStorage bool + func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() forever := make(chan bool) conf, err = config.NewConfig("finalize") if err != nil { @@ -37,22 +46,28 @@ func main() { if err != nil { log.Fatal(err) } - if db.Version < 21 { - log.Error("database schema v21 is required") - db.Close() - panic(err) + if db.Version < 23 { + log.Fatal("database schema v23 is required") } - if conf.Backup.Type != "" && conf.Archive.Type != "" { - log.Debugln("initiating storage backends") - backup, err = storage.NewBackend(conf.Backup) - if err != nil { - log.Fatal(err) - } - archive, err = storage.NewBackend(conf.Archive) - if err != nil { - log.Fatal(err) - } + lb, err := locationbroker.NewLocationBroker(db) + if err != nil { + log.Fatal("failed to init new location broker due to: %v", err) + } + + backupWriter, err = storage.NewWriter(ctx, "backup", lb) + if err != nil && !errors.Is(err, storageerrors.ErrorNoValidWriter) { + log.Fatalf("error creating backup writer: %v", err) + } + + archiveReader, err = storage.NewReader(ctx, "archive") + if err != nil && !errors.Is(err, storageerrors.ErrorNoValidReader) { + log.Fatalf("error creating archive reader: %v", err) + } + + if archiveReader != nil && backupWriter != nil { + backupInStorage = true + log.Info("will back up files in backup storage") } defer mq.Channel.Close() @@ -78,6 +93,7 @@ func main() { log.Fatal(err) } for delivered := range messages { + ctx := context.Background() log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { @@ -177,8 +193,8 @@ func main() { case "same": log.Infof("file already has a stable ID, marking it as ready, file-id: %s", fileID) default: - if conf.Backup.Type != "" && conf.Archive.Type != "" { - if err = backupFile(delivered); err != nil { + if backupInStorage { + if err = backupFile(ctx, delivered); err != nil { log.Errorf("failed to backup file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) @@ -226,17 +242,17 @@ func main() { <-forever } -func backupFile(delivered amqp.Delivery) error { +func backupFile(ctx context.Context, delivered amqp.Delivery) error { log.Debug("Backup initiated") fileID := delivered.CorrelationId - filePath, fileSize, err := db.GetArchived(fileID) + filePath, fileSize, archiveLocation, err := db.GetArchived(fileID) if err != nil { return fmt.Errorf("failed to get file archive information, reason: %v", err) } // Get size on disk, will also give some time for the file to appear if it has not already - diskFileSize, err := archive.GetFileSize(filePath, false) + diskFileSize, err := archiveReader.GetFileSize(ctx, archiveLocation, filePath) if err != nil { return fmt.Errorf("failed to get size info for archived file, reason: %v", err) } @@ -245,23 +261,27 @@ func backupFile(delivered amqp.Delivery) error { return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)", diskFileSize, fileSize) } - file, err := archive.NewFileReader(filePath) + file, err := archiveReader.NewFileReader(ctx, archiveLocation, filePath) if err != nil { return fmt.Errorf("failed to open archived file, reason: %v", err) } defer file.Close() - dest, err := backup.NewFileWriter(filePath) + contentReader, contentWriter := io.Pipe() + go func() { + // Copy the file and check is sizes match + copiedSize, err := io.Copy(contentWriter, file) + if err != nil || copiedSize != int64(fileSize) { + log.Errorf("failed to copy file, reason: %v)", err) + } + _ = contentWriter.Close() + }() + + _, err = backupWriter.WriteFile(ctx, filePath, contentReader) if err != nil { return fmt.Errorf("failed to open backup file for writing, reason: %v", err) } - defer dest.Close() - - // Copy the file and check is sizes match - copiedSize, err := io.Copy(dest, file) - if err != nil || copiedSize != int64(fileSize) { - log.Errorf("failed to copy file, reason: %v)", err) - } + _ = contentReader.Close() // Mark file as "backed up" if err := db.UpdateFileEventLog(fileID, "backed up", "finalize", "{}", string(delivered.Body)); err != nil { diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index f7c83f66a9..50c2674f99 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -481,34 +481,34 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec } // GetArchived retrieves the location and size of archive -func (dbs *SDAdb) GetArchived(fileID string) (string, int, error) { +func (dbs *SDAdb) GetArchived(fileID string) (string, int, string, error) { var ( - filePath string - fileSize int - err error - count int + archiveFilePath, archiveLocation string + archiveFileSize int + err error + count int ) for count == 0 || (err != nil && count < RetryTimes) { - filePath, fileSize, err = dbs.getArchived(fileID) + archiveFilePath, archiveFileSize, archiveLocation, err = dbs.getArchived(fileID) count++ } - return filePath, fileSize, err + return archiveFilePath, archiveFileSize, archiveLocation, err } -func (dbs *SDAdb) getArchived(fileID string) (string, int, error) { +func (dbs *SDAdb) getArchived(fileID string) (string, int, string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "SELECT archive_file_path, archive_file_size from sda.files WHERE id = $1;" + const query = "SELECT archive_file_path, archive_file_size, archive_location from sda.files WHERE id = $1;" - var filePath string - var fileSize int - if err := db.QueryRow(query, fileID).Scan(&filePath, &fileSize); err != nil { - return "", 0, err + var archiveFilePath, archiveLocation string + var archiveFileSize int + if err := db.QueryRow(query, fileID).Scan(&archiveFilePath, &archiveFileSize, &archiveLocation); err != nil { + return "", 0, "", err } - return filePath, fileSize, nil + return archiveFilePath, archiveFileSize, archiveLocation, nil } // CheckAccessionIdExists validates if an accessionID exists in the db diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index e764a9c755..384c65594a 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -289,7 +289,7 @@ func (suite *DatabaseTests) TestGetArchived() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestGetArchived.c4gh", "testuser") + fileID, err := db.RegisterFileWithLocation(nil, "/archive", "/testuser/TestGetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} @@ -299,10 +299,11 @@ func (suite *DatabaseTests) TestGetArchived() { err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) - filePath, fileSize, err := db.GetArchived(fileID) + filePath, fileSize, archiveLocation, err := db.GetArchived(fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) assert.Equal(suite.T(), 1000, fileSize) assert.Equal(suite.T(), "/tmp/TestGetArchived.c4gh", filePath) + assert.Equal(suite.T(), "/archive", archiveLocation) db.Close() } From f78e8aabec2c391b95b5578554a8873f32fa1f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 14:45:22 +0100 Subject: [PATCH 132/530] feat(finalize): update GetArchived to return new struct ArchiveData to not exceed lint rules about max return args, update READMD.md to now mention expected config, fix some linting --- sda/cmd/finalize/finalize.go | 42 ++++++++++++++-------- sda/cmd/finalize/finalize.md | 33 ++++++----------- sda/internal/database/database.go | 5 +++ sda/internal/database/db_functions.go | 24 ++++++------- sda/internal/database/db_functions_test.go | 12 +++---- 5 files changed, 60 insertions(+), 56 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index d4bb10f9e1..880e8d32dd 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -36,33 +36,47 @@ func main() { forever := make(chan bool) conf, err = config.NewConfig("finalize") if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } if db.Version < 23 { - log.Fatal("database schema v23 is required") + log.Error("database schema v23 is required") + + return } lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Fatal("failed to init new location broker due to: %v", err) + log.Errorf("failed to init new location broker due to: %v", err) + + return } backupWriter, err = storage.NewWriter(ctx, "backup", lb) if err != nil && !errors.Is(err, storageerrors.ErrorNoValidWriter) { - log.Fatalf("error creating backup writer: %v", err) + log.Errorf("error creating backup writer: %v", err) + + return } archiveReader, err = storage.NewReader(ctx, "archive") if err != nil && !errors.Is(err, storageerrors.ErrorNoValidReader) { - log.Fatalf("error creating archive reader: %v", err) + log.Errorf("error creating archive reader: %v", err) + + return } if archiveReader != nil && backupWriter != nil { @@ -246,22 +260,22 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { log.Debug("Backup initiated") fileID := delivered.CorrelationId - filePath, fileSize, archiveLocation, err := db.GetArchived(fileID) + archiveData, err := db.GetArchived(fileID) if err != nil { return fmt.Errorf("failed to get file archive information, reason: %v", err) } // Get size on disk, will also give some time for the file to appear if it has not already - diskFileSize, err := archiveReader.GetFileSize(ctx, archiveLocation, filePath) + diskFileSize, err := archiveReader.GetFileSize(ctx, archiveData.Location, archiveData.FilePath) if err != nil { return fmt.Errorf("failed to get size info for archived file, reason: %v", err) } - if diskFileSize != int64(fileSize) { - return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)", diskFileSize, fileSize) + if diskFileSize != int64(archiveData.FileSize) { + return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)", diskFileSize, archiveData.FileSize) } - file, err := archiveReader.NewFileReader(ctx, archiveLocation, filePath) + file, err := archiveReader.NewFileReader(ctx, archiveData.Location, archiveData.FilePath) if err != nil { return fmt.Errorf("failed to open archived file, reason: %v", err) } @@ -271,13 +285,13 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { go func() { // Copy the file and check is sizes match copiedSize, err := io.Copy(contentWriter, file) - if err != nil || copiedSize != int64(fileSize) { + if err != nil || copiedSize != int64(archiveData.FileSize) { log.Errorf("failed to copy file, reason: %v)", err) } _ = contentWriter.Close() }() - _, err = backupWriter.WriteFile(ctx, filePath, contentReader) + _, err = backupWriter.WriteFile(ctx, archiveData.FilePath, contentReader) if err != nil { return fmt.Errorf("failed to open backup file for writing, reason: %v", err) } diff --git a/sda/cmd/finalize/finalize.md b/sda/cmd/finalize/finalize.md index b3dd433f13..f92b15c332 100644 --- a/sda/cmd/finalize/finalize.md +++ b/sda/cmd/finalize/finalize.md @@ -101,28 +101,15 @@ These settings control how `finalize` connects to the RabbitMQ message broker. - `panic` ### Storage settings +If a storage backup is desired then the following storage configuration are needed: "archive", "backup". +```yaml +storage: + backup: + ${STORAGE_IMPLEMENTATION}: + archive: + ${STORAGE_IMPLEMENTATION}: +``` +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) -Storage backend is defined by the `ARCHIVE_TYPE`, and `BACKUP_TYPE` variables. -Valid values for these options are `S3` or `POSIX` -(Defaults to `POSIX` on unknown values). - -The value of these variables define what other variables are read. -The same variables are available for all storage types, differing by prefix (`ARCHIVE_`, or `BACKUP_`) - -if `*_TYPE` is `S3` then the following variables are available: - -- `*_URL`: URL to the S3 system -- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, -[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, -[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_BUCKET`: The S3 bucket to use as the storage root -- `*_PORT`: S3 connection port (default: `443`) -- `*_REGION`: S3 region (default: `us-east-1`) -- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. -- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity - -and if `*_TYPE` is `POSIX`: - -- `*_LOCATION`: POSIX path to use as storage root +if either "archive", or "backup" storage configuration, then storage backup will not take place. diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 5d12f041f9..fb53c9d860 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -46,6 +46,11 @@ type SyncData struct { FilePath string Checksum string } +type ArchiveData struct { + FilePath string + Location string + FileSize int +} type SubmissionFileInfo struct { AccessionID string `json:"accessionID,omitempty"` diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 50c2674f99..84551782d1 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -481,34 +481,32 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec } // GetArchived retrieves the location and size of archive -func (dbs *SDAdb) GetArchived(fileID string) (string, int, string, error) { +func (dbs *SDAdb) GetArchived(fileID string) (ArchiveData, error) { var ( - archiveFilePath, archiveLocation string - archiveFileSize int - err error - count int + archiveData ArchiveData + err error + count int ) for count == 0 || (err != nil && count < RetryTimes) { - archiveFilePath, archiveFileSize, archiveLocation, err = dbs.getArchived(fileID) + archiveData, err = dbs.getArchived(fileID) count++ } - return archiveFilePath, archiveFileSize, archiveLocation, err + return archiveData, err } -func (dbs *SDAdb) getArchived(fileID string) (string, int, string, error) { +func (dbs *SDAdb) getArchived(fileID string) (ArchiveData, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB const query = "SELECT archive_file_path, archive_file_size, archive_location from sda.files WHERE id = $1;" - var archiveFilePath, archiveLocation string - var archiveFileSize int - if err := db.QueryRow(query, fileID).Scan(&archiveFilePath, &archiveFileSize, &archiveLocation); err != nil { - return "", 0, "", err + var archiveData ArchiveData + if err := db.QueryRow(query, fileID).Scan(&archiveData.FilePath, &archiveData.FileSize, &archiveData.Location); err != nil { + return ArchiveData{}, err } - return archiveFilePath, archiveFileSize, archiveLocation, nil + return archiveData, nil } // CheckAccessionIdExists validates if an accessionID exists in the db diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 384c65594a..59ca23190a 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -289,21 +289,21 @@ func (suite *DatabaseTests) TestGetArchived() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFileWithLocation(nil, "/archive", "/testuser/TestGetArchived.c4gh", "testuser") + fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/testuser/TestGetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchivedWithLocation("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) - filePath, fileSize, archiveLocation, err := db.GetArchived(fileID) + archiveData, err := db.GetArchived(fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) - assert.Equal(suite.T(), 1000, fileSize) - assert.Equal(suite.T(), "/tmp/TestGetArchived.c4gh", filePath) - assert.Equal(suite.T(), "/archive", archiveLocation) + assert.Equal(suite.T(), 1000, archiveData.FileSize) + assert.Equal(suite.T(), "/tmp/TestGetArchived.c4gh", archiveData.FilePath) + assert.Equal(suite.T(), "/archive", archiveData.Location) db.Close() } From d3b24dfd26d9abc8f173dffc466713d22dd6211e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 13:35:18 +0100 Subject: [PATCH 133/530] feat(finalize): update config to not require and load in storage v1 for finalize service, and update log entry if archive or backup is not configured to be similar to current --- sda/cmd/finalize/finalize.go | 3 ++- sda/internal/config/config.go | 22 ---------------------- 2 files changed, 2 insertions(+), 23 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 880e8d32dd..d1c66656f9 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -81,7 +81,8 @@ func main() { if archiveReader != nil && backupWriter != nil { backupInStorage = true - log.Info("will back up files in backup storage") + } else { + log.Warn("archive or backup destination not configured, backup will not be performed.") } defer mq.Channel.Close() diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 21892c4e12..25c03cbd96 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -314,23 +314,6 @@ func NewConfig(app string) (*Config, error) { "db.database", } - switch viper.GetString("archive.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) - default: - log.Warnln(" archive not configured, backup will not be performed.") - } - - switch viper.GetString("backup.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"backup.url", "backup.accesskey", "backup.secretkey", "backup.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"backup.location"}...) - default: - log.Warnln(" backup destination not configured, backup will not be performed.") - } case "intercept": requiredConfVars = []string{ "broker.host", @@ -596,11 +579,6 @@ func NewConfig(app string) (*Config, error) { return nil, err } case "finalize": - if viper.GetString("archive.type") != "" && viper.GetString("backup.type") != "" { - c.configArchive() - c.configBackup() - } - err := c.configBroker() if err != nil { return nil, err From f9029189e7e8dbf75fd80118ebd81afd0efaed86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 08:48:03 +0100 Subject: [PATCH 134/530] feat(finalize): use fatal instead error log and return when main failed to initialize resources, remove ctx cancel, use bg ctx instead --- sda/cmd/finalize/finalize.go | 36 +++++++++--------------------------- 1 file changed, 9 insertions(+), 27 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index d1c66656f9..60b5f9eaf2 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -31,52 +31,34 @@ var message schema.IngestionAccession var backupInStorage bool func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() forever := make(chan bool) conf, err = config.NewConfig("finalize") if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to load config, due to: %v", err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize mq broker, due to: %v", err) } db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize sda db, due to: %v", err) } if db.Version < 23 { - log.Error("database schema v23 is required") - - return + log.Fatal("database schema v23 is required") } lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Errorf("failed to init new location broker due to: %v", err) - - return + log.Fatalf("failed to init new location broker, due to: %v", err) } - - backupWriter, err = storage.NewWriter(ctx, "backup", lb) + backupWriter, err = storage.NewWriter(context.Background(), "backup", lb) if err != nil && !errors.Is(err, storageerrors.ErrorNoValidWriter) { - log.Errorf("error creating backup writer: %v", err) - - return + log.Fatalf("failed to initialize backup writer, due to: %v", err) } - - archiveReader, err = storage.NewReader(ctx, "archive") + archiveReader, err = storage.NewReader(context.Background(), "archive") if err != nil && !errors.Is(err, storageerrors.ErrorNoValidReader) { - log.Errorf("error creating archive reader: %v", err) - - return + log.Fatalf("failed to initialize archive reader: %v", err) } if archiveReader != nil && backupWriter != nil { From 0ba2dffdc3b90bf2854ba5d789d0dca42e30b979 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 14:05:27 +0100 Subject: [PATCH 135/530] feat(finalize): update main shutdown and erroring logic --- sda/cmd/finalize/finalize.go | 303 +++++++++++++++++++---------------- 1 file changed, 163 insertions(+), 140 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 60b5f9eaf2..ae60557277 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -8,6 +8,9 @@ import ( "errors" "fmt" "io" + "os" + "os/signal" + "syscall" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" @@ -22,43 +25,60 @@ import ( ) var db *database.SDAdb +var mqBroker *broker.AMQPBroker var archiveReader storage.Reader var backupWriter storage.Writer -var conf *config.Config -var err error var message schema.IngestionAccession var backupInStorage bool func main() { - forever := make(chan bool) - conf, err = config.NewConfig("finalize") - if err != nil { - log.Fatalf("failed to load config, due to: %v", err) + if err := run(); err != nil { + log.Fatal(err) } - mq, err := broker.NewMQ(conf.Broker) +} +func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + conf, err := config.NewConfig("finalize") if err != nil { - log.Fatalf("failed to initialize mq broker, due to: %v", err) + return fmt.Errorf("failed to load config, due to: %v", err) } db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatalf("failed to initialize sda db, due to: %v", err) + return fmt.Errorf("failed to initialize sda db, due to: %v", err) } + defer db.Close() + if db.Version < 23 { - log.Fatal("database schema v23 is required") + return errors.New("database schema v23 is required") } + mqBroker, err = broker.NewMQ(conf.Broker) + if err != nil { + return fmt.Errorf("failed to initialize mq broker, due to: %v", err) + } + defer func() { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } + }() + lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Fatalf("failed to init new location broker, due to: %v", err) + return fmt.Errorf("failed to init new location broker, due to: %v", err) } - backupWriter, err = storage.NewWriter(context.Background(), "backup", lb) + backupWriter, err = storage.NewWriter(ctx, "backup", lb) if err != nil && !errors.Is(err, storageerrors.ErrorNoValidWriter) { - log.Fatalf("failed to initialize backup writer, due to: %v", err) + return fmt.Errorf("failed to initialize backup writer, due to: %v", err) } - archiveReader, err = storage.NewReader(context.Background(), "archive") + archiveReader, err = storage.NewReader(ctx, "archive") if err != nil && !errors.Is(err, storageerrors.ErrorNoValidReader) { - log.Fatalf("failed to initialize archive reader: %v", err) + return fmt.Errorf("failed to initialize archive reader: %v", err) } if archiveReader != nil && backupWriter != nil { @@ -67,142 +87,136 @@ func main() { log.Warn("archive or backup destination not configured, backup will not be performed.") } - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() - + log.Info("Starting finalize service") + consumeErr := make(chan error, 1) go func() { - connError := mq.ConnectionWatcher() - log.Error(connError) - forever <- false + consumeErr <- startConsumer() }() - go func() { - connError := mq.ChannelWatcher() - log.Error(connError) - forever <- false - }() + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + select { + case <-sigc: + case err := <-mqBroker.Connection.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-mqBroker.Channel.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-consumeErr: + return err + } - log.Info("Starting finalize service") - go func() { - messages, err := mq.GetMessages(conf.Broker.Queue) + return nil +} +func startConsumer() error { + messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) + if err != nil { + return err + } + for delivered := range messages { + ctx := context.Background() + log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", mqBroker.Conf.SchemasPath), delivered.Body) if err != nil { - log.Fatal(err) - } - for delivered := range messages { - ctx := context.Background() - log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } - - continue + log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) } - fileID := delivered.CorrelationId - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) - // If the file has been canceled by the uploader, don't spend time working on it. - status, err := db.GetFileStatus(fileID) - if err != nil { - log.Errorf("failed to get file status, file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } + continue + } - continue + fileID := delivered.CorrelationId + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + // If the file has been canceled by the uploader, don't spend time working on it. + status, err := db.GetFileStatus(fileID) + if err != nil { + log.Errorf("failed to get file status, file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) } - switch status { - case "disabled": - log.Infof("file with file-id: %s is disabled, aborting work", fileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } - - continue + continue + } - case "verified", "enabled": - case "ready": - log.Infof("File with file-id: %s is already marked as ready.", fileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking message, reason: %v", err) - } + switch status { + case "disabled": + log.Infof("file with file-id: %s is disabled, aborting work", fileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) + } - continue - default: - log.Warnf("file with file-id: %s is not verified yet, aborting work", fileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } + continue - continue + case "verified", "enabled": + case "ready": + log.Infof("File with file-id: %s is already marked as ready.", fileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking message, reason: %v", err) } - c := schema.IngestionCompletion{ - User: message.User, - FilePath: message.FilePath, - AccessionID: message.AccessionID, - DecryptedChecksums: message.DecryptedChecksums, + continue + default: + log.Warnf("file with file-id: %s is not verified yet, aborting work", fileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) } - completeMsg, _ := json.Marshal(&c) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", conf.Broker.SchemasPath), completeMsg) - if err != nil { - log.Errorf("Validation of outgoing message ingestion-completion failed, reason: (%v). Message body: %s\n", err, string(completeMsg)) - continue - } + continue + } - accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID, fileID) - if err != nil { - log.Errorf("CheckAccessionIdExists failed, file-id: %s, reason: %v ", fileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } + c := schema.IngestionCompletion{ + User: message.User, + FilePath: message.FilePath, + AccessionID: message.AccessionID, + DecryptedChecksums: message.DecryptedChecksums, + } + completeMsg, _ := json.Marshal(&c) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", mqBroker.Conf.SchemasPath), completeMsg) + if err != nil { + log.Errorf("Validation of outgoing message ingestion-completion failed, reason: (%v). Message body: %s\n", err, string(completeMsg)) - continue + continue + } + + accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID, fileID) + if err != nil { + log.Errorf("CheckAccessionIdExists failed, file-id: %s, reason: %v ", fileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) } - switch accessionIDExists { - case "duplicate": - log.Errorf("accession ID already exists in the system, file-id: %s, accession-id: %s\n", fileID, message.AccessionID) - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "There is a conflict regarding the file accessionID", - Reason: "The Accession ID already exists in the database, skipping marking it ready.", - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) + continue + } - // Send the message to an error queue so it can be analyzed. - if e := mq.SendMessage(fileID, conf.Broker.Exchange, "error", body); e != nil { - log.Errorf("failed to publish message, reason: %v", err) - } + switch accessionIDExists { + case "duplicate": + log.Errorf("accession ID already exists in the system, file-id: %s, accession-id: %s\n", fileID, message.AccessionID) + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "There is a conflict regarding the file accessionID", + Reason: "The Accession ID already exists in the database, skipping marking it ready.", + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: %v", err) - } + // Send the message to an error queue so it can be analyzed. + if e := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, "error", body); e != nil { + log.Errorf("failed to publish message, reason: %v", err) + } - continue - case "same": - log.Infof("file already has a stable ID, marking it as ready, file-id: %s", fileID) - default: - if backupInStorage { - if err = backupFile(ctx, delivered); err != nil { - log.Errorf("failed to backup file, file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } - - continue - } - } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: %v", err) + } - if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { - log.Errorf("failed to set accessionID for file, file-id: %s, reason: %v", fileID, err) + continue + case "same": + log.Infof("file already has a stable ID, marking it as ready, file-id: %s", fileID) + default: + if backupInStorage { + if err = backupFile(ctx, delivered); err != nil { + log.Errorf("failed to backup file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } @@ -211,32 +225,41 @@ func main() { } } - // Mark file as "ready" - if err := db.UpdateFileEventLog(fileID, "ready", "finalize", "{}", string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, file-id: %s, reason: %v", fileID, err) + if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { + log.Errorf("failed to set accessionID for file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } continue } + } - if err := mq.SendMessage(fileID, conf.Broker.Exchange, conf.Broker.RoutingKey, completeMsg); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } - - continue + // Mark file as "ready" + if err := db.UpdateFileEventLog(fileID, "ready", "finalize", "{}", string(delivered.Body)); err != nil { + log.Errorf("set status ready failed, file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: %v", err) + continue + } + + if err := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, mqBroker.Conf.RoutingKey, completeMsg); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) } + + continue } - }() - <-forever + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: %v", err) + } + } + + return nil } func backupFile(ctx context.Context, delivered amqp.Delivery) error { From 771edd5cf3ff177cd9ec49751353c0783ef5e19f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:16:02 +0100 Subject: [PATCH 136/530] fix(finalize): fix typo in finalize.md --- sda/cmd/finalize/finalize.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/finalize/finalize.md b/sda/cmd/finalize/finalize.md index f92b15c332..17d7281ae8 100644 --- a/sda/cmd/finalize/finalize.md +++ b/sda/cmd/finalize/finalize.md @@ -109,7 +109,7 @@ storage: archive: ${STORAGE_IMPLEMENTATION}: ``` -For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) if either "archive", or "backup" storage configuration, then storage backup will not take place. From 4fb331a98b766ce061e99bfac5f77876ab1e58ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:31:59 +0100 Subject: [PATCH 137/530] feat(finalize): update closing defer func of archive file reader to be explicit we ignore err from close --- sda/cmd/finalize/finalize.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index ae60557277..c758ab0e55 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -285,7 +285,9 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { if err != nil { return fmt.Errorf("failed to open archived file, reason: %v", err) } - defer file.Close() + defer func() { + _ = file.Close() + }() contentReader, contentWriter := io.Pipe() go func() { From 32c46a3987b27bbb20af2c4bfaf4c9a9aa13d9d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:51:28 +0100 Subject: [PATCH 138/530] feat(finalize): restructure handling of message to separate func with spawns content from main context so it can react to service shutting down and can gracefully shutdown --- sda/cmd/finalize/finalize.go | 223 ++++++++++++++++++----------------- 1 file changed, 113 insertions(+), 110 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index c758ab0e55..2a03a3bf0e 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -90,7 +90,7 @@ func run() error { log.Info("Starting finalize service") consumeErr := make(chan error, 1) go func() { - consumeErr <- startConsumer() + consumeErr <- startConsumer(ctx) }() sigc := make(chan os.Signal, 5) @@ -108,158 +108,161 @@ func run() error { return nil } -func startConsumer() error { +func startConsumer(ctx context.Context) error { messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) if err != nil { return err } for delivered := range messages { - ctx := context.Background() - log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", mqBroker.Conf.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } - - continue - } + handleMessage(ctx, delivered) + } - fileID := delivered.CorrelationId - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) - // If the file has been canceled by the uploader, don't spend time working on it. - status, err := db.GetFileStatus(fileID) - if err != nil { - log.Errorf("failed to get file status, file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } + return nil +} - continue +func handleMessage(ctx context.Context, delivered amqp.Delivery) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", mqBroker.Conf.SchemasPath), delivered.Body); err != nil { + log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) } - switch status { - case "disabled": - log.Infof("file with file-id: %s is disabled, aborting work", fileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } - - continue + return + } - case "verified", "enabled": - case "ready": - log.Infof("File with file-id: %s is already marked as ready.", fileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking message, reason: %v", err) - } + fileID := delivered.CorrelationId + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + // If the file has been canceled by the uploader, don't spend time working on it. + status, err := db.GetFileStatus(fileID) + if err != nil { + log.Errorf("failed to get file status, file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) + } - continue - default: - log.Warnf("file with file-id: %s is not verified yet, aborting work", fileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } + return + } - continue + switch status { + case "disabled": + log.Infof("file with file-id: %s is disabled, aborting work", fileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) } - c := schema.IngestionCompletion{ - User: message.User, - FilePath: message.FilePath, - AccessionID: message.AccessionID, - DecryptedChecksums: message.DecryptedChecksums, + return + case "verified", "enabled": + case "ready": + log.Infof("File with file-id: %s is already marked as ready.", fileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking message, reason: %v", err) } - completeMsg, _ := json.Marshal(&c) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", mqBroker.Conf.SchemasPath), completeMsg) - if err != nil { - log.Errorf("Validation of outgoing message ingestion-completion failed, reason: (%v). Message body: %s\n", err, string(completeMsg)) - continue + return + default: + log.Warnf("file with file-id: %s is not verified yet, aborting work", fileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) } - accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID, fileID) - if err != nil { - log.Errorf("CheckAccessionIdExists failed, file-id: %s, reason: %v ", fileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } + return + } + + c := schema.IngestionCompletion{ + User: message.User, + FilePath: message.FilePath, + AccessionID: message.AccessionID, + DecryptedChecksums: message.DecryptedChecksums, + } + completeMsg, _ := json.Marshal(&c) - continue + if err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", mqBroker.Conf.SchemasPath), completeMsg); err != nil { + log.Errorf("Validation of outgoing message ingestion-completion failed, reason: (%v). Message body: %s\n", err, string(completeMsg)) + + return + } + + accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID, fileID) + if err != nil { + log.Errorf("CheckAccessionIdExists failed, file-id: %s, reason: %v ", fileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) } - switch accessionIDExists { - case "duplicate": - log.Errorf("accession ID already exists in the system, file-id: %s, accession-id: %s\n", fileID, message.AccessionID) - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "There is a conflict regarding the file accessionID", - Reason: "The Accession ID already exists in the database, skipping marking it ready.", - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) + return + } - // Send the message to an error queue so it can be analyzed. - if e := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, "error", body); e != nil { - log.Errorf("failed to publish message, reason: %v", err) - } + switch accessionIDExists { + case "duplicate": + log.Errorf("accession ID already exists in the system, file-id: %s, accession-id: %s\n", fileID, message.AccessionID) + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "There is a conflict regarding the file accessionID", + Reason: "The Accession ID already exists in the database, skipping marking it ready.", + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: %v", err) - } + // Send the message to an error queue so it can be analyzed. + if e := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, "error", body); e != nil { + log.Errorf("failed to publish message, reason: %v", err) + } - continue - case "same": - log.Infof("file already has a stable ID, marking it as ready, file-id: %s", fileID) - default: - if backupInStorage { - if err = backupFile(ctx, delivered); err != nil { - log.Errorf("failed to backup file, file-id: %s, reason: %v", fileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } - - continue - } - } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: %v", err) + } - if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { - log.Errorf("failed to set accessionID for file, file-id: %s, reason: %v", fileID, err) + return + case "same": + log.Infof("file already has a stable ID, marking it as ready, file-id: %s", fileID) + default: + if backupInStorage { + if err = backupFile(ctx, delivered); err != nil { + log.Errorf("failed to backup file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } - continue + return } } - // Mark file as "ready" - if err := db.UpdateFileEventLog(fileID, "ready", "finalize", "{}", string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, file-id: %s, reason: %v", fileID, err) + if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { + log.Errorf("failed to set accessionID for file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } - continue + return } + } - if err := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, mqBroker.Conf.RoutingKey, completeMsg); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } - - continue + // Mark file as "ready" + if err := db.UpdateFileEventLog(fileID, "ready", "finalize", "{}", string(delivered.Body)); err != nil { + log.Errorf("set status ready failed, file-id: %s, reason: %v", fileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: %v", err) + return + } + + if err := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, mqBroker.Conf.RoutingKey, completeMsg); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) } + + return } - return nil + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: %v", err) + } } func backupFile(ctx context.Context, delivered amqp.Delivery) error { From 2b4ac26ff1bbdc6aa4a37fa0074d73be4211a832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 10:18:44 +0100 Subject: [PATCH 139/530] feat(finalize): update error handling when failing to write to writer during backup and close writer with err --- sda/cmd/finalize/finalize.go | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 2a03a3bf0e..2bfa1ef312 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -294,17 +294,20 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { contentReader, contentWriter := io.Pipe() go func() { - // Copy the file and check is sizes match - copiedSize, err := io.Copy(contentWriter, file) - if err != nil || copiedSize != int64(archiveData.FileSize) { - log.Errorf("failed to copy file, reason: %v)", err) + defer func() { + _ = contentWriter.Close() + }() + + if copiedSize, err := io.Copy(contentWriter, file); err != nil { + _ = contentWriter.CloseWithError(fmt.Errorf("failed to copy file, reason: %v)", err)) + } else if copiedSize != int64(archiveData.FileSize) { + _ = contentWriter.CloseWithError(errors.New("copied size does not match file size")) } - _ = contentWriter.Close() }() _, err = backupWriter.WriteFile(ctx, archiveData.FilePath, contentReader) if err != nil { - return fmt.Errorf("failed to open backup file for writing, reason: %v", err) + return fmt.Errorf("failed to write file to backup storage, reason: %v", err) } _ = contentReader.Close() From 1198adf59c579a1844491b8eb71fc4673ee9b5ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:23:05 +0100 Subject: [PATCH 140/530] feat(finalize): update sigc chan buffer size to 1 instead of 5 as we just care that any of the signals triggered --- sda/cmd/finalize/finalize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 2bfa1ef312..5617280a42 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -93,7 +93,7 @@ func run() error { consumeErr <- startConsumer(ctx) }() - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { From 1593d51d738b2e0be4c4ed774ed0455edf30c826 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 10 Feb 2026 09:34:42 +0100 Subject: [PATCH 141/530] feat(finalize): update finalize.md to clarify that both 'archive' and 'backup' are required to be configured for backup to take place. --- sda/cmd/finalize/finalize.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sda/cmd/finalize/finalize.md b/sda/cmd/finalize/finalize.md index 17d7281ae8..1c68e9433e 100644 --- a/sda/cmd/finalize/finalize.md +++ b/sda/cmd/finalize/finalize.md @@ -101,7 +101,10 @@ These settings control how `finalize` connects to the RabbitMQ message broker. - `panic` ### Storage settings -If a storage backup is desired then the following storage configuration are needed: "archive", "backup". +If a storage backup is desired then the following storage configurations are needed: "archive", "backup". + +[!IMPORTANT] Both `archive` and `backup` are required to be configured for backup to take place. If you omit one, storage backup will not take place. + ```yaml storage: backup: @@ -111,5 +114,3 @@ storage: ``` For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) -if either "archive", or "backup" storage configuration, then storage backup will not take place. - From 70cdf48da912d479002e696e64583576972bff58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 10 Feb 2026 09:38:14 +0100 Subject: [PATCH 142/530] feat(finalize): move message variable to handleMessage to eliminate risk of parallel message handling overwriting same variable --- sda/cmd/finalize/finalize.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 5617280a42..0a847283f0 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -28,7 +28,6 @@ var db *database.SDAdb var mqBroker *broker.AMQPBroker var archiveReader storage.Reader var backupWriter storage.Writer -var message schema.IngestionAccession var backupInStorage bool @@ -123,6 +122,7 @@ func startConsumer(ctx context.Context) error { func handleMessage(ctx context.Context, delivered amqp.Delivery) { ctx, cancel := context.WithCancel(ctx) defer cancel() + log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) if err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", mqBroker.Conf.SchemasPath), delivered.Body); err != nil { log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) @@ -134,6 +134,7 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { } fileID := delivered.CorrelationId + var message schema.IngestionAccession // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) // If the file has been canceled by the uploader, don't spend time working on it. From 45f2078fa0f3bea7cfb8ced5cb9af512213fdbb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 12 Jan 2026 13:57:06 +0100 Subject: [PATCH 143/530] feat(s3inbox): register uploaded files with location --- sda/cmd/s3inbox/proxy.go | 11 ++++++----- sda/cmd/s3inbox/proxy_test.go | 31 ++++--------------------------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 94697f6c30..5ecb8ae1e2 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -157,7 +157,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) anonymizedFilepath := helper.AnonymizeFilepath(rawFilepath, username) - filepath, err := formatUploadFilePath(anonymizedFilepath) + filePath, err := formatUploadFilePath(anonymizedFilepath) if err != nil { reportError(http.StatusNotAcceptable, err.Error(), w) @@ -166,14 +166,15 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw fileIdentifier := uniqueFileID{ username: username, - filePath: filepath, + filePath: filePath, } + location := fmt.Sprintf("%s:%d/%s", p.s3.URL, p.s3.Port, p.s3.Bucket) // if this is an upload request if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { // register file in database - log.Debugf("registering file %v in the database", r.URL.Path) - p.fileIDs[fileIdentifier], err = p.database.RegisterFile(nil, filepath, username) + log.Debugf("registering file %v in the database with location: %s", r.URL.Path, location) + p.fileIDs[fileIdentifier], err = p.database.RegisterFileWithLocation(nil, location, filePath, username) log.Debugf("fileId: %v", p.fileIDs[fileIdentifier]) if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to register file in database: %v", err)) @@ -226,7 +227,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // The following block is for treating the case when the client loses connection to the server and then it reconnects to a // different instance of s3inbox. For more details see #1358. if p.fileIDs[fileIdentifier] == "" { - p.fileIDs[fileIdentifier], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") + p.fileIDs[fileIdentifier], err = p.database.GetFileIDByUserPathAndStatus(username, filePath, "registered") if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 03e12747f4..6dfcf66463 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -526,11 +526,12 @@ func (s *ProxyTests) TestDatabaseConnection() { assert.Nil(s.T(), err, "Failed to connect to database") // Check that the file is in the database - var fileID string - query := "SELECT id FROM sda.files WHERE submission_file_path = $1;" - err = db.DB.QueryRow(query, anonymFilename).Scan(&fileID) + var fileID, location string + query := "SELECT id, submission_location FROM sda.files WHERE submission_file_path = $1;" + err = db.DB.QueryRow(query, anonymFilename).Scan(&fileID, &location) assert.Nil(s.T(), err, "Failed to query database") assert.NotNil(s.T(), fileID, "File not found in database") + assert.Equal(s.T(), fmt.Sprintf("%s:%d/%s", s.S3Fakeconf.URL, s.S3Fakeconf.Port, s.S3Fakeconf.Bucket), location) // Check that the "registered" status is in the database for this file for _, status := range []string{"registered", "uploaded"} { @@ -618,30 +619,6 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { assert.Contains(s.T(), err.Error(), "StatusCode: 403") } -func (s *ProxyTests) TestStoreObjectSizeInDB() { - db, err := database.NewSDAdb(s.DBConf) - assert.NoError(s.T(), err) - defer db.Close() - - mq, err := broker.NewMQ(s.MQConf) - assert.NoError(s.T(), err) - defer mq.Connection.Close() - - p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) - p.database = db - - fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") - assert.NoError(s.T(), err) - assert.NotNil(s.T(), fileID) - - assert.NoError(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) - - const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" - var objectSize int64 - assert.NoError(s.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) - assert.Equal(s.T(), int64(14), objectSize) -} - func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { db, err := database.NewSDAdb(s.DBConf) assert.NoError(s.T(), err) From c8736c486b362ea35abbf957e2a914653743fa1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:40:42 +0100 Subject: [PATCH 144/530] feat(s3inbox): update required db version check to v23 with new location columns --- sda/cmd/s3inbox/s3inbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index bac69ef7a1..f3c2caf59b 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -51,8 +51,8 @@ func main() { sigc <- syscall.SIGINT panic(err) } - if sdaDB.Version < 21 { - log.Error("database schema v21 is required") + if sdaDB.Version < 23 { + log.Error("database schema v23 is required") sigc <- syscall.SIGINT panic(err) } From e989b7ff2f48ed10e2857b62fd7c3e7635e1d427 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 15:34:02 +0100 Subject: [PATCH 145/530] feat(s3inbox): update s3inbox config to not use storage(v1)/S3Conf, instead introduce its own config struct S3InboxConf in config package, remove dependency on storage(v1) as s3inbox was only utilizing it to create s3 client --- sda/cmd/s3inbox/healthchecks.go | 11 +-- sda/cmd/s3inbox/healthchecks_test.go | 27 ++++--- sda/cmd/s3inbox/proxy.go | 101 ++++++++++++++++++++------- sda/cmd/s3inbox/proxy_test.go | 48 ++++++------- sda/cmd/s3inbox/s3inbox.go | 2 +- sda/internal/config/config.go | 28 ++++++-- 6 files changed, 138 insertions(+), 79 deletions(-) diff --git a/sda/cmd/s3inbox/healthchecks.go b/sda/cmd/s3inbox/healthchecks.go index 3c33b4e5a0..814f06592e 100644 --- a/sda/cmd/s3inbox/healthchecks.go +++ b/sda/cmd/s3inbox/healthchecks.go @@ -2,11 +2,9 @@ package main import ( "fmt" - "net" "net/http" "net/url" "path" - "strconv" "github.com/neicnordic/sensitive-data-archive/internal/broker" log "github.com/sirupsen/logrus" @@ -87,15 +85,12 @@ func (p *Proxy) httpsGetCheck(uri string) error { } func (p *Proxy) getS3ReadyPath() (string, error) { - s3URL, err := url.Parse(p.s3.URL) + s3URL, err := url.Parse(p.s3Conf.Endpoint) if err != nil { return "", err } - if p.s3.Port != 0 { - s3URL.Host = net.JoinHostPort(s3URL.Hostname(), strconv.Itoa(p.s3.Port)) - } - if p.s3.Readypath != "" { - s3URL.Path = path.Join(s3URL.Path, p.s3.Readypath) + if p.s3Conf.ReadyPath != "" { + s3URL.Path = path.Join(s3URL.Path, p.s3Conf.ReadyPath) } return s3URL.String(), nil diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index 870a3fdf9d..738394951e 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -7,18 +7,16 @@ import ( "testing" "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" - "github.com/neicnordic/sensitive-data-archive/internal/storage" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type HealthcheckTestSuite struct { suite.Suite - S3Fakeconf storage.S3Conf // fakeserver - S3conf storage.S3Conf // actual s3 container + mockS3Conf config.S3InboxConf // fakeserver DBConf database.DBConf fakeServer *FakeServer MQConf broker.MQConf @@ -34,9 +32,8 @@ func (ts *HealthcheckTestSuite) SetupTest() { ts.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - ts.S3Fakeconf = storage.S3Conf{ - URL: "http://127.0.0.1", - Port: 9024, + ts.mockS3Conf = config.S3InboxConf{ + Endpoint: "http://127.0.0.1:9024", AccessKey: "someAccess", SecretKey: "someSecret", Bucket: "buckbuck", @@ -77,7 +74,7 @@ func (ts *HealthcheckTestSuite) TearDownTest() { } func (ts *HealthcheckTestSuite) TestHttpsGetCheck() { - p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) url, _ := p.getS3ReadyPath() assert.NoError(ts.T(), p.httpsGetCheck(url)) @@ -85,12 +82,12 @@ func (ts *HealthcheckTestSuite) TestHttpsGetCheck() { } func (ts *HealthcheckTestSuite) TestS3URL() { - p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) _, err := p.getS3ReadyPath() assert.NoError(ts.T(), err) - p.s3.URL = "://badurl" + p.s3Conf.Endpoint = "://badurl" url, err := p.getS3ReadyPath() assert.Empty(ts.T(), url) assert.Error(ts.T(), err) @@ -101,7 +98,7 @@ func (ts *HealthcheckTestSuite) TestHealthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) @@ -115,7 +112,7 @@ func (ts *HealthcheckTestSuite) TestClosedDBHealthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Check that 200 is reported w := httptest.NewRecorder() @@ -138,11 +135,11 @@ func (ts *HealthcheckTestSuite) TestNoS3Healthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // S3 unavailable, check that 503 is reported w := httptest.NewRecorder() - p.s3.URL = "http://badurl" + p.s3Conf.Endpoint = "http://badurl" p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() @@ -154,7 +151,7 @@ func (ts *HealthcheckTestSuite) TestNoMQHealthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Messenger unavailable, check that 503 is reported p.messenger.Conf.Port = 123456 diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 5ecb8ae1e2..a5d5f82f4b 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "crypto/tls" + "crypto/x509" "encoding/json" "encoding/xml" "errors" @@ -11,20 +12,23 @@ import ( "io" "net/http" "net/url" + "os" "regexp" "strconv" "strings" "time" "github.com/aws/aws-sdk-go-v2/aws" + s3config "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/minio/minio-go/v6/pkg/signer" "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" ) @@ -35,7 +39,7 @@ type uniqueFileID struct { // Proxy represents the toplevel object in this application type Proxy struct { - s3 storage.S3Conf + s3Conf config.S3InboxConf auth userauth.Authenticator messenger *broker.AMQPBroker database *database.SDAdb @@ -82,7 +86,7 @@ const ( ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. -func NewProxy(s3conf storage.S3Conf, auth userauth.Authenticator, messenger *broker.AMQPBroker, db *database.SDAdb, tlsConf *tls.Config) *Proxy { +func NewProxy(s3conf config.S3InboxConf, auth userauth.Authenticator, messenger *broker.AMQPBroker, db *database.SDAdb, tlsConf *tls.Config) *Proxy { tr := &http.Transport{TLSClientConfig: tlsConf} client := &http.Client{Transport: tr, Timeout: 30 * time.Second} @@ -154,7 +158,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw } username := token.Subject() - rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) + rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3Conf.Bucket+"/", "", 1) anonymizedFilepath := helper.AnonymizeFilepath(rawFilepath, username) filePath, err := formatUploadFilePath(anonymizedFilepath) @@ -169,7 +173,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw filePath: filePath, } - location := fmt.Sprintf("%s:%d/%s", p.s3.URL, p.s3.Port, p.s3.Bucket) + location := p.s3Conf.Endpoint + "/" + p.s3Conf.Bucket // if this is an upload request if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { // register file in database @@ -340,10 +344,10 @@ func (p *Proxy) uploadFinishedSuccessfully(req *http.Request, response *http.Res } func (p *Proxy) forwardToBackend(r *http.Request) (*http.Response, error) { - p.resignHeader(r, p.s3.AccessKey, p.s3.SecretKey, fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)) + p.resignHeader(r) // Redirect request - nr, err := http.NewRequest(r.Method, fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)+r.URL.String(), r.Body) // #nosec G704 -- endpoint and port controlled by configuration, TODO verify if r.URL needs to be sanitized + nr, err := http.NewRequest(r.Method, p.s3Conf.Endpoint+r.URL.String(), r.Body) // #nosec G704 -- endpoint and port controlled by configuration, TODO verify if r.URL needs to be sanitized if err != nil { log.Debug("error when redirecting the request") log.Debug(err) @@ -359,7 +363,7 @@ func (p *Proxy) forwardToBackend(r *http.Request) (*http.Response, error) { // Add bucket to host path func (p *Proxy) prependBucketToHostPath(r *http.Request) error { - bucket := p.s3.Bucket + bucket := p.s3Conf.Bucket // Extract username for request's url path str, err := url.ParseRequestURI(r.URL.Path) @@ -422,8 +426,8 @@ func (p *Proxy) prependBucketToHostPath(r *http.Request) error { // Function for signing the headers of the s3 requests // Used for for creating a signature for with the default // credentials of the s3 service and the user's signature (authentication) -func (p *Proxy) resignHeader(r *http.Request, accessKey string, secretKey string, backendURL string) *http.Request { - log.Debugf("Generating resigning header for %s", backendURL) +func (p *Proxy) resignHeader(r *http.Request) *http.Request { + log.Debugf("Generating resigning header for %s", p.s3Conf.Endpoint) r.Header.Del("X-Amz-Security-Token") r.Header.Del("X-Forwarded-Port") r.Header.Del("X-Forwarded-Proto") @@ -433,12 +437,12 @@ func (p *Proxy) resignHeader(r *http.Request, accessKey string, secretKey string r.Header.Del("X-Real-Ip") r.Header.Del("X-Request-Id") r.Header.Del("X-Scheme") - if strings.Contains(backendURL, "//") { - host := strings.SplitN(backendURL, "//", 2) + if strings.Contains(p.s3Conf.Endpoint, "//") { + host := strings.SplitN(p.s3Conf.Endpoint, "//", 2) r.Host = host[1] } - return signer.SignV4(*r, accessKey, secretKey, "", p.s3.Region) + return signer.SignV4(*r, p.s3Conf.AccessKey, p.s3Conf.SecretKey, "", p.s3Conf.Region) } // Not necessarily a function on the struct since it does not use any of the @@ -527,16 +531,16 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anon // RequestInfo is a function that makes a request to the S3 and collects // the etag and size information for the uploaded document func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { - filePath := strings.Replace(fullPath, "/"+p.s3.Bucket+"/", "", 1) - client, err := storage.NewS3Client(p.s3) + filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) + client, err := newS3Client(p.s3Conf) if err != nil { return "", 0, err } input := &s3.ListObjectsV2Input{ - Bucket: &p.s3.Bucket, + Bucket: aws.String(p.s3Conf.Bucket), MaxKeys: aws.Int32(1), - Prefix: &filePath, + Prefix: aws.String(filePath), } result, err := client.ListObjectsV2(context.TODO(), input) @@ -549,18 +553,67 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.ReplaceAll(*result.Contents[0].ETag, "\"", "")))), *result.Contents[0].Size, nil } +func newS3Client(conf config.S3InboxConf) (*s3.Client, error) { + s3cfg, err := s3config.LoadDefaultConfig( + context.TODO(), + s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.SecretKey, "")), + s3config.WithHTTPClient(&http.Client{Transport: transportConfigS3(conf)}), + ) + if err != nil { + return nil, err + } + + s3Client := s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(conf.Endpoint) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(conf.Endpoint, "http:") + o.Region = conf.Region + o.UsePathStyle = true + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }, + ) + + return s3Client, nil +} +func transportConfigS3(conf config.S3InboxConf) http.RoundTripper { + cfg := new(tls.Config) + + // Read system CAs + systemCAs, err := x509.SystemCertPool() + if err != nil { + log.Errorf("failed to read system CAs: %v, using an empty pool as base", err) + systemCAs = x509.NewCertPool() + } + + cfg.RootCAs = systemCAs + + if conf.CAcert != "" { + cacert, e := os.ReadFile(conf.CAcert) // #nosec this file comes from our config + if e != nil { + log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main + } + if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { + log.Debug("no certs appended, using system certs only") + } + } + + return &http.Transport{TLSClientConfig: cfg, ForceAttemptHTTP2: true} +} + // checkFileExists makes a request to the S3 to check whether the file already // is uploaded. Returns a bool indicating whether the file was found. func (p *Proxy) checkFileExists(fullPath string) (bool, error) { - filePath := strings.Replace(fullPath, "/"+p.s3.Bucket+"/", "", 1) - client, err := storage.NewS3Client(p.s3) + filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) + client, err := newS3Client(p.s3Conf) if err != nil { return false, fmt.Errorf("could not connect to s3: %v", err) } result, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ - Bucket: &p.s3.Bucket, - Key: &filePath, + Bucket: aws.String(p.s3Conf.Bucket), + Key: aws.String(filePath), }) if err != nil && strings.Contains(err.Error(), "StatusCode: 404") { @@ -649,14 +702,14 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { } func (p *Proxy) storeObjectSizeInDB(path, fileID string) error { - client, err := storage.NewS3Client(p.s3) + client, err := newS3Client(p.s3Conf) if err != nil { return err } o, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ - Bucket: &p.s3.Bucket, - Key: &path, + Bucket: aws.String(p.s3Conf.Bucket), + Key: aws.String(path), }) if err != nil { return err diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 6dfcf66463..2285268bec 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -16,16 +16,16 @@ import ( "testing" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" + s3config "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/lestrrat-go/jwx/v2/jws" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/neicnordic/sensitive-data-archive/internal/broker" + "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" - "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -34,8 +34,8 @@ import ( type ProxyTests struct { suite.Suite - S3Fakeconf storage.S3Conf // fakeserver - S3conf storage.S3Conf // actual s3 container + S3Fakeconf config.S3InboxConf // fakeserver + S3conf config.S3InboxConf // actual s3 container DBConf database.DBConf fakeServer *FakeServer MQConf broker.MQConf @@ -54,18 +54,16 @@ func (s *ProxyTests) SetupTest() { s.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - s.S3Fakeconf = storage.S3Conf{ - URL: "http://127.0.0.1", - Port: 9024, + s.S3Fakeconf = config.S3InboxConf{ + Endpoint: "http://127.0.0.1:9024", AccessKey: "someAccess", SecretKey: "someSecret", Bucket: "buckbuck", Region: "us-east-1", } - s.S3conf = storage.S3Conf{ - URL: "http://127.0.0.1", - Port: s3Port, + s.S3conf = config.S3InboxConf{ + Endpoint: fmt.Sprintf("http://127.0.0.1:%d", s3Port), AccessKey: "access", SecretKey: "secretKey", Bucket: "buckbuck", @@ -125,15 +123,15 @@ func (s *ProxyTests) SetupTest() { s.T().FailNow() } - s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) + s3cfg, err := s3config.LoadDefaultConfig(context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) if err != nil { s.FailNow("bad") } s3Client := s3.NewFromConfig( s3cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", s.S3conf.URL, s.S3conf.Port)) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.URL, "http:") + o.BaseEndpoint = aws.String(s.S3conf.Endpoint) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.Endpoint, "http:") o.Region = s.S3conf.Region o.UsePathStyle = true }, @@ -227,7 +225,7 @@ func (s *ProxyTests) TestServeHTTP_disallowed() { r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() - log.Warnf("using proxy on port %d", s.S3Fakeconf.Port) + log.Warnf("using proxy at %s", s.S3Fakeconf.Endpoint) // Remove bucket disallowed r.Method = "DELETE" r.URL, _ = url.Parse("/asdf/") @@ -286,8 +284,8 @@ func (s *ProxyTests) TestServeHTTP_disallowed() { } func (s *ProxyTests) TestServeHTTPS3Unresponsive() { - s3conf := storage.S3Conf{ - URL: "http://localhost:40211", + s3conf := config.S3InboxConf{ + Endpoint: "http://localhost:40211", AccessKey: "someAccess", SecretKey: "someSecret", Bucket: "buckbuck", @@ -531,7 +529,7 @@ func (s *ProxyTests) TestDatabaseConnection() { err = db.DB.QueryRow(query, anonymFilename).Scan(&fileID, &location) assert.Nil(s.T(), err, "Failed to query database") assert.NotNil(s.T(), fileID, "File not found in database") - assert.Equal(s.T(), fmt.Sprintf("%s:%d/%s", s.S3Fakeconf.URL, s.S3Fakeconf.Port, s.S3Fakeconf.Bucket), location) + assert.Equal(s.T(), fmt.Sprintf("%s/%s", s.S3Fakeconf.Endpoint, s.S3Fakeconf.Bucket), location) // Check that the "registered" status is in the database for this file for _, status := range []string{"registered", "uploaded"} { @@ -603,7 +601,7 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { // Unaccessible S3 (wrong port) proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) - proxy.s3.Port = 1111 + proxy.s3Conf.Endpoint = "http://127.0.0.1:1111" res, err := proxy.checkFileExists("nonexistingfilepath") assert.False(s.T(), res) @@ -611,8 +609,8 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { assert.Contains(s.T(), err.Error(), "S3: HeadObject") // Bad access key gives 403 - proxy.s3.Port = s.S3conf.Port - proxy.s3.AccessKey = "invaild" + proxy.s3Conf.Endpoint = s.S3conf.Endpoint + proxy.s3Conf.AccessKey = "invaild" res, err = proxy.checkFileExists("nonexistingfilepath") assert.False(s.T(), res) assert.NotNil(s.T(), err) @@ -654,11 +652,11 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { assert.NotNil(s.T(), fileID) // Detect autentication failure - p.s3.AccessKey = "badKey" + p.s3Conf.AccessKey = "badKey" assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) // Detect unresponsive backend service - p.s3.Port = 1234 + p.s3Conf.Endpoint = "http://127.0.0.1:1234" assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) } @@ -680,15 +678,15 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) - s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) + s3cfg, err := s3config.LoadDefaultConfig(context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) if err != nil { s.FailNow("bad") } s3Client := s3.NewFromConfig( s3cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", s.S3conf.URL, s.S3conf.Port)) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.URL, "http:") + o.BaseEndpoint = aws.String(s.S3conf.Endpoint) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.Endpoint, "http:") o.Region = s.S3conf.Region o.UsePathStyle = true }, diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index f3c2caf59b..5821c27ab3 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -100,7 +100,7 @@ func main() { } } router := mux.NewRouter() - proxy := NewProxy(conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy) + proxy := NewProxy(conf.S3Inbox, auth, messenger, sdaDB, tlsProxy) router.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") router.HandleFunc("/health", proxy.CheckHealth) router.PathPrefix("/").Handler(proxy) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 25c03cbd96..5a715cb73b 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -43,6 +43,7 @@ type Config struct { Inbox storage.Conf Backup storage.Conf Server ServerConfig + S3Inbox S3InboxConf API APIConf Notify SMTPConf Orchestrator OrchestratorConf @@ -94,6 +95,15 @@ type SyncAPIConf struct { MappingRouting string `default:"mappings"` } +type S3InboxConf struct { + Endpoint string + AccessKey string + SecretKey string + Bucket string + Region string + CAcert string + ReadyPath string +} type APIConf struct { RBACpolicy []byte CACert string @@ -393,12 +403,12 @@ func NewConfig(app string) (*Config, error) { "broker.user", "broker.password", "broker.routingkey", - "inbox.url", - "inbox.accesskey", - "inbox.secretkey", - "inbox.bucket", + "s3inbox.endpoint", + "s3inbox.access_key", + "s3inbox.secret_key", + "s3inbox.bucket", + "s3inbox.region", } - viper.Set("inbox.type", S3) case "sync": requiredConfVars = []string{ "broker.host", @@ -672,7 +682,13 @@ func NewConfig(app string) (*Config, error) { return nil, err } - c.configInbox() + c.S3Inbox.Endpoint = viper.GetString("s3inbox.endpoint") + c.S3Inbox.AccessKey = viper.GetString("s3inbox.access_key") + c.S3Inbox.SecretKey = viper.GetString("s3inbox.secret_key") + c.S3Inbox.Bucket = viper.GetString("s3inbox.bucket") + c.S3Inbox.Region = viper.GetString("s3inbox.region") + c.S3Inbox.CAcert = viper.GetString("s3inbox.ca_cert") + c.S3Inbox.ReadyPath = viper.GetString("s3inbox.ready_path") err = c.configServer() if err != nil { From 0cb12432d0f65026f23f71418675fbc3b0fe7b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 16:06:42 +0100 Subject: [PATCH 146/530] feat(s3inbox): update requestInfo func to utilise HeadObjectInput to get ETag and size instead of ListObjectsV2 with prefix(which could in theory return the wrong information if another file has the current file name as a prefix) --- sda/cmd/s3inbox/proxy.go | 15 +++++++++------ sda/cmd/s3inbox/proxy_test.go | 23 ++++++++++++++++++----- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index a5d5f82f4b..50125ca760 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -537,20 +537,23 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { return "", 0, err } - input := &s3.ListObjectsV2Input{ - Bucket: aws.String(p.s3Conf.Bucket), - MaxKeys: aws.Int32(1), - Prefix: aws.String(filePath), + input := &s3.HeadObjectInput{ + Bucket: aws.String(p.s3Conf.Bucket), + Key: aws.String(filePath), } - result, err := client.ListObjectsV2(context.TODO(), input) + result, err := client.HeadObject(context.TODO(), input) if err != nil { log.Debug(err.Error()) return "", 0, err } - return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.ReplaceAll(*result.Contents[0].ETag, "\"", "")))), *result.Contents[0].Size, nil + if result == nil || result.ETag == nil || result.ContentLength == nil { + return "", 0, errors.New("unexpected response from s3, HeadObject response contains nil information") + } + + return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.ReplaceAll(*result.ETag, "\"", "")))), *result.ContentLength, nil } func newS3Client(conf config.S3InboxConf) (*s3.Client, error) { diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 2285268bec..71db501ac0 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -157,9 +157,10 @@ func (s *ProxyTests) TearDownTest() { } type FakeServer struct { - ts *httptest.Server - resp string - pinged bool + ts *httptest.Server + headHeaders map[string]string + resp string + pinged bool } func startFakeServer(port string) *FakeServer { @@ -171,9 +172,15 @@ func startFakeServer(port string) *FakeServer { f := FakeServer{} foo := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { f.pinged = true - log.Warnf("hello fake will return %s", f.resp) + if f.headHeaders != nil && r.Method == "HEAD" { + for k, v := range f.headHeaders { + w.Header().Set(k, v) + } + w.WriteHeader(http.StatusOK) + + return + } if f.resp != "" { - log.Warnf("fake writes %s", f.resp) _, _ = fmt.Fprint(w, f.resp) } }) @@ -316,6 +323,7 @@ func (s *ProxyTests) TestServeHTTP_MQConnectionClosed() { r, _ := http.NewRequest("PUT", "/dummy/connectionclosed-file", nil) w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} proxy.allowedResponse(w, r, s.token) assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose assert.False(s.T(), proxy.messenger.Connection.IsClosed()) @@ -334,6 +342,7 @@ func (s *ProxyTests) TestServeHTTP_MQChannelClosed() { r, _ := http.NewRequest("PUT", "/dummy/channelclosed-file", nil) w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} proxy.allowedResponse(w, r, s.token) assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose assert.False(s.T(), proxy.messenger.Channel.IsClosed()) @@ -353,6 +362,7 @@ func (s *ProxyTests) TestServeHTTP_MQ_Unavailable() { r, _ := http.NewRequest("PUT", "/dummy/mqunavailable-file", nil) w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} proxy.allowedResponse(w, r, s.token) assert.Equal(s.T(), 500, w.Result().StatusCode) // nolint:bodyclose } @@ -378,6 +388,7 @@ func (s *ProxyTests) TestServeHTTP_allowed() { r, err = http.NewRequest("PUT", "/dummy/file", nil) assert.NoError(s.T(), err) s.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\";", "Content-Length": "5"} proxy.allowedResponse(w, r, s.token) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) @@ -472,6 +483,7 @@ func (s *ProxyTests) TestMessageFormatting() { // start proxy that denies everything proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" + s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "1234"} msg, err := proxy.CreateMessageFromRequest(r, claims, "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) @@ -512,6 +524,7 @@ func (s *ProxyTests) TestDatabaseConnection() { r, _ := http.NewRequest("PUT", filename, stringReader) w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} proxy.allowedResponse(w, r, s.token) res := w.Result() defer res.Body.Close() From 3da220616b365ea2e7b2162f0bb7c4461d9c30d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 12:55:04 +0100 Subject: [PATCH 147/530] feat(s3inbox): update init logic to not use storage(v1) when creating client and checking that inbox bucket exists, also fix fataling since no point in sending to channel since listener hasnt started and it panics afterwards anyway --- sda/cmd/s3inbox/s3inbox.go | 61 ++++++++++++++++++++++---------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 5821c27ab3..72a91d231e 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -1,19 +1,24 @@ package main import ( + "context" + "errors" + "fmt" "net/http" "os" "os/signal" "syscall" "time" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" "github.com/gorilla/mux" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" - "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" @@ -33,51 +38,37 @@ func main() { conf, err := config.NewConfig("s3inbox") if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("failed to config due to: %v", err) } tlsProxy, err := config.TLSConfigProxy(conf) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("failed to setup tls config due to: %v", err) } sdaDB, err := database.NewSDAdb(conf.Database) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("failed to init new db due to: %v", err) } if sdaDB.Version < 23 { - log.Error("database schema v23 is required") - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("database schema v23 is required") } log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - - s3, err := storage.NewS3Client(conf.Inbox.S3) + log.Println(conf.S3Inbox.Endpoint + "/BUCKET == " + conf.S3Inbox.Bucket) + s3, err := newS3Client(conf.S3Inbox) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("failed to init new S3 client due to: %v", err) } - err = storage.CheckS3Bucket(conf.Inbox.S3.Bucket, s3) + err = checkS3Bucket(conf.S3Inbox.Bucket, s3) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("failed to check if inbox bucket exists due to: %v", err) } messenger, err := broker.NewMQ(conf.Broker) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("failed to init broker due to: %v", err) } go func() { @@ -124,3 +115,23 @@ func main() { } } } + +func checkS3Bucket(bucket string, s3Client *s3.Client) error { + _, err := s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: &bucket}) + if err != nil { + var apiErr smithy.APIError + if errors.As(err, &apiErr) { + var bae *types.BucketAlreadyExists + var baoby *types.BucketAlreadyOwnedByYou + if errors.As(err, &bae) || errors.As(err, &baoby) { + return nil + } + + return fmt.Errorf("unexpected issue while creating bucket: %s", err.Error()) + } + + return fmt.Errorf("verifying bucket failed, check S3 configuration: %s", err.Error()) + } + + return nil +} From 041936a9443ff6efd640346d7c4f06920fdb2562 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 16 Jan 2026 09:43:06 +0100 Subject: [PATCH 148/530] feat(s3inbox): update s3inbox.md to detail the now expected storage configuration --- sda/cmd/s3inbox/s3inbox.md | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.md b/sda/cmd/s3inbox/s3inbox.md index 15d4bfe90a..57af98add9 100644 --- a/sda/cmd/s3inbox/s3inbox.md +++ b/sda/cmd/s3inbox/s3inbox.md @@ -83,17 +83,15 @@ These settings control how verify connects to the RabbitMQ message broker. ### Storage settings -- `INBOX_TYPE`: Valid value is `S3` -- `INBOX_URL`: URL to the S3 system -- `INBOX_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, +- `S3INBOX_ENDPOINT`: Endpoint to the S3 system(including port) +- `S3INBOX_ACCESS_KEY`: The S3 access and secret key are used to authenticate to S3, [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `INBOX_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, +- `S3INBOX_SECRET_KEY`: The S3 access and secret key are used to authenticate to S3, [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `INBOX_BUCKET`: The S3 bucket to use as the storage root -- `INBOX_PORT`: S3 connection port (default: `443`) -- `INBOX_REGION`: S3 region (default: `us-east-1`) -- `INBOX_CHUNKSIZE`: S3 chunk size for multipart uploads. -- `INBOX_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity +- `S3INBOX_BUCKET`: The S3 bucket to use as the storage root +- `S3INBOX_REGION`: S3 region +- `S3INBOX_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity +- `S3INBOX_READY_PATH`: Path to use when pinging to check if the s3 bucket is healthy and ready for requests, final URL will be S3INBOX_ENDPOINT + S3INBOX_READY_PATH when calling ### Logging settings From 193a621616bded25ae3e41e135182df9b34dcaf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 11:47:12 +0100 Subject: [PATCH 149/530] feat(s3inbox): update main shutdown and error logic --- sda/cmd/s3inbox/s3inbox.go | 91 ++++++++++++++++++++++---------------- 1 file changed, 52 insertions(+), 39 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 72a91d231e..c136ec95ce 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -25,73 +25,68 @@ import ( ) func main() { - sigc := make(chan os.Signal, 5) - signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - - // Create a function to handle panic and exit gracefully - defer func() { - if err := recover(); err != nil { - log.Errorf("Could not recover from %v", err) - log.Fatal("Could not recover, exiting") - } - }() - + if err := run(); err != nil { + log.Fatal(err) + } +} +func run() error { conf, err := config.NewConfig("s3inbox") if err != nil { - log.Fatalf("failed to config due to: %v", err) + return fmt.Errorf("failed to load config due to: %v", err) } tlsProxy, err := config.TLSConfigProxy(conf) if err != nil { - log.Fatalf("failed to setup tls config due to: %v", err) + return fmt.Errorf("failed to setup tls config due to: %v", err) } sdaDB, err := database.NewSDAdb(conf.Database) if err != nil { - log.Fatalf("failed to init new db due to: %v", err) + return fmt.Errorf("failed to initialize sda db due to: %v", err) } + defer sdaDB.Close() if sdaDB.Version < 23 { - log.Fatalf("database schema v23 is required") + return errors.New("database schema v23 is required") } log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) log.Println(conf.S3Inbox.Endpoint + "/BUCKET == " + conf.S3Inbox.Bucket) - s3, err := newS3Client(conf.S3Inbox) + s3Client, err := newS3Client(conf.S3Inbox) if err != nil { - log.Fatalf("failed to init new S3 client due to: %v", err) + return fmt.Errorf("failed to initialize new S3 client due to: %v", err) } - err = checkS3Bucket(conf.S3Inbox.Bucket, s3) - if err != nil { - log.Fatalf("failed to check if inbox bucket exists due to: %v", err) + if err = checkS3Bucket(conf.S3Inbox.Bucket, s3Client); err != nil { + return fmt.Errorf("failed to check if inbox bucket exists due to: %v", err) } - messenger, err := broker.NewMQ(conf.Broker) + mqBroker, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatalf("failed to init broker due to: %v", err) + return fmt.Errorf("failed to initialize broker due to: %v", err) } - - go func() { - <-sigc - sdaDB.Close() - messenger.Channel.Close() - messenger.Connection.Close() - os.Exit(1) + defer func() { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } }() + auth := userauth.NewValidateFromToken(jwk.NewSet()) // Load keys for JWT verification if conf.Server.Jwtpubkeyurl != "" { if err := auth.FetchJwtPubKeyURL(conf.Server.Jwtpubkeyurl); err != nil { - log.Panicf("Error while getting key %s: %v", conf.Server.Jwtpubkeyurl, err) + return fmt.Errorf("failed to read jwt pub key from url: %s, due to %v", conf.Server.Jwtpubkeyurl, err) } } if conf.Server.Jwtpubkeypath != "" { if err := auth.ReadJwtPubKeyPath(conf.Server.Jwtpubkeypath); err != nil { - log.Panicf("Error while getting key %s: %v", conf.Server.Jwtpubkeypath, err) + return fmt.Errorf("failed to read jwt pub key from path: %s, due to %v", conf.Server.Jwtpubkeypath, err) } } router := mux.NewRouter() - proxy := NewProxy(conf.S3Inbox, auth, messenger, sdaDB, tlsProxy) + proxy := NewProxy(conf.S3Inbox, auth, mqBroker, sdaDB, tlsProxy) router.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") router.HandleFunc("/health", proxy.CheckHealth) router.PathPrefix("/").Handler(proxy) @@ -105,15 +100,33 @@ func main() { Handler: router, } - if conf.Server.Cert != "" && conf.Server.Key != "" { - if err := server.ListenAndServeTLS(conf.Server.Cert, conf.Server.Key); err != nil { - panic(err) - } - } else { - if err := server.ListenAndServe(); err != nil { - panic(err) + serverErr := make(chan error, 1) + go func() { + if conf.Server.Cert != "" && conf.Server.Key != "" { + if err := server.ListenAndServeTLS(conf.Server.Cert, conf.Server.Key); err != nil { + serverErr <- fmt.Errorf("failed to start https server, due to: %v", err) + } + } else { + if err := server.ListenAndServe(); err != nil { + serverErr <- fmt.Errorf("failed to start http server, due to: %v", err) + } } + }() + + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + select { + case <-sigc: + case err := <-serverErr: + return err } + + if err := server.Shutdown(context.Background()); err != nil { + log.Errorf("failed to close http/https server due to: %v", err) + } + + return nil } func checkS3Bucket(bucket string, s3Client *s3.Client) error { From e76a87f54b286942b78add724505ed77280b8c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 27 Jan 2026 10:20:26 +0100 Subject: [PATCH 150/530] feat(s3inbox): add s3inbox config to .github/integration/sda/config.yaml (as its used in unit tests), update unit tests to populate required configs, move s3 tls config func to s3inbox --- .github/integration/sda/config.yaml | 8 +++++ sda/cmd/s3inbox/s3inbox.go | 32 +++++++++++++++++++- sda/internal/config/config.go | 29 ------------------ sda/internal/config/config_test.go | 46 ++++++++++++++++++++--------- 4 files changed, 71 insertions(+), 44 deletions(-) diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 82733287c9..9768feacb9 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -52,6 +52,14 @@ inbox: bucket: "inbox" region: "us-east-1" +s3inbox: + endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + bucket: "inbox" + region: "us-east-1" + ready_path: "/minio/health/ready" + broker: host: "rabbitmq" port: "5672" diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index c136ec95ce..e417d57078 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -2,6 +2,8 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" "errors" "fmt" "net/http" @@ -35,7 +37,7 @@ func run() error { return fmt.Errorf("failed to load config due to: %v", err) } - tlsProxy, err := config.TLSConfigProxy(conf) + tlsProxy, err := configTLS(conf.S3Inbox) if err != nil { return fmt.Errorf("failed to setup tls config due to: %v", err) } @@ -148,3 +150,31 @@ func checkS3Bucket(bucket string, s3Client *s3.Client) error { return nil } + +func configTLS(c config.S3InboxConf) (*tls.Config, error) { + cfg := new(tls.Config) + + log.Debug("setting up TLS for S3 connection") + + // Read system CAs + systemCAs, err := x509.SystemCertPool() + if err != nil { + log.Errorf("failed to read system CAs: %v", err) + + return nil, err + } + + cfg.RootCAs = systemCAs + + if c.CAcert != "" { + cacert, e := os.ReadFile(c.CAcert) // #nosec this file comes from our configuration + if e != nil { + return nil, fmt.Errorf("failed to append %q to RootCAs: %v", cacert, e) + } + if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { + log.Debug("no certs appended, using system certs only") + } + } + + return cfg, nil +} diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 5a715cb73b..9565ab3cdc 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -1324,32 +1324,3 @@ func TLSConfigBroker(c *Config) (*tls.Config, error) { return cfg, nil } - -// TLSConfigProxy is a helper method to setup TLS for the S3 backend. -func TLSConfigProxy(c *Config) (*tls.Config, error) { - cfg := new(tls.Config) - - log.Debug("setting up TLS for S3 connection") - - // Read system CAs - systemCAs, err := x509.SystemCertPool() - if err != nil { - log.Errorf("failed to read system CAs: %v", err) - - return nil, err - } - - cfg.RootCAs = systemCAs - - if c.Inbox.S3.CAcert != "" { - cacert, e := os.ReadFile(c.Inbox.S3.CAcert) // #nosec this file comes from our configuration - if e != nil { - return nil, fmt.Errorf("failed to append %q to RootCAs: %v", cacert, e) - } - if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { - log.Debug("no certs appended, using system certs only") - } - } - - return cfg, nil -} diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 04f2003891..5dc76d967c 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -129,6 +129,12 @@ func (ts *ConfigTestSuite) TestNoConfig() { } func (ts *ConfigTestSuite) TestMissingRequiredConfVar() { + viper.Set("s3inbox.endpoint", "mock-value") + viper.Set("s3inbox.access_key", "mock-value") + viper.Set("s3inbox.secret_key", "mock-value") + viper.Set("s3inbox.bucket", "mock-value") + viper.Set("s3inbox.region", "mock-value") + for _, requiredConfVar := range requiredConfVars { requiredConfVarValue := viper.Get(requiredConfVar) viper.Set(requiredConfVar, nil) @@ -143,17 +149,28 @@ func (ts *ConfigTestSuite) TestMissingRequiredConfVar() { } func (ts *ConfigTestSuite) TestConfigS3Storage() { + viper.Set("s3inbox.endpoint", "testurl") + viper.Set("s3inbox.access_key", "testaccess") + viper.Set("s3inbox.secret_key", "testsecret") + viper.Set("s3inbox.bucket", "testbucket") + viper.Set("s3inbox.region", "testregion") config, err := NewConfig("s3inbox") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) assert.NotNil(ts.T(), config.Inbox.S3) - assert.Equal(ts.T(), "testurl", config.Inbox.S3.URL) - assert.Equal(ts.T(), "testaccess", config.Inbox.S3.AccessKey) - assert.Equal(ts.T(), "testsecret", config.Inbox.S3.SecretKey) - assert.Equal(ts.T(), "testbucket", config.Inbox.S3.Bucket) + assert.Equal(ts.T(), "testurl", config.S3Inbox.Endpoint) + assert.Equal(ts.T(), "testaccess", config.S3Inbox.AccessKey) + assert.Equal(ts.T(), "testsecret", config.S3Inbox.SecretKey) + assert.Equal(ts.T(), "testbucket", config.S3Inbox.Bucket) + assert.Equal(ts.T(), "testregion", config.S3Inbox.Region) } func (ts *ConfigTestSuite) TestConfigBroker() { + viper.Set("s3inbox.endpoint", "mock-value") + viper.Set("s3inbox.access_key", "mock-value") + viper.Set("s3inbox.secret_key", "mock-value") + viper.Set("s3inbox.bucket", "mock-value") + viper.Set("s3inbox.region", "mock-value") config, err := NewConfig("s3inbox") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) @@ -181,6 +198,12 @@ func (ts *ConfigTestSuite) TestTLSConfigBroker() { viper.Set("broker.serverName", "broker") viper.Set("broker.ssl", true) viper.Set("broker.cacert", certPath+"/ca.crt") + viper.Set("s3inbox.endpoint", "mock-value") + viper.Set("s3inbox.access_key", "mock-value") + viper.Set("s3inbox.secret_key", "mock-value") + viper.Set("s3inbox.bucket", "mock-value") + viper.Set("s3inbox.region", "mock-value") + config, err := NewConfig("s3inbox") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) @@ -208,18 +231,13 @@ func (ts *ConfigTestSuite) TestTLSConfigBroker() { assert.Error(ts.T(), err) } -func (ts *ConfigTestSuite) TestTLSConfigProxy() { - viper.Set("inbox.cacert", certPath+"/ca.crt") - config, err := NewConfig("s3inbox") - assert.NotNil(ts.T(), config) - assert.NoError(ts.T(), err) - tlsProxy, err := TLSConfigProxy(config) - assert.NotNil(ts.T(), tlsProxy) - assert.NoError(ts.T(), err) -} - func (ts *ConfigTestSuite) TestDefaultLogLevel() { viper.Set("log.level", "test") + viper.Set("s3inbox.endpoint", "mock-value") + viper.Set("s3inbox.access_key", "mock-value") + viper.Set("s3inbox.secret_key", "mock-value") + viper.Set("s3inbox.bucket", "mock-value") + viper.Set("s3inbox.region", "mock-value") config, err := NewConfig("s3inbox") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) From df22b219133e239ad45d368a8e7b7fcfc46b65a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 08:53:13 +0100 Subject: [PATCH 151/530] feat(s3inbox): defer the http/https server shutdown to ensure it closed incase of error --- sda/cmd/s3inbox/s3inbox.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index e417d57078..3df25b2615 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -114,21 +114,21 @@ func run() error { } } }() + defer func() { + if err := server.Shutdown(context.Background()); err != nil { + log.Errorf("failed to close http/https server due to: %v", err) + } + }() sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { case <-sigc: + return nil case err := <-serverErr: return err } - - if err := server.Shutdown(context.Background()); err != nil { - log.Errorf("failed to close http/https server due to: %v", err) - } - - return nil } func checkS3Bucket(bucket string, s3Client *s3.Client) error { From 1006b5e15fc80f4a756a116a4f53572345f24083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 11:33:59 +0100 Subject: [PATCH 152/530] feat(s3inbox): remove debug log statement --- sda/cmd/s3inbox/s3inbox.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 3df25b2615..46c4c624e8 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -52,7 +52,6 @@ func run() error { } log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - log.Println(conf.S3Inbox.Endpoint + "/BUCKET == " + conf.S3Inbox.Bucket) s3Client, err := newS3Client(conf.S3Inbox) if err != nil { return fmt.Errorf("failed to initialize new S3 client due to: %v", err) From e478f14cccbfc6f55f41d0ee39c4a53d418625b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 11:40:16 +0100 Subject: [PATCH 153/530] feat(s3inbox): remove duplicated code for tls config setup --- sda/cmd/s3inbox/proxy.go | 33 ++++++--------------------------- sda/cmd/s3inbox/s3inbox.go | 9 +++------ 2 files changed, 9 insertions(+), 33 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 50125ca760..7ca9677449 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -4,7 +4,6 @@ import ( "context" "crypto/sha256" "crypto/tls" - "crypto/x509" "encoding/json" "encoding/xml" "errors" @@ -12,7 +11,6 @@ import ( "io" "net/http" "net/url" - "os" "regexp" "strconv" "strings" @@ -557,10 +555,15 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { } func newS3Client(conf config.S3InboxConf) (*s3.Client, error) { + tlsConfig, err := configTLS(conf) + if err != nil { + return nil, err + } + s3cfg, err := s3config.LoadDefaultConfig( context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.SecretKey, "")), - s3config.WithHTTPClient(&http.Client{Transport: transportConfigS3(conf)}), + s3config.WithHTTPClient(&http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig, ForceAttemptHTTP2: true}}), ) if err != nil { return nil, err @@ -580,30 +583,6 @@ func newS3Client(conf config.S3InboxConf) (*s3.Client, error) { return s3Client, nil } -func transportConfigS3(conf config.S3InboxConf) http.RoundTripper { - cfg := new(tls.Config) - - // Read system CAs - systemCAs, err := x509.SystemCertPool() - if err != nil { - log.Errorf("failed to read system CAs: %v, using an empty pool as base", err) - systemCAs = x509.NewCertPool() - } - - cfg.RootCAs = systemCAs - - if conf.CAcert != "" { - cacert, e := os.ReadFile(conf.CAcert) // #nosec this file comes from our config - if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main - } - if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { - log.Debug("no certs appended, using system certs only") - } - } - - return &http.Transport{TLSClientConfig: cfg, ForceAttemptHTTP2: true} -} // checkFileExists makes a request to the S3 to check whether the file already // is uploaded. Returns a bool indicating whether the file was found. diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 46c4c624e8..623d0fad76 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -153,20 +153,17 @@ func checkS3Bucket(bucket string, s3Client *s3.Client) error { func configTLS(c config.S3InboxConf) (*tls.Config, error) { cfg := new(tls.Config) - log.Debug("setting up TLS for S3 connection") - // Read system CAs systemCAs, err := x509.SystemCertPool() if err != nil { - log.Errorf("failed to read system CAs: %v", err) - - return nil, err + log.Errorf("failed to read system CAs: %v, using an empty pool as base", err) + systemCAs = x509.NewCertPool() } cfg.RootCAs = systemCAs if c.CAcert != "" { - cacert, e := os.ReadFile(c.CAcert) // #nosec this file comes from our configuration + cacert, e := os.ReadFile(c.CAcert) if e != nil { return nil, fmt.Errorf("failed to append %q to RootCAs: %v", cacert, e) } From 017e0f16bc4463a8c11b3885c6cf7fee65c6d23e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 11:44:04 +0100 Subject: [PATCH 154/530] feat(s3inbox): create s3 client with ctx either from request or main, and use caller ctx when using s3Client to do s3 requests, add timeout on shutting down http/https server at 10 sec --- sda/cmd/s3inbox/proxy.go | 30 +++++++++++++++--------------- sda/cmd/s3inbox/proxy_test.go | 16 ++++++++-------- sda/cmd/s3inbox/s3inbox.go | 14 +++++++++----- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 7ca9677449..765171d9d7 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -239,7 +239,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw log.Debugf("resuming work on file with fileId: %v", p.fileIDs[fileIdentifier]) } - if err := p.storeObjectSizeInDB(rawFilepath, p.fileIDs[fileIdentifier]); err != nil { + if err := p.storeObjectSizeInDB(r.Context(), rawFilepath, p.fileIDs[fileIdentifier]); err != nil { log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) p.internalServerError(w, r, "storeObjectSizeInDB failed") @@ -508,7 +508,7 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anon checksum := Checksum{} var err error - checksum.Value, event.Filesize, err = p.requestInfo(r.URL.Path) + checksum.Value, event.Filesize, err = p.requestInfo(r.Context(), r.URL.Path) if err != nil { return event, fmt.Errorf("could not get checksum information: %s", err) } @@ -528,9 +528,9 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anon // RequestInfo is a function that makes a request to the S3 and collects // the etag and size information for the uploaded document -func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { +func (p *Proxy) requestInfo(ctx context.Context, fullPath string) (string, int64, error) { filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) - client, err := newS3Client(p.s3Conf) + client, err := newS3Client(ctx, p.s3Conf) if err != nil { return "", 0, err } @@ -540,7 +540,7 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { Key: aws.String(filePath), } - result, err := client.HeadObject(context.TODO(), input) + result, err := client.HeadObject(ctx, input) if err != nil { log.Debug(err.Error()) @@ -554,14 +554,14 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.ReplaceAll(*result.ETag, "\"", "")))), *result.ContentLength, nil } -func newS3Client(conf config.S3InboxConf) (*s3.Client, error) { +func newS3Client(ctx context.Context, conf config.S3InboxConf) (*s3.Client, error) { tlsConfig, err := configTLS(conf) if err != nil { return nil, err } - + s3cfg, err := s3config.LoadDefaultConfig( - context.TODO(), + ctx, s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.SecretKey, "")), s3config.WithHTTPClient(&http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig, ForceAttemptHTTP2: true}}), ) @@ -586,14 +586,14 @@ func newS3Client(conf config.S3InboxConf) (*s3.Client, error) { // checkFileExists makes a request to the S3 to check whether the file already // is uploaded. Returns a bool indicating whether the file was found. -func (p *Proxy) checkFileExists(fullPath string) (bool, error) { +func (p *Proxy) checkFileExists(ctx context.Context, fullPath string) (bool, error) { filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) - client, err := newS3Client(p.s3Conf) + client, err := newS3Client(ctx, p.s3Conf) if err != nil { return false, fmt.Errorf("could not connect to s3: %v", err) } - result, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + result, err := client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(filePath), }) @@ -608,7 +608,7 @@ func (p *Proxy) checkFileExists(fullPath string) (bool, error) { } func (p *Proxy) sendMessageOnOverwrite(fileID string, r *http.Request, rawFilepath string, token jwt.Token) error { - exist, err := p.checkFileExists(r.URL.Path) + exist, err := p.checkFileExists(r.Context(), r.URL.Path) if err != nil { return err } @@ -683,13 +683,13 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { } } -func (p *Proxy) storeObjectSizeInDB(path, fileID string) error { - client, err := newS3Client(p.s3Conf) +func (p *Proxy) storeObjectSizeInDB(ctx context.Context, path, fileID string) error { + client, err := newS3Client(ctx, p.s3Conf) if err != nil { return err } - o, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + o, err := client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(path), }) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 71db501ac0..75d94eea02 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -581,7 +581,7 @@ func (s *ProxyTests) TestCheckFileExists() { defer messenger.Connection.Close() proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) - res, err := proxy.checkFileExists("/dummy/file") + res, err := proxy.checkFileExists(context.TODO(), "/dummy/file") assert.True(s.T(), res) assert.Nil(s.T(), err) } @@ -597,7 +597,7 @@ func (s *ProxyTests) TestCheckFileExists_nonExistingFile() { defer messenger.Connection.Close() proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) - res, err := proxy.checkFileExists("nonexistingfilepath") + res, err := proxy.checkFileExists(context.TODO(), "nonexistingfilepath") assert.False(s.T(), res) assert.Nil(s.T(), err) } @@ -616,7 +616,7 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.s3Conf.Endpoint = "http://127.0.0.1:1111" - res, err := proxy.checkFileExists("nonexistingfilepath") + res, err := proxy.checkFileExists(context.TODO(), "nonexistingfilepath") assert.False(s.T(), res) assert.NotNil(s.T(), err) assert.Contains(s.T(), err.Error(), "S3: HeadObject") @@ -624,7 +624,7 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { // Bad access key gives 403 proxy.s3Conf.Endpoint = s.S3conf.Endpoint proxy.s3Conf.AccessKey = "invaild" - res, err = proxy.checkFileExists("nonexistingfilepath") + res, err = proxy.checkFileExists(context.TODO(), "nonexistingfilepath") assert.False(s.T(), res) assert.NotNil(s.T(), err) assert.Contains(s.T(), err.Error(), "StatusCode: 403") @@ -646,7 +646,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { assert.NotNil(s.T(), fileID) db.Close() - assert.NoError(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + assert.NoError(s.T(), p.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) } func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { @@ -666,11 +666,11 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { // Detect autentication failure p.s3Conf.AccessKey = "badKey" - assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + assert.Error(s.T(), p.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) // Detect unresponsive backend service p.s3Conf.Endpoint = "http://127.0.0.1:1234" - assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + assert.Error(s.T(), p.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) } // This test is intended to try to catch some issues we sometimes see when a query to the S3 backend @@ -714,7 +714,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { assert.NoError(s.T(), err) assert.NotNil(s.T(), output, output) - assert.NoError(s.T(), p.storeObjectSizeInDB("/test/new_file", fileID)) + assert.NoError(s.T(), p.storeObjectSizeInDB(context.TODO(), "/test/new_file", fileID)) const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" var objectSize int64 diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 623d0fad76..16b5e2330b 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -32,6 +32,8 @@ func main() { } } func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() conf, err := config.NewConfig("s3inbox") if err != nil { return fmt.Errorf("failed to load config due to: %v", err) @@ -52,12 +54,12 @@ func run() error { } log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - s3Client, err := newS3Client(conf.S3Inbox) + s3Client, err := newS3Client(ctx, conf.S3Inbox) if err != nil { return fmt.Errorf("failed to initialize new S3 client due to: %v", err) } - if err = checkS3Bucket(conf.S3Inbox.Bucket, s3Client); err != nil { + if err = checkS3Bucket(ctx, conf.S3Inbox.Bucket, s3Client); err != nil { return fmt.Errorf("failed to check if inbox bucket exists due to: %v", err) } @@ -114,9 +116,11 @@ func run() error { } }() defer func() { - if err := server.Shutdown(context.Background()); err != nil { + serverShutdownCtx, serverShutdownCancel := context.WithTimeout(ctx, 10*time.Second) + if err := server.Shutdown(serverShutdownCtx); err != nil { log.Errorf("failed to close http/https server due to: %v", err) } + serverShutdownCancel() }() sigc := make(chan os.Signal, 5) @@ -130,8 +134,8 @@ func run() error { } } -func checkS3Bucket(bucket string, s3Client *s3.Client) error { - _, err := s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: &bucket}) +func checkS3Bucket(ctx context.Context, bucket string, s3Client *s3.Client) error { + _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &bucket}) if err != nil { var apiErr smithy.APIError if errors.As(err, &apiErr) { From 53a8a8de3e2b8c0e6016a97271d441247985a07e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 11:55:28 +0100 Subject: [PATCH 155/530] feat(config): populate S3InboxConf by viper.UnmarshalKey instead of with viper.Get statements --- sda/cmd/s3inbox/s3inbox.go | 21 +++++++++------------ sda/internal/config/config.go | 24 ++++++++++-------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 16b5e2330b..76a70d8df8 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -13,7 +13,6 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" "github.com/gorilla/mux" @@ -135,20 +134,18 @@ func run() error { } func checkS3Bucket(ctx context.Context, bucket string, s3Client *s3.Client) error { - _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &bucket}) + _, err := s3Client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) if err != nil { var apiErr smithy.APIError if errors.As(err, &apiErr) { - var bae *types.BucketAlreadyExists - var baoby *types.BucketAlreadyOwnedByYou - if errors.As(err, &bae) || errors.As(err, &baoby) { - return nil + if apiErr.ErrorCode() == "NotFound" { + return fmt.Errorf("bucket: %s does not exists at the configured s3 endpoint", bucket) } - return fmt.Errorf("unexpected issue while creating bucket: %s", err.Error()) + return fmt.Errorf("unexpected issue while checking bucket: %s exists, due to %v", bucket, err) } - return fmt.Errorf("verifying bucket failed, check S3 configuration: %s", err.Error()) + return fmt.Errorf("verifying bucket failed, check S3 configuration: %v", err) } return nil @@ -166,12 +163,12 @@ func configTLS(c config.S3InboxConf) (*tls.Config, error) { cfg.RootCAs = systemCAs - if c.CAcert != "" { - cacert, e := os.ReadFile(c.CAcert) + if c.CaCert != "" { + caCert, e := os.ReadFile(c.CaCert) if e != nil { - return nil, fmt.Errorf("failed to append %q to RootCAs: %v", cacert, e) + return nil, fmt.Errorf("failed to append %q to RootCAs: %v", caCert, e) } - if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { + if ok := cfg.RootCAs.AppendCertsFromPEM(caCert); !ok { log.Debug("no certs appended, using system certs only") } } diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 9565ab3cdc..8c09e91b7e 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -96,13 +96,13 @@ type SyncAPIConf struct { } type S3InboxConf struct { - Endpoint string - AccessKey string - SecretKey string - Bucket string - Region string - CAcert string - ReadyPath string + Endpoint string `mapstructure:"endpoint"` + AccessKey string `mapstructure:"access_key"` + SecretKey string `mapstructure:"secret_key"` + Bucket string `mapstructure:"bucket"` + Region string `mapstructure:"region"` + CaCert string `mapstructure:"ca_cert"` + ReadyPath string `mapstructure:"ready_path"` } type APIConf struct { RBACpolicy []byte @@ -682,13 +682,9 @@ func NewConfig(app string) (*Config, error) { return nil, err } - c.S3Inbox.Endpoint = viper.GetString("s3inbox.endpoint") - c.S3Inbox.AccessKey = viper.GetString("s3inbox.access_key") - c.S3Inbox.SecretKey = viper.GetString("s3inbox.secret_key") - c.S3Inbox.Bucket = viper.GetString("s3inbox.bucket") - c.S3Inbox.Region = viper.GetString("s3inbox.region") - c.S3Inbox.CAcert = viper.GetString("s3inbox.ca_cert") - c.S3Inbox.ReadyPath = viper.GetString("s3inbox.ready_path") + if err := viper.UnmarshalKey("s3inbox", &c.S3Inbox); err != nil { + return nil, fmt.Errorf("failed to parse key configurations: %v", err) + } err = c.configServer() if err != nil { From c93a412d114cb28ed4b8b72e82246f8dfd47b125 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:17:38 +0100 Subject: [PATCH 156/530] feat(s3inbox): do not interpet http.ErrServerClosed as a server error --- sda/cmd/s3inbox/s3inbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 76a70d8df8..f70b24de97 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -105,11 +105,11 @@ func run() error { serverErr := make(chan error, 1) go func() { if conf.Server.Cert != "" && conf.Server.Key != "" { - if err := server.ListenAndServeTLS(conf.Server.Cert, conf.Server.Key); err != nil { + if err := server.ListenAndServeTLS(conf.Server.Cert, conf.Server.Key); err != nil && !errors.Is(err, http.ErrServerClosed) { serverErr <- fmt.Errorf("failed to start https server, due to: %v", err) } } else { - if err := server.ListenAndServe(); err != nil { + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { serverErr <- fmt.Errorf("failed to start http server, due to: %v", err) } } From 12b2ec58e1bd85e545122854cf152d24350a7ea0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:22:15 +0100 Subject: [PATCH 157/530] feat(s3inbox): update sigc chan buffer size to 1 instead of 5 as we just care that any of the signals triggered --- sda/cmd/s3inbox/s3inbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index f70b24de97..a90d5d4a74 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -122,7 +122,7 @@ func run() error { serverShutdownCancel() }() - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { From 5b06067449967c2f46c0fd71d43e5cf9a18279f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Feb 2026 13:40:01 +0100 Subject: [PATCH 158/530] feat(s3inbox): fix some typos in error messages from checkS3Bucket --- sda/cmd/s3inbox/s3inbox.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index a90d5d4a74..1978d52d7d 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -139,10 +139,10 @@ func checkS3Bucket(ctx context.Context, bucket string, s3Client *s3.Client) erro var apiErr smithy.APIError if errors.As(err, &apiErr) { if apiErr.ErrorCode() == "NotFound" { - return fmt.Errorf("bucket: %s does not exists at the configured s3 endpoint", bucket) + return fmt.Errorf("bucket: %s does not exist at the configured s3 endpoint", bucket) } - return fmt.Errorf("unexpected issue while checking bucket: %s exists, due to %v", bucket, err) + return fmt.Errorf("unexpected issue while checking that bucket: %s exists, due to %v", bucket, err) } return fmt.Errorf("verifying bucket failed, check S3 configuration: %v", err) From b212060cbaad7215d8d73b59d7d45aeb0209baf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:14:51 +0100 Subject: [PATCH 159/530] feat(s3inbox): update s3inbox.md to clarify storage config descriptions --- sda/cmd/s3inbox/s3inbox.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.md b/sda/cmd/s3inbox/s3inbox.md index 57af98add9..9b7d1335e6 100644 --- a/sda/cmd/s3inbox/s3inbox.md +++ b/sda/cmd/s3inbox/s3inbox.md @@ -83,14 +83,14 @@ These settings control how verify connects to the RabbitMQ message broker. ### Storage settings -- `S3INBOX_ENDPOINT`: Endpoint to the S3 system(including port) -- `S3INBOX_ACCESS_KEY`: The S3 access and secret key are used to authenticate to S3, +- `S3INBOX_ENDPOINT`: Endpoint to the S3 service including the port (e.g., https://s3.example.com:9000) +- `S3INBOX_ACCESS_KEY`: The S3 access key (username equivalent), used to authenticate to S3 [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `S3INBOX_SECRET_KEY`: The S3 access and secret key are used to authenticate to S3, +- `S3INBOX_SECRET_KEY`: The S3 access key (username equivalent), used to authenticate to S3 [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `S3INBOX_BUCKET`: The S3 bucket to use as the storage root +- `S3INBOX_BUCKET`: The S3 bucket to be used for storage of uploaded files when forwarding upload requests - `S3INBOX_REGION`: S3 region -- `S3INBOX_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity +- `S3INBOX_CACERT`: Path to the Certificate Authority (CA) certificate file for the storage system, this is only needed if the S3 server has a certificate signed by a private entity - `S3INBOX_READY_PATH`: Path to use when pinging to check if the s3 bucket is healthy and ready for requests, final URL will be S3INBOX_ENDPOINT + S3INBOX_READY_PATH when calling ### Logging settings From b9fd35f10d8ce5dfe25a5b45f5ff2e07aab0daa1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:21:23 +0100 Subject: [PATCH 160/530] feat(s3inbox): fix error msg when failing to read ca cert to log ca cert file path --- sda/cmd/s3inbox/s3inbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 1978d52d7d..0220a8ba61 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -166,7 +166,7 @@ func configTLS(c config.S3InboxConf) (*tls.Config, error) { if c.CaCert != "" { caCert, e := os.ReadFile(c.CaCert) if e != nil { - return nil, fmt.Errorf("failed to append %q to RootCAs: %v", caCert, e) + return nil, fmt.Errorf("failed to append %q to RootCAs: %v", c.CaCert, e) } if ok := cfg.RootCAs.AppendCertsFromPEM(caCert); !ok { log.Debug("no certs appended, using system certs only") From 026f2cfaca668979cd2ef2d130f8017d8be469c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:25:29 +0100 Subject: [PATCH 161/530] feat(s3inbox): update requestInfo to just strip quotes from ETag instead of taking the sha256 of it --- sda/cmd/s3inbox/proxy.go | 3 +-- sda/cmd/s3inbox/proxy_test.go | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 765171d9d7..14c430a654 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -2,7 +2,6 @@ package main import ( "context" - "crypto/sha256" "crypto/tls" "encoding/json" "encoding/xml" @@ -551,7 +550,7 @@ func (p *Proxy) requestInfo(ctx context.Context, fullPath string) (string, int64 return "", 0, errors.New("unexpected response from s3, HeadObject response contains nil information") } - return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.ReplaceAll(*result.ETag, "\"", "")))), *result.ContentLength, nil + return strings.Trim(*result.ETag, "\""), *result.ContentLength, nil } func newS3Client(ctx context.Context, conf config.S3InboxConf) (*s3.Client, error) { diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 75d94eea02..539cfc93f2 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -496,7 +496,7 @@ func (s *ProxyTests) TestMessageFormatting() { checksum := Checksum{} _ = json.Unmarshal(c, &checksum) assert.Equal(s.T(), "sha256", checksum.Type) - assert.Equal(s.T(), "5b233b981dc12e7ccf4c242b99c042b7842b73b956ad662e4fe0f8354151538b", checksum.Value) + assert.Equal(s.T(), "0a44282bd39178db9680f24813c41aec-1", checksum.Value) // Test single shot upload r.Method = "PUT" From 4f8734c4506f2b78f93b3c3f41aa8aec66aad1be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:28:21 +0100 Subject: [PATCH 162/530] feat(s3inbox): update defer closing of rabbit mq connection and channel to check for nil before close --- sda/cmd/s3inbox/s3inbox.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 0220a8ba61..9ebee2c6f1 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -67,11 +67,18 @@ func run() error { return fmt.Errorf("failed to initialize broker due to: %v", err) } defer func() { - if err := mqBroker.Channel.Close(); err != nil { - log.Errorf("failed to close mq broker channel due to: %v", err) + if mqBroker == nil { + return } - if err := mqBroker.Connection.Close(); err != nil { - log.Errorf("failed to close mq broker connection due to: %v", err) + if mqBroker.Channel != nil { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + } + if mqBroker.Connection != nil { + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } } }() From 89764720707e4fc2097c27cd229b6b5917c507c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 10:53:30 +0100 Subject: [PATCH 163/530] feat(s3inbox): update request handling to only create one s3 client per request instead of 3+ --- sda/cmd/s3inbox/proxy.go | 46 +++++++++++++---------------- sda/cmd/s3inbox/proxy_test.go | 54 ++++++++++++++++++++++------------- sda/cmd/s3inbox/s3inbox.go | 4 +-- 3 files changed, 56 insertions(+), 48 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 14c430a654..3a838fc158 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -170,6 +170,13 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw filePath: filePath, } + s3Client, err := newS3Client(r.Context(), p.s3Conf) + if err != nil { + reportError(http.StatusInternalServerError, fmt.Sprintf("failed to connect to s3, reason: %v", err), w) + + return + } + location := p.s3Conf.Endpoint + "/" + p.s3Conf.Bucket // if this is an upload request if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { @@ -185,7 +192,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // check if the file already exists, in that case send an overwrite message, // so that the FEGA portal is informed that a new version of the file exists. - err = p.sendMessageOnOverwrite(p.fileIDs[fileIdentifier], r, rawFilepath, token) + err = p.sendMessageOnOverwrite(r, s3Client, p.fileIDs[fileIdentifier], rawFilepath, token) if err != nil { p.internalServerError(w, r, err.Error()) @@ -205,7 +212,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") - message, err := p.CreateMessageFromRequest(r, token, anonymizedFilepath) + message, err := p.CreateMessageFromRequest(r, token, s3Client, anonymizedFilepath) if err != nil { p.internalServerError(w, r, err.Error()) @@ -238,7 +245,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw log.Debugf("resuming work on file with fileId: %v", p.fileIDs[fileIdentifier]) } - if err := p.storeObjectSizeInDB(r.Context(), rawFilepath, p.fileIDs[fileIdentifier]); err != nil { + if err := p.storeObjectSizeInDB(r.Context(), s3Client, rawFilepath, p.fileIDs[fileIdentifier]); err != nil { log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) p.internalServerError(w, r, "storeObjectSizeInDB failed") @@ -502,12 +509,12 @@ func (p *Proxy) detectRequestType(r *http.Request) S3RequestType { // CreateMessageFromRequest is a function that can take a http request and // figure out the correct rabbitmq message to send from it. -func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anonymizedFilepath string) (Event, error) { +func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, s3Client *s3.Client, anonymizedFilepath string) (Event, error) { event := Event{} checksum := Checksum{} var err error - checksum.Value, event.Filesize, err = p.requestInfo(r.Context(), r.URL.Path) + checksum.Value, event.Filesize, err = p.requestInfo(r.Context(), s3Client, r.URL.Path) if err != nil { return event, fmt.Errorf("could not get checksum information: %s", err) } @@ -527,19 +534,15 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anon // RequestInfo is a function that makes a request to the S3 and collects // the etag and size information for the uploaded document -func (p *Proxy) requestInfo(ctx context.Context, fullPath string) (string, int64, error) { +func (p *Proxy) requestInfo(ctx context.Context, s3Client *s3.Client, fullPath string) (string, int64, error) { filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) - client, err := newS3Client(ctx, p.s3Conf) - if err != nil { - return "", 0, err - } input := &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(filePath), } - result, err := client.HeadObject(ctx, input) + result, err := s3Client.HeadObject(ctx, input) if err != nil { log.Debug(err.Error()) @@ -585,14 +588,10 @@ func newS3Client(ctx context.Context, conf config.S3InboxConf) (*s3.Client, erro // checkFileExists makes a request to the S3 to check whether the file already // is uploaded. Returns a bool indicating whether the file was found. -func (p *Proxy) checkFileExists(ctx context.Context, fullPath string) (bool, error) { +func (p *Proxy) checkFileExists(ctx context.Context, s3Client *s3.Client, fullPath string) (bool, error) { filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) - client, err := newS3Client(ctx, p.s3Conf) - if err != nil { - return false, fmt.Errorf("could not connect to s3: %v", err) - } - result, err := client.HeadObject(ctx, &s3.HeadObjectInput{ + result, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(filePath), }) @@ -606,8 +605,8 @@ func (p *Proxy) checkFileExists(ctx context.Context, fullPath string) (bool, err return result != nil, err } -func (p *Proxy) sendMessageOnOverwrite(fileID string, r *http.Request, rawFilepath string, token jwt.Token) error { - exist, err := p.checkFileExists(r.Context(), r.URL.Path) +func (p *Proxy) sendMessageOnOverwrite(r *http.Request, s3Client *s3.Client, fileID, rawFilepath string, token jwt.Token) error { + exist, err := p.checkFileExists(r.Context(), s3Client, r.URL.Path) if err != nil { return err } @@ -682,13 +681,8 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { } } -func (p *Proxy) storeObjectSizeInDB(ctx context.Context, path, fileID string) error { - client, err := newS3Client(ctx, p.s3Conf) - if err != nil { - return err - } - - o, err := client.HeadObject(ctx, &s3.HeadObjectInput{ +func (p *Proxy) storeObjectSizeInDB(ctx context.Context, s3Client *s3.Client, path, fileID string) error { + o, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(path), }) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 539cfc93f2..72e65929e8 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -484,7 +484,9 @@ func (s *ProxyTests) TestMessageFormatting() { proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "1234"} - msg, err := proxy.CreateMessageFromRequest(r, claims, "new_file.txt") + s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) + msg, err := proxy.CreateMessageFromRequest(r, claims, s3Client, "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) @@ -500,7 +502,7 @@ func (s *ProxyTests) TestMessageFormatting() { // Test single shot upload r.Method = "PUT" - msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), "new_file.txt") + msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), s3Client, "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), "upload", msg.Operation) @@ -580,8 +582,9 @@ func (s *ProxyTests) TestCheckFileExists() { assert.NoError(s.T(), err) defer messenger.Connection.Close() proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) - - res, err := proxy.checkFileExists(context.TODO(), "/dummy/file") + s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) + res, err := proxy.checkFileExists(context.TODO(), s3Client, "/dummy/file") assert.True(s.T(), res) assert.Nil(s.T(), err) } @@ -596,8 +599,9 @@ func (s *ProxyTests) TestCheckFileExists_nonExistingFile() { assert.NoError(s.T(), err) defer messenger.Connection.Close() proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) - - res, err := proxy.checkFileExists(context.TODO(), "nonexistingfilepath") + s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) + res, err := proxy.checkFileExists(context.TODO(), s3Client, "nonexistingfilepath") assert.False(s.T(), res) assert.Nil(s.T(), err) } @@ -615,8 +619,9 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { // Unaccessible S3 (wrong port) proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.s3Conf.Endpoint = "http://127.0.0.1:1111" - - res, err := proxy.checkFileExists(context.TODO(), "nonexistingfilepath") + s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) + res, err := proxy.checkFileExists(context.TODO(), s3Client, "nonexistingfilepath") assert.False(s.T(), res) assert.NotNil(s.T(), err) assert.Contains(s.T(), err.Error(), "S3: HeadObject") @@ -624,7 +629,9 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { // Bad access key gives 403 proxy.s3Conf.Endpoint = s.S3conf.Endpoint proxy.s3Conf.AccessKey = "invaild" - res, err = proxy.checkFileExists(context.TODO(), "nonexistingfilepath") + s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) + res, err = proxy.checkFileExists(context.TODO(), s3Client, "nonexistingfilepath") assert.False(s.T(), res) assert.NotNil(s.T(), err) assert.Contains(s.T(), err.Error(), "StatusCode: 403") @@ -638,15 +645,17 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { assert.NoError(s.T(), err) defer mq.Connection.Close() - p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) - p.database = db + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + proxy.database = db + s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) db.Close() - assert.NoError(s.T(), p.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) + assert.NoError(s.T(), proxy.storeObjectSizeInDB(context.TODO(), s3Client, "/dummy/file", fileID)) } func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { @@ -657,20 +666,25 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { assert.NoError(s.T(), err) defer mq.Connection.Close() - p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) - p.database = db - + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + proxy.database = db + s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) // Detect autentication failure - p.s3Conf.AccessKey = "badKey" - assert.Error(s.T(), p.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) + proxy.s3Conf.AccessKey = "badKey" + s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) + assert.Error(s.T(), proxy.storeObjectSizeInDB(context.TODO(), s3Client, "/dummy/file", fileID)) // Detect unresponsive backend service - p.s3Conf.Endpoint = "http://127.0.0.1:1234" - assert.Error(s.T(), p.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) + proxy.s3Conf.Endpoint = "http://127.0.0.1:1234" + s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) + assert.NoError(s.T(), err) + assert.Error(s.T(), proxy.storeObjectSizeInDB(context.TODO(), s3Client, "/dummy/file", fileID)) } // This test is intended to try to catch some issues we sometimes see when a query to the S3 backend @@ -714,7 +728,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { assert.NoError(s.T(), err) assert.NotNil(s.T(), output, output) - assert.NoError(s.T(), p.storeObjectSizeInDB(context.TODO(), "/test/new_file", fileID)) + assert.NoError(s.T(), p.storeObjectSizeInDB(context.TODO(), s3Client, "/test/new_file", fileID)) const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" var objectSize int64 diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 9ebee2c6f1..5df0868b43 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -58,7 +58,7 @@ func run() error { return fmt.Errorf("failed to initialize new S3 client due to: %v", err) } - if err = checkS3Bucket(ctx, conf.S3Inbox.Bucket, s3Client); err != nil { + if err = checkS3Bucket(ctx, s3Client, conf.S3Inbox.Bucket); err != nil { return fmt.Errorf("failed to check if inbox bucket exists due to: %v", err) } @@ -140,7 +140,7 @@ func run() error { } } -func checkS3Bucket(ctx context.Context, bucket string, s3Client *s3.Client) error { +func checkS3Bucket(ctx context.Context, s3Client *s3.Client, bucket string) error { _, err := s3Client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) if err != nil { var apiErr smithy.APIError From 6c7d7b716d25f147fecff21e448c0dc7486928f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:59:35 +0100 Subject: [PATCH 164/530] feat(verify): update to use storage/v2 instead of v1, add db func GetArchiveLocation, update db version check to v22 --- sda/cmd/verify/verify.go | 40 +++++++++++++++++++++++---- sda/internal/database/db_functions.go | 14 ++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 04e202f308..909b8009e1 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -4,6 +4,7 @@ package main import ( "bytes" + "context" "crypto/md5" "crypto/sha256" "encoding/json" @@ -17,12 +18,14 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" log "github.com/sirupsen/logrus" ) func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() forever := make(chan bool) conf, err := config.NewConfig("verify") if err != nil { @@ -36,12 +39,13 @@ func main() { if err != nil { log.Fatal(err) } - if db.Version < 21 { - log.Error("database schema v21 is required") + if db.Version < 23 { + log.Error("database schema v23 is required") db.Close() panic(err) } - archive, err := storage.NewBackend(conf.Archive) + + archiveReader, err := storage.NewReader(ctx, "archive") if err != nil { log.Fatal(err) } @@ -76,6 +80,7 @@ func main() { err) } for delivered := range messages { + ctx := context.Background() log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { @@ -160,8 +165,31 @@ func main() { continue } + archiveLocation, err := db.GetArchiveLocation(message.FileID) + if err != nil { + log.Errorf("failed to get archive location of file: %s, error: %v", message.FileID, err) + + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "GetArchiveLocation failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } + + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } + + continue + } + var file database.FileInfo - file.Size, err = archive.GetFileSize(message.ArchivePath, false) + file.Size, err = archiveReader.GetFileSize(ctx, archiveLocation, message.ArchivePath) if err != nil { //nolint:nestif log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { @@ -190,7 +218,7 @@ func main() { } archiveFileHash := sha256.New() - f, err := archive.NewFileReader(message.ArchivePath) + f, err := archiveReader.NewFileReader(ctx, archiveLocation, message.ArchivePath) if err != nil { log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 84551782d1..3c67806129 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -911,6 +911,20 @@ func (dbs *SDAdb) getArchivePath(stableID string) (string, error) { return archivePath, nil } +func (dbs *SDAdb) GetArchiveLocation(fileID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + const getFileID = "SELECT archive_location from sda.files WHERE id = $1;" + + var archiveLocation string + err := db.QueryRow(getFileID, fileID).Scan(&archiveLocation) + if err != nil { + return "", err + } + + return archiveLocation, nil +} + // SetSubmissionFileSize sets the submission file size for a file func (dbs *SDAdb) SetSubmissionFileSize(fileID string, submissionFileSize int64) error { var err error From 2fc4c239ed5dd0d78fac57e8d0fb220ebb1eed85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 15:09:59 +0100 Subject: [PATCH 165/530] feat(verify): fix some linting issues, and update README.md to now mention expected storage config --- sda/cmd/verify/verify.go | 24 +++++++++++++++++------- sda/cmd/verify/verify.md | 31 +++++++------------------------ 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 909b8009e1..0538a00996 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -29,29 +29,39 @@ func main() { forever := make(chan bool) conf, err := config.NewConfig("verify") if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } db, err := database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } if db.Version < 23 { log.Error("database schema v23 is required") - db.Close() - panic(err) + + return } archiveReader, err := storage.NewReader(ctx, "archive") if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } archiveKeyList, err := config.GetC4GHprivateKeys() if err != nil || len(archiveKeyList) == 0 { - log.Fatal("no C4GH private keys configured") + log.Error("no C4GH private keys configured") + + return } defer mq.Channel.Close() diff --git a/sda/cmd/verify/verify.md b/sda/cmd/verify/verify.md index 4fcb13c8f4..a2ada9f3f1 100644 --- a/sda/cmd/verify/verify.md +++ b/sda/cmd/verify/verify.md @@ -106,30 +106,13 @@ These settings control how `verify` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings - -Storage backend is defined by the `ARCHIVE_TYPE` variable. -Valid values for these options are `S3` or `POSIX` -(Defaults to `POSIX` on unknown values). - -The value of these variables define what other variables are read. -The same variables are available for all storage types, differing by prefix (`ARCHIVE_`) - -if `*_TYPE` is `S3` then the following variables are available: - -- `*_URL`: URL to the S3 system -- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, - [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, - [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_BUCKET`: The S3 bucket to use as the storage root -- `*_PORT`: S3 connection port (default: `443`) -- `*_REGION`: S3 region (default: `us-east-1`) -- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. -- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity - -and if `*_TYPE` is `POSIX`: - -- `*_LOCATION`: POSIX path to use as storage root +The verify service requires access to the "archive" storage, to configure such the following config is needed: +```yaml +storage: + archive: + ${STORAGE_IMPLEMENTATION}: +``` +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) ### Logging settings From aba47f56d92f05a75642b63313f8e088b4555afa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 16:46:53 +0100 Subject: [PATCH 166/530] feat(verify): update config to not require and load in storage v1 for verify service --- sda/internal/config/config.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 8c09e91b7e..a0b9dbd841 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -474,15 +474,6 @@ func NewConfig(app string) (*Config, error) { "db.database", } - switch viper.GetString("archive.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) - default: - return nil, errors.New("archive.type not set") - } - default: return nil, fmt.Errorf("application '%s' doesn't exist", app) } @@ -717,7 +708,6 @@ func NewConfig(app string) (*Config, error) { c.configSyncAPI() c.configSchemas() case "verify": - c.configArchive() err := c.configBroker() if err != nil { From a97f48ea218dfc6dcf0672d209a7164cc4e97949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 14:43:26 +0100 Subject: [PATCH 167/530] feat(verify): fix scanning in GetArchiveLocation incase archive_location is null, add check if archive location is set and add event log error if its not known --- sda/cmd/verify/verify.go | 17 ++++++++++++++--- sda/internal/database/db_functions.go | 11 +++++++---- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 0538a00996..8ed9d50f7b 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -179,10 +179,23 @@ func main() { if err != nil { log.Errorf("failed to get archive location of file: %s, error: %v", message.FileID, err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } + + continue + } + if archiveLocation == "" { + log.Errorf("archive location for file: %s, not known in database", message.FileID) + jsonMsg, _ := json.Marshal(map[string]string{"error": "archive location for file not known in database"}) + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { + log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) + } + // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "GetArchiveLocation failed", - Reason: err.Error(), + Reason: "archive location for file not known in database", OriginalMessage: message, } @@ -194,8 +207,6 @@ func main() { if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - - continue } var file database.FileInfo diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 3c67806129..c723461810 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -916,13 +916,16 @@ func (dbs *SDAdb) GetArchiveLocation(fileID string) (string, error) { db := dbs.DB const getFileID = "SELECT archive_location from sda.files WHERE id = $1;" - var archiveLocation string - err := db.QueryRow(getFileID, fileID).Scan(&archiveLocation) - if err != nil { + var archiveLocation sql.NullString + + if err := db.QueryRow(getFileID, fileID).Scan(&archiveLocation); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } return "", err } - return archiveLocation, nil + return archiveLocation.String, nil } // SetSubmissionFileSize sets the submission file size for a file From dafb7b9ccae473db07c0979b210fe2335e44d867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 09:05:19 +0100 Subject: [PATCH 168/530] feat(verify): use fatal instead error log and return when main failed to initialize resources and remove context.WithCancel and just use context.Background() when calling NewReader --- sda/cmd/verify/verify.go | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 8ed9d50f7b..18f4566b63 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -24,44 +24,30 @@ import ( ) func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() forever := make(chan bool) conf, err := config.NewConfig("verify") if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to load config, due to: %v", err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize mq broker, due to: %v", err) } db, err := database.NewSDAdb(conf.Database) if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize sda db, due to: %v", err) } if db.Version < 23 { - log.Error("database schema v23 is required") - - return + log.Fatal("database schema v23 is required") } - archiveReader, err := storage.NewReader(ctx, "archive") + archiveReader, err := storage.NewReader(context.Background(), "archive") if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize archive reader, due to: %v", err) } archiveKeyList, err := config.GetC4GHprivateKeys() if err != nil || len(archiveKeyList) == 0 { - log.Error("no C4GH private keys configured") - - return + log.Fatal("no C4GH private keys configured") } defer mq.Channel.Close() From 11dc76be1747b9de673f0819758eee6cb9e6c512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 15:05:56 +0100 Subject: [PATCH 169/530] feat(verify): update main shutdown and erroring logic --- sda/cmd/verify/verify.go | 645 ++++++++++++++------------ sda/internal/database/db_functions.go | 1 + 2 files changed, 340 insertions(+), 306 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 18f4566b63..04a828a70f 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -8,9 +8,13 @@ import ( "crypto/md5" "crypto/sha256" "encoding/json" + "errors" "fmt" "io" + "os" + "os/signal" "strings" + "syscall" "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/crypt4gh/streaming" @@ -19,428 +23,457 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" ) +var ( + message schema.IngestionVerification + db *database.SDAdb + mqBroker *broker.AMQPBroker + archiveReader storage.Reader + + archiveKeyList []*[32]byte +) + func main() { - forever := make(chan bool) - conf, err := config.NewConfig("verify") - if err != nil { - log.Fatalf("failed to load config, due to: %v", err) + if err := run(); err != nil { + log.Fatal(err) } - mq, err := broker.NewMQ(conf.Broker) +} +func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + conf, err := config.NewConfig("verify") if err != nil { - log.Fatalf("failed to initialize mq broker, due to: %v", err) + return fmt.Errorf("failed to load config, due to: %v", err) } - db, err := database.NewSDAdb(conf.Database) + db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatalf("failed to initialize sda db, due to: %v", err) + return fmt.Errorf("failed to initialize sda db, due to: %v", err) } + defer db.Close() + if db.Version < 23 { - log.Fatal("database schema v23 is required") + return errors.New("database schema v23 is required") + } + mqBroker, err = broker.NewMQ(conf.Broker) + if err != nil { + return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } + defer func() { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } + }() - archiveReader, err := storage.NewReader(context.Background(), "archive") + archiveReader, err = storage.NewReader(ctx, "archive") if err != nil { - log.Fatalf("failed to initialize archive reader, due to: %v", err) + return fmt.Errorf("failed to initialize archive reader, due to: %v", err) } - archiveKeyList, err := config.GetC4GHprivateKeys() + archiveKeyList, err = config.GetC4GHprivateKeys() if err != nil || len(archiveKeyList) == 0 { - log.Fatal("no C4GH private keys configured") + return errors.New("no C4GH private keys configured") } - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() - - go func() { - connError := mq.ConnectionWatcher() - log.Error(connError) - forever <- false - }() - + consumerErr := make(chan error, 1) + log.Info("starting verify service") go func() { - connError := mq.ChannelWatcher() - log.Error(connError) - forever <- false + consumerErr <- startConsumer() }() - log.Info("starting verify service") - var message schema.IngestionVerification + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + select { + case <-sigc: + case err := <-mqBroker.Connection.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-mqBroker.Channel.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-consumerErr: + return err + } - go func() { - messages, err := mq.GetMessages(conf.Broker.Queue) + return nil +} +func startConsumer() error { + messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) + if err != nil { + return fmt.Errorf("failed to get messages (error: %v) ", err) + } + for delivered := range messages { + ctx := context.Background() + log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", mqBroker.Conf.SchemasPath), delivered.Body) if err != nil { - log.Fatalf("Failed to get messages (error: %v) ", - err) + log.Errorf("validation of incoming message (ingestion-verification) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: %v", err) + } + + // Restart on new message + continue } - for delivered := range messages { - ctx := context.Background() - log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (ingestion-verification) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Message validation failed", - Reason: err.Error(), - OriginalMessage: message, - } + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: %v", err) - } + log.Infof( + "Received work (message.correlation-id: %s, file-id: %s, filepath: %s, user: %s)", + delivered.CorrelationId, message.FileID, message.FilePath, message.User, + ) - // Restart on new message - continue + // If the file has been canceled by the uploader, don't spend time working on it. + status, err := db.GetFileStatus(message.FileID) + if err != nil { + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, } - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) - log.Infof( - "Received work (message.correlation-id: %s, file-id: %s, filepath: %s, user: %s)", - delivered.CorrelationId, message.FileID, message.FilePath, message.User, - ) + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } - // If the file has been canceled by the uploader, don't spend time working on it. - status, err := db.GetFileStatus(message.FileID) - if err != nil { - log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", - Reason: err.Error(), - OriginalMessage: message, - } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + continue + } + if status == "disabled" { + log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + continue + } - continue + header, err := db.GetHeader(message.FileID) + if err != nil { + log.Errorf("GetHeader failed for file with ID: %v, reason: %v", message.FileID, err.Error()) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to nack following getheader error message") + } + // store full message info in case we want to fix the db entry and retry + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, } - if status == "disabled" { - log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } - continue + body, _ := json.Marshal(infoErrorMessage) + + // Send the message to an error queue so it can be analyzed. + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } - header, err := db.GetHeader(message.FileID) - if err != nil { - log.Errorf("GetHeader failed for file with ID: %v, reason: %v", message.FileID, err.Error()) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to nack following getheader error message") - } - // store full message info in case we want to fix the db entry and retry - infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", - Reason: err.Error(), - OriginalMessage: message, - } + continue + } - body, _ := json.Marshal(infoErrorMessage) + archiveLocation, err := db.GetArchiveLocation(message.FileID) + if err != nil { + log.Errorf("failed to get archive location of file: %s, error: %v", message.FileID, err) - // Send the message to an error queue so it can be analyzed. - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + if err := delivered.Nack(false, false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } - continue + continue + } + if archiveLocation == "" { + log.Errorf("archive location for file: %s, not known in database", message.FileID) + jsonMsg, _ := json.Marshal(map[string]string{"error": "archive location for file not known in database"}) + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { + log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) } - archiveLocation, err := db.GetArchiveLocation(message.FileID) - if err != nil { - log.Errorf("failed to get archive location of file: %s, error: %v", message.FileID, err) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "GetArchiveLocation failed", + Reason: "archive location for file not known in database", + OriginalMessage: message, + } - if err := delivered.Nack(false, false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } - continue + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - if archiveLocation == "" { - log.Errorf("archive location for file: %s, not known in database", message.FileID) - jsonMsg, _ := json.Marshal(map[string]string{"error": "archive location for file not known in database"}) + + continue + } + + var file database.FileInfo + file.Size, err = archiveReader.GetFileSize(ctx, archiveLocation, message.ArchivePath) + if err != nil { //nolint:nestif + log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) + if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { + jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) } + } - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "GetArchiveLocation failed", - Reason: "archive location for file not known in database", - OriginalMessage: message, - } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) + } - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to get archived file size", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + continue + } + + archiveFileHash := sha256.New() + f, err := archiveReader.NewFileReader(ctx, archiveLocation, message.ArchivePath) + if err != nil { + log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to open archived file", + Reason: err.Error(), + OriginalMessage: message, } - var file database.FileInfo - file.Size, err = archiveReader.GetFileSize(ctx, archiveLocation, message.ArchivePath) - if err != nil { //nolint:nestif - log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) - if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { - jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) - } - } + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) - } + // Restart on new message + continue + } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to get archived file size", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + var key *[32]byte + for _, k := range archiveKeyList { + size, err := headers.EncryptedSegmentSize(header, *k) + if (err == nil) && (size != 0) { + key = k - continue + break } + } - archiveFileHash := sha256.New() - f, err := archiveReader.NewFileReader(ctx, archiveLocation, message.ArchivePath) - if err != nil { - log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Failed to open archived file", - Reason: err.Error(), - OriginalMessage: message, - } + if key == nil { + log.Errorf("no matching key found for file, file-id: %s, archive-path: %s", message.FileID, message.ArchivePath) - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + continue + } - // Restart on new message - continue - } + mr := io.MultiReader(bytes.NewReader(header), io.TeeReader(f, archiveFileHash)) + c4ghr, err := streaming.NewCrypt4GHReader(mr, *key, nil) + if err != nil { + log.Errorf("failed to open c4gh decryptor stream, file-id: %s, archive-path: %s, reason: %s", message.FileID, message.ArchivePath, err.Error()) - var key *[32]byte - for _, k := range archiveKeyList { - size, err := headers.EncryptedSegmentSize(header, *k) - if (err == nil) && (size != 0) { - key = k + continue + } - break - } - } + md5hash := md5.New() + sha256hash := sha256.New() + stream := io.TeeReader(c4ghr, md5hash) - if key == nil { - log.Errorf("no matching key found for file, file-id: %s, archive-path: %s", message.FileID, message.ArchivePath) + if file.DecryptedSize, err = io.Copy(sha256hash, stream); err != nil { + log.Errorf("failed to copy decrypted data, file-id: %s, reason: (%s)", message.FileID, err.Error()) - continue + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to verify archived file", + Reason: err.Error(), + OriginalMessage: message, } - mr := io.MultiReader(bytes.NewReader(header), io.TeeReader(f, archiveFileHash)) - c4ghr, err := streaming.NewCrypt4GHReader(mr, *key, nil) - if err != nil { - log.Errorf("failed to open c4gh decryptor stream, file-id: %s, archive-path: %s, reason: %s", message.FileID, message.ArchivePath, err.Error()) + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("Failed to publish error message, reason: (%s)", err.Error()) + } - continue + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } - md5hash := md5.New() - sha256hash := sha256.New() - stream := io.TeeReader(c4ghr, md5hash) + continue + } - if file.DecryptedSize, err = io.Copy(sha256hash, stream); err != nil { - log.Errorf("failed to copy decrypted data, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // At this point we should do checksum comparison - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Failed to verify archived file", - Reason: err.Error(), - OriginalMessage: message, - } - - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("Failed to publish error message, reason: (%s)", err.Error()) - } + file.ArchiveChecksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) + file.DecryptedChecksum = fmt.Sprintf("%x", sha256hash.Sum(nil)) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, reason: (%s)", err.Error()) + switch { + case message.ReVerify: + decrypted, err := db.GetDecryptedChecksum(message.FileID) + if err != nil { + log.Errorf("failed to get unencrypted checksum for file, file-id: %s, reason: %s", message.FileID, err.Error()) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue } - // At this point we should do checksum comparison - - file.ArchiveChecksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) - file.DecryptedChecksum = fmt.Sprintf("%x", sha256hash.Sum(nil)) - - switch { - case message.ReVerify: - decrypted, err := db.GetDecryptedChecksum(message.FileID) - if err != nil { - log.Errorf("failed to get unencrypted checksum for file, file-id: %s, reason: %s", message.FileID, err.Error()) + if file.DecryptedChecksum != decrypted { + log.Errorf("encrypted checksum don't match for file, file-id: %s", message.FileID) + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("failed to Nack message, reason: (%v)", err) } continue } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) + } - if file.DecryptedChecksum != decrypted { - log.Errorf("encrypted checksum don't match for file, file-id: %s", message.FileID) - if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } + continue + } - continue - } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, reason: (%s)", err.Error()) + if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { + log.Errorf("encrypted checksum mismatch for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) } continue } + } - if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { - log.Errorf("encrypted checksum mismatch for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) - if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) + } - continue - } - } + continue + default: + c := schema.IngestionAccessionRequest{ + User: message.User, + FilePath: message.FilePath, + DecryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fmt.Sprintf("%x", sha256hash.Sum(nil))}, + {Type: "md5", Value: fmt.Sprintf("%x", md5hash.Sum(nil))}, + }, + } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, reason: (%s)", err.Error()) - } + verifiedMessage, _ := json.Marshal(&c) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession-request.json", mqBroker.Conf.SchemasPath), verifiedMessage) + if err != nil { + log.Errorf("Validation of outgoing (ingestion-accession-request) failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // Logging is in ValidateJSON so just restart on new message continue - default: - c := schema.IngestionAccessionRequest{ - User: message.User, - FilePath: message.FilePath, - DecryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fmt.Sprintf("%x", sha256hash.Sum(nil))}, - {Type: "md5", Value: fmt.Sprintf("%x", md5hash.Sum(nil))}, - }, + } + status, err := db.GetFileStatus(message.FileID) + if err != nil { + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, } - verifiedMessage, _ := json.Marshal(&c) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession-request.json", conf.Broker.SchemasPath), verifiedMessage) - if err != nil { - log.Errorf("Validation of outgoing (ingestion-accession-request) failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) - - // Logging is in ValidateJSON so just restart on new message - continue + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } - status, err := db.GetFileStatus(message.FileID) - if err != nil { - log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + continue + } - continue + if status == "disabled" { + log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - if status == "disabled" { - log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + continue + } - continue + fileInfo, err := db.GetFileInfo(message.FileID) + if err != nil { + log.Errorf("failed to get info for file, file-id: %s", message.FileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - fileInfo, err := db.GetFileInfo(message.FileID) - if err != nil { - log.Errorf("failed to get info for file, file-id: %s", message.FileID) + continue + } + + if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { + if err := db.SetVerified(file, message.FileID); err != nil { + log.Errorf("SetVerified failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue } + } else { + log.Infof("file is already verified, file-id: %s", message.FileID) + } - if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { - if err := db.SetVerified(file, message.FileID); err != nil { - log.Errorf("SetVerified failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - } else { - log.Infof("file is already verified, file-id: %s", message.FileID) + if err := db.UpdateFileEventLog(message.FileID, "verified", "ingest", "{}", string(verifiedMessage)); err != nil { + log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - if err := db.UpdateFileEventLog(message.FileID, "verified", "ingest", "{}", string(verifiedMessage)); err != nil { - log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } + continue + } - // Send message to verified queue - if err := mq.SendMessage(message.FileID, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { - // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + // Send message to verified queue + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, mqBroker.Conf.RoutingKey, verifiedMessage); err != nil { + // TODO fix resend mechanism + log.Errorf("failed to publish message, reason: (%s)", err.Error()) - continue - } + continue + } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } - log.Infof("Successfully verified the file, file-id: %s, filepath: %s", message.FileID, message.FilePath) } - }() + log.Infof("Successfully verified the file, file-id: %s, filepath: %s", message.FileID, message.FilePath) + } - <-forever + return nil } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index c723461810..b439621fcb 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -922,6 +922,7 @@ func (dbs *SDAdb) GetArchiveLocation(fileID string) (string, error) { if errors.Is(err, sql.ErrNoRows) { return "", nil } + return "", err } From 4ac24adc2d96ffa3409d4da369289767f3a47191 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 08:40:42 +0100 Subject: [PATCH 170/530] feat(verify): add new db func GetArchiveLocation to Database Interface for the sake of eventual completion --- sda/internal/database/database_interface.go | 1 + sda/internal/storage/v2/locationbroker/location_broker_test.go | 3 +++ 2 files changed, 4 insertions(+) diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index b35163d509..d440cfad23 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -7,4 +7,5 @@ import ( type Database interface { GetSizeAndObjectCountOfLocation(ctx context.Context, location string) (uint64, uint64, error) GetUploadedSubmissionFilePathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) + GetArchiveLocation(fileID string) (string, error) } diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index f530923c3b..9fce88ce8d 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -26,6 +26,9 @@ func (m *mockDatabase) GetSizeAndObjectCountOfLocation(_ context.Context, locati func (m *mockDatabase) GetUploadedSubmissionFilePathAndLocation(_ context.Context, _, _ string) (string, string, error) { panic("function not expected to be called in unit tests") } +func (m *mockDatabase) GetArchiveLocation(_ string) (string, error) { + panic("function not expected to be called in unit tests") +} func TestLocationBrokerTestSuite(t *testing.T) { suite.Run(t, new(LocationBrokerTestSuite)) From a5dc17be1a4f37ccfcbf8e1da25d4638f71b3519 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:13:56 +0100 Subject: [PATCH 171/530] fix(verify): fix typo in verify.md --- sda/cmd/verify/verify.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/verify/verify.md b/sda/cmd/verify/verify.md index a2ada9f3f1..17766eb1fd 100644 --- a/sda/cmd/verify/verify.md +++ b/sda/cmd/verify/verify.md @@ -112,7 +112,7 @@ storage: archive: ${STORAGE_IMPLEMENTATION}: ``` -For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) ### Logging settings From 14f1cc078a682e3602c2e3569138cbba93e66bb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:24:19 +0100 Subject: [PATCH 172/530] feat(verify): restrucutre handling of message func so we can defer closing of readers to not need closing statement on each continue case --- sda/cmd/verify/verify.go | 563 ++++++++++++++++++++------------------- 1 file changed, 285 insertions(+), 278 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 04a828a70f..6dc2467196 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -108,372 +108,379 @@ func startConsumer() error { return fmt.Errorf("failed to get messages (error: %v) ", err) } for delivered := range messages { - ctx := context.Background() - log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", mqBroker.Conf.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (ingestion-verification) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Message validation failed", - Reason: err.Error(), - OriginalMessage: message, - } + handleMessage(delivered) + } - body, _ := json.Marshal(infoErrorMessage) - if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: %v", err) - } + return nil +} - // Restart on new message - continue +func handleMessage(delivered amqp.Delivery) { + ctx := context.Background() + log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", mqBroker.Conf.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (ingestion-verification) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed", + Reason: err.Error(), + OriginalMessage: message, } - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) - log.Infof( - "Received work (message.correlation-id: %s, file-id: %s, filepath: %s, user: %s)", - delivered.CorrelationId, message.FileID, message.FilePath, message.User, - ) + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: %v", err) + } - // If the file has been canceled by the uploader, don't spend time working on it. - status, err := db.GetFileStatus(message.FileID) - if err != nil { - log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", - Reason: err.Error(), - OriginalMessage: message, - } + // Restart on new message + return + } + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) - body, _ := json.Marshal(infoErrorMessage) - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + log.Infof( + "Received work (message.correlation-id: %s, file-id: %s, filepath: %s, user: %s)", + delivered.CorrelationId, message.FileID, message.FilePath, message.User, + ) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + // If the file has been canceled by the uploader, don't spend time working on it. + status, err := db.GetFileStatus(message.FileID) + if err != nil { + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, + } - continue + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } - if status == "disabled" { - log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } - continue + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - header, err := db.GetHeader(message.FileID) - if err != nil { - log.Errorf("GetHeader failed for file with ID: %v, reason: %v", message.FileID, err.Error()) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to nack following getheader error message") - } - // store full message info in case we want to fix the db entry and retry - infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", - Reason: err.Error(), - OriginalMessage: message, - } + return + } + if status == "disabled" { + log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } - body, _ := json.Marshal(infoErrorMessage) + return + } - // Send the message to an error queue so it can be analyzed. - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + header, err := db.GetHeader(message.FileID) + if err != nil { + log.Errorf("GetHeader failed for file with ID: %v, reason: %v", message.FileID, err.Error()) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to nack following getheader error message") + } + // store full message info in case we want to fix the db entry and retry + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) - continue + // Send the message to an error queue so it can be analyzed. + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } - archiveLocation, err := db.GetArchiveLocation(message.FileID) - if err != nil { - log.Errorf("failed to get archive location of file: %s, error: %v", message.FileID, err) + return + } - if err := delivered.Nack(false, false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + archiveLocation, err := db.GetArchiveLocation(message.FileID) + if err != nil { + log.Errorf("failed to get archive location of file: %s, error: %v", message.FileID, err) + + if err := delivered.Nack(false, false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + } + } + if archiveLocation == "" { + log.Errorf("archive location for file: %s, not known in database", message.FileID) + jsonMsg, _ := json.Marshal(map[string]string{"error": "archive location for file not known in database"}) + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { + log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) + } + + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "GetArchiveLocation failed", + Reason: "archive location for file not known in database", + OriginalMessage: message, + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } - continue + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - if archiveLocation == "" { - log.Errorf("archive location for file: %s, not known in database", message.FileID) - jsonMsg, _ := json.Marshal(map[string]string{"error": "archive location for file not known in database"}) + + return + } + + var file database.FileInfo + file.Size, err = archiveReader.GetFileSize(ctx, archiveLocation, message.ArchivePath) + if err != nil { //nolint:nestif + log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) + if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { + jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) } + } - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "GetArchiveLocation failed", - Reason: "archive location for file not known in database", - OriginalMessage: message, - } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) + } - body, _ := json.Marshal(infoErrorMessage) - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to get archived file size", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + return + } - continue + archiveFileHash := sha256.New() + f, err := archiveReader.NewFileReader(ctx, archiveLocation, message.ArchivePath) + if err != nil { + log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to open archived file", + Reason: err.Error(), + OriginalMessage: message, } - var file database.FileInfo - file.Size, err = archiveReader.GetFileSize(ctx, archiveLocation, message.ArchivePath) - if err != nil { //nolint:nestif - log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) - if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { - jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - if err := db.UpdateFileEventLog(message.FileID, "error", "verify", string(jsonMsg), string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) - } - } + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) - } + // Restart on new message + return + } + defer func() { + _ = f.Close() + }() - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to get archived file size", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + var key *[32]byte + for _, k := range archiveKeyList { + size, err := headers.EncryptedSegmentSize(header, *k) + if (err == nil) && (size != 0) { + key = k - continue + break } + } - archiveFileHash := sha256.New() - f, err := archiveReader.NewFileReader(ctx, archiveLocation, message.ArchivePath) - if err != nil { - log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Failed to open archived file", - Reason: err.Error(), - OriginalMessage: message, - } + if key == nil { + log.Errorf("no matching key found for file, file-id: %s, archive-path: %s", message.FileID, message.ArchivePath) - body, _ := json.Marshal(infoErrorMessage) - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } + return + } - // Restart on new message - continue - } + mr := io.MultiReader(bytes.NewReader(header), io.TeeReader(f, archiveFileHash)) + c4ghr, err := streaming.NewCrypt4GHReader(mr, *key, nil) + if err != nil { + log.Errorf("failed to open c4gh decryptor stream, file-id: %s, archive-path: %s, reason: %s", message.FileID, message.ArchivePath, err.Error()) - var key *[32]byte - for _, k := range archiveKeyList { - size, err := headers.EncryptedSegmentSize(header, *k) - if (err == nil) && (size != 0) { - key = k + return + } - break - } - } + md5hash := md5.New() + sha256hash := sha256.New() + stream := io.TeeReader(c4ghr, md5hash) - if key == nil { - log.Errorf("no matching key found for file, file-id: %s, archive-path: %s", message.FileID, message.ArchivePath) + if file.DecryptedSize, err = io.Copy(sha256hash, stream); err != nil { + log.Errorf("failed to copy decrypted data, file-id: %s, reason: (%s)", message.FileID, err.Error()) - continue + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Failed to verify archived file", + Reason: err.Error(), + OriginalMessage: message, } - mr := io.MultiReader(bytes.NewReader(header), io.TeeReader(f, archiveFileHash)) - c4ghr, err := streaming.NewCrypt4GHReader(mr, *key, nil) - if err != nil { - log.Errorf("failed to open c4gh decryptor stream, file-id: %s, archive-path: %s, reason: %s", message.FileID, message.ArchivePath, err.Error()) + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("Failed to publish error message, reason: (%s)", err.Error()) + } - continue + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } - md5hash := md5.New() - sha256hash := sha256.New() - stream := io.TeeReader(c4ghr, md5hash) + return + } + defer func() { + _ = c4ghr.Close() + }() - if file.DecryptedSize, err = io.Copy(sha256hash, stream); err != nil { - log.Errorf("failed to copy decrypted data, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // At this point we should do checksum comparison - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Failed to verify archived file", - Reason: err.Error(), - OriginalMessage: message, - } + file.ArchiveChecksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) + file.DecryptedChecksum = fmt.Sprintf("%x", sha256hash.Sum(nil)) - body, _ := json.Marshal(infoErrorMessage) - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("Failed to publish error message, reason: (%s)", err.Error()) + switch { + case message.ReVerify: + decrypted, err := db.GetDecryptedChecksum(message.FileID) + if err != nil { + log.Errorf("failed to get unencrypted checksum for file, file-id: %s, reason: %s", message.FileID, err.Error()) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } + return + } + + if file.DecryptedChecksum != decrypted { + log.Errorf("encrypted checksum don't match for file, file-id: %s", message.FileID) + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) + } + + return + } if err := delivered.Ack(false); err != nil { log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } - continue + return } - // At this point we should do checksum comparison - - file.ArchiveChecksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) - file.DecryptedChecksum = fmt.Sprintf("%x", sha256hash.Sum(nil)) - - switch { - case message.ReVerify: - decrypted, err := db.GetDecryptedChecksum(message.FileID) - if err != nil { - log.Errorf("failed to get unencrypted checksum for file, file-id: %s, reason: %s", message.FileID, err.Error()) + if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { + log.Errorf("encrypted checksum mismatch for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) + if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("failed to Nack message, reason: (%v)", err) } - continue + return } + } - if file.DecryptedChecksum != decrypted { - log.Errorf("encrypted checksum don't match for file, file-id: %s", message.FileID) - if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) + } - continue - } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, reason: (%s)", err.Error()) - } + return + default: + c := schema.IngestionAccessionRequest{ + User: message.User, + FilePath: message.FilePath, + DecryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fmt.Sprintf("%x", sha256hash.Sum(nil))}, + {Type: "md5", Value: fmt.Sprintf("%x", md5hash.Sum(nil))}, + }, + } - continue + verifiedMessage, _ := json.Marshal(&c) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession-request.json", mqBroker.Conf.SchemasPath), verifiedMessage) + if err != nil { + log.Errorf("Validation of outgoing (ingestion-accession-request) failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // Logging is in ValidateJSON so just restart on new message + return + } + status, err := db.GetFileStatus(message.FileID) + if err != nil { + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Getheader failed", + Reason: err.Error(), + OriginalMessage: message, } - if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { - log.Errorf("encrypted checksum mismatch for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) - if err := db.UpdateFileEventLog(message.FileID, "error", "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } - - continue - } + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, reason: (%s)", err.Error()) - } - - continue - default: - c := schema.IngestionAccessionRequest{ - User: message.User, - FilePath: message.FilePath, - DecryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fmt.Sprintf("%x", sha256hash.Sum(nil))}, - {Type: "md5", Value: fmt.Sprintf("%x", md5hash.Sum(nil))}, - }, + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - verifiedMessage, _ := json.Marshal(&c) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession-request.json", mqBroker.Conf.SchemasPath), verifiedMessage) - if err != nil { - log.Errorf("Validation of outgoing (ingestion-accession-request) failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) + return + } - // Logging is in ValidateJSON so just restart on new message - continue + if status == "disabled" { + log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - status, err := db.GetFileStatus(message.FileID) - if err != nil { - log.Errorf("failed to get file status, file-id: %s, reason: (%s)", message.FileID, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(infoErrorMessage) - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + return + } - continue + fileInfo, err := db.GetFileInfo(message.FileID) + if err != nil { + log.Errorf("failed to get info for file, file-id: %s", message.FileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - if status == "disabled" { - log.Infof("file with file-id: %s is disabled, stopping verification", message.FileID) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } - - continue - } + return + } - fileInfo, err := db.GetFileInfo(message.FileID) - if err != nil { - log.Errorf("failed to get info for file, file-id: %s", message.FileID) + if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { + if err := db.SetVerified(file, message.FileID); err != nil { + log.Errorf("SetVerified failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - continue + return } + } else { + log.Infof("file is already verified, file-id: %s", message.FileID) + } - if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { - if err := db.SetVerified(file, message.FileID); err != nil { - log.Errorf("SetVerified failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - } else { - log.Infof("file is already verified, file-id: %s", message.FileID) + if err := db.UpdateFileEventLog(message.FileID, "verified", "ingest", "{}", string(verifiedMessage)); err != nil { + log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - if err := db.UpdateFileEventLog(message.FileID, "verified", "ingest", "{}", string(verifiedMessage)); err != nil { - log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } + return + } - // Send message to verified queue - if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, mqBroker.Conf.RoutingKey, verifiedMessage); err != nil { - // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + // Send message to verified queue + if err := mqBroker.SendMessage(message.FileID, mqBroker.Conf.Exchange, mqBroker.Conf.RoutingKey, verifiedMessage); err != nil { + // TODO fix resend mechanism + log.Errorf("failed to publish message, reason: (%s)", err.Error()) - continue - } + return + } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } - log.Infof("Successfully verified the file, file-id: %s, filepath: %s", message.FileID, message.FilePath) } - - return nil + log.Infof("Successfully verified the file, file-id: %s, filepath: %s", message.FileID, message.FilePath) } From bc788f80260ea4124a4e882ee6c44c2c1b8fe83e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:26:28 +0100 Subject: [PATCH 173/530] feat(verify): requeue message if GetArchiveLocation failes for message to reconsumed --- sda/cmd/verify/verify.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 6dc2467196..8241c0b162 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -204,8 +204,8 @@ func handleMessage(delivered amqp.Delivery) { if err != nil { log.Errorf("failed to get archive location of file: %s, error: %v", message.FileID, err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } } if archiveLocation == "" { From 7164c25f8fde97e4c5094b68228fe815f4debf92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:27:43 +0100 Subject: [PATCH 174/530] feat(database): rename query const name for GetArchiveLocation --- sda/internal/database/db_functions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index b439621fcb..840845ceb4 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -914,11 +914,11 @@ func (dbs *SDAdb) getArchivePath(stableID string) (string, error) { func (dbs *SDAdb) GetArchiveLocation(fileID string) (string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFileID = "SELECT archive_location from sda.files WHERE id = $1;" + const getArchiveLocation = "SELECT archive_location from sda.files WHERE id = $1;" var archiveLocation sql.NullString - if err := db.QueryRow(getFileID, fileID).Scan(&archiveLocation); err != nil { + if err := db.QueryRow(getArchiveLocation, fileID).Scan(&archiveLocation); err != nil { if errors.Is(err, sql.ErrNoRows) { return "", nil } From 37c8969f1f1a82c7ccec1077bcac03ebca4c5328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:42:41 +0100 Subject: [PATCH 175/530] feat(verify): pass main ctx down do handleMessage such that it can react to service shutting down and react to ctx being cancelled and gracefully shutdown --- sda/cmd/verify/verify.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 8241c0b162..46a39414b4 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -84,7 +84,7 @@ func run() error { consumerErr := make(chan error, 1) log.Info("starting verify service") go func() { - consumerErr <- startConsumer() + consumerErr <- startConsumer(ctx) }() sigc := make(chan os.Signal, 5) @@ -102,20 +102,21 @@ func run() error { return nil } -func startConsumer() error { +func startConsumer(ctx context.Context) error { messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) if err != nil { return fmt.Errorf("failed to get messages (error: %v) ", err) } for delivered := range messages { - handleMessage(delivered) + handleMessage(ctx, delivered) } return nil } -func handleMessage(delivered amqp.Delivery) { - ctx := context.Background() +func handleMessage(ctx context.Context, delivered amqp.Delivery) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", mqBroker.Conf.SchemasPath), delivered.Body) if err != nil { From ca771c1e8fb896fb1584eabf4553ab4729aeb164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:25:53 +0100 Subject: [PATCH 176/530] feat(verify): update sigc chan buffer size to 1 instead of 5 as we just care that any of the signals triggered --- sda/cmd/verify/verify.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 46a39414b4..a1e528b833 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -87,7 +87,7 @@ func run() error { consumerErr <- startConsumer(ctx) }() - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { From 6d224a307ee6cc53ec44d699be43e33a28982f0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 15:03:55 +0100 Subject: [PATCH 177/530] feat(verify): fix some sentance in verify.md --- sda/cmd/verify/verify.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/verify/verify.md b/sda/cmd/verify/verify.md index 17766eb1fd..3940a7ef7d 100644 --- a/sda/cmd/verify/verify.md +++ b/sda/cmd/verify/verify.md @@ -106,7 +106,7 @@ These settings control how `verify` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings -The verify service requires access to the "archive" storage, to configure such the following config is needed: +The verify service requires access to the "archive" storage. To configure that, the following configuration is required: ```yaml storage: archive: From eb65382d15014e95a6745a68e001b97c864ee1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:36:28 +0100 Subject: [PATCH 178/530] feat(verify): update defer closing of rabbit mq connection and channel to check for nil before close --- sda/cmd/verify/verify.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index a1e528b833..fb0d05cbaf 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -64,11 +64,18 @@ func run() error { return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } defer func() { - if err := mqBroker.Channel.Close(); err != nil { - log.Errorf("failed to close mq broker channel due to: %v", err) + if mqBroker == nil { + return + } + if mqBroker.Channel != nil { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } } - if err := mqBroker.Connection.Close(); err != nil { - log.Errorf("failed to close mq broker connection due to: %v", err) + if mqBroker.Connection != nil { + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } } }() From 511798ac4d80d2262f23bcb5c40a5094fec17a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 10:17:05 +0100 Subject: [PATCH 179/530] feat(verify): move message variable to handleMessage instead of global variable --- sda/cmd/verify/verify.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index fb0d05cbaf..5becddb275 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -29,11 +29,9 @@ import ( ) var ( - message schema.IngestionVerification - db *database.SDAdb - mqBroker *broker.AMQPBroker - archiveReader storage.Reader - + db *database.SDAdb + mqBroker *broker.AMQPBroker + archiveReader storage.Reader archiveKeyList []*[32]byte ) @@ -132,7 +130,7 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { infoErrorMessage := broker.InfoError{ Error: "Message validation failed", Reason: err.Error(), - OriginalMessage: message, + OriginalMessage: delivered, } body, _ := json.Marshal(infoErrorMessage) @@ -146,6 +144,8 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { // Restart on new message return } + + var message schema.IngestionVerification // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) From f78935d557648f266a6626b12a44d3a4200f37b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 10:30:22 +0100 Subject: [PATCH 180/530] feat(verify): move defer close of c4gh reader to ensure its closed if io.copy failes --- sda/cmd/verify/verify.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 5becddb275..8c0fc73cd4 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -317,6 +317,11 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { return } + defer func() { + if err := c4ghr.Close(); err != nil { + log.Errorf("failed to close crypt4gh reader, file-id %s, reason: %v", message.FileID, err) + } + }() md5hash := md5.New() sha256hash := sha256.New() @@ -343,12 +348,8 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { return } - defer func() { - _ = c4ghr.Close() - }() // At this point we should do checksum comparison - file.ArchiveChecksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) file.DecryptedChecksum = fmt.Sprintf("%x", sha256hash.Sum(nil)) From f1f563f391a2704a00eec56c001114c247a613be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:38:17 +0100 Subject: [PATCH 181/530] feat(sync): update to use storage/v2 instead of v1, update db func GetArchivePath -> GetArchivePathAndLocation --- sda/cmd/sync/sync.go | 60 +++++++++++++++++---------- sda/internal/database/db_functions.go | 27 ++++++------ 2 files changed, 51 insertions(+), 36 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 18e2be26d7..38286ec497 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -4,6 +4,7 @@ package main import ( "bytes" + "context" "encoding/json" "errors" "fmt" @@ -18,20 +19,24 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" log "github.com/sirupsen/logrus" "golang.org/x/crypto/chacha20poly1305" ) var ( - err error - key *[32]byte - db *database.SDAdb - conf *config.Config - archive, syncDestination storage.Backend + err error + key *[32]byte + db *database.SDAdb + conf *config.Config + archiveReader storage.Reader + syncWriter storage.Writer ) func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() forever := make(chan bool) conf, err = config.NewConfig("sync") if err != nil { @@ -46,11 +51,15 @@ func main() { log.Fatal(err) } - syncDestination, err = storage.NewBackend(conf.Sync.Destination) + lb, err := locationbroker.NewLocationBroker(db) + if err != nil { + log.Fatalf("failed to init new location broker due to: %v", err) + } + syncWriter, err = storage.NewWriter(ctx, "sync", lb) if err != nil { log.Fatal(err) } - archive, err = storage.NewBackend(conf.Archive) + archiveReader, err = storage.NewReader(ctx, "archive") if err != nil { log.Fatal(err) } @@ -85,6 +94,7 @@ func main() { log.Fatal(err) } for delivered := range messages { + ctx := context.Background() log.Debugf("Received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) @@ -123,7 +133,7 @@ func main() { } for _, aID := range message.AccessionIDs { - if err := syncFiles(aID); err != nil { + if err := syncFiles(ctx, aID); err != nil { log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) if err := delivered.Nack(false, false); err != nil { log.Errorf("failed to nack following GetFileSize error message") @@ -156,35 +166,31 @@ func main() { <-forever } -func syncFiles(stableID string) error { +func syncFiles(ctx context.Context, stableID string) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() log.Debugf("syncing file %s", stableID) inboxPath, err := db.GetInboxPath(stableID) if err != nil { return fmt.Errorf("failed to get inbox path for file with stable ID: %s, reason: %v", stableID, err) } - archivePath, err := db.GetArchivePath(stableID) + archivePath, archiveLocation, err := db.GetArchivePathAndLocation(stableID) if err != nil { return fmt.Errorf("failed to get archive path for file with stable ID: %s, reason: %v", stableID, err) } - fileSize, err := archive.GetFileSize(archivePath, false) + fileSize, err := archiveReader.GetFileSize(ctx, archiveLocation, archivePath) if err != nil { return fmt.Errorf("failed to get filesize for file with archivepath: %s, reason: %v", archivePath, err) } - file, err := archive.NewFileReader(archivePath) + file, err := archiveReader.NewFileReader(ctx, archiveLocation, archivePath) if err != nil { return err } defer file.Close() - dest, err := syncDestination.NewFileWriter(inboxPath) - if err != nil { - return err - } - defer dest.Close() - header, err := db.GetHeaderForStableID(stableID) if err != nil { return err @@ -197,21 +203,29 @@ func syncFiles(stableID string) error { return err } - _, err = dest.Write(newHeader) + contentReader, contentWriter := io.Pipe() + _, err = contentWriter.Write(newHeader) if err != nil { return err } // Copy the file and check is sizes match - copiedSize, err := io.Copy(dest, file) - if err != nil || copiedSize != int64(fileSize) { + copiedSize, err := io.Copy(contentWriter, file) + if err != nil || copiedSize != fileSize { switch { - case copiedSize != int64(fileSize): + case copiedSize != fileSize: return errors.New("copied size does not match file size") default: return err } } + _ = contentWriter.Close() + + _, err = syncWriter.WriteFile(ctx, inboxPath, contentReader) + if err != nil { + return err + } + defer contentReader.Close() return nil } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 840845ceb4..8793013bed 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -882,33 +882,34 @@ func (dbs *SDAdb) checkIfDatasetExists(datasetID string) (bool, error) { return yesNo, nil } -// GetInboxPath retrieves the submission_fie_path for a file with a given accessionID -func (dbs *SDAdb) GetArchivePath(stableID string) (string, error) { +// GetArchivePathAndLocation retrieves the submission_fie_path for a file with a given accessionID +func (dbs *SDAdb) GetArchivePathAndLocation(stableID string) (string, string, error) { var ( - err error - count int - archivePath string + err error + count int + archivePath string + archiveLocation string ) for count == 0 || (err != nil && count < RetryTimes) { - archivePath, err = dbs.getArchivePath(stableID) + archivePath, archiveLocation, err = dbs.getArchivePath(stableID) count++ } - return archivePath, err + return archivePath, archiveLocation, err } -func (dbs *SDAdb) getArchivePath(stableID string) (string, error) { +func (dbs *SDAdb) getArchivePath(stableID string) (string, string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFileID = "SELECT archive_file_path from sda.files WHERE stable_id = $1;" + const getFileID = "SELECT archive_file_path, archive_location from sda.files WHERE stable_id = $1;" - var archivePath string - err := db.QueryRow(getFileID, stableID).Scan(&archivePath) + var archivePath, archiveLocation string + err := db.QueryRow(getFileID, stableID).Scan(&archivePath, &archiveLocation) if err != nil { - return "", err + return "", "", err } - return archivePath, nil + return archivePath, archiveLocation, nil } func (dbs *SDAdb) GetArchiveLocation(fileID string) (string, error) { From 655715bcab8f4066c193712022f295529cc687c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:44:38 +0100 Subject: [PATCH 182/530] feat(sync): add required db version check at v22 with new location columns --- sda/cmd/sync/sync.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 38286ec497..0a56b5e586 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -50,6 +50,11 @@ func main() { if err != nil { log.Fatal(err) } + if db.Version < 23 { + log.Error("database schema v23 is required") + db.Close() + panic(err) + } lb, err := locationbroker.NewLocationBroker(db) if err != nil { From 07e6816c0c43074d1e176c7c5f2a1a27159ac89c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 15:04:44 +0100 Subject: [PATCH 183/530] feat(sync): fix database unit test, some linting issues, and update README.md to now mention expected storage config --- sda/cmd/sync/sync.go | 28 +++++++++++----- sda/cmd/sync/sync.md | 37 ++++------------------ sda/internal/database/db_functions_test.go | 7 ++-- 3 files changed, 31 insertions(+), 41 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 0a56b5e586..19b4c7d03a 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -40,20 +40,26 @@ func main() { forever := make(chan bool) conf, err = config.NewConfig("sync") if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } if db.Version < 23 { log.Error("database schema v23 is required") - db.Close() - panic(err) + + return } lb, err := locationbroker.NewLocationBroker(db) @@ -62,16 +68,22 @@ func main() { } syncWriter, err = storage.NewWriter(ctx, "sync", lb) if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } archiveReader, err = storage.NewReader(ctx, "archive") if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } key, err = config.GetC4GHKey() if err != nil { - log.Fatal(err) + log.Error(err.Error()) + + return } defer mq.Channel.Close() diff --git a/sda/cmd/sync/sync.md b/sda/cmd/sync/sync.md index 46e74c36e3..699fe62cf8 100644 --- a/sda/cmd/sync/sync.md +++ b/sda/cmd/sync/sync.md @@ -107,36 +107,13 @@ These settings control how sync connects to the RabbitMQ message broker. ### Storage settings - -Storage backend is defined by the `ARCHIVE_TYPE`, and `SYNC_DESTINATION_TYPE` variables. -Valid values for these options are `S3` or `POSIX` for `ARCHIVE_TYPE` and `POSIX`, `S3` or `SFTP` for `SYNC_DESTINATION_TYPE`. - -The value of these variables define what other variables are read. -The same variables are available for all storage types, differing by prefix (`ARCHIVE_`, or `SYNC_DESTINATION_`) - -if `*_TYPE` is `S3` then the following variables are available: - -- `*_URL`: URL to the S3 system -- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_BUCKET`: The S3 bucket to use as the storage root -- `*_PORT`: S3 connection port (default: `443`) -- `*_REGION`: S3 region (default: `us-east-1`) -- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. -- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, CA certificate is only needed if the S3 server has a certificate signed by a private entity - -if `*_TYPE` is `POSIX`: - -- `*_LOCATION`: POSIX path to use as storage root - -and if `*_TYPE` is `SFTP`: - -- `*_HOST`: URL to the SFTP server -- `*_PORT`: Port of the SFTP server to connect to -- `*_USERNAME`: Username connection to the SFTP server -- `*_HOSTKEY`: The SFTP server's public key -- `*_PEMKEYPATH`: Path to the ssh private key used to connect to the SFTP server -- `*_PEMKEYPASS`: Passphrase for the ssh private key +The sync service requires access to the "archive" storage, to configure such the following config is needed: +```yaml +storage: + inbox: + ${STORAGE_IMPLEMENTATION}: +``` +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) ### Logging settings diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 59ca23190a..717f82c019 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -633,7 +633,7 @@ func (suite *DatabaseTests) TestGetArchivePath() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile(nil, "/testuser/TestGetArchivePath-001.c4gh", "testuser") + fileID, err := db.RegisterFileWithLocation(nil, "/archive", "/testuser/TestGetArchivePath-001.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New()) @@ -645,9 +645,10 @@ func (suite *DatabaseTests) TestGetArchivePath() { err = db.SetAccessionID("acession-0001", fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) - path, err := db.getArchivePath("acession-0001") + path, location, err := db.getArchivePath("acession-0001") assert.NoError(suite.T(), err, "getArchivePath failed") - assert.Equal(suite.T(), path, corrID) + assert.Equal(suite.T(), corrID, path) + assert.Equal(suite.T(), "/archive", location) db.Close() } From f189a3afa39ce5e36b8a7a9d37b74a20d2301144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 16:44:11 +0100 Subject: [PATCH 184/530] feat(sync): fix sync.md storage settings to mention "sync" storage and fix example --- sda/cmd/sync/sync.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sda/cmd/sync/sync.md b/sda/cmd/sync/sync.md index 699fe62cf8..6d3cd0be3b 100644 --- a/sda/cmd/sync/sync.md +++ b/sda/cmd/sync/sync.md @@ -107,10 +107,12 @@ These settings control how sync connects to the RabbitMQ message broker. ### Storage settings -The sync service requires access to the "archive" storage, to configure such the following config is needed: +The sync service requires access to the "archive", and "sync" storage, to configure such the following config is needed: ```yaml storage: - inbox: + archive: + ${STORAGE_IMPLEMENTATION}: + sync: ${STORAGE_IMPLEMENTATION}: ``` For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) From cb582a91f336a3c4530dc215d2c97e0742acfe90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 16:46:16 +0100 Subject: [PATCH 185/530] feat(sync): update config to not require and load in storage v1 for sync service --- sda/cmd/sync/sync.go | 4 +++- sda/internal/config/config.go | 35 ------------------------------ sda/internal/config/config_test.go | 2 -- 3 files changed, 3 insertions(+), 38 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 19b4c7d03a..579c1becc4 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -64,7 +64,9 @@ func main() { lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Fatalf("failed to init new location broker due to: %v", err) + log.Errorf("failed to init new location broker due to: %v", err) + + return } syncWriter, err = storage.NewWriter(ctx, "sync", lb) if err != nil { diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index a0b9dbd841..9c03131a6b 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -79,7 +79,6 @@ type RotateKeyConf struct { type Sync struct { CenterPrefix string - Destination storage.Conf RemoteHost string RemotePassword string RemotePort int @@ -430,25 +429,6 @@ func NewConfig(app string) (*Config, error) { "sync.remote.password", } - switch viper.GetString("archive.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) - default: - return nil, errors.New("archive.type not set") - } - - switch viper.GetString("sync.destination.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"sync.destination.url", "sync.destination.accesskey", "sync.destination.secretkey", "sync.destination.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"sync.destination.location"}...) - case SFTP: - requiredConfVars = append(requiredConfVars, []string{"sync.destination.sftp.host", "sync.destination.sftp.port", "sync.destination.sftp.userName", "sync.destination.sftp.pemKeyPath", "sync.destination.sftp.pemKeyPass"}...) - default: - return nil, errors.New("sync.destination.type not set") - } case "sync-api": requiredConfVars = []string{ "broker.exchange", @@ -691,7 +671,6 @@ func NewConfig(app string) (*Config, error) { return nil, err } - c.configArchive() if err := c.configSync(); err != nil { return nil, err } @@ -1120,20 +1099,6 @@ func (c *Config) configSMTP() { // configSync provides configuration for the sync destination storage func (c *Config) configSync() error { - switch viper.GetString("sync.destination.type") { - case S3: - c.Sync.Destination.Type = S3 - c.Sync.Destination.S3 = configS3Storage("sync.destination") - case SFTP: - c.Sync.Destination.Type = SFTP - c.Sync.Destination.SFTP = configSFTP("sync.destination") - case POSIX: - c.Sync.Destination.Type = POSIX - c.Sync.Destination.Posix.Location = viper.GetString("sync.destination.location") - default: - return errors.New("sync.destination.type not set") - } - c.Sync.RemoteHost = viper.GetString("sync.remote.host") if viper.IsSet("sync.remote.port") { c.Sync.RemotePort = viper.GetInt("sync.remote.port") diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 5dc76d967c..aced31d246 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -347,8 +347,6 @@ func (ts *ConfigTestSuite) TestSyncConfig() { assert.NotNil(ts.T(), config.Archive.Posix) assert.Equal(ts.T(), "test", config.Archive.Posix.Location) assert.NotNil(ts.T(), config.Sync) - assert.NotNil(ts.T(), config.Sync.Destination.Posix) - assert.Equal(ts.T(), "test", config.Sync.Destination.Posix.Location) defer os.RemoveAll(ts.pubKeyPath) } From 02a404f4adc4e8eb8233821c4ba44a29aa62e8d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 21 Jan 2026 10:53:13 +0100 Subject: [PATCH 186/530] feat(sync): update syncFiles handling, update to avoid dubble(or more) nacking a message if syncFiles return error, ensure closing of readers / writers in syncFiles, and update io.Copy error check to capture err before copiedSize != fileSize --- sda/cmd/sync/sync.go | 62 +++++++++++++++++++++++++------------------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 579c1becc4..e7e82f2696 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -151,16 +151,22 @@ func main() { continue } + var syncFilesErr error for _, aID := range message.AccessionIDs { if err := syncFiles(ctx, aID); err != nil { log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following GetFileSize error message") - } + syncFilesErr = err - continue + break } } + if syncFilesErr != nil { + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following GetFileSize error message") + } + + continue + } log.Infoln("buildSyncDatasetJSON") blob, err := buildSyncDatasetJSON(delivered.Body) @@ -191,60 +197,64 @@ func syncFiles(ctx context.Context, stableID string) error { log.Debugf("syncing file %s", stableID) inboxPath, err := db.GetInboxPath(stableID) if err != nil { - return fmt.Errorf("failed to get inbox path for file with stable ID: %s, reason: %v", stableID, err) + return fmt.Errorf("failed to get inbox path, reason: %v", err) } archivePath, archiveLocation, err := db.GetArchivePathAndLocation(stableID) if err != nil { - return fmt.Errorf("failed to get archive path for file with stable ID: %s, reason: %v", stableID, err) + return fmt.Errorf("failed to get archive path and location, reason: %v", err) } fileSize, err := archiveReader.GetFileSize(ctx, archiveLocation, archivePath) if err != nil { - return fmt.Errorf("failed to get filesize for file with archivepath: %s, reason: %v", archivePath, err) + return fmt.Errorf("failed to get file size from archive storage, location: %s, path: %s, reason: %v", archiveLocation, archivePath, err) } file, err := archiveReader.NewFileReader(ctx, archiveLocation, archivePath) if err != nil { - return err + return fmt.Errorf("failed to read file from archive storage, location: %s, path: %s, reason: %v", archiveLocation, archivePath, err) } - defer file.Close() + defer func() { + _ = file.Close() + }() header, err := db.GetHeaderForStableID(stableID) if err != nil { - return err + return fmt.Errorf("failed to get header from db, reason: %v", err) } - pubkeyList := [][chacha20poly1305.KeySize]byte{} - pubkeyList = append(pubkeyList, *conf.Sync.PublicKey) - newHeader, err := headers.ReEncryptHeader(header, *key, pubkeyList) + newHeader, err := headers.ReEncryptHeader(header, *key, [][chacha20poly1305.KeySize]byte{*conf.Sync.PublicKey}) if err != nil { - return err + return fmt.Errorf("failed to reencrypt header, reason: %v", err) } contentReader, contentWriter := io.Pipe() - _, err = contentWriter.Write(newHeader) - if err != nil { - return err + // Ensure contentReader, contentWriter are closed in case error + defer func() { + _ = contentWriter.Close() + _ = contentReader.Close() + }() + + if _, err := contentWriter.Write(newHeader); err != nil { + return fmt.Errorf("failed to write header, reason: %v", err) } // Copy the file and check is sizes match copiedSize, err := io.Copy(contentWriter, file) - if err != nil || copiedSize != fileSize { - switch { - case copiedSize != fileSize: - return errors.New("copied size does not match file size") - default: - return err - } + switch { + case err != nil: + return fmt.Errorf("failed to write file content, reason: %v", err) + case copiedSize != fileSize: + return errors.New("copied size does not match file size") + default: } _ = contentWriter.Close() _, err = syncWriter.WriteFile(ctx, inboxPath, contentReader) if err != nil { - return err + return fmt.Errorf("failed to upload file to storage, reason: %v", err) } - defer contentReader.Close() + _ = contentReader.Close() return nil } From ddd39aa961de31f60f292b068840e026782e8f07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 09:02:42 +0100 Subject: [PATCH 187/530] feat(sync): use fatal instead error log and return when main failed to initialize resources, remove context.WithCancel and just use bg ctx --- sda/cmd/sync/sync.go | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index e7e82f2696..924c80014a 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -35,57 +35,39 @@ var ( ) func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() forever := make(chan bool) conf, err = config.NewConfig("sync") if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to load config, due to: %v", err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize mq broker, due to: %v", err) } db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize sda db, due to: %v", err) } if db.Version < 23 { - log.Error("database schema v23 is required") - - return + log.Fatal("database schema v23 is required") } lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Errorf("failed to init new location broker due to: %v", err) - - return + log.Fatalf("failed to initialize location broker, due to: %v", err) } - syncWriter, err = storage.NewWriter(ctx, "sync", lb) + syncWriter, err = storage.NewWriter(context.Background(), "sync", lb) if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize sync writer, due to: %v", err) } - archiveReader, err = storage.NewReader(ctx, "archive") + archiveReader, err = storage.NewReader(context.Background(), "archive") if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to initialize archive reader, due to: %v", err) } key, err = config.GetC4GHKey() if err != nil { - log.Error(err.Error()) - - return + log.Fatalf("failed to get c4gh key from config, due to: %v", err) } defer mq.Channel.Close() From c3f8f952ef56cd78507b5a7a43ba9a78094a43fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 14:56:13 +0100 Subject: [PATCH 188/530] feat(sync): update main shutdown and erroring logic --- sda/cmd/sync/sync.go | 213 ++++++++++++++++++++++++------------------- 1 file changed, 119 insertions(+), 94 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 924c80014a..d644821f97 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -11,7 +11,10 @@ import ( "io" "net/http" "net/url" + "os" + "os/signal" "strings" + "syscall" "time" "github.com/neicnordic/crypt4gh/model/headers" @@ -21,156 +24,178 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" + amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" "golang.org/x/crypto/chacha20poly1305" ) var ( - err error key *[32]byte db *database.SDAdb conf *config.Config + mqBroker *broker.AMQPBroker archiveReader storage.Reader syncWriter storage.Writer + message schema.DatasetMapping ) func main() { - forever := make(chan bool) - conf, err = config.NewConfig("sync") - if err != nil { - log.Fatalf("failed to load config, due to: %v", err) + if err := run(); err != nil { + log.Fatal(err) } - mq, err := broker.NewMQ(conf.Broker) +} +func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var err error + conf, err = config.NewConfig("sync") if err != nil { - log.Fatalf("failed to initialize mq broker, due to: %v", err) + return fmt.Errorf("failed to load config, due to: %v", err) } + db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatalf("failed to initialize sda db, due to: %v", err) + return fmt.Errorf("failed to initialize sda db, due to: %v", err) } + defer db.Close() if db.Version < 23 { - log.Fatal("database schema v23 is required") + return errors.New("database schema v23 is required") } + mqBroker, err = broker.NewMQ(mqBroker.Conf) + if err != nil { + return fmt.Errorf("failed to initialize mq broker, due to: %v", err) + } + defer func() { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } + }() + lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Fatalf("failed to initialize location broker, due to: %v", err) + return fmt.Errorf("failed to initialize location broker, due to: %v", err) } - syncWriter, err = storage.NewWriter(context.Background(), "sync", lb) + syncWriter, err = storage.NewWriter(ctx, "sync", lb) if err != nil { - log.Fatalf("failed to initialize sync writer, due to: %v", err) + return fmt.Errorf("failed to initialize sync writer, due to: %v", err) } - archiveReader, err = storage.NewReader(context.Background(), "archive") + archiveReader, err = storage.NewReader(ctx, "archive") if err != nil { - log.Fatalf("failed to initialize archive reader, due to: %v", err) + return fmt.Errorf("failed to initialize archive reader, due to: %v", err) } key, err = config.GetC4GHKey() if err != nil { - log.Fatalf("failed to get c4gh key from config, due to: %v", err) + return fmt.Errorf("failed to get c4gh key from config, due to: %v", err) } - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() + log.Info("Starting sync service") + consumeErr := make(chan error, 1) go func() { - connError := mq.ConnectionWatcher() - log.Error(connError) - forever <- false + consumeErr <- startConsumer() }() - go func() { - connError := mq.ChannelWatcher() - log.Error(connError) - forever <- false - }() + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - log.Info("Starting sync service") - var message schema.DatasetMapping + select { + case <-sigc: + case err := <-mqBroker.Connection.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-mqBroker.Channel.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-consumeErr: + return err + } - go func() { - messages, err := mq.GetMessages(conf.Broker.Queue) + return nil +} +func startConsumer() error { + messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) + if err != nil { + return err + } + for delivered := range messages { + ctx := context.Background() + log.Debugf("Received a message (correlation-id: %s, message: %s)", + delivered.CorrelationId, + delivered.Body) + + err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", mqBroker.Conf.SchemasPath), delivered.Body) if err != nil { - log.Fatal(err) - } - for delivered := range messages { - ctx := context.Background() - log.Debugf("Received a message (correlation-id: %s, message: %s)", - delivered.CorrelationId, - delivered.Body) - - err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", conf.Broker.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (dataset-mapping) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Message validation failed in sync service", - Reason: err.Error(), - OriginalMessage: string(delivered.Body), - } - - body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%v)", err) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } - - continue + log.Errorf("validation of incoming message (dataset-mapping) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed in sync service", + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } + + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%v)", err) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) + continue + } - if !strings.HasPrefix(message.DatasetID, conf.Sync.CenterPrefix) { - log.Infoln("external dataset") - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) - continue + if !strings.HasPrefix(message.DatasetID, conf.Sync.CenterPrefix) { + log.Infoln("external dataset") + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } - var syncFilesErr error - for _, aID := range message.AccessionIDs { - if err := syncFiles(ctx, aID); err != nil { - log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) - syncFilesErr = err + continue + } - break - } - } - if syncFilesErr != nil { - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following GetFileSize error message") - } + var syncFilesErr error + for _, aID := range message.AccessionIDs { + if err := syncFiles(ctx, aID); err != nil { + log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) + syncFilesErr = err - continue + break } - - log.Infoln("buildSyncDatasetJSON") - blob, err := buildSyncDatasetJSON(delivered.Body) - if err != nil { - log.Errorf("failed to build SyncDatasetJSON, Reason: %v", err) + } + if syncFilesErr != nil { + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following GetFileSize error message") } - if err := sendPOST(blob); err != nil { - log.Errorf("failed to send POST, Reason: %v", err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following sendPOST error message") - } - continue - } + continue + } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + log.Infoln("buildSyncDatasetJSON") + blob, err := buildSyncDatasetJSON(delivered.Body) + if err != nil { + log.Errorf("failed to build SyncDatasetJSON, Reason: %v", err) + } + if err := sendPOST(blob); err != nil { + log.Errorf("failed to send POST, Reason: %v", err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following sendPOST error message") } + + continue } - }() - <-forever + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + } + + return nil } func syncFiles(ctx context.Context, stableID string) error { From 77363dd732e10815376c97c9ae4857eeff69dd40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 27 Jan 2026 11:03:27 +0100 Subject: [PATCH 189/530] feat(sync): fix TestSyncConfig, sync no longer has destination config, expected to be given as a "sync" storage/v2 --- sda/internal/config/config_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index aced31d246..35a299a397 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -320,8 +320,6 @@ func (ts *ConfigTestSuite) TestSyncConfig() { viper.Set("archive.type", "posix") viper.Set("archive.location", "test") viper.Set("sync.centerPrefix", "prefix") - viper.Set("sync.destination.type", "posix") - viper.Set("sync.destination.location", "test") viper.Set("sync.remote.host", "https://test.org") viper.Set("sync.remote.user", "test") viper.Set("sync.remote.password", "test") @@ -343,9 +341,6 @@ func (ts *ConfigTestSuite) TestSyncConfig() { assert.Equal(ts.T(), "test", config.Database.User) assert.Equal(ts.T(), "test", config.Database.Password) assert.Equal(ts.T(), "test", config.Database.Database) - assert.NotNil(ts.T(), config.Archive) - assert.NotNil(ts.T(), config.Archive.Posix) - assert.Equal(ts.T(), "test", config.Archive.Posix.Location) assert.NotNil(ts.T(), config.Sync) defer os.RemoveAll(ts.pubKeyPath) From b7e1c5b9dcb7dca172456ded188a66fa3377d3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 27 Jan 2026 14:04:49 +0100 Subject: [PATCH 190/530] feat(database): fix TestGetArchivePath unit test, use SetArchivedWithLocation to set archived_location --- sda/internal/database/db_functions_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 717f82c019..53170c61c1 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -633,13 +633,13 @@ func (suite *DatabaseTests) TestGetArchivePath() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFileWithLocation(nil, "/archive", "/testuser/TestGetArchivePath-001.c4gh", "testuser") + fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/testuser/TestGetArchivePath-001.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New()) corrID := uuid.New().String() fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1234, corrID, checksum, 999, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchivedWithLocation("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = db.SetAccessionID("acession-0001", fileID) From 5f1200829df8e3012094c2caa5425424c7d3f89e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 08:45:40 +0100 Subject: [PATCH 191/530] feat(sync): align private db func name with public, add updated db func GetArchivePathAndLocation to database interface for the sake of eventual completion --- sda/internal/database/database_interface.go | 1 + sda/internal/database/db_functions.go | 4 ++-- sda/internal/database/db_functions_test.go | 4 ++-- .../storage/v2/locationbroker/location_broker_test.go | 3 +++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index d440cfad23..0455277ffa 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -8,4 +8,5 @@ type Database interface { GetSizeAndObjectCountOfLocation(ctx context.Context, location string) (uint64, uint64, error) GetUploadedSubmissionFilePathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) GetArchiveLocation(fileID string) (string, error) + GetArchivePathAndLocation(stableID string) (string, string, error) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 8793013bed..11b0e7a185 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -892,13 +892,13 @@ func (dbs *SDAdb) GetArchivePathAndLocation(stableID string) (string, string, er ) for count == 0 || (err != nil && count < RetryTimes) { - archivePath, archiveLocation, err = dbs.getArchivePath(stableID) + archivePath, archiveLocation, err = dbs.getArchivePathAndLocation(stableID) count++ } return archivePath, archiveLocation, err } -func (dbs *SDAdb) getArchivePath(stableID string) (string, string, error) { +func (dbs *SDAdb) getArchivePathAndLocation(stableID string) (string, string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB const getFileID = "SELECT archive_file_path, archive_location from sda.files WHERE stable_id = $1;" diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 53170c61c1..4126437382 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -645,8 +645,8 @@ func (suite *DatabaseTests) TestGetArchivePath() { err = db.SetAccessionID("acession-0001", fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) - path, location, err := db.getArchivePath("acession-0001") - assert.NoError(suite.T(), err, "getArchivePath failed") + path, location, err := db.getArchivePathAndLocation("acession-0001") + assert.NoError(suite.T(), err, "getArchivePathAndLocation failed") assert.Equal(suite.T(), corrID, path) assert.Equal(suite.T(), "/archive", location) diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index 9fce88ce8d..f5aa6a00b7 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -29,6 +29,9 @@ func (m *mockDatabase) GetUploadedSubmissionFilePathAndLocation(_ context.Contex func (m *mockDatabase) GetArchiveLocation(_ string) (string, error) { panic("function not expected to be called in unit tests") } +func (m *mockDatabase) GetArchivePathAndLocation(_ string) (string, string, error) { + panic("function not expected to be called in unit tests") +} func TestLocationBrokerTestSuite(t *testing.T) { suite.Run(t, new(LocationBrokerTestSuite)) From d9337757f11fcf157504d3d708f32a27dd408a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:14:35 +0100 Subject: [PATCH 192/530] fix(sync): fix typo in sync.md --- sda/cmd/sync/sync.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/sync/sync.md b/sda/cmd/sync/sync.md index 6d3cd0be3b..9f9f2e0c92 100644 --- a/sda/cmd/sync/sync.md +++ b/sda/cmd/sync/sync.md @@ -115,7 +115,7 @@ storage: sync: ${STORAGE_IMPLEMENTATION}: ``` -For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) ### Logging settings From 9a2413b8044e37a04644a021c735e47b42076d40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:27:13 +0100 Subject: [PATCH 193/530] feat(sync): fix mq broker init --- sda/cmd/sync/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index d644821f97..3a0c08449a 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -62,7 +62,7 @@ func run() error { return errors.New("database schema v23 is required") } - mqBroker, err = broker.NewMQ(mqBroker.Conf) + mqBroker, err = broker.NewMQ(conf.Broker) if err != nil { return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } From 1e479724f1fa68fbedec832f9c84e577c9f03909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:34:47 +0100 Subject: [PATCH 194/530] feat(sync): fix syncFiles, write to writer in go func so we can read and upload to storage concurrently --- sda/cmd/sync/sync.go | 32 ++++++++++++++------------------ 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 3a0c08449a..0627e534d8 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -236,26 +236,22 @@ func syncFiles(ctx context.Context, stableID string) error { } contentReader, contentWriter := io.Pipe() - // Ensure contentReader, contentWriter are closed in case error - defer func() { - _ = contentWriter.Close() - _ = contentReader.Close() - }() - if _, err := contentWriter.Write(newHeader); err != nil { - return fmt.Errorf("failed to write header, reason: %v", err) - } + go func() { + defer func() { + _ = contentWriter.Close() + }() + if _, err := contentWriter.Write(newHeader); err != nil { + _ = contentWriter.CloseWithError(fmt.Errorf("failed to write header, reason: %v", err)) - // Copy the file and check is sizes match - copiedSize, err := io.Copy(contentWriter, file) - switch { - case err != nil: - return fmt.Errorf("failed to write file content, reason: %v", err) - case copiedSize != fileSize: - return errors.New("copied size does not match file size") - default: - } - _ = contentWriter.Close() + return + } + if copiedSize, err := io.Copy(contentWriter, file); err != nil { + _ = contentWriter.CloseWithError(fmt.Errorf("failed to write file content, reason: %v", err)) + } else if copiedSize != fileSize { + _ = contentWriter.CloseWithError(errors.New("copied size does not match file size")) + } + }() _, err = syncWriter.WriteFile(ctx, inboxPath, contentReader) if err != nil { From 092c4a131a80c3f03ffa76861cb3eb79524a8591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:37:01 +0100 Subject: [PATCH 195/530] feat(database): update GetArchivePathAndLocation comment to describe actual use --- sda/internal/database/db_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 11b0e7a185..17cefe15f9 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -882,7 +882,7 @@ func (dbs *SDAdb) checkIfDatasetExists(datasetID string) (bool, error) { return yesNo, nil } -// GetArchivePathAndLocation retrieves the submission_fie_path for a file with a given accessionID +// GetArchivePathAndLocation retrieves the archive_file_path and archive_location for a file with a given stableID(accessionID) func (dbs *SDAdb) GetArchivePathAndLocation(stableID string) (string, string, error) { var ( err error From 686d5da8b200c9418183305d9fe6cdfd6dc6d57c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:41:12 +0100 Subject: [PATCH 196/530] feat(sync): restructure handling of message to separate func with spawns content from main context so it can react to service shutting down and can gracefully shutdown --- sda/cmd/sync/sync.go | 127 ++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 61 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 0627e534d8..3aee0ae021 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -97,7 +97,7 @@ func run() error { consumeErr := make(chan error, 1) go func() { - consumeErr <- startConsumer() + consumeErr <- startConsumer(ctx) }() sigc := make(chan os.Signal, 5) @@ -115,87 +115,92 @@ func run() error { return nil } -func startConsumer() error { +func startConsumer(ctx context.Context) error { messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) if err != nil { return err } for delivered := range messages { - ctx := context.Background() - log.Debugf("Received a message (correlation-id: %s, message: %s)", - delivered.CorrelationId, - delivered.Body) + handleMessage(ctx, delivered) + } - err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", mqBroker.Conf.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (dataset-mapping) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Message validation failed in sync service", - Reason: err.Error(), - OriginalMessage: string(delivered.Body), - } - - body, _ := json.Marshal(infoErrorMessage) - if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%v)", err) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } - - continue - } + return nil +} - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) +func handleMessage(ctx context.Context, delivered amqp.Delivery) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + log.Debugf("Received a message (correlation-id: %s, message: %s)", + delivered.CorrelationId, + delivered.Body) - if !strings.HasPrefix(message.DatasetID, conf.Sync.CenterPrefix) { - log.Infoln("external dataset") - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } + err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", mqBroker.Conf.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (dataset-mapping) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed in sync service", + Reason: err.Error(), + OriginalMessage: string(delivered.Body), + } - continue + body, _ := json.Marshal(infoErrorMessage) + if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%v)", err) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } - var syncFilesErr error - for _, aID := range message.AccessionIDs { - if err := syncFiles(ctx, aID); err != nil { - log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) - syncFilesErr = err + return + } - break - } - } - if syncFilesErr != nil { - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following GetFileSize error message") - } + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) - continue + if !strings.HasPrefix(message.DatasetID, conf.Sync.CenterPrefix) { + log.Infoln("external dataset") + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } - log.Infoln("buildSyncDatasetJSON") - blob, err := buildSyncDatasetJSON(delivered.Body) - if err != nil { - log.Errorf("failed to build SyncDatasetJSON, Reason: %v", err) - } - if err := sendPOST(blob); err != nil { - log.Errorf("failed to send POST, Reason: %v", err) - if err := delivered.Nack(false, false); err != nil { - log.Errorf("failed to nack following sendPOST error message") - } + return + } + + var syncFilesErr error + for _, aID := range message.AccessionIDs { + if err := syncFiles(ctx, aID); err != nil { + log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) + syncFilesErr = err - continue + break + } + } + if syncFilesErr != nil { + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following GetFileSize error message") } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + return + } + + log.Infoln("buildSyncDatasetJSON") + blob, err := buildSyncDatasetJSON(delivered.Body) + if err != nil { + log.Errorf("failed to build SyncDatasetJSON, Reason: %v", err) + } + if err := sendPOST(blob); err != nil { + log.Errorf("failed to send POST, Reason: %v", err) + if err := delivered.Nack(false, false); err != nil { + log.Errorf("failed to nack following sendPOST error message") } + + return } - return nil + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } } func syncFiles(ctx context.Context, stableID string) error { From ad0ede350d8c0412953b170bc414ce28b76b57b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:24:19 +0100 Subject: [PATCH 197/530] feat(sync): update sigc chan buffer size to 1 instead of 5 as we just care that any of the signals triggered --- sda/cmd/sync/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 3aee0ae021..284c9f6e60 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -100,7 +100,7 @@ func run() error { consumeErr <- startConsumer(ctx) }() - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { From d362afca39bac4ae75a4d3c110a8633c2a5b153d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 15:03:29 +0100 Subject: [PATCH 198/530] feat(sync): fix some sentance in sync.md --- sda/cmd/sync/sync.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/sync/sync.md b/sda/cmd/sync/sync.md index 9f9f2e0c92..edd557398b 100644 --- a/sda/cmd/sync/sync.md +++ b/sda/cmd/sync/sync.md @@ -107,7 +107,7 @@ These settings control how sync connects to the RabbitMQ message broker. ### Storage settings -The sync service requires access to the "archive", and "sync" storage, to configure such the following config is needed: +The sync service requires access to the "archive", and "sync" storage. To configure that, the following configuration is required: ```yaml storage: archive: From 8565d104b9869183c14b85dc9cd0e89aaaff84b0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 10 Feb 2026 09:30:05 +0100 Subject: [PATCH 199/530] feat(sync): update sync.md to add some information about the 'sync' storages --- sda/cmd/sync/sync.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/cmd/sync/sync.md b/sda/cmd/sync/sync.md index edd557398b..32b50197d3 100644 --- a/sda/cmd/sync/sync.md +++ b/sda/cmd/sync/sync.md @@ -117,6 +117,8 @@ storage: ``` For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) +Sync will read from the "archive", and backup files that have been added to a dataset in the "sync" storage. + ### Logging settings - `LOG_FORMAT` can be set to “json” to get logs in json format. All other values result in text logging From 6951a11fffce0fbb58f696bb250222ac5f053ee2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:35:42 +0100 Subject: [PATCH 200/530] feat(sync): update defer closing of rabbit mq connection and channel to check for nil before close --- sda/cmd/sync/sync.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 284c9f6e60..283aa5bed6 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -67,11 +67,18 @@ func run() error { return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } defer func() { - if err := mqBroker.Channel.Close(); err != nil { - log.Errorf("failed to close mq broker channel due to: %v", err) + if mqBroker == nil { + return + } + if mqBroker.Channel != nil { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } } - if err := mqBroker.Connection.Close(); err != nil { - log.Errorf("failed to close mq broker connection due to: %v", err) + if mqBroker.Connection != nil { + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } } }() From d6f4daee6934e758709a57d6b30785b22e2ffc56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 10:06:53 +0100 Subject: [PATCH 201/530] feat(sync): update some sentancing in sync.md for better clarity --- sda/cmd/sync/sync.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/sync/sync.md b/sda/cmd/sync/sync.md index 32b50197d3..982fe5c9f3 100644 --- a/sda/cmd/sync/sync.md +++ b/sda/cmd/sync/sync.md @@ -107,7 +107,7 @@ These settings control how sync connects to the RabbitMQ message broker. ### Storage settings -The sync service requires access to the "archive", and "sync" storage. To configure that, the following configuration is required: +The sync service requires access to the "archive" and "sync" storage backends. To configure these, the following configuration is required: ```yaml storage: archive: @@ -117,7 +117,7 @@ storage: ``` For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) -Sync will read from the "archive", and backup files that have been added to a dataset in the "sync" storage. +Sync operates by reading file data from the "archive" backend and replicating it to the "sync" backend for all files associated with a dataset. ### Logging settings From bb311f8c76652121d35065a19257e19fb5786836 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 10:07:18 +0100 Subject: [PATCH 202/530] feat(sync): move message variable to handleMessage instead of global variable --- sda/cmd/sync/sync.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 283aa5bed6..fe3e8c6576 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -36,7 +36,6 @@ var ( mqBroker *broker.AMQPBroker archiveReader storage.Reader syncWriter storage.Writer - message schema.DatasetMapping ) func main() { @@ -162,6 +161,7 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { return } + var message schema.DatasetMapping // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) From 5572f73dd08259b47d37276f0040efbae066e9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:13:11 +0100 Subject: [PATCH 203/530] feat(mapper): update to use storage/v2 instead of v1, replace db func GetFileInfoFromAccessionID with GetFileInfoFromAccessionID that returns new struct MappingData instead of SyncData --- sda/cmd/mapper/mapper.go | 17 +++++++++++------ sda/internal/database/database.go | 7 +++++++ sda/internal/database/db_functions.go | 20 ++++++++++---------- 3 files changed, 28 insertions(+), 16 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index c4a675aac0..dcbb906ab5 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -3,6 +3,7 @@ package main import ( + "context" "encoding/json" "errors" "fmt" @@ -12,8 +13,8 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" - + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" log "github.com/sirupsen/logrus" ) @@ -31,7 +32,11 @@ func main() { if err != nil { log.Fatal(err) } - inbox, err := storage.NewBackend(conf.Inbox) + lb, err := locationbroker.NewLocationBroker(db) + if err != nil { + log.Fatalf("failed to init new location broker due to: %v", err) + } + inboxWriter, err := storage.NewWriter(context.Background(), "inbox", lb) if err != nil { log.Fatal(err) } @@ -105,14 +110,14 @@ func main() { for _, aID := range mappings.AccessionIDs { log.Debugf("Mapped file to dataset (correlation-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) - fileInfo, err := db.GetFileInfoFromAccessionID(aID) + fileMappingData, err := db.GetMappingData(aID) if err != nil { log.Errorf("failed to get file info for file with stable ID: %s", aID) } - err = inbox.RemoveFile(helper.UnanonymizeFilepath(fileInfo.FilePath, fileInfo.User)) + err = inboxWriter.RemoveFile(context.Background(), fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)) if err != nil { - log.Errorf("Remove file from inbox %s failed, reason: %v", fileInfo.FilePath, err) + log.Errorf("Remove file: %s failed, reason: %v", fileMappingData.FileID, err) } } diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index fb53c9d860..6d673b9984 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -41,6 +41,13 @@ type FileInfo struct { UploadedChecksum string } +type MappingData struct { + FileID string + User string + SubmissionFilePath string + SubmissionLocation string +} + type SyncData struct { User string FilePath string diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 17cefe15f9..2591c071d5 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -782,15 +782,15 @@ func (dbs *SDAdb) GetHeaderForStableID(stableID string) ([]byte, error) { return header, nil } -// GetFileInfoFromAccessionID retrieves the file information needed for mapping -func (dbs *SDAdb) GetFileInfoFromAccessionID(accessionID string) (SyncData, error) { +// GetMappingData retrieves the file information needed for mapping +func (dbs *SDAdb) GetMappingData(accessionID string) (MappingData, error) { var ( - s SyncData + s MappingData err error ) for count := 1; count <= RetryTimes; count++ { - s, err = dbs.getFileInfoFromAccessionID(accessionID) + s, err = dbs.getMappingData(accessionID) if err == nil { break } @@ -800,16 +800,16 @@ func (dbs *SDAdb) GetFileInfoFromAccessionID(accessionID string) (SyncData, erro return s, err } -// getFileInfoFromAccessionID is the actual function performing work for GetFileInfoFromAccessionID -func (dbs *SDAdb) getFileInfoFromAccessionID(accessionID string) (SyncData, error) { +// getMappingData is the actual function performing work for GetFileInfoFromAccessionID +func (dbs *SDAdb) getMappingData(accessionID string) (MappingData, error) { dbs.checkAndReconnectIfNeeded() - const query = "SELECT submission_user, submission_file_path from sda.files WHERE stable_id = $1;" - var data SyncData - if err := dbs.DB.QueryRow(query, accessionID).Scan(&data.User, &data.FilePath); err != nil { + const query = "SELECT id, submission_user, submission_file_path, submission_location FROM sda.files WHERE stable_id = $1;" + var data MappingData + if err := dbs.DB.QueryRow(query, accessionID).Scan(&data.FileID, &data.User, &data.SubmissionFilePath, &data.SubmissionLocation); err != nil { log.Warnf("Error while searching for id %s: %v", accessionID, err) - return SyncData{}, err + return MappingData{}, err } return data, nil From eaa4958e76f5543e07657d9d5e8ccbd4424f845b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:45:29 +0100 Subject: [PATCH 204/530] feat(mapper): add required db version check at v23 with new location columns --- sda/cmd/mapper/mapper.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index dcbb906ab5..bdd00b1fd3 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -32,6 +32,10 @@ func main() { if err != nil { log.Fatal(err) } + if db.Version < 23 { + log.Fatal("database schema v23 is required") + } + lb, err := locationbroker.NewLocationBroker(db) if err != nil { log.Fatalf("failed to init new location broker due to: %v", err) From 1fcedf672209098453b8f0667f1a64efec525017 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 14:57:30 +0100 Subject: [PATCH 205/530] feat(mapper): update README.md to now mention expected storage config --- sda/cmd/mapper/mapper.md | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/sda/cmd/mapper/mapper.md b/sda/cmd/mapper/mapper.md index 8cf78b7d1f..d65f44b0b9 100644 --- a/sda/cmd/mapper/mapper.md +++ b/sda/cmd/mapper/mapper.md @@ -80,30 +80,14 @@ These settings control how `mapper` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings - -Storage backend is defined by the `INBOX_TYPE` variable. -Valid values for these options are `S3` or `POSIX` -(Defaults to `POSIX` on unknown values). - -The value of these variables define what other variables are read. -The same variables are available for all storage types, differing by prefix (`INBOX_`) - -if `*_TYPE` is `S3` then the following variables are available: - -- `*_URL`: URL to the S3 system -- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, - [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, - [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_BUCKET`: The S3 bucket to use as the storage root -- `*_PORT`: S3 connection port (default: `443`) -- `*_REGION`: S3 region (default: `us-east-1`) -- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. -- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity - -and if `*_TYPE` is `POSIX`: - -- `*_LOCATION`: POSIX path to use as storage root +The mapper service requires access to the "inbox" storage, to configure such the following config is needed: +Such as +```yaml +storage: + inbox: + ${STORAGE_IMPLEMENTATION}: +``` +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) ### Logging settings From f06303b8214293da4d3d6dbe259d8cdd7ef70857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 13:36:46 +0100 Subject: [PATCH 206/530] feat(mapper): update config to not require and load in storage v1 for mapper service --- sda/internal/config/config.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 9c03131a6b..1dfb29406f 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -346,14 +346,6 @@ func NewConfig(app string) (*Config, error) { "db.database", } - switch viper.GetString("inbox.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) - default: - return nil, errors.New("inbox.type not set") - } case "notify": requiredConfVars = []string{ "broker.host", @@ -601,7 +593,6 @@ func NewConfig(app string) (*Config, error) { return nil, err } - c.configInbox() c.configSchemas() case "notify": c.configSMTP() From e0c26b853ebd802613481da5de257259ccfae092 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 08:58:44 +0100 Subject: [PATCH 207/530] feat(mapper): use fatal instead error log and return when main failed to initialize resources, remove context cancel, just use bg ctx --- sda/cmd/mapper/mapper.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index bdd00b1fd3..c630d677c9 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -22,15 +22,15 @@ func main() { forever := make(chan bool) conf, err := config.NewConfig("mapper") if err != nil { - log.Fatal(err) + log.Fatalf("failed to load config, due to: %v", err) } mq, err := broker.NewMQ(conf.Broker) if err != nil { - log.Fatal(err) + log.Fatalf("failed to initalize mq broker, due to: %v", err) } db, err := database.NewSDAdb(conf.Database) if err != nil { - log.Fatal(err) + log.Fatalf("failed to initalize sda db, due to: %v", err) } if db.Version < 23 { log.Fatal("database schema v23 is required") @@ -38,11 +38,11 @@ func main() { lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Fatalf("failed to init new location broker due to: %v", err) + log.Fatalf("failed to initialize location broker, due to: %v", err) } inboxWriter, err := storage.NewWriter(context.Background(), "inbox", lb) if err != nil { - log.Fatal(err) + log.Fatalf("failed to initialize inbox writer, due to: %v", err) } defer mq.Channel.Close() From e9c1ce434f0dae5b2b9ec50dca4b21553d80899c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 14:28:26 +0100 Subject: [PATCH 208/530] feat(mapper): update main shutdown and erroring logic --- sda/cmd/mapper/mapper.go | 236 ++++++++++++++++++++++----------------- 1 file changed, 131 insertions(+), 105 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index c630d677c9..7e945684e5 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -7,6 +7,9 @@ import ( "encoding/json" "errors" "fmt" + "os" + "os/signal" + "syscall" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" @@ -15,155 +18,178 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" + amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" ) +var db *database.SDAdb +var inboxWriter storage.Writer +var mappings schema.DatasetMapping +var mqBroker *broker.AMQPBroker + func main() { - forever := make(chan bool) - conf, err := config.NewConfig("mapper") - if err != nil { - log.Fatalf("failed to load config, due to: %v", err) + if err := run(); err != nil { + log.Fatal(err) } - mq, err := broker.NewMQ(conf.Broker) +} +func run() error { + var err error + conf, err := config.NewConfig("mapper") if err != nil { - log.Fatalf("failed to initalize mq broker, due to: %v", err) + return fmt.Errorf("failed to load config, due to: %v", err) } - db, err := database.NewSDAdb(conf.Database) + + db, err = database.NewSDAdb(conf.Database) if err != nil { - log.Fatalf("failed to initalize sda db, due to: %v", err) + return fmt.Errorf("failed to initalize sda db, due to: %v", err) } + defer db.Close() if db.Version < 23 { - log.Fatal("database schema v23 is required") + return errors.New("database schema v23 is required") } + mqBroker, err = broker.NewMQ(conf.Broker) + if err != nil { + return fmt.Errorf("failed to initalize mq broker, due to: %v", err) + } + defer func() { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } + }() + lb, err := locationbroker.NewLocationBroker(db) if err != nil { - log.Fatalf("failed to initialize location broker, due to: %v", err) + return fmt.Errorf("failed to initialize location broker, due to: %v", err) } - inboxWriter, err := storage.NewWriter(context.Background(), "inbox", lb) + inboxWriter, err = storage.NewWriter(context.Background(), "inbox", lb) if err != nil { - log.Fatalf("failed to initialize inbox writer, due to: %v", err) + return fmt.Errorf("failed to initialize inbox writer, due to: %v", err) } - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() - + log.Info("Starting mapper service") + consumeErr := make(chan error, 1) go func() { - connError := mq.ConnectionWatcher() - log.Error(connError) - forever <- false + consumeErr <- startConsumer() }() - go func() { - connError := mq.ChannelWatcher() - log.Error(connError) - forever <- false - }() + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + select { + case <-sigc: + case err := <-mqBroker.Connection.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-mqBroker.Channel.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-consumeErr: + return err + } - log.Info("Starting mapper service") - var mappings schema.DatasetMapping + return nil +} +func startConsumer() error { + messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) + if err != nil { + return fmt.Errorf("failed to get message from mq (error: %v)", err) + } - go func() { - messages, err := mq.GetMessages(conf.Broker.Queue) + for delivered := range messages { + log.Debugf("received a message: %s", delivered.Body) + schemaType, err := schemaFromDatasetOperation(delivered.Body) if err != nil { - log.Fatalf("Failed to get message from mq (error: %v)", err) - } - - for delivered := range messages { - log.Debugf("received a message: %s", delivered.Body) - schemaType, err := schemaFromDatasetOperation(delivered.Body) - if err != nil { - log.Errorf("%s", err.Error()) - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message: %v", err) - } - if err := mq.SendMessage(delivered.CorrelationId, mq.Conf.Exchange, "error", delivered.Body); err != nil { - log.Errorf("failed to send error message: %v", err) - } - - continue + log.Errorf("%s", err.Error()) + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message: %v", err) + } + if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", delivered.Body); err != nil { + log.Errorf("failed to send error message: %v", err) } - err = schema.ValidateJSON(fmt.Sprintf("%s/%s.json", conf.Broker.SchemasPath, schemaType), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (%s) failed, reason: %v ", schemaType, err) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } + continue + } - continue + err = schema.ValidateJSON(fmt.Sprintf("%s/%s.json", mqBroker.Conf.SchemasPath, schemaType), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (%s) failed, reason: %v ", schemaType, err) + if err := delivered.Ack(false); err != nil { + log.Errorf("failed acking canceled work, reason: %v", err) } - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &mappings) + continue + } - switch mappings.Type { - case "mapping": - log.Debug("Mapping type operation, mapping files to dataset") - if err := db.MapFilesToDataset(mappings.DatasetID, mappings.AccessionIDs); err != nil { - log.Errorf("failed to map files to dataset, dataset-id: %s, reason: %v", mappings.DatasetID, err) + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &mappings) - // Nack message so the server gets notified that something is wrong and requeue the message - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } + switch mappings.Type { + case "mapping": + log.Debug("mapping type operation, mapping files to dataset") + if err := db.MapFilesToDataset(mappings.DatasetID, mappings.AccessionIDs); err != nil { + log.Errorf("failed to map files to dataset, dataset-id: %s, reason: %v", mappings.DatasetID, err) - continue + // Nack message so the server gets notified that something is wrong and requeue the message + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) } - for _, aID := range mappings.AccessionIDs { - log.Debugf("Mapped file to dataset (correlation-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) - fileMappingData, err := db.GetMappingData(aID) - if err != nil { - log.Errorf("failed to get file info for file with stable ID: %s", aID) - } - - err = inboxWriter.RemoveFile(context.Background(), fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)) - if err != nil { - log.Errorf("Remove file: %s failed, reason: %v", fileMappingData.FileID, err) - } - } + continue + } - if err := db.UpdateDatasetEvent(mappings.DatasetID, "registered", string(delivered.Body)); err != nil { - log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } + for _, aID := range mappings.AccessionIDs { + log.Debugf("Mapped file to dataset (correlation-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) + fileMappingData, err := db.GetMappingData(aID) + if err != nil { + log.Errorf("failed to get file info for file with stable ID: %s", aID) + } - continue + err = inboxWriter.RemoveFile(context.Background(), fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)) + if err != nil { + log.Errorf("remove file: %s failed, reason: %v", fileMappingData.FileID, err) } - case "release": - log.Debug("Release type operation, marking dataset as released") - if err := db.UpdateDatasetEvent(mappings.DatasetID, "released", string(delivered.Body)); err != nil { - log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue + } + + if err := db.UpdateDatasetEvent(mappings.DatasetID, "registered", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - case "deprecate": - log.Debug("Deprecate type operation, marking dataset as deprecated") - if err := db.UpdateDatasetEvent(mappings.DatasetID, "deprecated", string(delivered.Body)); err != nil { - log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue + + continue + } + case "release": + log.Debug("release type operation, marking dataset as released") + if err := db.UpdateDatasetEvent(mappings.DatasetID, "released", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - default: - log.Errorf("unknown mapping type, %s", mappings.Type) + + continue } + case "deprecate": + log.Debug("deprecate type operation, marking dataset as deprecated") + if err := db.UpdateDatasetEvent(mappings.DatasetID, "deprecated", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%v)", err) + continue } + default: + log.Errorf("unknown mapping type, %s", mappings.Type) } - }() - <-forever + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%v)", err) + } + } + + return nil } // schemaFromDatasetOperation returns the operation done with dataset supplied in body of the message From eaeecea440d63b0a30d35d0aa0b06d2f19406626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Feb 2026 10:05:03 +0100 Subject: [PATCH 209/530] feat(sync): update GetMappingData func, ensure we dont scan null submission location to string, add check if submission location is empty before removing inbox file, add updated db func GetMappingData to database interface for the sake of eventual completion --- sda/cmd/mapper/mapper.go | 10 +++++++++- sda/internal/database/database_interface.go | 1 + sda/internal/database/db_functions.go | 20 +++++++++++++------ .../v2/locationbroker/location_broker_test.go | 4 ++++ 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 7e945684e5..e91d253681 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -143,7 +143,15 @@ func startConsumer() error { log.Debugf("Mapped file to dataset (correlation-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) fileMappingData, err := db.GetMappingData(aID) if err != nil { - log.Errorf("failed to get file info for file with stable ID: %s", aID) + log.Errorf("failed to get file info for file with stable ID: %s, can not remove file from inbox", aID) + + continue + } + + if fileMappingData == nil || fileMappingData.SubmissionLocation == "" { + log.Errorf("failed to find submission location for file with stable ID: %s, can not remove file from inbox", aID) + + continue } err = inboxWriter.RemoveFile(context.Background(), fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)) diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index 0455277ffa..6de9ed60a6 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -9,4 +9,5 @@ type Database interface { GetUploadedSubmissionFilePathAndLocation(ctx context.Context, submissionUser, fileID string) (string, string, error) GetArchiveLocation(fileID string) (string, error) GetArchivePathAndLocation(stableID string) (string, string, error) + GetMappingData(accessionID string) (*MappingData, error) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 2591c071d5..dd2e023009 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -783,9 +783,9 @@ func (dbs *SDAdb) GetHeaderForStableID(stableID string) ([]byte, error) { } // GetMappingData retrieves the file information needed for mapping -func (dbs *SDAdb) GetMappingData(accessionID string) (MappingData, error) { +func (dbs *SDAdb) GetMappingData(accessionID string) (*MappingData, error) { var ( - s MappingData + s *MappingData err error ) @@ -801,15 +801,23 @@ func (dbs *SDAdb) GetMappingData(accessionID string) (MappingData, error) { } // getMappingData is the actual function performing work for GetFileInfoFromAccessionID -func (dbs *SDAdb) getMappingData(accessionID string) (MappingData, error) { +func (dbs *SDAdb) getMappingData(accessionID string) (*MappingData, error) { dbs.checkAndReconnectIfNeeded() const query = "SELECT id, submission_user, submission_file_path, submission_location FROM sda.files WHERE stable_id = $1;" - var data MappingData - if err := dbs.DB.QueryRow(query, accessionID).Scan(&data.FileID, &data.User, &data.SubmissionFilePath, &data.SubmissionLocation); err != nil { + data := &MappingData{} + var submissionLocation sql.NullString + if err := dbs.DB.QueryRow(query, accessionID).Scan(&data.FileID, &data.User, &data.SubmissionFilePath, &submissionLocation); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } log.Warnf("Error while searching for id %s: %v", accessionID, err) - return MappingData{}, err + return nil, err + } + + if submissionLocation.Valid { + data.SubmissionLocation = submissionLocation.String } return data, nil diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index f5aa6a00b7..0902886cfd 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" ) @@ -32,6 +33,9 @@ func (m *mockDatabase) GetArchiveLocation(_ string) (string, error) { func (m *mockDatabase) GetArchivePathAndLocation(_ string) (string, string, error) { panic("function not expected to be called in unit tests") } +func (m *mockDatabase) GetMappingData(_ string) (*database.MappingData, error) { + panic("function not expected to be called in unit tests") +} func TestLocationBrokerTestSuite(t *testing.T) { suite.Run(t, new(LocationBrokerTestSuite)) From 3431cd2b8f1b70eecd6139fb7239b4d2fff87406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:15:06 +0100 Subject: [PATCH 210/530] fix(mapper): fix typo in mapper.md --- sda/cmd/mapper/mapper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/mapper/mapper.md b/sda/cmd/mapper/mapper.md index d65f44b0b9..a855fc8e6b 100644 --- a/sda/cmd/mapper/mapper.md +++ b/sda/cmd/mapper/mapper.md @@ -87,7 +87,7 @@ storage: inbox: ${STORAGE_IMPLEMENTATION}: ``` -For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) ### Logging settings From 1decd86bb83060d82085e22eb49578c74c867a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:46:52 +0100 Subject: [PATCH 211/530] feat(mapper): restructure handling of message to separate func with spawns content from main context so it can react to service shutting down and can gracefully shutdown --- sda/cmd/mapper/mapper.go | 161 +++++++++++++++++++++------------------ 1 file changed, 85 insertions(+), 76 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index e91d253681..3fcb8af74e 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -33,6 +33,9 @@ func main() { } } func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var err error conf, err := config.NewConfig("mapper") if err != nil { @@ -65,7 +68,7 @@ func run() error { if err != nil { return fmt.Errorf("failed to initialize location broker, due to: %v", err) } - inboxWriter, err = storage.NewWriter(context.Background(), "inbox", lb) + inboxWriter, err = storage.NewWriter(ctx, "inbox", lb) if err != nil { return fmt.Errorf("failed to initialize inbox writer, due to: %v", err) } @@ -73,7 +76,7 @@ func run() error { log.Info("Starting mapper service") consumeErr := make(chan error, 1) go func() { - consumeErr <- startConsumer() + consumeErr <- startConsumer(ctx) }() sigc := make(chan os.Signal, 5) @@ -91,113 +94,119 @@ func run() error { return nil } -func startConsumer() error { +func startConsumer(ctx context.Context) error { messages, err := mqBroker.GetMessages(mqBroker.Conf.Queue) if err != nil { return fmt.Errorf("failed to get message from mq (error: %v)", err) } for delivered := range messages { - log.Debugf("received a message: %s", delivered.Body) - schemaType, err := schemaFromDatasetOperation(delivered.Body) - if err != nil { - log.Errorf("%s", err.Error()) - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message: %v", err) - } - if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", delivered.Body); err != nil { - log.Errorf("failed to send error message: %v", err) - } + handleMessage(ctx, delivered) + } - continue + return nil +} + +func handleMessage(ctx context.Context, delivered amqp.Delivery) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + log.Debugf("received a message: %s", delivered.Body) + schemaType, err := schemaFromDatasetOperation(delivered.Body) + if err != nil { + log.Errorf("%s", err.Error()) + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message: %v", err) + } + if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", delivered.Body); err != nil { + log.Errorf("failed to send error message: %v", err) } - err = schema.ValidateJSON(fmt.Sprintf("%s/%s.json", mqBroker.Conf.SchemasPath, schemaType), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (%s) failed, reason: %v ", schemaType, err) - if err := delivered.Ack(false); err != nil { - log.Errorf("failed acking canceled work, reason: %v", err) - } + return + } - continue + err = schema.ValidateJSON(fmt.Sprintf("%s/%s.json", mqBroker.Conf.SchemasPath, schemaType), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (%s) failed, reason: %v ", schemaType, err) + if err := delivered.Ack(false); err != nil { + log.Errorf("failed acking canceled work, reason: %v", err) } - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &mappings) + return + } - switch mappings.Type { - case "mapping": - log.Debug("mapping type operation, mapping files to dataset") - if err := db.MapFilesToDataset(mappings.DatasetID, mappings.AccessionIDs); err != nil { - log.Errorf("failed to map files to dataset, dataset-id: %s, reason: %v", mappings.DatasetID, err) + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &mappings) - // Nack message so the server gets notified that something is wrong and requeue the message - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) - } + switch mappings.Type { + case "mapping": + log.Debug("mapping type operation, mapping files to dataset") + if err := db.MapFilesToDataset(mappings.DatasetID, mappings.AccessionIDs); err != nil { + log.Errorf("failed to map files to dataset, dataset-id: %s, reason: %v", mappings.DatasetID, err) - continue + // Nack message so the server gets notified that something is wrong and requeue the message + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%v)", err) } - for _, aID := range mappings.AccessionIDs { - log.Debugf("Mapped file to dataset (correlation-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) - fileMappingData, err := db.GetMappingData(aID) - if err != nil { - log.Errorf("failed to get file info for file with stable ID: %s, can not remove file from inbox", aID) - - continue - } - - if fileMappingData == nil || fileMappingData.SubmissionLocation == "" { - log.Errorf("failed to find submission location for file with stable ID: %s, can not remove file from inbox", aID) + return + } - continue - } + for _, aID := range mappings.AccessionIDs { + log.Debugf("Mapped file to dataset (correlation-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) + fileMappingData, err := db.GetMappingData(aID) + if err != nil { + log.Errorf("failed to get file info for file with stable ID: %s, can not remove file from inbox", aID) - err = inboxWriter.RemoveFile(context.Background(), fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)) - if err != nil { - log.Errorf("remove file: %s failed, reason: %v", fileMappingData.FileID, err) - } + continue } - if err := db.UpdateDatasetEvent(mappings.DatasetID, "registered", string(delivered.Body)); err != nil { - log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } + if fileMappingData == nil || fileMappingData.SubmissionLocation == "" { + log.Errorf("failed to find submission location for file with stable ID: %s, can not remove file from inbox", aID) continue } - case "release": - log.Debug("release type operation, marking dataset as released") - if err := db.UpdateDatasetEvent(mappings.DatasetID, "released", string(delivered.Body)); err != nil { - log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - continue + err = inboxWriter.RemoveFile(ctx, fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)) + if err != nil { + log.Errorf("remove file: %s failed, reason: %v", fileMappingData.FileID, err) } - case "deprecate": - log.Debug("deprecate type operation, marking dataset as deprecated") - if err := db.UpdateDatasetEvent(mappings.DatasetID, "deprecated", string(delivered.Body)); err != nil { - log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } + } - continue + if err := db.UpdateDatasetEvent(mappings.DatasetID, "registered", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - default: - log.Errorf("unknown mapping type, %s", mappings.Type) + + return } + case "release": + log.Debug("release type operation, marking dataset as released") + if err := db.UpdateDatasetEvent(mappings.DatasetID, "released", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%v)", err) + return + } + case "deprecate": + log.Debug("deprecate type operation, marking dataset as deprecated") + if err := db.UpdateDatasetEvent(mappings.DatasetID, "deprecated", string(delivered.Body)); err != nil { + log.Errorf("failed to set dataset status for dataset: %s", mappings.DatasetID) + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + } + + return } + default: + log.Errorf("unknown mapping type, %s", mappings.Type) } - return nil + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%v)", err) + } } // schemaFromDatasetOperation returns the operation done with dataset supplied in body of the message From 2aebfe207d5f6f43438980bcae92547ece85180f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:23:48 +0100 Subject: [PATCH 212/530] feat(mapper): update sigc chan buffer size to 1 instead of 5 as we just care that any of the signals triggered --- sda/cmd/mapper/mapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 3fcb8af74e..48dc1b9129 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -79,7 +79,7 @@ func run() error { consumeErr <- startConsumer(ctx) }() - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { From 2e4bc40a6abe0f915a2d09ab8171126faa81662b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:43:12 +0100 Subject: [PATCH 213/530] feat(mapper): if message is a unknown mapping type, ack and send to error queue --- sda/cmd/mapper/mapper.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 48dc1b9129..29b1572ebf 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -166,8 +166,7 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { continue } - err = inboxWriter.RemoveFile(ctx, fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)) - if err != nil { + if err = inboxWriter.RemoveFile(ctx, fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)); err != nil { log.Errorf("remove file: %s failed, reason: %v", fileMappingData.FileID, err) } } @@ -202,6 +201,14 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { } default: log.Errorf("unknown mapping type, %s", mappings.Type) + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message: %v", err) + } + if err := mqBroker.SendMessage(delivered.CorrelationId, mqBroker.Conf.Exchange, "error", delivered.Body); err != nil { + log.Errorf("failed to send error message: %v", err) + } + + return } if err := delivered.Ack(false); err != nil { From 823b8d48666052c45f62ac8bd886196d929acafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:55:12 +0100 Subject: [PATCH 214/530] feat(mapper): fix typo in error message initalize -> initialize --- sda/cmd/mapper/mapper.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 29b1572ebf..40ddf94ac9 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -44,7 +44,7 @@ func run() error { db, err = database.NewSDAdb(conf.Database) if err != nil { - return fmt.Errorf("failed to initalize sda db, due to: %v", err) + return fmt.Errorf("failed to initialize sda db, due to: %v", err) } defer db.Close() if db.Version < 23 { @@ -53,7 +53,7 @@ func run() error { mqBroker, err = broker.NewMQ(conf.Broker) if err != nil { - return fmt.Errorf("failed to initalize mq broker, due to: %v", err) + return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } defer func() { if err := mqBroker.Channel.Close(); err != nil { From 37c9c1b076e0dfb735f5fa4df1e1b15bfbb8f254 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 13:04:34 +0100 Subject: [PATCH 215/530] feat(mapper): update inboxWriter.RemoveFile error message to include location and unanonymized file path --- sda/cmd/mapper/mapper.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 40ddf94ac9..036501caa9 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -166,8 +166,9 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { continue } - if err = inboxWriter.RemoveFile(ctx, fileMappingData.SubmissionLocation, helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User)); err != nil { - log.Errorf("remove file: %s failed, reason: %v", fileMappingData.FileID, err) + unanonymizedSubmissionFilePath := helper.UnanonymizeFilepath(fileMappingData.SubmissionFilePath, fileMappingData.User) + if err := inboxWriter.RemoveFile(ctx, fileMappingData.SubmissionLocation, unanonymizedSubmissionFilePath); err != nil { + log.Errorf("removal of file id: %s at location: %s, path: %s failed, reason: %v", fileMappingData.FileID, fileMappingData.SubmissionLocation, unanonymizedSubmissionFilePath, err) } } From 01d99ea76bf3c5b5741e909aab5037df1cb8be08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 15:02:59 +0100 Subject: [PATCH 216/530] feat(mapper): fix some sentance in mapper.md --- sda/cmd/mapper/mapper.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sda/cmd/mapper/mapper.md b/sda/cmd/mapper/mapper.md index a855fc8e6b..557bca6661 100644 --- a/sda/cmd/mapper/mapper.md +++ b/sda/cmd/mapper/mapper.md @@ -80,8 +80,7 @@ These settings control how `mapper` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings -The mapper service requires access to the "inbox" storage, to configure such the following config is needed: -Such as +The mapper service requires access to the "inbox" storage. To configure that, the following configuration is required: ```yaml storage: inbox: From f00c84f814065d392fcde374efd8cd2b1a2d0c25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:11:27 +0100 Subject: [PATCH 217/530] feat(database): remove retry for GetMappingData --- sda/internal/database/db_functions.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index dd2e023009..a6aef29a9a 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -784,24 +784,6 @@ func (dbs *SDAdb) GetHeaderForStableID(stableID string) ([]byte, error) { // GetMappingData retrieves the file information needed for mapping func (dbs *SDAdb) GetMappingData(accessionID string) (*MappingData, error) { - var ( - s *MappingData - err error - ) - - for count := 1; count <= RetryTimes; count++ { - s, err = dbs.getMappingData(accessionID) - if err == nil { - break - } - time.Sleep(time.Duration(math.Pow(3, float64(count))) * time.Second) - } - - return s, err -} - -// getMappingData is the actual function performing work for GetFileInfoFromAccessionID -func (dbs *SDAdb) getMappingData(accessionID string) (*MappingData, error) { dbs.checkAndReconnectIfNeeded() const query = "SELECT id, submission_user, submission_file_path, submission_location FROM sda.files WHERE stable_id = $1;" From cbd0af09e13cea39f85263669f99b62ff57376e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:29:13 +0100 Subject: [PATCH 218/530] feat(mapper): update defer closing of rabbit mq connection and channel to check for nil before close --- sda/cmd/mapper/mapper.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 036501caa9..0a4cb96bb7 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -56,11 +56,18 @@ func run() error { return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } defer func() { - if err := mqBroker.Channel.Close(); err != nil { - log.Errorf("failed to close mq broker channel due to: %v", err) + if mqBroker == nil { + return + } + if mqBroker.Channel != nil { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } } - if err := mqBroker.Connection.Close(); err != nil { - log.Errorf("failed to close mq broker connection due to: %v", err) + if mqBroker.Connection != nil { + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } } }() From c86077c4ebdb2b356ed4ca95848b406f47a8862b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 10:03:17 +0100 Subject: [PATCH 219/530] feat(mapper): move message variable to handleMessage instead of global variable --- sda/cmd/mapper/mapper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 0a4cb96bb7..dcdd941b07 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -24,7 +24,6 @@ import ( var db *database.SDAdb var inboxWriter storage.Writer -var mappings schema.DatasetMapping var mqBroker *broker.AMQPBroker func main() { @@ -141,6 +140,7 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { return } + var mappings schema.DatasetMapping // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &mappings) From 1c30affd2541e60857503d58c247cd7a4446c73c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 11:11:35 +0100 Subject: [PATCH 220/530] feat(ingest): update to use storage/v2 instead of v1, add new db func GetSubmissionLocation to be used to find the submission location of a file by its id --- sda/cmd/ingest/ingest.go | 446 +++++++++++--------- sda/cmd/ingest/ingest_test.go | 151 ++++--- sda/internal/database/database_interface.go | 1 + sda/internal/database/db_functions.go | 17 + 4 files changed, 363 insertions(+), 252 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 27b30e0c02..4e211bc4f7 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -5,7 +5,9 @@ package main import ( "bytes" + "context" "crypto/sha256" + "database/sql" "encoding/hex" "encoding/json" "errors" @@ -24,21 +26,25 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" - + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" log "github.com/sirupsen/logrus" ) type Ingest struct { - Archive storage.Backend + ArchiveWriter storage.Writer + ArchiveReader storage.Reader ArchiveKeyList []*[32]byte - Conf *config.Config DB *database.SDAdb - Inbox storage.Backend + InboxReader storage.Reader + InboxWriter storage.Writer // Used to delete files from inbox + brokerConf broker.MQConf MQ *broker.AMQPBroker } func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() app := Ingest{} var err error sigc := make(chan os.Signal, 5) @@ -52,19 +58,20 @@ func main() { forever := make(chan bool) - app.Conf, err = config.NewConfig("ingest") + ingestConf, err := config.NewConfig("ingest") if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - app.MQ, err = broker.NewMQ(app.Conf.Broker) + app.brokerConf = ingestConf.Broker + app.MQ, err = broker.NewMQ(app.brokerConf) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - app.DB, err = database.NewSDAdb(app.Conf.Database) + app.DB, err = database.NewSDAdb(ingestConf.Database) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -85,13 +92,24 @@ func main() { panic(err) } - app.Archive, err = storage.NewBackend(app.Conf.Archive) + storageLocationBroker, err := locationbroker.NewLocationBroker(app.DB) + if err != nil { + log.Fatalf("failed to init new location broker due to: %v", err) + } + + app.ArchiveWriter, err = storage.NewWriter(ctx, "archive", storageLocationBroker) + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } + app.InboxReader, err = storage.NewReader(ctx, "inbox") if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - app.Inbox, err = storage.NewBackend(app.Conf.Inbox) + app.InboxWriter, err = storage.NewWriter(ctx, "inbox", storageLocationBroker) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -116,72 +134,75 @@ func main() { log.Info("starting ingest service") - go func() { - messages, err := app.MQ.GetMessages(app.Conf.Broker.Queue) - if err != nil { - log.Panic(err) - } + go app.run() - for delivered := range messages { - log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - message := schema.IngestionTrigger{} - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.Conf.Broker.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Message validation failed", - Reason: err.Error(), - OriginalMessage: message, - } + <-forever +} - body, _ := json.Marshal(infoErrorMessage) - if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } +func (app *Ingest) run() { + messages, err := app.MQ.GetMessages(app.brokerConf.Queue) + if err != nil { + log.Panic(err) + } - continue + for delivered := range messages { + ctx := context.Background() + log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + message := schema.IngestionTrigger{} + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.brokerConf.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed", + Reason: err.Error(), + OriginalMessage: message, } - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) - log.Infof("Received work (correlation-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) - - ackNack := "" - switch message.Type { - case "cancel": - ackNack = app.cancelFile(delivered.CorrelationId, message) - case "ingest": - ackNack = app.ingestFile(delivered.CorrelationId, message) - default: - log.Errorln("unexpected ingest message type") - if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message, reason: %v", err) - } + body, _ := json.Marshal(infoErrorMessage) + if err := app.MQ.SendMessage(delivered.CorrelationId, app.brokerConf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) } - switch ackNack { - case "ack": - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message, reason: %v", err) - } - case "nack": - if err = delivered.Nack(false, false); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } - default: - // will catch `reject`s, failures that should not be requeued. - if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message, reason: %v", err) - } + continue + } + + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + log.Infof("Received work (correlation-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) + + ackNack := "" + switch message.Type { + case "cancel": + ackNack = app.cancelFile(delivered.CorrelationId, message) + case "ingest": + ackNack = app.ingestFile(ctx, delivered.CorrelationId, message) + default: + log.Errorln("unexpected ingest message type") + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message, reason: %v", err) } } - }() - <-forever + switch ackNack { + case "ack": + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message, reason: %v", err) + } + case "nack": + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) + } + default: + // will catch `reject`s, failures that should not be requeued. + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message, reason: %v", err) + } + } + } } func (app *Ingest) registerC4GHKey() error { @@ -215,14 +236,27 @@ func (app *Ingest) cancelFile(fileID string, message schema.IngestionTrigger) st return "ack" } -func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) string { +func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema.IngestionTrigger) string { status, err := app.DB.GetFileStatus(fileID) - if err != nil && err.Error() != "sql: no rows in result set" { + if err != nil && !errors.Is(sql.ErrNoRows, err) { log.Errorf("failed to get status for file, fileID: %s, reason: (%s)", fileID, err.Error()) return "nack" } + submissionLocation, err := app.DB.GetSubmissionLocation(ctx, fileID) + if err != nil { + log.Errorf("failed to get submission location for file, fileID: %s, reason: (%s)", fileID, err.Error()) + + return "nack" + } + + if status != "" && submissionLocation == "" { + log.Errorf("file %s has been registered but has no submission location", fileID) + + return "nack" + } + switch status { case "disabled": @@ -236,7 +270,7 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st // What if the file in the inbox is different this time? // Check uploaded checksum in the DB against the checksum of the file. // Would be easy if the inbox message had the checksum or that the s3inbox added the checksum to the DB. - file, err := app.Inbox.NewFileReader(helper.UnanonymizeFilepath(message.FilePath, message.User)) + file, err := app.InboxReader.NewFileReader(ctx, submissionLocation, helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { switch { case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): @@ -253,7 +287,7 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st OriginalMessage: message, } body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) return "reject" @@ -285,7 +319,7 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st }, } archivedMsg, _ := json.Marshal(&msg) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.brokerConf.SchemasPath), archivedMsg) if err != nil { log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) @@ -299,7 +333,7 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st return "nack" } - if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, app.brokerConf.RoutingKey, archivedMsg); err != nil { log.Errorf("failed to publish message, reason: %v", err) return "reject" @@ -308,27 +342,18 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st return "ack" } case "": - // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox - log.Infof("registering file, file-id: %s", fileID) fileID, err = app.DB.RegisterFile(&fileID, message.FilePath, message.User) if err != nil { log.Errorf("failed to register file, fileID: %s, reason: (%s)", fileID, err.Error()) return "nack" } - case "uploaded": - - default: - log.Warnf("unsupported file status: %s, file-id: %s", status, fileID) - return "reject" - } - - file, err := app.Inbox.NewFileReader(helper.UnanonymizeFilepath(message.FilePath, message.User)) - if err != nil { - switch { - case (strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:")): - log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox + // Since we dont have the submission location in storage, we need to look through all configured storage locations. + submissionLocation, err = app.InboxReader.FindFile(ctx, message.FilePath) + if err != nil { + log.Errorf("failed to find submission location for file in all configured storage locations, file-id: %s", fileID) jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { @@ -341,31 +366,39 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st OriginalMessage: message, } body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) return "reject" } return "ack" - default: - log.Errorf("unexpected error when opening file for reading, file-id: %s, filepath: %s, reason: %s", fileID, message.FilePath, err.Error()) - - return "nack" } + log.Infof("registering file, file-id: %s", fileID) + + case "uploaded": + + default: + log.Warnf("unsupported file status: %s, file-id: %s", status, fileID) + + return "reject" } - fileSize, err := app.Inbox.GetFileSize(helper.UnanonymizeFilepath(message.FilePath, message.User), false) + file, err := app.InboxReader.NewFileReader(ctx, submissionLocation, helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { - log.Errorf("Failed to get file size of file to ingest, file-id: %s, filepath: %s, reason: (%s)", fileID, message.FilePath, err.Error()) - // Since reading the file worked, this should eventually succeed so it is ok to requeue. + log.Errorf("unexpected error when opening file for reading, file-id: %s, filepath: %s, reason: %s", fileID, message.FilePath, err.Error()) + return "nack" } + // Ensure file is closed in case we encounter error, etc + defer func() { + _ = file.Close() + }() - dest, err := app.Archive.NewFileWriter(fileID) + fileSize, err := app.InboxReader.GetFileSize(ctx, submissionLocation, helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { - log.Errorf("Failed to create archive file, file-id: %s, reason: (%s)", fileID, err.Error()) - // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. + log.Errorf("Failed to get file size of file to ingest, file-id: %s, filepath: %s, reason: (%s)", fileID, message.FilePath, err.Error()) + // Since reading the file worked, this should eventually succeed so it is ok to requeue. return "nack" } @@ -374,140 +407,161 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st log.Errorf("failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } - // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock - var bufSize int - if bufSize = 4 * 1024 * 1024; app.Conf.Inbox.S3.Chunksize > 4*1024*1024 { - bufSize = app.Conf.Inbox.S3.Chunksize - } + // 50MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock + bufSize := 50 * 1024 * 1024 readBuffer := make([]byte, bufSize) hash := sha256.New() var bytesRead int64 var byteBuf bytes.Buffer + contentReader, contentWriter := io.Pipe() + // Ensure these are closed in case we encounter error + defer func() { + _ = contentReader.Close() + _ = contentWriter.Close() + }() - for bytesRead < fileSize { - i, _ := io.ReadFull(file, readBuffer) - if i == 0 { - log.Errorf("readBuffer returned 0 bytes, this should not happen, file-id: %s", fileID) - - return "reject" - } - // truncate the readbuffer if the file is smaller than the buffer size - if i < len(readBuffer) { - readBuffer = readBuffer[:i] - } + uploadCtx, uploadCancel := context.WithCancel(ctx) - bytesRead += int64(i) + go func() { - h := bytes.NewReader(readBuffer) - if _, err = io.Copy(hash, h); err != nil { - log.Errorf("Copy to hash failed while reading file, file-id: %s, reason: (%s)", fileID, err.Error()) + for bytesRead < fileSize { + i, _ := io.ReadFull(file, readBuffer) + if i == 0 { + log.Errorf("readBuffer returned 0 bytes, this should not happen, file-id: %s", fileID) + uploadCancel() - return "nack" - } + return + } + // truncate the readbuffer if the file is smaller than the buffer size + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } - switch { - case bytesRead <= int64(len(readBuffer)): - var privateKey *[32]byte - var header []byte + bytesRead += int64(i) - // Iterate over the key list to try decryption - for _, key := range app.ArchiveKeyList { - header, err = tryDecrypt(key, readBuffer) - if err == nil { - privateKey = key + h := bytes.NewReader(readBuffer) + if _, err = io.Copy(hash, h); err != nil { + log.Errorf("Copy to hash failed while reading file, file-id: %s, reason: (%s)", fileID, err.Error()) + uploadCancel() - break - } - log.Warnf("Decryption failed with key, trying next key. file-id: %s, reason: (%s)", fileID, err.Error()) + return } - - // Check if decryption was successful with any key - if privateKey == nil { - log.Errorf("All keys failed to decrypt the submitted file, file-id: %s", fileID) - m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { - log.Errorf("Failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) + switch { + case bytesRead <= int64(len(readBuffer)): + var privateKey *[32]byte + var header []byte + + // Iterate over the key list to try decryption + for _, key := range app.ArchiveKeyList { + header, err = tryDecrypt(key, readBuffer) + if err == nil { + privateKey = key + + break + } + log.Warnf("Decryption failed with key, trying next key. file-id: %s, reason: (%s)", fileID, err.Error()) } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Trying to decrypt the submitted file failed", - Reason: "Decryption failed with the available key(s)", - OriginalMessage: message, + // Check if decryption was successful with any key + if privateKey == nil { + log.Errorf("All keys failed to decrypt the submitted file, file-id: %s", fileID) + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { + log.Errorf("Failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) + } + + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Trying to decrypt the submitted file failed", + Reason: "Decryption failed with the available key(s)", + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + } + uploadCancel() + + return } - body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) + + // Proceed with the successful key + // Set the file's hex encoded public key + publicKey := keys.DerivePublicKey(*privateKey) + keyhash := hex.EncodeToString(publicKey[:]) + err = app.DB.SetKeyHash(keyhash, fileID) + if err != nil { + log.Errorf("Key hash %s could not be set for file, file-id: %s, reason: (%s)", keyhash, fileID, err.Error()) + uploadCancel() + + return } - return "ack" - } + log.Debugln("store header") + if err := app.DB.StoreHeader(header, fileID); err != nil { + log.Errorf("StoreHeader failed, file-id: %s, reason: (%s)", fileID, err.Error()) + uploadCancel() - // Proceed with the successful key - // Set the file's hex encoded public key - publicKey := keys.DerivePublicKey(*privateKey) - keyhash := hex.EncodeToString(publicKey[:]) - err = app.DB.SetKeyHash(keyhash, fileID) - if err != nil { - log.Errorf("Key hash %s could not be set for file, file-id: %s, reason: (%s)", keyhash, fileID, err.Error()) + return + } - return "nack" - } + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for header read, file-id: %s, reason: %v)", fileID, err.Error()) + uploadCancel() - log.Debugln("store header") - if err := app.DB.StoreHeader(header, fileID); err != nil { - log.Errorf("StoreHeader failed, file-id: %s, reason: (%s)", fileID, err.Error()) + return + } - return "nack" - } + // Strip header from buffer + h := make([]byte, len(header)) + if _, err = byteBuf.Read(h); err != nil { + log.Errorf("Failed to strip header from buffer, file-id: %s, reason: (%s)", fileID, err.Error()) + uploadCancel() - if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for header read, file-id: %s, reason: %v)", fileID, err.Error()) + return + } + default: + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for full read, file-id: %s, reason: (%s)", fileID, err.Error()) + uploadCancel() - return "nack" + return + } } - // Strip header from buffer - h := make([]byte, len(header)) - if _, err = byteBuf.Read(h); err != nil { - log.Errorf("Failed to strip header from buffer, file-id: %s, reason: (%s)", fileID, err.Error()) + // Write data to file + if _, err = byteBuf.WriteTo(contentWriter); err != nil { + log.Errorf("Failed to write to archive file, file-id: %s, reason: (%s)", fileID, err.Error()) - return "nack" - } - default: - if i < len(readBuffer) { - readBuffer = readBuffer[:i] - } - if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for full read, file-id: %s, reason: (%s)", fileID, err.Error()) + _ = file.Close() + uploadCancel() - return "nack" + return } } - // Write data to file - if _, err = byteBuf.WriteTo(dest); err != nil { - log.Errorf("Failed to write to archive file, file-id: %s, reason: (%s)", fileID, err.Error()) - - _ = file.Close() - _ = dest.Close() - if err := app.Archive.RemoveFile(fileID); err != nil { - log.Errorf("error when removing uncompleted file, file-id: %s, reason: %s", fileID, err.Error()) - } + _ = contentWriter.Close() + _ = file.Close() + }() - return "nack" - } + location, err := app.ArchiveWriter.WriteFile(uploadCtx, fileID, contentReader) + if err != nil { + log.Errorf("Failed to upload archive file, file-id: %s, reason: (%s)", fileID, err.Error()) + // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. + return "nack" } - - file.Close() - dest.Close() + // As we are done with uploadCtx now, we cancel it + uploadCancel() + _ = contentReader.Close() // At this point we should do checksum comparison, but that requires updating the AWS library - fileInfo := database.FileInfo{} fileInfo.Path = fileID fileInfo.UploadedChecksum = fmt.Sprintf("%x", hash.Sum(nil)) - fileInfo.Size, err = app.Archive.GetFileSize(fileID, true) + fileInfo.Size, err = app.ArchiveReader.GetFileSize(ctx, location, fileID) if err != nil { log.Errorf("Couldn't get file size from archive, file-id: %s, reason: %v)", fileID, err.Error()) @@ -554,14 +608,14 @@ func (app *Ingest) ingestFile(fileID string, message schema.IngestionTrigger) st } archivedMsg, _ := json.Marshal(&msg) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.brokerConf.SchemasPath), archivedMsg) if err != nil { log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } - if err := app.MQ.SendMessage(fileID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, app.brokerConf.RoutingKey, archivedMsg); err != nil { // TODO fix resend mechanism log.Errorf("failed to publish message, reason: %v", err) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 23f77b7f2b..8f4a961960 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/rand" "crypto/sha256" "database/sql" @@ -13,9 +14,11 @@ import ( "path/filepath" "runtime" "strconv" + "strings" "testing" "time" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" log "github.com/sirupsen/logrus" "github.com/google/uuid" @@ -26,7 +29,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" - "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" "github.com/spf13/viper" @@ -171,15 +174,15 @@ type TestSuite struct { ingest Ingest tempDir string UserName string + + archiveDir string + inboxDir string } func (ts *TestSuite) SetupSuite() { var err error viper.Set("log.level", "debug") - ts.tempDir, err = os.MkdirTemp("", "c4gh-keys") - if err != nil { - ts.FailNow("Failed to create temp directory") - } + ts.tempDir = ts.T().TempDir() keyFile1 := fmt.Sprintf("%s/c4gh1.key", ts.tempDir) keyFile2 := fmt.Sprintf("%s/c4gh2.key", ts.tempDir) @@ -218,15 +221,16 @@ func (ts *TestSuite) SetupSuite() { viper.Set("inbox.location", "/tmp/") viper.Set("schema.path", "../../schemas/isolated/") - ts.ingest.Conf, err = config.NewConfig("ingest") + ingestConf, err := config.NewConfig("ingest") if err != nil { ts.FailNowf("failed to init config: %s", err.Error()) } - ts.ingest.DB, err = database.NewSDAdb(ts.ingest.Conf.Database) + ts.ingest.DB, err = database.NewSDAdb(ingestConf.Database) if err != nil { ts.FailNowf("failed to setup database connection: %s", err.Error()) } - ts.ingest.MQ, err = broker.NewMQ(ts.ingest.Conf.Broker) + ts.ingest.brokerConf = ingestConf.Broker + ts.ingest.MQ, err = broker.NewMQ(ts.ingest.brokerConf) if err != nil { ts.FailNowf("failed to setup rabbitMQ connection: %s", err.Error()) } @@ -242,31 +246,17 @@ func (ts *TestSuite) SetupSuite() { ts.UserName = "test-ingest" } -func (ts *TestSuite) TearDownSuite() { - _ = os.RemoveAll(ts.ingest.Conf.Archive.Posix.Location) - _ = os.RemoveAll(ts.ingest.Conf.Inbox.Posix.Location) - _ = os.RemoveAll(ts.tempDir) -} - func (ts *TestSuite) SetupTest() { - var err error - ts.ingest.Conf.Archive.Posix.Location, err = os.MkdirTemp("", "archive") - if err != nil { - ts.FailNow("failed to create temp folder") - } - - ts.ingest.Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") - if err != nil { - ts.FailNow("failed to create temp folder") - } + ts.archiveDir = ts.T().TempDir() + ts.inboxDir = ts.T().TempDir() // Ensure a folder with the user name exists - err = os.Mkdir(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName), 0750) + err := os.Mkdir(path.Join(ts.inboxDir, ts.UserName), 0750) if err != nil { - ts.FailNow("failed to create temp folder") + ts.FailNow("failed to create user folder in inbox directory") } - f, err := os.CreateTemp(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName), "") + f, err := os.CreateTemp(path.Join(ts.inboxDir, ts.UserName), "") if err != nil { ts.FailNow("failed to create test file") } @@ -302,13 +292,42 @@ func (ts *TestSuite) SetupTest() { ts.filePath = filepath.Base(outFileName) - ts.ingest.Archive, err = storage.NewBackend(ts.ingest.Conf.Archive) + if err := os.WriteFile(filepath.Join(ts.tempDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + inbox: + posix: + - path: %s + archive: + posix: + - path: %s +`, ts.inboxDir, ts.archiveDir)), 0600); err != nil { + ts.FailNow(err.Error()) + } + + viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + viper.SetConfigType("yaml") + viper.SetConfigFile(filepath.Join(ts.tempDir, "config.yaml")) + if err := viper.ReadInConfig(); err != nil { + ts.FailNow(err.Error()) + } + + lb, err := locationbroker.NewLocationBroker(ts.ingest.DB) + ts.NoError(err) + ts.ingest.ArchiveWriter, err = storage.NewWriter(context.TODO(), "archive", lb) + if err != nil { + ts.FailNow("failed to setup archive writer") + } + ts.ingest.ArchiveReader, err = storage.NewReader(context.TODO(), "archive") + if err != nil { + ts.FailNow("failed to setup archive reader") + } + ts.ingest.InboxWriter, err = storage.NewWriter(context.TODO(), "inbox", lb) if err != nil { - ts.FailNow("failed to setup archive backend") + ts.FailNow("failed to setup inbox writer") } - ts.ingest.Inbox, err = storage.NewBackend(ts.ingest.Conf.Inbox) + ts.ingest.InboxReader, err = storage.NewReader(context.TODO(), "inbox") if err != nil { - ts.FailNow("failed to setup inbox backend") + ts.FailNow("failed to setup inbox reader") } viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ @@ -400,6 +419,7 @@ func (ts *TestSuite) TestCancelFile() { assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(fileID, message)) } + func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // prepare the DB entries userName := "test-cancel" @@ -422,6 +442,24 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // messages of type `ingest` func (ts *TestSuite) TestIngestFile() { + // prepare the DB entries + fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + assert.NoError(ts.T(), err, "failed to register file in database") + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: ts.UserName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) +} + +func (ts *TestSuite) TestNoSubmissionLocation() { // prepare the DB entries fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") @@ -436,11 +474,12 @@ func (ts *TestSuite) TestIngestFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "nack", ts.ingest.ingestFile(context.TODO(), fileID, message)) } + func (ts *TestSuite) TestIngestFile_secondTime() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -453,10 +492,10 @@ func (ts *TestSuite) TestIngestFile_secondTime() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) // file is already in `archived` state - assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(context.TODO(), fileID, message)) } func (ts *TestSuite) TestIngestFile_unknownInboxType() { message := schema.IngestionTrigger{ @@ -465,11 +504,11 @@ func (ts *TestSuite) TestIngestFile_unknownInboxType() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(uuid.New().String(), message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), uuid.New().String(), message)) } func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -482,17 +521,17 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) } func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -505,14 +544,14 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } // over write the encrypted file to generate new checksum - f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") + f, err := os.CreateTemp(ts.inboxDir, "") if err != nil { ts.FailNow("failed to create test file") } @@ -528,7 +567,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName, ts.filePath)) + outFile, err := os.Create(path.Join(ts.inboxDir, ts.UserName, ts.filePath)) if err != nil { ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } @@ -549,7 +588,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { crypt4GHWriter.Close() // reingestion should work - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) // DB should have the new checksum var dbChecksum string @@ -562,7 +601,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -575,7 +614,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) // fake file verification sha256hash := sha256.New() @@ -588,11 +627,11 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { ts.Fail("failed to mark file as verified") } - assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(context.TODO(), fileID, message)) } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -605,7 +644,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) // fake file verification sha256hash := sha256.New() @@ -622,11 +661,11 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { ts.Fail("failed to update file event log") } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -639,7 +678,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) var firstDbChecksum string const q1 = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" @@ -663,7 +702,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { } // over write the encrypted file to generate new checksum - f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") + f, err := os.CreateTemp(ts.inboxDir, "") if err != nil { ts.FailNow("failed to create test file") } @@ -679,7 +718,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName, ts.filePath)) + outFile, err := os.Create(path.Join(ts.inboxDir, ts.UserName, ts.filePath)) if err != nil { ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } @@ -700,7 +739,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { crypt4GHWriter.Close() // reingestion should work - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), fileID, message)) // DB should have the new checksum var dbChecksum string @@ -726,7 +765,7 @@ func (ts *TestSuite) TestIngestFile_missingFile() { User: ts.UserName, } - assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(newFileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(context.TODO(), newFileID, message)) } func (ts *TestSuite) TestDetectMisingC4GHKeys() { viper.Set("c4gh.privateKeys", "") diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index 6de9ed60a6..64fa6c0c4c 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -10,4 +10,5 @@ type Database interface { GetArchiveLocation(fileID string) (string, error) GetArchivePathAndLocation(stableID string) (string, string, error) GetMappingData(accessionID string) (*MappingData, error) + GetSubmissionLocation(ctx context.Context, fileID string) (string, error) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index a6aef29a9a..1a0b29bbda 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -765,6 +765,23 @@ MAX(checksum) FILTER(where source = 'UPLOADED') as Uploaded from sda.checksums w return info, nil } +func (dbs *SDAdb) GetSubmissionLocation(ctx context.Context, fileID string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const getFileID = "SELECT submission_location FROM sda.files WHERE id = $1;" + + var submissionLocation string + if err := db.QueryRowContext(ctx, getFileID, fileID).Scan(&submissionLocation); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return "", err + } + + return submissionLocation, nil +} + // GetHeaderForStableID retrieves the file header by using stable id func (dbs *SDAdb) GetHeaderForStableID(stableID string) ([]byte, error) { dbs.checkAndReconnectIfNeeded() From 99a380769c312e609727c5f7ad8d78ac4ffdcf11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 13:41:17 +0100 Subject: [PATCH 221/530] feat(ingest): update required db version check to v23 with new location columns --- sda/cmd/ingest/ingest.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 4e211bc4f7..d658c9ce5a 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -77,8 +77,8 @@ func main() { sigc <- syscall.SIGINT panic(err) } - if app.DB.Version < 21 { - log.Error("database schema v21 is required") + if app.DB.Version < 23 { + log.Error("database schema v23 is required") sigc <- syscall.SIGINT panic(err) } From 0467eaac87170c76f86055d779850fca00ff8f2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 13 Jan 2026 14:53:15 +0100 Subject: [PATCH 222/530] feat(ingest): update READMD.md to now mention expected storage config, fix linting issues --- sda/cmd/ingest/ingest.go | 15 ++++++++----- sda/cmd/ingest/ingest.md | 32 ++++++++------------------- sda/internal/database/db_functions.go | 1 + 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index d658c9ce5a..4be8972ffb 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -134,15 +134,19 @@ func main() { log.Info("starting ingest service") - go app.run() + go func() { + if err := app.run(); err != nil { + panic(err) + } + }() <-forever } -func (app *Ingest) run() { +func (app *Ingest) run() error { messages, err := app.MQ.GetMessages(app.brokerConf.Queue) if err != nil { - log.Panic(err) + return err } for delivered := range messages { @@ -203,6 +207,8 @@ func (app *Ingest) run() { } } } + + return nil } func (app *Ingest) registerC4GHKey() error { @@ -238,7 +244,7 @@ func (app *Ingest) cancelFile(fileID string, message schema.IngestionTrigger) st func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema.IngestionTrigger) string { status, err := app.DB.GetFileStatus(fileID) - if err != nil && !errors.Is(sql.ErrNoRows, err) { + if err != nil && !errors.Is(err, sql.ErrNoRows) { log.Errorf("failed to get status for file, fileID: %s, reason: (%s)", fileID, err.Error()) return "nack" @@ -423,7 +429,6 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema uploadCtx, uploadCancel := context.WithCancel(ctx) go func() { - for bytesRead < fileSize { i, _ := io.ReadFull(file, readBuffer) if i == 0 { diff --git a/sda/cmd/ingest/ingest.md b/sda/cmd/ingest/ingest.md index e044434f7e..ad67d6bb5b 100644 --- a/sda/cmd/ingest/ingest.md +++ b/sda/cmd/ingest/ingest.md @@ -104,29 +104,15 @@ These settings control how `ingest` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings - -Storage backend is defined by the `ARCHIVE_TYPE`, and `INBOX_TYPE` variables. -Valid values for these options are `S3` or `POSIX` -(Defaults to `POSIX` on unknown values). - -The value of these variables define what other variables are read. -The same variables are available for all storage types, differing by prefix (`ARCHIVE_`, or `INBOX_`) - -if `*_TYPE` is `S3` then the following variables are available: - -- `*_URL`: URL to the S3 system -- `*_ACCESSKEY`: The S3 access and secret key are used to authenticate to S3, -[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_SECRETKEY`: The S3 access and secret key are used to authenticate to S3, -[more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `*_BUCKET`: The S3 bucket to use as the storage root -- `*_PORT`: S3 connection port (default: `443`) -- `*_REGION`: S3 region (default: `us-east-1`) -- `*_CHUNKSIZE`: S3 chunk size for multipart uploads. -- `*_CACERT`: Certificate Authority (CA) certificate for the storage system, this is only needed if the S3 server has a certificate signed by a private entity - -and if `*_TYPE` is `POSIX`: - - `*_LOCATION`: POSIX path to use as storage root +The ingest service requires access to the "inbox", and "archive" storages, to configure such the following config is needed: +```yaml +storage: + inbox: + ${STORAGE_IMPLEMENTATION}: + archive: + ${STORAGE_IMPLEMENTATION}: +``` +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) ### Logging settings: diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 1a0b29bbda..e7ace90568 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -776,6 +776,7 @@ func (dbs *SDAdb) GetSubmissionLocation(ctx context.Context, fileID string) (str if errors.Is(err, sql.ErrNoRows) { return "", nil } + return "", err } From 0cc5484b592f92e8919fb0678a231118374f240b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 14 Jan 2026 13:36:15 +0100 Subject: [PATCH 223/530] feat(ingest): update config to not require and load in storage v1 for ingest service --- sda/internal/config/config.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 1dfb29406f..62f018d434 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -291,23 +291,6 @@ func NewConfig(app string) (*Config, error) { "db.database", } - switch viper.GetString("archive.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) - default: - return nil, errors.New("archive.type not set") - } - - switch viper.GetString("inbox.type") { - case S3: - requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) - case POSIX: - requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) - default: - return nil, errors.New("inbox.type not set") - } case "finalize": requiredConfVars = []string{ "broker.host", @@ -563,7 +546,6 @@ func NewConfig(app string) (*Config, error) { c.configSchemas() case "ingest": - c.configArchive() err := c.configBroker() if err != nil { return nil, err @@ -573,7 +555,6 @@ func NewConfig(app string) (*Config, error) { return nil, err } - c.configInbox() c.configSchemas() case "intercept": err := c.configBroker() From 7d220f470aa5fdf0f08531bdbe771d2b1bc0fd8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 09:50:22 +0100 Subject: [PATCH 224/530] feat(ingest): update cancelFile check if file can be canelled, also remove file from archive(if its been uploaded there), unset file as archived, and unset accession id, added db funcs to support such --- sda/cmd/ingest/ingest.go | 59 ++++++++++- sda/cmd/ingest/ingest_test.go | 4 +- sda/internal/database/db_functions.go | 137 ++++++++++++++++++++++++-- 3 files changed, 187 insertions(+), 13 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 4be8972ffb..200aa057b5 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -181,7 +181,7 @@ func (app *Ingest) run() error { ackNack := "" switch message.Type { case "cancel": - ackNack = app.cancelFile(delivered.CorrelationId, message) + ackNack = app.cancelFile(ctx, delivered.CorrelationId, message) case "ingest": ackNack = app.ingestFile(ctx, delivered.CorrelationId, message) default: @@ -228,10 +228,63 @@ func (app *Ingest) registerC4GHKey() error { return nil } -func (app *Ingest) cancelFile(fileID string, message schema.IngestionTrigger) string { +func (app *Ingest) cancelFile(ctx context.Context, fileID string, message schema.IngestionTrigger) string { m, _ := json.Marshal(message) + + // Check if file can be cancelled + inDataset, err := app.DB.IsFileInDataset(ctx, fileID) + if err != nil { + log.Errorf("failed to check if file with id: %s is in a dataset, due to %v", fileID, err) + + return "nack" + } + if inDataset { + log.Warnf("can not cancel file with id: %s, as it has been added to a dataset", fileID) + + fileError := broker.InfoError{ + Error: "Cancel of file not possible", + Reason: "File has been added to a dataset", + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + + return "reject" + } + + return "ack" + } + + archiveData, err := app.DB.GetArchived(fileID) + if err != nil { + log.Errorf("failed to get archive data for file with id: %s, due to %v", fileID, err) + + return "nack" + } + + if archiveData != nil && archiveData.Location != "" { + if err := app.ArchiveWriter.RemoveFile(ctx, archiveData.Location, archiveData.FilePath); err != nil { + log.Errorf("failed to remove file with id %s from archive due to %v", fileID, err) + + return "nack" + } + } + + if err := app.DB.UnsetArchived(ctx, fileID); err != nil { + log.Errorf("failed to unset file with id: %s as archived due to %v", fileID, err) + + return "nack" + } + + if err := app.DB.UnsetAccessionID(ctx, fileID); err != nil { + log.Errorf("failed to unset accession id of file with id: %s due to %v", fileID, err) + + return "nack" + } + if err := app.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to update event log for file with id : %s", fileID) + log.Errorf("failed to update event log for file with id: %s, due to %v", fileID, err) if strings.Contains(err.Error(), "sql: no rows in result set") { return "reject" } diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 8f4a961960..8b106b6c16 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -417,7 +417,7 @@ func (ts *TestSuite) TestCancelFile() { User: userName, } - assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(fileID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(context.TODO(), fileID, message)) } func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { @@ -437,7 +437,7 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { User: userName, } - assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(uuid.NewString(), message)) + assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(context.TODO(), uuid.NewString(), message)) } // messages of type `ingest` diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index e7ace90568..0c6bc98284 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -337,6 +337,115 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec return nil } +// UnsetArchived unsets the file as 'ARCHIVED' happens if file is cancelled +func (dbs *SDAdb) UnsetArchived(ctx context.Context, fileID string) error { + var err error + + for count := 1; count <= RetryTimes; count++ { + err = dbs.unsetArchived(ctx, fileID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return err +} + +func (dbs *SDAdb) unsetArchived(ctx context.Context, fileID string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const unsetArchived = ` +UPDATE sda.files +SET archive_location = NULL, + archive_file_path = NULL, + archive_file_size = NULL +WHERE id = $1;` + + if _, err := db.ExecContext(ctx, unsetArchived, fileID); err != nil { + return err + } + + const addChecksum = ` +DELETE FROM sda.checksums +WHERE file_id = $1 +AND source = 'UPLOADED'` + + if _, err := db.ExecContext(ctx, addChecksum, fileID); err != nil { + return err + } + + return nil +} + +// UnsetAccessionID unsets the accession id from a file if the file is cancelled +func (dbs *SDAdb) UnsetAccessionID(ctx context.Context, fileID string) error { + var err error + + for count := 1; count <= RetryTimes; count++ { + err = dbs.unsetAccessionID(ctx, fileID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return err +} + +func (dbs *SDAdb) unsetAccessionID(ctx context.Context, fileID string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const unsetAccessionID = ` +UPDATE sda.files +SET stable_id = NULL +WHERE id = $1;` + + if _, err := db.ExecContext(ctx, unsetAccessionID, fileID); err != nil { + return err + } + + return nil +} + +// IsFileInDataset checks if a file has been added to a dataset +func (dbs *SDAdb) IsFileInDataset(ctx context.Context, fileID string) (bool, error) { + var err error + var inDataset bool + + for count := 1; count <= RetryTimes; count++ { + inDataset, err = dbs.isFileInDataset(ctx, fileID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return inDataset, err +} + +func (dbs *SDAdb) isFileInDataset(ctx context.Context, fileID string) (bool, error) { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const isFileInDataset = ` +SELECT true +FROM sda.file_dataset +WHERE file_id = $1;` + + var inDataset bool + if err := db.QueryRowContext(ctx, isFileInDataset, fileID).Scan(&inDataset); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return false, nil + } + return false, err + } + + return inDataset, nil +} + func (dbs *SDAdb) GetFileStatus(fileID string) (string, error) { var ( err error @@ -481,9 +590,9 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec } // GetArchived retrieves the location and size of archive -func (dbs *SDAdb) GetArchived(fileID string) (ArchiveData, error) { +func (dbs *SDAdb) GetArchived(fileID string) (*ArchiveData, error) { var ( - archiveData ArchiveData + archiveData *ArchiveData err error count int ) @@ -495,21 +604,33 @@ func (dbs *SDAdb) GetArchived(fileID string) (ArchiveData, error) { return archiveData, err } -func (dbs *SDAdb) getArchived(fileID string) (ArchiveData, error) { +func (dbs *SDAdb) getArchived(fileID string) (*ArchiveData, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB const query = "SELECT archive_file_path, archive_file_size, archive_location from sda.files WHERE id = $1;" - var archiveData ArchiveData - if err := db.QueryRow(query, fileID).Scan(&archiveData.FilePath, &archiveData.FileSize, &archiveData.Location); err != nil { - return ArchiveData{}, err + var archiveFilePath, archiveLocation sql.NullString + var archiveFileSize sql.Null[int] + if err := db.QueryRow(query, fileID).Scan(&archiveFilePath, &archiveFileSize, &archiveLocation); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, err + } + if archiveFilePath.Valid || archiveFileSize.Valid || archiveLocation.Valid { + return &ArchiveData{ + FilePath: archiveFilePath.String, + Location: archiveLocation.String, + FileSize: archiveFileSize.V, + }, nil } - return archiveData, nil + // We have a files table entry but archive data has not been set + return nil, nil } -// CheckAccessionIdExists validates if an accessionID exists in the db +// CheckAccessionIDExists validates if an accessionID exists in the db func (dbs *SDAdb) CheckAccessionIDExists(accessionID, fileID string) (string, error) { var err error var exists string From efff8f160c610648e9c44646e689f6ee0585c9f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 13:11:04 +0100 Subject: [PATCH 225/530] feat(ingest): add initializing of the archive reader --- sda/cmd/ingest/ingest.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 200aa057b5..ad175abcfe 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -96,13 +96,18 @@ func main() { if err != nil { log.Fatalf("failed to init new location broker due to: %v", err) } - app.ArchiveWriter, err = storage.NewWriter(ctx, "archive", storageLocationBroker) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } + app.ArchiveReader, err = storage.NewReader(ctx, "archive") + if err != nil { + log.Error(err) + sigc <- syscall.SIGINT + panic(err) + } app.InboxReader, err = storage.NewReader(ctx, "inbox") if err != nil { log.Error(err) From f722dd1023ce03125e43e24d091f731bbf1cae51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 13:14:46 +0100 Subject: [PATCH 226/530] feat(ingest): remove inbox writer as not used, also update main error handling as no point in sending to signal channel since we dont listen to it yet and panic afterwards anyway, so replaced it with fataling --- sda/cmd/ingest/ingest.go | 58 ++++++++++++++++------------------- sda/cmd/ingest/ingest_test.go | 4 --- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index ad175abcfe..5cea934c38 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -37,7 +37,6 @@ type Ingest struct { ArchiveKeyList []*[32]byte DB *database.SDAdb InboxReader storage.Reader - InboxWriter storage.Writer // Used to delete files from inbox brokerConf broker.MQConf MQ *broker.AMQPBroker } @@ -60,65 +59,62 @@ func main() { ingestConf, err := config.NewConfig("ingest") if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Errorf("failed to load config due to: %v", err) + + return } app.brokerConf = ingestConf.Broker app.MQ, err = broker.NewMQ(app.brokerConf) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Fatalf("failed to init broker due to: %v", err) } app.DB, err = database.NewSDAdb(ingestConf.Database) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Errorf("failed to init db due to: %v", err) + + return } if app.DB.Version < 23 { log.Error("database schema v23 is required") - sigc <- syscall.SIGINT - panic(err) + + return } app.ArchiveKeyList, err = config.GetC4GHprivateKeys() if err != nil || len(app.ArchiveKeyList) == 0 { - sigc <- syscall.SIGINT - panic(errors.New("no C4GH private keys configured")) + log.Error("no C4GH private keys configured") + + return } if err := app.registerC4GHKey(); err != nil { - panic(err) + log.Error("failed to register c4gh key due to: %v", err) + + return } storageLocationBroker, err := locationbroker.NewLocationBroker(app.DB) if err != nil { - log.Fatalf("failed to init new location broker due to: %v", err) + log.Errorf("failed to init new location broker due to: %v", err) + + return } app.ArchiveWriter, err = storage.NewWriter(ctx, "archive", storageLocationBroker) if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Errorf("failed to init archive writer due to: %v", err) + + return } app.ArchiveReader, err = storage.NewReader(ctx, "archive") if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Errorf("failed to init archive reader due to: %v", err) + + return } app.InboxReader, err = storage.NewReader(ctx, "inbox") if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) - } - app.InboxWriter, err = storage.NewWriter(ctx, "inbox", storageLocationBroker) - if err != nil { - log.Error(err) - sigc <- syscall.SIGINT - panic(err) + log.Errorf("failed to init inbox reader due to: %v", err) + + return } defer app.MQ.Channel.Close() diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 8b106b6c16..52aa71d576 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -321,10 +321,6 @@ storage: if err != nil { ts.FailNow("failed to setup archive reader") } - ts.ingest.InboxWriter, err = storage.NewWriter(context.TODO(), "inbox", lb) - if err != nil { - ts.FailNow("failed to setup inbox writer") - } ts.ingest.InboxReader, err = storage.NewReader(context.TODO(), "inbox") if err != nil { ts.FailNow("failed to setup inbox reader") From 290a35be5189c72e6216fe4f34798ce17602bb17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 13:23:36 +0100 Subject: [PATCH 227/530] feat(ingest): use SetArchivedWithLocation instead of SetArchived --- sda/cmd/ingest/ingest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 5cea934c38..8974cea33b 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -642,7 +642,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema return "ack" } - if err := app.DB.SetArchived(fileInfo, fileID); err != nil { + if err := app.DB.SetArchivedWithLocation(location, fileInfo, fileID); err != nil { log.Errorf("SetArchived failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" From 251101ddcae4cf2bc76e420c62282b08926966a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 13:34:35 +0100 Subject: [PATCH 228/530] feat(postgres): grant SELECT permission to ingest user --- postgresql/initdb.d/04_grants.sql | 1 + ...gest_select_permission_to_file_dataset.sql | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+) create mode 100644 postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index ee6eff27ad..09de0d50d8 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -46,6 +46,7 @@ GRANT SELECT ON sda.file_event_log TO ingest; GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO ingest; GRANT SELECT ON sda.encryption_keys TO ingest; GRANT INSERT ON sda.encryption_keys TO ingest; +GRANT SELECT ON sda.file_dataset TO ingest; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO ingest; diff --git a/postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql b/postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql new file mode 100644 index 0000000000..12b09dde30 --- /dev/null +++ b/postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql @@ -0,0 +1,21 @@ + +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 22; + changes VARCHAR := 'Grant ingest user SELECT access to file_dataset'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + GRANT SELECT ON sda.file_dataset TO ingest; + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ From 25f24f5871da1fa1a0f9a69e0ca8b21ffc1736e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 13:43:15 +0100 Subject: [PATCH 229/530] feat(database): update unsetArchived to set archive_file_path as empty string instead of null due to not null constraint --- sda/internal/database/db_functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 0c6bc98284..328722f94c 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -359,7 +359,7 @@ func (dbs *SDAdb) unsetArchived(ctx context.Context, fileID string) error { const unsetArchived = ` UPDATE sda.files SET archive_location = NULL, - archive_file_path = NULL, + archive_file_path = '', archive_file_size = NULL WHERE id = $1;` From 67d5b481224ad2bdb939ed2c3c026aa004a3466b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 13:55:55 +0100 Subject: [PATCH 230/530] feat(ingest): also grant ingest user DELETE permission to checksums table --- postgresql/initdb.d/01_main.sql | 1 + postgresql/initdb.d/04_grants.sql | 2 +- ...gest_select_permission_to_file_dataset.sql | 21 ------------------- 3 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index 6c6daf0466..fff86824b7 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -39,6 +39,7 @@ VALUES (0, now(), 'Created with version'), (22, now(), 'Add file_headers_backup table for key rotation safekeeping'), (23, now(), 'Expand files table with storage locations'); + -- Datasets are used to group files, and permissions are set on the dataset -- level CREATE TABLE datasets ( diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index 09de0d50d8..e552bc1e4a 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -39,6 +39,7 @@ GRANT SELECT ON sda.file_dataset TO ingest; GRANT INSERT ON sda.checksums TO ingest; GRANT UPDATE ON sda.checksums TO ingest; GRANT SELECT ON sda.checksums TO ingest; +GRANT DELETE ON sda.checksums TO ingest; GRANT INSERT ON sda.file_events TO ingest; GRANT USAGE, SELECT ON SEQUENCE sda.checksums_id_seq TO ingest; GRANT INSERT ON sda.file_event_log TO ingest; @@ -46,7 +47,6 @@ GRANT SELECT ON sda.file_event_log TO ingest; GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO ingest; GRANT SELECT ON sda.encryption_keys TO ingest; GRANT INSERT ON sda.encryption_keys TO ingest; -GRANT SELECT ON sda.file_dataset TO ingest; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO ingest; diff --git a/postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql b/postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql deleted file mode 100644 index 12b09dde30..0000000000 --- a/postgresql/migratedb.d/23_grant_ingest_select_permission_to_file_dataset.sql +++ /dev/null @@ -1,21 +0,0 @@ - -DO -$$ -DECLARE --- The version we know how to do migration from, at the end of a successful migration --- we will no longer be at this version. - sourcever INTEGER := 22; - changes VARCHAR := 'Grant ingest user SELECT access to file_dataset'; -BEGIN - IF (select max(version) from sda.dbschema_version) = sourcever then - RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; - RAISE NOTICE 'Changes: %', changes; - INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); - - GRANT SELECT ON sda.file_dataset TO ingest; - - ELSE - RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; - END IF; -END -$$ From 95f7b4a3b59aa498fdfc3191e2e89bebaa3ad87e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 14:11:31 +0100 Subject: [PATCH 231/530] feat(ingest): rename UnsetArchived func -> CancelFile, also delete all checksums, and set decrypted_file_size, stable_id to NULL, remove UnsetAccessionID func --- sda/cmd/ingest/ingest.go | 8 +---- sda/internal/database/db_functions.go | 50 ++++++--------------------- 2 files changed, 11 insertions(+), 47 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 8974cea33b..d0119c732d 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -272,18 +272,12 @@ func (app *Ingest) cancelFile(ctx context.Context, fileID string, message schema } } - if err := app.DB.UnsetArchived(ctx, fileID); err != nil { + if err := app.DB.CancelFile(ctx, fileID); err != nil { log.Errorf("failed to unset file with id: %s as archived due to %v", fileID, err) return "nack" } - if err := app.DB.UnsetAccessionID(ctx, fileID); err != nil { - log.Errorf("failed to unset accession id of file with id: %s due to %v", fileID, err) - - return "nack" - } - if err := app.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", string(m)); err != nil { log.Errorf("failed to update event log for file with id: %s, due to %v", fileID, err) if strings.Contains(err.Error(), "sql: no rows in result set") { diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 328722f94c..027bd59709 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -337,12 +337,12 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec return nil } -// UnsetArchived unsets the file as 'ARCHIVED' happens if file is cancelled -func (dbs *SDAdb) UnsetArchived(ctx context.Context, fileID string) error { +// CancelFile cancels the file and all actions that have been taken (eg, setting checksums, archiving, etc) +func (dbs *SDAdb) CancelFile(ctx context.Context, fileID string) error { var err error for count := 1; count <= RetryTimes; count++ { - err = dbs.unsetArchived(ctx, fileID) + err = dbs.cancelFile(ctx, fileID) if err == nil { break } @@ -352,7 +352,7 @@ func (dbs *SDAdb) UnsetArchived(ctx context.Context, fileID string) error { return err } -func (dbs *SDAdb) unsetArchived(ctx context.Context, fileID string) error { +func (dbs *SDAdb) cancelFile(ctx context.Context, fileID string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB @@ -360,50 +360,20 @@ func (dbs *SDAdb) unsetArchived(ctx context.Context, fileID string) error { UPDATE sda.files SET archive_location = NULL, archive_file_path = '', - archive_file_size = NULL + archive_file_size = NULL, + decrypted_file_size = NULL, + stable_id = NULL WHERE id = $1;` if _, err := db.ExecContext(ctx, unsetArchived, fileID); err != nil { return err } - const addChecksum = ` + const deleteChecksums = ` DELETE FROM sda.checksums -WHERE file_id = $1 -AND source = 'UPLOADED'` +WHERE file_id = $1` - if _, err := db.ExecContext(ctx, addChecksum, fileID); err != nil { - return err - } - - return nil -} - -// UnsetAccessionID unsets the accession id from a file if the file is cancelled -func (dbs *SDAdb) UnsetAccessionID(ctx context.Context, fileID string) error { - var err error - - for count := 1; count <= RetryTimes; count++ { - err = dbs.unsetAccessionID(ctx, fileID) - if err == nil { - break - } - time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) - } - - return err -} - -func (dbs *SDAdb) unsetAccessionID(ctx context.Context, fileID string) error { - dbs.checkAndReconnectIfNeeded() - - db := dbs.DB - const unsetAccessionID = ` -UPDATE sda.files -SET stable_id = NULL -WHERE id = $1;` - - if _, err := db.ExecContext(ctx, unsetAccessionID, fileID); err != nil { + if _, err := db.ExecContext(ctx, deleteChecksums, fileID); err != nil { return err } From f715c8f80fea538e783ca6262d771931547f1070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 14:12:48 +0100 Subject: [PATCH 232/530] feat(ingest): update "disabled" case to be the same as "uploaded" due to the changes to cancel file handling --- sda/cmd/ingest/ingest.go | 85 +--------------------------------------- 1 file changed, 1 insertion(+), 84 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index d0119c732d..2de12c95c8 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -312,89 +312,6 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema } switch status { - case "disabled": - - fileInfo, err := app.DB.GetFileInfo(fileID) - if err != nil { - log.Errorf("failed to get info for file, file-id: %s, reason: %s", fileID, err.Error()) - - return "nack" - } - - // What if the file in the inbox is different this time? - // Check uploaded checksum in the DB against the checksum of the file. - // Would be easy if the inbox message had the checksum or that the s3inbox added the checksum to the DB. - file, err := app.InboxReader.NewFileReader(ctx, submissionLocation, helper.UnanonymizeFilepath(message.FilePath, message.User)) - if err != nil { - switch { - case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): - log.Errorf("Failed to open file to ingest, file-id: %s, inbox path: %s, reason: (%s)", fileID, message.FilePath, err.Error()) - jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { - log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) - } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to open file to ingest", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - - return "reject" - } - - return "ack" - default: - return "nack" - } - } - defer file.Close() - - inboxChecksum := sha256.New() - _, err = io.Copy(inboxChecksum, file) - if err != nil { - log.Errorf("failed to calculate the checksum for file, file-id: %s, reason: %s", fileID, err.Error()) - - return "nack" - } - - if fileInfo.UploadedChecksum == string(inboxChecksum.Sum(nil)) && fileInfo.ArchiveChecksum != "" { - msg := schema.IngestionVerification{ - User: message.User, - FilePath: message.FilePath, - FileID: fileID, - ArchivePath: fileInfo.Path, - EncryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fileInfo.ArchiveChecksum}, - }, - } - archivedMsg, _ := json.Marshal(&msg) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.brokerConf.SchemasPath), archivedMsg) - if err != nil { - log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) - - return "nack" - } - - m, _ := json.Marshal(message) - if err = app.DB.UpdateFileEventLog(fileInfo.Path, "enabled", "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set ingestion status for file from message, file-id: %s", fileID) - - return "nack" - } - - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, app.brokerConf.RoutingKey, archivedMsg); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - - return "reject" - } - - return "ack" - } case "": fileID, err = app.DB.RegisterFile(&fileID, message.FilePath, message.User) if err != nil { @@ -430,7 +347,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema } log.Infof("registering file, file-id: %s", fileID) - case "uploaded": + case "uploaded", "disabled": default: log.Warnf("unsupported file status: %s, file-id: %s", status, fileID) From efacb82eb912373f73c1f001ea3d910ac1973d8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 14:33:28 +0100 Subject: [PATCH 233/530] feat(ingest): add event log error if submission location known but file not found in storage --- sda/cmd/ingest/ingest.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 2de12c95c8..54199c316f 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -28,6 +28,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" log "github.com/sirupsen/logrus" ) @@ -357,6 +358,28 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema file, err := app.InboxReader.NewFileReader(ctx, submissionLocation, helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { + if errors.Is(err, storageerrors.ErrorFileNotFoundInLocation) { + log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { + log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) + } + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to open file to ingest", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + + return "reject" + } + + return "ack" + } log.Errorf("unexpected error when opening file for reading, file-id: %s, filepath: %s, reason: %s", fileID, message.FilePath, err.Error()) return "nack" From ff35f83f356f152a9e8e68fd536a275abed28d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 15 Jan 2026 16:37:55 +0100 Subject: [PATCH 234/530] feat(ingest): update uploading of file to be able to handle error from both reader and storagewriter whichever occurs first --- sda/cmd/ingest/ingest.go | 45 ++++++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 54199c316f..80c48e66c0 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -88,7 +88,7 @@ func main() { } if err := app.registerC4GHKey(); err != nil { - log.Error("failed to register c4gh key due to: %v", err) + log.Errorf("failed to register c4gh key due to: %v", err) return } @@ -141,7 +141,6 @@ func main() { panic(err) } }() - <-forever } @@ -415,12 +414,19 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema }() uploadCtx, uploadCancel := context.WithCancel(ctx) + defer uploadCancel() + readFileAck := make(chan string, 1) go func() { for bytesRead < fileSize { + // If storageWriter has encountered an error, and we've exited, we want to stop this goroutine as well + if uploadCtx.Err() != nil { + return + } i, _ := io.ReadFull(file, readBuffer) if i == 0 { log.Errorf("readBuffer returned 0 bytes, this should not happen, file-id: %s", fileID) + readFileAck <- "reject" uploadCancel() return @@ -435,6 +441,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema h := bytes.NewReader(readBuffer) if _, err = io.Copy(hash, h); err != nil { log.Errorf("Copy to hash failed while reading file, file-id: %s, reason: (%s)", fileID, err.Error()) + readFileAck <- "nack" uploadCancel() return @@ -473,6 +480,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) } + readFileAck <- "ack" uploadCancel() return @@ -485,6 +493,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema err = app.DB.SetKeyHash(keyhash, fileID) if err != nil { log.Errorf("Key hash %s could not be set for file, file-id: %s, reason: (%s)", keyhash, fileID, err.Error()) + readFileAck <- "nack" uploadCancel() return @@ -493,6 +502,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema log.Debugln("store header") if err := app.DB.StoreHeader(header, fileID); err != nil { log.Errorf("StoreHeader failed, file-id: %s, reason: (%s)", fileID, err.Error()) + readFileAck <- "nack" uploadCancel() return @@ -500,6 +510,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema if _, err = byteBuf.Write(readBuffer); err != nil { log.Errorf("Failed to write to read buffer for header read, file-id: %s, reason: %v)", fileID, err.Error()) + readFileAck <- "nack" uploadCancel() return @@ -509,6 +520,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema h := make([]byte, len(header)) if _, err = byteBuf.Read(h); err != nil { log.Errorf("Failed to strip header from buffer, file-id: %s, reason: (%s)", fileID, err.Error()) + readFileAck <- "nack" uploadCancel() return @@ -519,6 +531,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema } if _, err = byteBuf.Write(readBuffer); err != nil { log.Errorf("Failed to write to read buffer for full read, file-id: %s, reason: (%s)", fileID, err.Error()) + readFileAck <- "nack" uploadCancel() return @@ -528,8 +541,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema // Write data to file if _, err = byteBuf.WriteTo(contentWriter); err != nil { log.Errorf("Failed to write to archive file, file-id: %s, reason: (%s)", fileID, err.Error()) - - _ = file.Close() + readFileAck <- "nack" uploadCancel() return @@ -540,11 +552,26 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema _ = file.Close() }() - location, err := app.ArchiveWriter.WriteFile(uploadCtx, fileID, contentReader) - if err != nil { - log.Errorf("Failed to upload archive file, file-id: %s, reason: (%s)", fileID, err.Error()) - // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. - return "nack" + uploadErr := make(chan error, 1) + var location string + go func() { + var err error + location, err = app.ArchiveWriter.WriteFile(uploadCtx, fileID, contentReader) + uploadErr <- err + }() + + // React to first issue, either from storage writer of file reader + select { + case ack := <-readFileAck: + // if ack != "" the reading of data has encountered an error and we should ack this message with the code + if ack != "" { + return ack + } + case err := <-uploadErr: + if err != nil { + log.Errorf("Failed to upload archive file, file-id: %s, reason: (%s)", fileID, err.Error()) + return "nack" + } } // As we are done with uploadCtx now, we cancel it uploadCancel() From eb2df785f150d3106af5a5c107ba4cc739bc4e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 19 Jan 2026 16:04:59 +0100 Subject: [PATCH 235/530] feat(ingest): update cancelFile to also delete file from backup if backup storage is configured(mention optional backup config in ingest.md), update GetArchived to also return backup path and location if set in database --- sda/cmd/ingest/ingest.go | 20 ++++++++++++++++++++ sda/cmd/ingest/ingest.md | 5 ++++- sda/internal/database/database.go | 3 +++ sda/internal/database/db_functions.go | 27 +++++++++++++++++++-------- 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 80c48e66c0..a2c41d32df 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -34,6 +34,7 @@ import ( type Ingest struct { ArchiveWriter storage.Writer + BackupWriter storage.Writer ArchiveReader storage.Reader ArchiveKeyList []*[32]byte DB *database.SDAdb @@ -118,6 +119,17 @@ func main() { return } + backupWriter, err := storage.NewWriter(ctx, "backup", storageLocationBroker) + if err != nil && errors.Is(err, storageerrors.ErrorNoValidWriter) { + log.Fatalf("failed to init backup writer due to: %v", err) + } + if backupWriter != nil { + log.Info("backup writer initialized, will clean cancelled files from backup storage") + app.BackupWriter = backupWriter + } else { + log.Info("no backup writer initialized, will NOT clean cancelled files from backup storage") + } + defer app.MQ.Channel.Close() defer app.MQ.Connection.Close() defer app.DB.Close() @@ -272,6 +284,14 @@ func (app *Ingest) cancelFile(ctx context.Context, fileID string, message schema } } + if app.BackupWriter != nil && archiveData != nil && archiveData.BackupFilePath != "" && archiveData.BackupLocation != "" { + if err := app.BackupWriter.RemoveFile(ctx, archiveData.BackupLocation, archiveData.BackupFilePath); err != nil { + log.Errorf("failed to remove file with id %s from backup due to %v", fileID, err) + + return "nack" + } + } + if err := app.DB.CancelFile(ctx, fileID); err != nil { log.Errorf("failed to unset file with id: %s as archived due to %v", fileID, err) diff --git a/sda/cmd/ingest/ingest.md b/sda/cmd/ingest/ingest.md index ad67d6bb5b..1db3eb6620 100644 --- a/sda/cmd/ingest/ingest.md +++ b/sda/cmd/ingest/ingest.md @@ -104,13 +104,16 @@ These settings control how `ingest` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings -The ingest service requires access to the "inbox", and "archive" storages, to configure such the following config is needed: +The ingest service requires access to the "inbox", and "archive" storages, "backup" storage is optional if to also delete files from backup storage when a file is cancelled, +to configure such the following config is needed: ```yaml storage: inbox: ${STORAGE_IMPLEMENTATION}: archive: ${STORAGE_IMPLEMENTATION}: + backup: # Exclude if no backup storage + ${STORAGE_IMPLEMENTATION}: ``` For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 6d673b9984..e9a7a69a8e 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -57,6 +57,9 @@ type ArchiveData struct { FilePath string Location string FileSize int + + BackupFilePath string + BackupLocation string } type SubmissionFileInfo struct { diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 027bd59709..0876395235 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -410,6 +410,7 @@ WHERE file_id = $1;` if errors.Is(err, sql.ErrNoRows) { return false, nil } + return false, err } @@ -578,22 +579,32 @@ func (dbs *SDAdb) getArchived(fileID string) (*ArchiveData, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "SELECT archive_file_path, archive_file_size, archive_location from sda.files WHERE id = $1;" + const query = "SELECT archive_file_path, archive_file_size, archive_location, backup_path, backup_location from sda.files WHERE id = $1;" - var archiveFilePath, archiveLocation sql.NullString + var archiveFilePath, archiveLocation, backupFilePath, backupLocation sql.NullString var archiveFileSize sql.Null[int] - if err := db.QueryRow(query, fileID).Scan(&archiveFilePath, &archiveFileSize, &archiveLocation); err != nil { + if err := db.QueryRow(query, fileID).Scan(&archiveFilePath, &archiveFileSize, &archiveLocation, &backupFilePath, &backupLocation); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, err } if archiveFilePath.Valid || archiveFileSize.Valid || archiveLocation.Valid { - return &ArchiveData{ - FilePath: archiveFilePath.String, - Location: archiveLocation.String, - FileSize: archiveFileSize.V, - }, nil + ad := &ArchiveData{ + FilePath: archiveFilePath.String, + Location: archiveLocation.String, + FileSize: archiveFileSize.V, + BackupFilePath: backupFilePath.String, + BackupLocation: backupLocation.String, + } + if backupFilePath.Valid { + ad.BackupFilePath = backupFilePath.String + } + if backupLocation.Valid { + ad.BackupLocation = backupLocation.String + } + + return ad, nil } // We have a files table entry but archive data has not been set From 94bee7c04e926589d5dc0ec8ec060c451b709e06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 20 Jan 2026 16:15:26 +0100 Subject: [PATCH 236/530] feat(storage-v2): update location broker unit test mock database to implement new func GetSubmissionLocation with panic as not expected to be used in those unit tests --- .../storage/v2/locationbroker/location_broker_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index 0902886cfd..d5bfb74b67 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -37,6 +37,10 @@ func (m *mockDatabase) GetMappingData(_ string) (*database.MappingData, error) { panic("function not expected to be called in unit tests") } +func (m *mockDatabase) GetSubmissionLocation(_ context.Context, _ string) (string, error) { + panic("function not expected to be called in unit tests") +} + func TestLocationBrokerTestSuite(t *testing.T) { suite.Run(t, new(LocationBrokerTestSuite)) } From d159d0f8308a353ad77f4154499d5dc5807a21b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 20 Jan 2026 16:39:50 +0100 Subject: [PATCH 237/530] feat(ingest): fix linter issues, spacing, fatalling before defer cancel(), add setFileEventErrorAndSendToErrorQueue func to reduce complex nestedif when there is a unexpected not retryable error for a file --- sda/cmd/ingest/ingest.go | 72 +++++++++++++-------------- sda/internal/database/db_functions.go | 1 + 2 files changed, 35 insertions(+), 38 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index a2c41d32df..d6e2572130 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -68,7 +68,9 @@ func main() { app.brokerConf = ingestConf.Broker app.MQ, err = broker.NewMQ(app.brokerConf) if err != nil { - log.Fatalf("failed to init broker due to: %v", err) + log.Errorf("failed to init broker due to: %v", err) + + return } app.DB, err = database.NewSDAdb(ingestConf.Database) if err != nil { @@ -121,7 +123,9 @@ func main() { backupWriter, err := storage.NewWriter(ctx, "backup", storageLocationBroker) if err != nil && errors.Is(err, storageerrors.ErrorNoValidWriter) { - log.Fatalf("failed to init backup writer due to: %v", err) + log.Errorf("failed to init backup writer due to: %v", err) + + return } if backupWriter != nil { log.Info("backup writer initialized, will clean cancelled files from backup storage") @@ -345,21 +349,11 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema submissionLocation, err = app.InboxReader.FindFile(ctx, message.FilePath) if err != nil { log.Errorf("failed to find submission location for file in all configured storage locations, file-id: %s", fileID) - jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { - log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) - } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to open file to ingest", + if err := app.setFileEventErrorAndSendToErrorQueue(fileID, &broker.InfoError{ + Error: "Failed to open file to ingest, file not found in any of the configured storage locations", Reason: err.Error(), OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - + }); err != nil { return "reject" } @@ -379,21 +373,11 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema if err != nil { if errors.Is(err, storageerrors.ErrorFileNotFoundInLocation) { log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) - jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { - log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) - } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ + if err := app.setFileEventErrorAndSendToErrorQueue(fileID, &broker.InfoError{ Error: "Failed to open file to ingest", Reason: err.Error(), OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - + }); err != nil { return "reject" } @@ -485,20 +469,15 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema // Check if decryption was successful with any key if privateKey == nil { log.Errorf("All keys failed to decrypt the submitted file, file-id: %s", fileID) - m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { - log.Errorf("Failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) - } - - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ + if err := app.setFileEventErrorAndSendToErrorQueue(fileID, &broker.InfoError{ Error: "Trying to decrypt the submitted file failed", Reason: "Decryption failed with the available key(s)", OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) + }); err != nil { + readFileAck <- "reject" + uploadCancel() + + return } readFileAck <- "ack" uploadCancel() @@ -590,6 +569,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema case err := <-uploadErr: if err != nil { log.Errorf("Failed to upload archive file, file-id: %s, reason: (%s)", fileID, err.Error()) + return "nack" } } @@ -692,3 +672,19 @@ func tryDecrypt(key *[32]byte, buf []byte) ([]byte, error) { return header, nil } + +func (app *Ingest) setFileEventErrorAndSendToErrorQueue(fileID string, infoError *broker.InfoError) error { + jsonMsg, _ := json.Marshal(map[string]string{"error": infoError.Error, "reason": infoError.Reason}) + m, _ := json.Marshal(infoError.OriginalMessage) + if err := app.DB.UpdateFileEventLog(fileID, "error", "ingest", string(jsonMsg), string(m)); err != nil { + log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) + } + body, _ := json.Marshal(infoError) + if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + + return err + } + + return nil +} diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 0876395235..a1bcc74b62 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -587,6 +587,7 @@ func (dbs *SDAdb) getArchived(fileID string) (*ArchiveData, error) { if errors.Is(err, sql.ErrNoRows) { return nil, nil } + return nil, err } if archiveFilePath.Valid || archiveFileSize.Valid || archiveLocation.Valid { From faef4eb6d01efc76a3ba07be4987f643154cd4ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 08:52:10 +0100 Subject: [PATCH 238/530] feat(ingest): use fatal instead error log and return when main failed to initialize resources, remove context cancel, use bg ctx instead, remove defer recover, as we do not expect to recover from a panic in main --- sda/cmd/ingest/ingest.go | 73 ++++++++++++---------------------------- 1 file changed, 21 insertions(+), 52 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index d6e2572130..26e8b19a4e 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -44,88 +44,52 @@ type Ingest struct { } func main() { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() app := Ingest{} - var err error - sigc := make(chan os.Signal, 5) - signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - // Create a function to handle panic and exit gracefully - defer func() { - if err := recover(); err != nil { - log.Fatal("Could not recover, exiting") - } - }() - - forever := make(chan bool) - ingestConf, err := config.NewConfig("ingest") if err != nil { - log.Errorf("failed to load config due to: %v", err) - - return + log.Fatalf("failed to load config, due to: %v", err) } app.brokerConf = ingestConf.Broker app.MQ, err = broker.NewMQ(app.brokerConf) if err != nil { - log.Errorf("failed to init broker due to: %v", err) - - return + log.Fatalf("failed to initialize mq broker, due to: %v", err) } app.DB, err = database.NewSDAdb(ingestConf.Database) if err != nil { - log.Errorf("failed to init db due to: %v", err) - - return + log.Fatalf("failed to initialize sda db due to: %v", err) } if app.DB.Version < 23 { - log.Error("database schema v23 is required") - - return + log.Fatal("database schema v23 is required") } app.ArchiveKeyList, err = config.GetC4GHprivateKeys() if err != nil || len(app.ArchiveKeyList) == 0 { - log.Error("no C4GH private keys configured") - - return + log.Fatal("no C4GH private keys configured") } if err := app.registerC4GHKey(); err != nil { - log.Errorf("failed to register c4gh key due to: %v", err) - - return + log.Fatalf("failed to register c4gh key, due to: %v", err) } storageLocationBroker, err := locationbroker.NewLocationBroker(app.DB) if err != nil { - log.Errorf("failed to init new location broker due to: %v", err) - - return + log.Fatalf("failed to initialize location broker, due to: %v", err) } - app.ArchiveWriter, err = storage.NewWriter(ctx, "archive", storageLocationBroker) + app.ArchiveWriter, err = storage.NewWriter(context.Background(), "archive", storageLocationBroker) if err != nil { - log.Errorf("failed to init archive writer due to: %v", err) - - return + log.Fatalf("failed to initialize archive writer, due to: %v", err) } - app.ArchiveReader, err = storage.NewReader(ctx, "archive") + app.ArchiveReader, err = storage.NewReader(context.Background(), "archive") if err != nil { - log.Errorf("failed to init archive reader due to: %v", err) - - return + log.Fatalf("failed to initialize archive reader, due to: %v", err) } - app.InboxReader, err = storage.NewReader(ctx, "inbox") + app.InboxReader, err = storage.NewReader(context.Background(), "inbox") if err != nil { - log.Errorf("failed to init inbox reader due to: %v", err) - - return + log.Fatalf("failed to initialize inbox reader, due to: %v", err) } - backupWriter, err := storage.NewWriter(ctx, "backup", storageLocationBroker) + backupWriter, err := storage.NewWriter(context.Background(), "backup", storageLocationBroker) if err != nil && errors.Is(err, storageerrors.ErrorNoValidWriter) { - log.Errorf("failed to init backup writer due to: %v", err) - - return + log.Fatalf("failed to initialize backup writer due to: %v", err) } if backupWriter != nil { log.Info("backup writer initialized, will clean cancelled files from backup storage") @@ -138,6 +102,10 @@ func main() { defer app.MQ.Connection.Close() defer app.DB.Close() + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + forever := make(chan bool) + go func() { connError := app.MQ.ConnectionWatcher() log.Error(connError) @@ -154,7 +122,8 @@ func main() { go func() { if err := app.run(); err != nil { - panic(err) + log.Error(err) + forever <- false } }() <-forever From b4f22f8f4696d89c602cc81c5a8ea0e9a0b1faf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 08:55:17 +0100 Subject: [PATCH 239/530] feat(finalize): fix GetArchived usage after method return was changed to *ArchiveData --- postgresql/initdb.d/01_main.sql | 1 - sda/cmd/finalize/finalize.go | 4 ++++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index fff86824b7..6c6daf0466 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -39,7 +39,6 @@ VALUES (0, now(), 'Created with version'), (22, now(), 'Add file_headers_backup table for key rotation safekeeping'), (23, now(), 'Expand files table with storage locations'); - -- Datasets are used to group files, and permissions are set on the dataset -- level CREATE TABLE datasets ( diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 0a847283f0..0108220d21 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -275,6 +275,10 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { return fmt.Errorf("failed to get file archive information, reason: %v", err) } + if archiveData == nil { + return fmt.Errorf("file archive data not found in database, file-id: %s", fileID) + } + // Get size on disk, will also give some time for the file to appear if it has not already diskFileSize, err := archiveReader.GetFileSize(ctx, archiveData.Location, archiveData.FilePath) if err != nil { From 0d7e26fecae40df3a785c353fa68846034ebffee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 26 Jan 2026 14:37:39 +0100 Subject: [PATCH 240/530] feat(ingest): update main shutdown and erroring logic --- sda/cmd/ingest/ingest.go | 107 ++++++++++++++++++---------------- sda/cmd/ingest/ingest_test.go | 3 +- 2 files changed, 58 insertions(+), 52 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 26e8b19a4e..411f993e03 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -29,6 +29,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" + amqp "github.com/rabbitmq/amqp091-go" log "github.com/sirupsen/logrus" ) @@ -39,57 +40,72 @@ type Ingest struct { ArchiveKeyList []*[32]byte DB *database.SDAdb InboxReader storage.Reader - brokerConf broker.MQConf MQ *broker.AMQPBroker } func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} +func run() error { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + app := Ingest{} ingestConf, err := config.NewConfig("ingest") if err != nil { - log.Fatalf("failed to load config, due to: %v", err) + return fmt.Errorf("failed to load config, due to: %v", err) } - app.brokerConf = ingestConf.Broker - app.MQ, err = broker.NewMQ(app.brokerConf) + app.MQ, err = broker.NewMQ(ingestConf.Broker) if err != nil { - log.Fatalf("failed to initialize mq broker, due to: %v", err) + return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } + defer func() { + if err := app.MQ.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + if err := app.MQ.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } + }() app.DB, err = database.NewSDAdb(ingestConf.Database) if err != nil { - log.Fatalf("failed to initialize sda db due to: %v", err) + return fmt.Errorf("failed to initialize sda db due to: %v", err) } + defer app.DB.Close() if app.DB.Version < 23 { - log.Fatal("database schema v23 is required") + return errors.New("database schema v23 is required") } app.ArchiveKeyList, err = config.GetC4GHprivateKeys() if err != nil || len(app.ArchiveKeyList) == 0 { - log.Fatal("no C4GH private keys configured") + return errors.New("no C4GH private keys configured") } if err := app.registerC4GHKey(); err != nil { - log.Fatalf("failed to register c4gh key, due to: %v", err) + return fmt.Errorf("failed to register c4gh key, due to: %v", err) } storageLocationBroker, err := locationbroker.NewLocationBroker(app.DB) if err != nil { - log.Fatalf("failed to initialize location broker, due to: %v", err) + return fmt.Errorf("failed to initialize location broker, due to: %v", err) } - app.ArchiveWriter, err = storage.NewWriter(context.Background(), "archive", storageLocationBroker) + app.ArchiveWriter, err = storage.NewWriter(ctx, "archive", storageLocationBroker) if err != nil { - log.Fatalf("failed to initialize archive writer, due to: %v", err) + return fmt.Errorf("failed to initialize archive writer, due to: %v", err) } - app.ArchiveReader, err = storage.NewReader(context.Background(), "archive") + app.ArchiveReader, err = storage.NewReader(ctx, "archive") if err != nil { - log.Fatalf("failed to initialize archive reader, due to: %v", err) + return fmt.Errorf("failed to initialize archive reader, due to: %v", err) } - app.InboxReader, err = storage.NewReader(context.Background(), "inbox") + app.InboxReader, err = storage.NewReader(ctx, "inbox") if err != nil { - log.Fatalf("failed to initialize inbox reader, due to: %v", err) + return fmt.Errorf("failed to initialize inbox reader, due to: %v", err) } - backupWriter, err := storage.NewWriter(context.Background(), "backup", storageLocationBroker) + backupWriter, err := storage.NewWriter(ctx, "backup", storageLocationBroker) if err != nil && errors.Is(err, storageerrors.ErrorNoValidWriter) { - log.Fatalf("failed to initialize backup writer due to: %v", err) + return fmt.Errorf("failed to initialize backup writer due to: %v", err) } if backupWriter != nil { log.Info("backup writer initialized, will clean cancelled files from backup storage") @@ -97,40 +113,31 @@ func main() { } else { log.Info("no backup writer initialized, will NOT clean cancelled files from backup storage") } + log.Info("starting ingest service") - defer app.MQ.Channel.Close() - defer app.MQ.Connection.Close() - defer app.DB.Close() - - sigc := make(chan os.Signal, 5) - signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - forever := make(chan bool) - + consumeErr := make(chan error, 1) go func() { - connError := app.MQ.ConnectionWatcher() - log.Error(connError) - forever <- false + consumeErr <- app.startConsumer() }() - go func() { - connError := app.MQ.ChannelWatcher() - log.Error(connError) - forever <- false - }() + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) - log.Info("starting ingest service") + select { + case <-sigc: + case err := <-app.MQ.Connection.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-app.MQ.Channel.NotifyClose(make(chan *amqp.Error)): + return err + case err := <-consumeErr: + return err + } - go func() { - if err := app.run(); err != nil { - log.Error(err) - forever <- false - } - }() - <-forever + return nil } -func (app *Ingest) run() error { - messages, err := app.MQ.GetMessages(app.brokerConf.Queue) +func (app *Ingest) startConsumer() error { + messages, err := app.MQ.GetMessages(app.MQ.Conf.Queue) if err != nil { return err } @@ -139,7 +146,7 @@ func (app *Ingest) run() error { ctx := context.Background() log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) message := schema.IngestionTrigger{} - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.brokerConf.SchemasPath), delivered.Body) + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.MQ.Conf.SchemasPath), delivered.Body) if err != nil { log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. @@ -150,7 +157,7 @@ func (app *Ingest) run() error { } body, _ := json.Marshal(infoErrorMessage) - if err := app.MQ.SendMessage(delivered.CorrelationId, app.brokerConf.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(delivered.CorrelationId, app.MQ.Conf.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) } if err := delivered.Ack(false); err != nil { @@ -233,7 +240,7 @@ func (app *Ingest) cancelFile(ctx context.Context, fileID string, message schema OriginalMessage: message, } body, _ := json.Marshal(fileError) - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(fileID, app.MQ.Conf.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) return "reject" @@ -597,14 +604,14 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema } archivedMsg, _ := json.Marshal(&msg) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.brokerConf.SchemasPath), archivedMsg) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.MQ.Conf.SchemasPath), archivedMsg) if err != nil { log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, app.brokerConf.RoutingKey, archivedMsg); err != nil { + if err := app.MQ.SendMessage(fileID, app.MQ.Conf.Exchange, app.MQ.Conf.RoutingKey, archivedMsg); err != nil { // TODO fix resend mechanism log.Errorf("failed to publish message, reason: %v", err) @@ -649,7 +656,7 @@ func (app *Ingest) setFileEventErrorAndSendToErrorQueue(fileID string, infoError log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } body, _ := json.Marshal(infoError) - if err := app.MQ.SendMessage(fileID, app.brokerConf.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(fileID, app.MQ.Conf.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) return err diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 52aa71d576..a6feb9ebe0 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -229,8 +229,7 @@ func (ts *TestSuite) SetupSuite() { if err != nil { ts.FailNowf("failed to setup database connection: %s", err.Error()) } - ts.ingest.brokerConf = ingestConf.Broker - ts.ingest.MQ, err = broker.NewMQ(ts.ingest.brokerConf) + ts.ingest.MQ, err = broker.NewMQ(ingestConf.Broker) if err != nil { ts.FailNowf("failed to setup rabbitMQ connection: %s", err.Error()) } From c17571de8868697ac459db869b7403480cdcb1c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 28 Jan 2026 15:49:54 +0100 Subject: [PATCH 241/530] feat(database): add CancelFile, and IsFileInDataset to the database interface, add unit test for those, fix issue in IsFileInDataset with archive_file_path isnt nullable --- sda/internal/database/database_interface.go | 2 + sda/internal/database/db_functions.go | 36 ++-------- sda/internal/database/db_functions_test.go | 65 +++++++++++++++++++ .../v2/locationbroker/location_broker_test.go | 8 +++ 4 files changed, 79 insertions(+), 32 deletions(-) diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index 64fa6c0c4c..09fb9cba4a 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -11,4 +11,6 @@ type Database interface { GetArchivePathAndLocation(stableID string) (string, string, error) GetMappingData(accessionID string) (*MappingData, error) GetSubmissionLocation(ctx context.Context, fileID string) (string, error) + CancelFile(ctx context.Context, fileID string) error + IsFileInDataset(ctx context.Context, fileID string) (bool, error) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index a1bcc74b62..af9b79f8ed 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -339,20 +339,6 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec // CancelFile cancels the file and all actions that have been taken (eg, setting checksums, archiving, etc) func (dbs *SDAdb) CancelFile(ctx context.Context, fileID string) error { - var err error - - for count := 1; count <= RetryTimes; count++ { - err = dbs.cancelFile(ctx, fileID) - if err == nil { - break - } - time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) - } - - return err -} - -func (dbs *SDAdb) cancelFile(ctx context.Context, fileID string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB @@ -382,21 +368,6 @@ WHERE file_id = $1` // IsFileInDataset checks if a file has been added to a dataset func (dbs *SDAdb) IsFileInDataset(ctx context.Context, fileID string) (bool, error) { - var err error - var inDataset bool - - for count := 1; count <= RetryTimes; count++ { - inDataset, err = dbs.isFileInDataset(ctx, fileID) - if err == nil { - break - } - time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) - } - - return inDataset, err -} - -func (dbs *SDAdb) isFileInDataset(ctx context.Context, fileID string) (bool, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB @@ -581,7 +552,8 @@ func (dbs *SDAdb) getArchived(fileID string) (*ArchiveData, error) { db := dbs.DB const query = "SELECT archive_file_path, archive_file_size, archive_location, backup_path, backup_location from sda.files WHERE id = $1;" - var archiveFilePath, archiveLocation, backupFilePath, backupLocation sql.NullString + var archiveFilePath string + var archiveLocation, backupFilePath, backupLocation sql.NullString var archiveFileSize sql.Null[int] if err := db.QueryRow(query, fileID).Scan(&archiveFilePath, &archiveFileSize, &archiveLocation, &backupFilePath, &backupLocation); err != nil { if errors.Is(err, sql.ErrNoRows) { @@ -590,9 +562,9 @@ func (dbs *SDAdb) getArchived(fileID string) (*ArchiveData, error) { return nil, err } - if archiveFilePath.Valid || archiveFileSize.Valid || archiveLocation.Valid { + if archiveFilePath != "" || archiveFileSize.Valid || archiveLocation.Valid { ad := &ArchiveData{ - FilePath: archiveFilePath.String, + FilePath: archiveFilePath, Location: archiveLocation.String, FileSize: archiveFileSize.V, BackupFilePath: backupFilePath.String, diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 4126437382..53ea2df8bb 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1803,3 +1803,68 @@ func (suite *DatabaseTests) TestGetSizeAndObjectCountOfLocation() { }) } } + +func (suite *DatabaseTests) TestCancelFile() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "failed to create new connection") + + fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/test.file", "user") + if err != nil { + suite.FailNow("failed to register file", err) + } + + assert.NoError(suite.T(), db.SetArchivedWithLocation("/archive", FileInfo{ + ArchiveChecksum: "123", + Size: 500, + Path: "/test.file3", + DecryptedChecksum: "321", + DecryptedSize: 550, + UploadedChecksum: "abc", + }, fileID)) + + assert.NoError(suite.T(), db.CancelFile(context.TODO(), fileID)) + + // Check that data has been unset + archiveData, err := db.getArchived(fileID) + assert.NoError(suite.T(), err) + assert.Nil(suite.T(), archiveData) +} + +func (suite *DatabaseTests) TestIsFileInDataset_No() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "failed to create new connection") + + fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/test.file", "user") + if err != nil { + suite.FailNow("failed to register file", err) + } + + inDataset, err := db.IsFileInDataset(context.TODO(), fileID) + assert.NoError(suite.T(), err) + assert.False(suite.T(), inDataset) +} + +func (suite *DatabaseTests) TestIsFileInDataset_Yes() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "failed to create new connection") + + fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/test.file", "user") + if err != nil { + suite.FailNow("failed to register file", err) + } + assert.NoError(suite.T(), db.SetArchivedWithLocation("/archive", FileInfo{ + ArchiveChecksum: "123", + Size: 500, + Path: "/test.file3", + DecryptedChecksum: "321", + DecryptedSize: 550, + UploadedChecksum: "abc", + }, fileID)) + + assert.NoError(suite.T(), db.setAccessionID("accessionID-1", fileID)) + assert.NoError(suite.T(), db.mapFilesToDataset("unit-test-dataset-id", []string{"accessionID-1"})) + + inDataset, err := db.IsFileInDataset(context.TODO(), fileID) + assert.NoError(suite.T(), err) + assert.True(suite.T(), inDataset) +} diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index d5bfb74b67..006c75192b 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -41,6 +41,14 @@ func (m *mockDatabase) GetSubmissionLocation(_ context.Context, _ string) (strin panic("function not expected to be called in unit tests") } +func (m *mockDatabase) IsFileInDataset(_ context.Context, _ string) (bool, error) { + panic("function not expected to be called in unit tests") +} + +func (m *mockDatabase) CancelFile(_ context.Context, _ string) error { + panic("function not expected to be called in unit tests") +} + func TestLocationBrokerTestSuite(t *testing.T) { suite.Run(t, new(LocationBrokerTestSuite)) } From 7c0784ee811b16b93fa465261d06b6b2479f0f5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Feb 2026 08:59:05 +0100 Subject: [PATCH 242/530] feat(database): update type of ArchiveData.FileSize from int -> int64 --- sda/cmd/finalize/finalize.go | 4 ++-- sda/internal/database/database.go | 2 +- sda/internal/database/db_functions.go | 4 ++-- sda/internal/database/db_functions_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 0108220d21..8e08c877ea 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -285,7 +285,7 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { return fmt.Errorf("failed to get size info for archived file, reason: %v", err) } - if diskFileSize != int64(archiveData.FileSize) { + if diskFileSize != archiveData.FileSize { return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)", diskFileSize, archiveData.FileSize) } @@ -305,7 +305,7 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { if copiedSize, err := io.Copy(contentWriter, file); err != nil { _ = contentWriter.CloseWithError(fmt.Errorf("failed to copy file, reason: %v)", err)) - } else if copiedSize != int64(archiveData.FileSize) { + } else if copiedSize != archiveData.FileSize { _ = contentWriter.CloseWithError(errors.New("copied size does not match file size")) } }() diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index e9a7a69a8e..4b513cc598 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -56,7 +56,7 @@ type SyncData struct { type ArchiveData struct { FilePath string Location string - FileSize int + FileSize int64 BackupFilePath string BackupLocation string diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index af9b79f8ed..3f1e4a8ac5 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -554,7 +554,7 @@ func (dbs *SDAdb) getArchived(fileID string) (*ArchiveData, error) { var archiveFilePath string var archiveLocation, backupFilePath, backupLocation sql.NullString - var archiveFileSize sql.Null[int] + var archiveFileSize sql.NullInt64 if err := db.QueryRow(query, fileID).Scan(&archiveFilePath, &archiveFileSize, &archiveLocation, &backupFilePath, &backupLocation); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil @@ -566,7 +566,7 @@ func (dbs *SDAdb) getArchived(fileID string) (*ArchiveData, error) { ad := &ArchiveData{ FilePath: archiveFilePath, Location: archiveLocation.String, - FileSize: archiveFileSize.V, + FileSize: archiveFileSize.Int64, BackupFilePath: backupFilePath.String, BackupLocation: backupLocation.String, } diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 53ea2df8bb..a6f9ea8724 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -301,7 +301,7 @@ func (suite *DatabaseTests) TestGetArchived() { archiveData, err := db.GetArchived(fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) - assert.Equal(suite.T(), 1000, archiveData.FileSize) + assert.Equal(suite.T(), int64(1000), archiveData.FileSize) assert.Equal(suite.T(), "/tmp/TestGetArchived.c4gh", archiveData.FilePath) assert.Equal(suite.T(), "/archive", archiveData.Location) From 5aeb3cb79abf092c625c8213578a7f7a86880f64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 4 Feb 2026 13:15:32 +0100 Subject: [PATCH 243/530] fix(ingest): fix typo in ingest.md --- sda/cmd/ingest/ingest.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.md b/sda/cmd/ingest/ingest.md index 1db3eb6620..729b03c49c 100644 --- a/sda/cmd/ingest/ingest.md +++ b/sda/cmd/ingest/ingest.md @@ -115,7 +115,7 @@ storage: backup: # Exclude if no backup storage ${STORAGE_IMPLEMENTATION}: ``` -For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/READMD.md) +For more details on available configuration see [storage/v2 README.md](../../internal/storage/v2/README.md) ### Logging settings: From 44eeab6eb87040f74d786fa0adf6fd844782b36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 08:48:44 +0100 Subject: [PATCH 244/530] feat(ingest): restructure handling of message to separate func with spawns content from main context so it can react to service shutting down and can gracefully shutdown --- sda/cmd/ingest/ingest.go | 109 ++++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 52 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 411f993e03..532c1abff0 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -117,7 +117,7 @@ func run() error { consumeErr := make(chan error, 1) go func() { - consumeErr <- app.startConsumer() + consumeErr <- app.startConsumer(ctx) }() sigc := make(chan os.Signal, 5) @@ -136,72 +136,77 @@ func run() error { return nil } -func (app *Ingest) startConsumer() error { +func (app *Ingest) startConsumer(ctx context.Context) error { messages, err := app.MQ.GetMessages(app.MQ.Conf.Queue) if err != nil { return err } for delivered := range messages { - ctx := context.Background() - log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - message := schema.IngestionTrigger{} - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.MQ.Conf.SchemasPath), delivered.Body) - if err != nil { - log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) - // Send the message to an error queue so it can be analyzed. - infoErrorMessage := broker.InfoError{ - Error: "Message validation failed", - Reason: err.Error(), - OriginalMessage: message, - } + app.handleMessage(ctx, delivered) + } - body, _ := json.Marshal(infoErrorMessage) - if err := app.MQ.SendMessage(delivered.CorrelationId, app.MQ.Conf.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: %v", err) - } - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) - } + return nil +} - continue +func (app *Ingest) handleMessage(ctx context.Context, delivered amqp.Delivery) { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) + message := schema.IngestionTrigger{} + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.MQ.Conf.SchemasPath), delivered.Body) + if err != nil { + log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + // Send the message to an error queue so it can be analyzed. + infoErrorMessage := broker.InfoError{ + Error: "Message validation failed", + Reason: err.Error(), + OriginalMessage: message, } - // we unmarshal the message in the validation step so this is safe to do - _ = json.Unmarshal(delivered.Body, &message) - log.Infof("Received work (correlation-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) - - ackNack := "" - switch message.Type { - case "cancel": - ackNack = app.cancelFile(ctx, delivered.CorrelationId, message) - case "ingest": - ackNack = app.ingestFile(ctx, delivered.CorrelationId, message) - default: - log.Errorln("unexpected ingest message type") - if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message, reason: %v", err) - } + body, _ := json.Marshal(infoErrorMessage) + if err := app.MQ.SendMessage(delivered.CorrelationId, app.MQ.Conf.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: %v", err) + } + if err := delivered.Ack(false); err != nil { + log.Errorf("Failed acking canceled work, reason: %v", err) } - switch ackNack { - case "ack": - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message, reason: %v", err) - } - case "nack": - if err = delivered.Nack(false, false); err != nil { - log.Errorf("failed to Nack message, reason: %v", err) - } - default: - // will catch `reject`s, failures that should not be requeued. - if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message, reason: %v", err) - } + return + } + + // we unmarshal the message in the validation step so this is safe to do + _ = json.Unmarshal(delivered.Body, &message) + log.Infof("Received work (correlation-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) + + ackNack := "" + switch message.Type { + case "cancel": + ackNack = app.cancelFile(ctx, delivered.CorrelationId, message) + case "ingest": + ackNack = app.ingestFile(ctx, delivered.CorrelationId, message) + default: + log.Errorln("unexpected ingest message type") + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message, reason: %v", err) } } - return nil + switch ackNack { + case "ack": + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to ack message, reason: %v", err) + } + case "nack": + if err = delivered.Nack(false, false); err != nil { + log.Errorf("failed to Nack message, reason: %v", err) + } + default: + // will catch `reject`s, failures that should not be requeued. + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message, reason: %v", err) + } + } } func (app *Ingest) registerC4GHKey() error { From 001bc0d84acb89f3fd11a6d72551a6a9374782eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Feb 2026 12:23:25 +0100 Subject: [PATCH 245/530] feat(ingest): update sigc chan buffer size to 1 instead of 5 as we just care that any of the signals triggered --- sda/cmd/ingest/ingest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 532c1abff0..0ad9e968ce 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -120,7 +120,7 @@ func run() error { consumeErr <- app.startConsumer(ctx) }() - sigc := make(chan os.Signal, 5) + sigc := make(chan os.Signal, 1) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) select { From b0dfb6af7ab5e380089209af20a11404a062835e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Feb 2026 15:02:28 +0100 Subject: [PATCH 246/530] feat(ingest): fix some sentance in ingest.md --- sda/cmd/ingest/ingest.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/ingest/ingest.md b/sda/cmd/ingest/ingest.md index 729b03c49c..d8b136a2d6 100644 --- a/sda/cmd/ingest/ingest.md +++ b/sda/cmd/ingest/ingest.md @@ -104,8 +104,8 @@ These settings control how `ingest` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings -The ingest service requires access to the "inbox", and "archive" storages, "backup" storage is optional if to also delete files from backup storage when a file is cancelled, -to configure such the following config is needed: +The ingest service requires access to the "inbox", and "archive" storages, "backup" storage is optional if to also delete files from backup storage when a file is cancelled. +To configure that the following configuration is required: ```yaml storage: inbox: From fb3cdfc4282bb704e0a953fb07913ec20c4da384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 10 Feb 2026 08:59:19 +0100 Subject: [PATCH 247/530] feat(ingest): update sentence in README.md to clarify the optional 'backup' storage --- sda/cmd/ingest/ingest.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sda/cmd/ingest/ingest.md b/sda/cmd/ingest/ingest.md index d8b136a2d6..a991816175 100644 --- a/sda/cmd/ingest/ingest.md +++ b/sda/cmd/ingest/ingest.md @@ -104,8 +104,7 @@ These settings control how `ingest` connects to the RabbitMQ message broker. - `DB_CACERT`: Certificate Authority (CA) certificate for the database to use ### Storage settings -The ingest service requires access to the "inbox", and "archive" storages, "backup" storage is optional if to also delete files from backup storage when a file is cancelled. -To configure that the following configuration is required: +The ingest service requires access to the "inbox", and "archive" storages, "backup" storage is optional if cancelled files are to be automatically deleted from the backup location as well. ```yaml storage: inbox: From 25d395a025fc4ca39fc230364bc61818f5a44765 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 10 Feb 2026 09:12:58 +0100 Subject: [PATCH 248/530] feat(database): update IsFileInDataset query to use EXISTS to simplify error handling and query efficency --- sda/internal/database/db_functions.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 3f1e4a8ac5..33287bc7d7 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -371,21 +371,12 @@ func (dbs *SDAdb) IsFileInDataset(ctx context.Context, fileID string) (bool, err dbs.checkAndReconnectIfNeeded() db := dbs.DB - const isFileInDataset = ` -SELECT true -FROM sda.file_dataset -WHERE file_id = $1;` + const query = `SELECT EXISTS(SELECT 1 FROM sda.file_dataset WHERE file_id = $1);` var inDataset bool - if err := db.QueryRowContext(ctx, isFileInDataset, fileID).Scan(&inDataset); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return false, nil - } - - return false, err - } + err := db.QueryRowContext(ctx, query, fileID).Scan(&inDataset) - return inDataset, nil + return inDataset, err } func (dbs *SDAdb) GetFileStatus(fileID string) (string, error) { From 2cdc729a6a7291e427a75c440f0bb22ff216f34f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 10 Feb 2026 09:27:31 +0100 Subject: [PATCH 249/530] feat(database): update CancelFile to use tx, and update to also set file event log as part of same tx --- sda/cmd/ingest/ingest.go | 14 +----- sda/internal/database/database_interface.go | 2 +- sda/internal/database/db_functions.go | 47 ++++++++++++------- sda/internal/database/db_functions_test.go | 2 +- .../v2/locationbroker/location_broker_test.go | 2 +- 5 files changed, 34 insertions(+), 33 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 0ad9e968ce..f539b06414 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -15,7 +15,6 @@ import ( "io" "os" "os/signal" - "strings" "syscall" "github.com/neicnordic/crypt4gh/keys" @@ -277,17 +276,8 @@ func (app *Ingest) cancelFile(ctx context.Context, fileID string, message schema } } - if err := app.DB.CancelFile(ctx, fileID); err != nil { - log.Errorf("failed to unset file with id: %s as archived due to %v", fileID, err) - - return "nack" - } - - if err := app.DB.UpdateFileEventLog(fileID, "disabled", "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to update event log for file with id: %s, due to %v", fileID, err) - if strings.Contains(err.Error(), "sql: no rows in result set") { - return "reject" - } + if err := app.DB.CancelFile(ctx, fileID, string(m)); err != nil { + log.Errorf("failed to cancel file with id: %s, due to %v", fileID, err) return "nack" } diff --git a/sda/internal/database/database_interface.go b/sda/internal/database/database_interface.go index 09fb9cba4a..8fff7bada8 100644 --- a/sda/internal/database/database_interface.go +++ b/sda/internal/database/database_interface.go @@ -11,6 +11,6 @@ type Database interface { GetArchivePathAndLocation(stableID string) (string, string, error) GetMappingData(accessionID string) (*MappingData, error) GetSubmissionLocation(ctx context.Context, fileID string) (string, error) - CancelFile(ctx context.Context, fileID string) error + CancelFile(ctx context.Context, fileID, message string) error IsFileInDataset(ctx context.Context, fileID string) (bool, error) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 33287bc7d7..c588373eb8 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -338,32 +338,43 @@ ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.chec } // CancelFile cancels the file and all actions that have been taken (eg, setting checksums, archiving, etc) -func (dbs *SDAdb) CancelFile(ctx context.Context, fileID string) error { +func (dbs *SDAdb) CancelFile(ctx context.Context, fileID string, message string) error { dbs.checkAndReconnectIfNeeded() - db := dbs.DB - const unsetArchived = ` -UPDATE sda.files -SET archive_location = NULL, - archive_file_path = '', - archive_file_size = NULL, - decrypted_file_size = NULL, - stable_id = NULL -WHERE id = $1;` - - if _, err := db.ExecContext(ctx, unsetArchived, fileID); err != nil { + tx, err := dbs.DB.BeginTx(ctx, nil) + if err != nil { return err } + defer func() { + if err := tx.Rollback(); err != nil { + log.Errorf("failed to rollback CancelFile transaction, due to: %v", err) + } + }() - const deleteChecksums = ` -DELETE FROM sda.checksums -WHERE file_id = $1` + const unsetArchived = ` + UPDATE sda.files SET archive_location = NULL, archive_file_path = '', + archive_file_size = NULL, decrypted_file_size = NULL, stable_id = NULL + WHERE id = $1;` - if _, err := db.ExecContext(ctx, deleteChecksums, fileID); err != nil { - return err + if _, err := tx.ExecContext(ctx, unsetArchived, fileID); err != nil { + return fmt.Errorf("failed to unset file data (file-id: %s): %v", fileID, err) } - return nil + const deleteChecksums = `DELETE FROM sda.checksums WHERE file_id = $1` + + if _, err := tx.ExecContext(ctx, deleteChecksums, fileID); err != nil { + return fmt.Errorf("failed to delete checksums (file-id: %s): %v", fileID, err) + } + + const logEvent = ` + INSERT INTO sda.file_event_log (file_id, event, user_id, details, message) + VALUES ($1, 'disabled', 'system', '{"reason": "file cancelled"}', $2);` + + if _, err := tx.ExecContext(ctx, logEvent, fileID, message); err != nil { + return fmt.Errorf("failed to log cancel file event (file-id: %s): %v", fileID, err) + } + + return tx.Commit() } // IsFileInDataset checks if a file has been added to a dataset diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index a6f9ea8724..7662abebdd 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1822,7 +1822,7 @@ func (suite *DatabaseTests) TestCancelFile() { UploadedChecksum: "abc", }, fileID)) - assert.NoError(suite.T(), db.CancelFile(context.TODO(), fileID)) + assert.NoError(suite.T(), db.CancelFile(context.TODO(), fileID, "")) // Check that data has been unset archiveData, err := db.getArchived(fileID) diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index 006c75192b..fd554bdd8f 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -45,7 +45,7 @@ func (m *mockDatabase) IsFileInDataset(_ context.Context, _ string) (bool, error panic("function not expected to be called in unit tests") } -func (m *mockDatabase) CancelFile(_ context.Context, _ string) error { +func (m *mockDatabase) CancelFile(_ context.Context, _, _ string) error { panic("function not expected to be called in unit tests") } From df93433ed0c9b875d16122736e66f12b92060614 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 11 Feb 2026 12:55:15 +0100 Subject: [PATCH 250/530] feat(config): merge identical switch cases for ingest and finalize --- sda/internal/config/config.go | 30 ++---------------------------- 1 file changed, 2 insertions(+), 28 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 62f018d434..4c113f2e6d 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -276,22 +276,7 @@ func NewConfig(app string) (*Config, error) { if viper.GetBool("auth.resignJwt") { requiredConfVars = append(requiredConfVars, []string{"auth.jwt.issuer", "auth.jwt.privateKey", "auth.jwt.signatureAlg", "auth.jwt.tokenTTL"}...) } - case "ingest": - requiredConfVars = []string{ - "broker.host", - "broker.port", - "broker.user", - "broker.password", - "broker.queue", - "broker.routingkey", - "db.host", - "db.port", - "db.user", - "db.password", - "db.database", - } - - case "finalize": + case "ingest", "finalize": requiredConfVars = []string{ "broker.host", "broker.port", @@ -534,18 +519,7 @@ func NewConfig(app string) (*Config, error) { if err != nil { return nil, err } - case "finalize": - err := c.configBroker() - if err != nil { - return nil, err - } - err = c.configDatabase() - if err != nil { - return nil, err - } - - c.configSchemas() - case "ingest": + case "finalize", "ingest": err := c.configBroker() if err != nil { return nil, err From 50c1a7bd13abe2856ddc4f04f051b62da7ae80b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:06:25 +0100 Subject: [PATCH 251/530] feat(ingest): update CancelFile defer tx rollback to check if tx has been commited before logging error, update CancelFile to reject if file archive data does not exists, fix CancelFile unit test --- sda/cmd/ingest/ingest.go | 10 ++++++++-- sda/internal/database/db_functions.go | 2 +- sda/internal/database/db_functions_test.go | 2 +- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index f539b06414..6ab4942169 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -260,7 +260,13 @@ func (app *Ingest) cancelFile(ctx context.Context, fileID string, message schema return "nack" } - if archiveData != nil && archiveData.Location != "" { + if archiveData == nil { + log.Warnf("file with id: %s, could not be cancelled, as it has not yet been archived", fileID) + + return "reject" + } + + if archiveData.Location != "" { if err := app.ArchiveWriter.RemoveFile(ctx, archiveData.Location, archiveData.FilePath); err != nil { log.Errorf("failed to remove file with id %s from archive due to %v", fileID, err) @@ -268,7 +274,7 @@ func (app *Ingest) cancelFile(ctx context.Context, fileID string, message schema } } - if app.BackupWriter != nil && archiveData != nil && archiveData.BackupFilePath != "" && archiveData.BackupLocation != "" { + if app.BackupWriter != nil && archiveData.BackupFilePath != "" && archiveData.BackupLocation != "" { if err := app.BackupWriter.RemoveFile(ctx, archiveData.BackupLocation, archiveData.BackupFilePath); err != nil { log.Errorf("failed to remove file with id %s from backup due to %v", fileID, err) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index c588373eb8..c8a50fdd74 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -346,7 +346,7 @@ func (dbs *SDAdb) CancelFile(ctx context.Context, fileID string, message string) return err } defer func() { - if err := tx.Rollback(); err != nil { + if err := tx.Rollback(); err != nil && !errors.Is(err, sql.ErrTxDone) { log.Errorf("failed to rollback CancelFile transaction, due to: %v", err) } }() diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 7662abebdd..9a010a601f 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1822,7 +1822,7 @@ func (suite *DatabaseTests) TestCancelFile() { UploadedChecksum: "abc", }, fileID)) - assert.NoError(suite.T(), db.CancelFile(context.TODO(), fileID, "")) + assert.NoError(suite.T(), db.CancelFile(context.TODO(), fileID, "{}")) // Check that data has been unset archiveData, err := db.getArchived(fileID) From ef46d0e1d17f74695d1ae28a68465d84f613da33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 09:30:26 +0100 Subject: [PATCH 252/530] feat(ingest): update defer closing of rabbit mq connection and channel to check for nil before close --- sda/cmd/ingest/ingest.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 6ab4942169..e884ffb8ed 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -61,11 +61,18 @@ func run() error { return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } defer func() { - if err := app.MQ.Channel.Close(); err != nil { - log.Errorf("failed to close mq broker channel due to: %v", err) + if app.MQ == nil { + return } - if err := app.MQ.Connection.Close(); err != nil { - log.Errorf("failed to close mq broker connection due to: %v", err) + if app.MQ.Channel != nil { + if err := app.MQ.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } + } + if app.MQ.Connection != nil { + if err := app.MQ.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } } }() app.DB, err = database.NewSDAdb(ingestConf.Database) From afdfdd574959ad35e929e84e9b68fac19076d986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 13:54:57 +0100 Subject: [PATCH 253/530] feat(config): merge mapper and verify config setup with finalize and ingest given the identical switch cases --- sda/cmd/s3inbox/proxy_test.go | 4 +-- sda/internal/config/config.go | 59 ++--------------------------------- 2 files changed, 3 insertions(+), 60 deletions(-) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 72e65929e8..120e068dde 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -668,15 +668,13 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.database = db - s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) - assert.NoError(s.T(), err) fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) // Detect autentication failure proxy.s3Conf.AccessKey = "badKey" - s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) + s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) assert.NoError(s.T(), err) assert.Error(s.T(), proxy.storeObjectSizeInDB(context.TODO(), s3Client, "/dummy/file", fileID)) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 4c113f2e6d..96d7fcee8a 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -276,7 +276,7 @@ func NewConfig(app string) (*Config, error) { if viper.GetBool("auth.resignJwt") { requiredConfVars = append(requiredConfVars, []string{"auth.jwt.issuer", "auth.jwt.privateKey", "auth.jwt.signatureAlg", "auth.jwt.tokenTTL"}...) } - case "ingest", "finalize": + case "ingest", "finalize", "mapper", "verify": requiredConfVars = []string{ "broker.host", "broker.port", @@ -299,21 +299,6 @@ func NewConfig(app string) (*Config, error) { "broker.password", "broker.queue", } - case "mapper": - // Mapper does not require broker.routingkey thus we remove it - requiredConfVars = []string{ - "broker.host", - "broker.port", - "broker.user", - "broker.password", - "broker.queue", - "db.host", - "db.port", - "db.user", - "db.password", - "db.database", - } - case "notify": requiredConfVars = []string{ "broker.host", @@ -399,21 +384,6 @@ func NewConfig(app string) (*Config, error) { "sync.api.user", "sync.api.password", } - case "verify": - requiredConfVars = []string{ - "broker.host", - "broker.port", - "broker.user", - "broker.password", - "broker.queue", - "broker.routingkey", - "db.host", - "db.port", - "db.user", - "db.password", - "db.database", - } - default: return nil, fmt.Errorf("application '%s' doesn't exist", app) } @@ -519,7 +489,7 @@ func NewConfig(app string) (*Config, error) { if err != nil { return nil, err } - case "finalize", "ingest": + case "finalize", "ingest", "mapper", "verify": err := c.configBroker() if err != nil { return nil, err @@ -536,18 +506,6 @@ func NewConfig(app string) (*Config, error) { return nil, err } - c.configSchemas() - case "mapper": - err := c.configBroker() - if err != nil { - return nil, err - } - - err = c.configDatabase() - if err != nil { - return nil, err - } - c.configSchemas() case "notify": c.configSMTP() @@ -631,19 +589,6 @@ func NewConfig(app string) (*Config, error) { } c.configSyncAPI() - c.configSchemas() - case "verify": - - err := c.configBroker() - if err != nil { - return nil, err - } - - err = c.configDatabase() - if err != nil { - return nil, err - } - c.configSchemas() default: return nil, errors.New("unknown app name") From 5027505a7a96145cd6bbf56ed4e7c415ae5390ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 14:47:39 +0100 Subject: [PATCH 254/530] feat(ingest): fix CancelFile unit tests, add happy path scenario --- sda/cmd/ingest/ingest_test.go | 34 ++++++++++++++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index a6feb9ebe0..35c091fa2f 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -217,8 +217,6 @@ func (ts *TestSuite) SetupSuite() { viper.Set("db.password", "rootpasswd") viper.Set("db.database", "sda") viper.Set("db.sslMode", "disable") - viper.Set("inbox.type", "posix") - viper.Set("inbox.location", "/tmp/") viper.Set("schema.path", "../../schemas/isolated/") ingestConf, err := config.NewConfig("ingest") @@ -394,6 +392,27 @@ func (ts *TestSuite) TestTryDecrypt() { } } +// messages of type `cancel` +func (ts *TestSuite) TestCancelFile_NotYetArchived() { + // prepare the DB entries + userName := "test-cancel" + file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", userName) + fileID, err := ts.ingest.DB.RegisterFile(nil, file1, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "cancel", + FilePath: file1, + User: userName, + } + + assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(context.TODO(), fileID, message)) +} + // messages of type `cancel` func (ts *TestSuite) TestCancelFile() { // prepare the DB entries @@ -406,6 +425,17 @@ func (ts *TestSuite) TestCancelFile() { ts.Fail("failed to update file event log") } + assert.NoError(ts.T(), ts.ingest.DB.SetArchivedWithLocation(ts.archiveDir, database.FileInfo{ + ArchiveChecksum: "123", + Size: 500, + Path: fileID, + DecryptedChecksum: "321", + DecryptedSize: 550, + UploadedChecksum: "abc", + }, fileID)) + + ts.NoError(os.WriteFile(filepath.Join(ts.archiveDir, fileID), []byte("unit testing file"), 0600)) + message := schema.IngestionTrigger{ Type: "cancel", FilePath: file1, From 7faf05f00b7fdb267dd263a03bfa2e338ca34370 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 14:48:00 +0100 Subject: [PATCH 255/530] feat(config): remove deprecated storage(v1) configs --- sda/internal/config/config.go | 28 ---------------------------- sda/internal/config/config_test.go | 2 -- 2 files changed, 30 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 96d7fcee8a..75b7332d5d 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -40,8 +40,6 @@ type Config struct { Archive storage.Conf Broker broker.MQConf Database database.DBConf - Inbox storage.Conf - Backup storage.Conf Server ServerConfig S3Inbox S3InboxConf API APIConf @@ -647,21 +645,6 @@ func (c *Config) configArchive() { } } -// configBackup provides configuration for the backup storage -func (c *Config) configBackup() { - switch viper.GetString("backup.type") { - case S3: - c.Backup.Type = S3 - c.Backup.S3 = configS3Storage("backup") - case SFTP: - c.Backup.Type = SFTP - c.Backup.SFTP = configSFTP("backup") - default: - c.Backup.Type = POSIX - c.Backup.Posix.Location = viper.GetString("backup.location") - } -} - // configBroker provides configuration for the message broker func (c *Config) configBroker() error { // Setup broker @@ -759,17 +742,6 @@ func (c *Config) configDatabase() error { return nil } -// configInbox provides configuration for the inbox storage -func (c *Config) configInbox() { - if viper.GetString("inbox.type") == S3 { - c.Inbox.Type = S3 - c.Inbox.S3 = configS3Storage("inbox") - } else { - c.Inbox.Type = POSIX - c.Inbox.Posix.Location = viper.GetString("inbox.location") - } -} - // configOrchestrator provides the configuration for the standalone orchestator. func (c *Config) configOrchestrator() { c.Orchestrator = OrchestratorConf{} diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 35a299a397..b00d728d96 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -157,7 +157,6 @@ func (ts *ConfigTestSuite) TestConfigS3Storage() { config, err := NewConfig("s3inbox") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) - assert.NotNil(ts.T(), config.Inbox.S3) assert.Equal(ts.T(), "testurl", config.S3Inbox.Endpoint) assert.Equal(ts.T(), "testaccess", config.S3Inbox.AccessKey) assert.Equal(ts.T(), "testsecret", config.S3Inbox.SecretKey) @@ -174,7 +173,6 @@ func (ts *ConfigTestSuite) TestConfigBroker() { config, err := NewConfig("s3inbox") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) - assert.NotNil(ts.T(), config.Inbox.S3) assert.Equal(ts.T(), "/testvhost", config.Broker.Vhost) assert.Equal(ts.T(), false, config.Broker.Ssl) From d79db3fa98f1117a6f43ca13d2610a9a41243437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 15:08:33 +0100 Subject: [PATCH 256/530] feat(database): replace deprecated functions RegisterFile, SetArchived without location with the respective WithLocation function, update ingest file registring when to ensure file is registered with location or registered withou if error when finding where file is --- sda/cmd/api/api_test.go | 68 +++++----- sda/cmd/ingest/ingest.go | 19 +-- sda/cmd/ingest/ingest_test.go | 24 ++-- sda/cmd/rotatekey/rotatekey_test.go | 2 +- sda/cmd/s3inbox/proxy.go | 2 +- sda/cmd/s3inbox/proxy_test.go | 6 +- sda/cmd/sync/sync_test.go | 4 +- sda/internal/database/db_functions.go | 60 +-------- sda/internal/database/db_functions_test.go | 144 ++++++++++----------- 9 files changed, 139 insertions(+), 190 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 182a88b403..87507a6391 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -302,7 +302,7 @@ type TestSuite struct { } func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, hash.Hash) { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update status of file in database") @@ -322,7 +322,7 @@ func helperCreateVerifiedTestFile(s *TestSuite, user, filePath string) (string, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID) + err = Conf.API.DB.SetArchived("/archive", fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Verified") @@ -657,7 +657,7 @@ func (s *TestSuite) TestAPIGetFiles() { // Insert a file and make sure it is listed file1 := fmt.Sprintf("/%v/TestAPIGetFiles.c4gh", s.User) - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, file1, s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, file1, s.User) assert.NoError(s.T(), err, "failed to register file in database") latestStatus := "uploaded" @@ -701,7 +701,7 @@ func (s *TestSuite) TestAPIGetFiles() { // Insert a second file and make sure it is listed file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", s.User) - _, err = Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, file2, s.User) + _, err = Conf.API.DB.RegisterFile(nil, s.inboxDir, file2, s.User) assert.NoError(s.T(), err, "failed to register file in database") resp, err = client.Do(req) // #nosec G704 -- request controlled by unit test @@ -727,7 +727,7 @@ func (s *TestSuite) TestAPIGetFiles() { func (s *TestSuite) TestAPIGetFiles_SubmissionFileSize() { for i := 1; i <= 2; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", "submission_b", i), "dummy") + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", "submission_b", i), "dummy") if err != nil { s.FailNow("failed to register file in database") } @@ -795,7 +795,7 @@ func (s *TestSuite) TestAPIGetFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -1145,7 +1145,7 @@ func (s *TestSuite) TestIngestFile_WithPayload() { s.FailNow("failed to setup RBAC enforcer") } - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1204,7 +1204,7 @@ func (s *TestSuite) TestIngestFile_WithPayload_NoUser() { s.FailNow("failed to setup RBAC enforcer") } - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1238,7 +1238,7 @@ func (s *TestSuite) TestIngestFile_WithPayload_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1271,7 +1271,7 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1305,7 +1305,7 @@ func (s *TestSuite) TestIngestFile_WrongFilePath() { func (s *TestSuite) TestIngestFile_WithFileID() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) @@ -1350,7 +1350,7 @@ func (s *TestSuite) TestIngestFile_WithFileID() { func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) @@ -1376,7 +1376,7 @@ func (s *TestSuite) TestIngestFile_WithFileID_WrongID() { func (s *TestSuite) TestIngestFile_BothFileIDAndPayloadProvided() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err) err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err) @@ -1713,7 +1713,7 @@ func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update status of file in database") @@ -1733,7 +1733,7 @@ func (s *TestSuite) TestCreateDataset() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID) + err = Conf.API.DB.SetArchived("/archive", fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) @@ -1795,7 +1795,7 @@ func (s *TestSuite) TestCreateDataset_BadFormat() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1815,7 +1815,7 @@ func (s *TestSuite) TestCreateDataset_BadFormat() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID) + err = Conf.API.DB.SetArchived("/archive", fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) @@ -1917,7 +1917,7 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", user, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -1937,7 +1937,7 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID) + err = Conf.API.DB.SetArchived("/archive", fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) @@ -1975,7 +1975,7 @@ func (s *TestSuite) TestCreateDataset_WrongUser() { func (s *TestSuite) TestReleaseDataset() { user := "TestReleaseDataset" for i := 0; i < 3; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2108,7 +2108,7 @@ func (s *TestSuite) TestReleaseDataset_DeprecatedDataset() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2167,7 +2167,7 @@ func (s *TestSuite) TestListActiveUsers() { testUsers := []string{"User-A", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 3; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) if err != nil { s.FailNow("failed to register file in database") } @@ -2226,7 +2226,7 @@ func (s *TestSuite) TestListUserFiles() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2292,7 +2292,7 @@ func (s *TestSuite) TestListUserFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2553,7 +2553,7 @@ func (s *TestSuite) TestDeprecateC4ghHash_wrongHash() { func (s *TestSuite) TestListDatasets() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") if err != nil { s.FailNow("failed to register file in database") } @@ -2611,7 +2611,7 @@ func (s *TestSuite) TestListDatasets() { func (s *TestSuite) TestListUserDatasets() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -2668,7 +2668,7 @@ func (s *TestSuite) TestListUserDatasets() { func (s *TestSuite) TestListDatasetsAsUser() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) if err != nil { s.FailNow("failed to register file in database") } @@ -2727,7 +2727,7 @@ func (s *TestSuite) TestReVerifyFile() { user := "TestReVerify" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerify-00%d.c4gh", user, i) - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) if err != nil { s.FailNow("failed to register file in database") } @@ -2755,7 +2755,7 @@ func (s *TestSuite) TestReVerifyFile() { Size: 1000, UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } - if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { + if err := Conf.API.DB.SetArchived("/archive", fileInfo, fileID); err != nil { s.FailNow("failed to mark file as Archived") } @@ -2829,7 +2829,7 @@ func (s *TestSuite) TestReVerifyDataset() { user := "TestReVerifyDataset" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerifyDataset-00%d.c4gh", user, i) - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, user) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, user) if err != nil { s.FailNow("failed to register file in database") } @@ -2857,7 +2857,7 @@ func (s *TestSuite) TestReVerifyDataset() { Size: 1000, UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } - if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { + if err := Conf.API.DB.SetArchived("/archive", fileInfo, fileID); err != nil { s.FailNow("failed to mark file as Archived") } @@ -2949,7 +2949,7 @@ func (s *TestSuite) TestDownloadFile() { defer ts.Close() // Register the file in the database - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filepath.Base(s.GoodC4ghFile), s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filepath.Base(s.GoodC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -3036,7 +3036,7 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { // Register a file in the database (but don't create the actual file) filePath := fmt.Sprintf("/%v/nonexistent.c4gh", s.User) - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filePath, s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filePath, s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -3068,7 +3068,7 @@ func (s *TestSuite) TestDownloadFile_badC4ghFile() { defer ts.Close() // Register a file in the database (but don't create the actual file) - fileID, err := Conf.API.DB.RegisterFileWithLocation(nil, s.inboxDir, filepath.Base(s.BadC4ghFile), s.User) + fileID, err := Conf.API.DB.RegisterFile(nil, s.inboxDir, filepath.Base(s.BadC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index e884ffb8ed..4d25b06be2 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -321,21 +321,23 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema switch status { case "": - fileID, err = app.DB.RegisterFile(&fileID, message.FilePath, message.User) - if err != nil { + // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox + // Since we dont have the submission location in storage, we need to look through all configured storage locations. + var findFileErr, registerErr error + submissionLocation, findFileErr = app.InboxReader.FindFile(ctx, message.FilePath) + // Register file even if FindFile didnt succeed with submissionLocation == "", as we will add an error file event log to it in that case + fileID, registerErr = app.DB.RegisterFile(&fileID, submissionLocation, message.FilePath, message.User) + if registerErr != nil { log.Errorf("failed to register file, fileID: %s, reason: (%s)", fileID, err.Error()) return "nack" } - // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox - // Since we dont have the submission location in storage, we need to look through all configured storage locations. - submissionLocation, err = app.InboxReader.FindFile(ctx, message.FilePath) - if err != nil { + if findFileErr != nil { log.Errorf("failed to find submission location for file in all configured storage locations, file-id: %s", fileID) if err := app.setFileEventErrorAndSendToErrorQueue(fileID, &broker.InfoError{ Error: "Failed to open file to ingest, file not found in any of the configured storage locations", - Reason: err.Error(), + Reason: findFileErr.Error(), OriginalMessage: message, }); err != nil { return "reject" @@ -343,7 +345,6 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema return "ack" } - log.Infof("registering file, file-id: %s", fileID) case "uploaded", "disabled": @@ -587,7 +588,7 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema return "ack" } - if err := app.DB.SetArchivedWithLocation(location, fileInfo, fileID); err != nil { + if err := app.DB.SetArchived(location, fileInfo, fileID); err != nil { log.Errorf("SetArchived failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 35c091fa2f..96c1553410 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -397,7 +397,7 @@ func (ts *TestSuite) TestCancelFile_NotYetArchived() { // prepare the DB entries userName := "test-cancel" file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", userName) - fileID, err := ts.ingest.DB.RegisterFile(nil, file1, userName) + fileID, err := ts.ingest.DB.RegisterFile(nil, "/inbox", file1, userName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", userName, "{}", "{}"); err != nil { @@ -418,14 +418,14 @@ func (ts *TestSuite) TestCancelFile() { // prepare the DB entries userName := "test-cancel" file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", userName) - fileID, err := ts.ingest.DB.RegisterFile(nil, file1, userName) + fileID, err := ts.ingest.DB.RegisterFile(nil, "/inbox", file1, userName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } - assert.NoError(ts.T(), ts.ingest.DB.SetArchivedWithLocation(ts.archiveDir, database.FileInfo{ + assert.NoError(ts.T(), ts.ingest.DB.SetArchived(ts.archiveDir, database.FileInfo{ ArchiveChecksum: "123", Size: 500, Path: fileID, @@ -449,7 +449,7 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // prepare the DB entries userName := "test-cancel" file1 := fmt.Sprintf("/%v/TestCancelMessage_wrongCorrelationID.c4gh", userName) - fileID, err := ts.ingest.DB.RegisterFile(nil, file1, userName) + fileID, err := ts.ingest.DB.RegisterFile(nil, "/inbox", file1, userName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", userName, "{}", "{}"); err != nil { @@ -468,7 +468,7 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // messages of type `ingest` func (ts *TestSuite) TestIngestFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -486,7 +486,7 @@ func (ts *TestSuite) TestIngestFile() { func (ts *TestSuite) TestNoSubmissionLocation() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFile(nil, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, "/inbox", ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -504,7 +504,7 @@ func (ts *TestSuite) TestNoSubmissionLocation() { func (ts *TestSuite) TestIngestFile_secondTime() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -533,7 +533,7 @@ func (ts *TestSuite) TestIngestFile_unknownInboxType() { } func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -556,7 +556,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -626,7 +626,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -656,7 +656,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { @@ -690,7 +690,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { // prepare the DB entries - fileID, err := ts.ingest.DB.RegisterFileWithLocation(nil, ts.inboxDir, ts.filePath, ts.UserName) + fileID, err := ts.ingest.DB.RegisterFile(nil, ts.inboxDir, ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", ts.UserName, "{}", "{}"); err != nil { diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go index b862195f7b..dc1a1906c6 100644 --- a/sda/cmd/rotatekey/rotatekey_test.go +++ b/sda/cmd/rotatekey/rotatekey_test.go @@ -212,7 +212,7 @@ func (ts *TestSuite) SetupSuite() { ts.app.Conf.RotateKey.PublicKey = &publicKey - ts.fileID, err = ts.app.DB.RegisterFile(nil, "rotate-key-test/data.c4gh", "tester_example.org") + ts.fileID, err = ts.app.DB.RegisterFile(nil, "/inbox", "rotate-key-test/data.c4gh", "tester_example.org") if err != nil { ts.FailNow("Failed to register file in DB") } diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 3a838fc158..c4d5a40647 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -182,7 +182,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { // register file in database log.Debugf("registering file %v in the database with location: %s", r.URL.Path, location) - p.fileIDs[fileIdentifier], err = p.database.RegisterFileWithLocation(nil, location, filePath, username) + p.fileIDs[fileIdentifier], err = p.database.RegisterFile(nil, location, filePath, username) log.Debugf("fileId: %v", p.fileIDs[fileIdentifier]) if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to register file in database: %v", err)) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 120e068dde..e519a11999 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -650,7 +650,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) assert.NoError(s.T(), err) - fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") + fileID, err := db.RegisterFile(nil, "/inbox", "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) @@ -668,7 +668,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.database = db - fileID, err := db.RegisterFile(nil, "/dummy/file", "test-user") + fileID, err := db.RegisterFile(nil, "/inbox", "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) @@ -699,7 +699,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db - fileID, err := db.RegisterFile(nil, "/test/new_file", "test-user") + fileID, err := db.RegisterFile(nil, "/inbox", "/test/new_file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index c3c10d2f64..7b1db529a6 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -159,7 +159,7 @@ func (s *SyncTest) TestBuildSyncDatasetJSON() { db, err = database.NewSDAdb(conf.Database) assert.NoError(s.T(), err) - fileID, err := db.RegisterFile(nil, "dummy.user/test/file1.c4gh", "dummy.user") + fileID, err := db.RegisterFile(nil, "/inbox", "dummy.user/test/file1.c4gh", "dummy.user") assert.NoError(s.T(), err, "failed to register file in database") err = db.SetAccessionID("ed6af454-d910-49e3-8cda-488a6f246e67", fileID) assert.NoError(s.T(), err) @@ -167,7 +167,7 @@ func (s *SyncTest) TestBuildSyncDatasetJSON() { checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := database.FileInfo{ArchiveChecksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(s.T(), err, "failed to mark file as Verified") diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index c8a50fdd74..6c6477ddbf 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -17,37 +17,13 @@ import ( log "github.com/sirupsen/logrus" ) -// RegisterFile inserts a file in the database, along with a "registered" log +// RegisterFile inserts a file in the database with its inbox location, along with a "registered" log // event. If the file already exists in the database, the entry is updated, but // a new file event is always inserted. // If fileId is provided the new files table row will have that id, otherwise a new uuid will be generated // If the unique unique_ingested constraint(submission_file_path, archive_file_path, submission_user) already exists // and a different fileId is provided, the fileId in the database will NOT be updated. -func (dbs *SDAdb) RegisterFile(fileID *string, uploadPath, uploadUser string) (string, error) { - dbs.checkAndReconnectIfNeeded() - - query := "SELECT sda.register_file($1, $2, $3, $4);" - - var createdFileID string - - fileIDArg := sql.NullString{} - if fileID != nil { - fileIDArg.Valid = true - fileIDArg.String = *fileID - } - - err := dbs.DB.QueryRow(query, fileIDArg, "", uploadPath, uploadUser).Scan(&createdFileID) - - return createdFileID, err -} - -// RegisterFileWithLocation inserts a file in the database with its inbox location, along with a "registered" log -// event. If the file already exists in the database, the entry is updated, but -// a new file event is always inserted. -// If fileId is provided the new files table row will have that id, otherwise a new uuid will be generated -// If the unique unique_ingested constraint(submission_file_path, archive_file_path, submission_user) already exists -// and a different fileId is provided, the fileId in the database will NOT be updated. -func (dbs *SDAdb) RegisterFileWithLocation(fileID *string, inboxLocation, uploadPath, uploadUser string) (string, error) { +func (dbs *SDAdb) RegisterFile(fileID *string, inboxLocation, uploadPath, uploadUser string) (string, error) { dbs.checkAndReconnectIfNeeded() query := "SELECT sda.register_file($1, $2, $3, $4);" @@ -287,36 +263,8 @@ func (dbs *SDAdb) rotateHeaderKey(header []byte, keyHash, fileID string) error { return nil } -// SetArchived marks the file as 'ARCHIVED' -func (dbs *SDAdb) SetArchived(file FileInfo, fileID string) error { - var err error - - for count := 1; count <= RetryTimes; count++ { - err = dbs.setArchived("", file, fileID) - if err == nil { - break - } - time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) - } - - return err -} - -// SetArchivedWithLocation marks the file as 'ARCHIVED' with its archive location -func (dbs *SDAdb) SetArchivedWithLocation(location string, file FileInfo, fileID string) error { - var err error - - for count := 1; count <= RetryTimes; count++ { - err = dbs.setArchived(location, file, fileID) - if err == nil { - break - } - time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) - } - - return err -} -func (dbs *SDAdb) setArchived(location string, file FileInfo, fileID string) error { +// SetArchived marks the file as 'ARCHIVED' with its archive location +func (dbs *SDAdb) SetArchived(location string, file FileInfo, fileID string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 9a010a601f..da1ab2d8f0 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -41,7 +41,7 @@ func (suite *DatabaseTests) TestRegisterFile() { }, } { suite.T().Run(step.name, func(t *testing.T) { - fileID, err := db.RegisterFile(&step.id, step.filePath, step.userName) + fileID, err := db.RegisterFile(&step.id, "/inbox", step.filePath, step.userName) assert.NoError(suite.T(), err, "RegisterFile encountered an unexpected error: ", err) assert.NoError(suite.T(), uuid.Validate(fileID), "RegisterFile did not return a UUID") }) @@ -54,7 +54,7 @@ func (suite *DatabaseTests) TestRegisterFileWithID() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) insertedFileID := uuid.New().String() - fileID, err := db.RegisterFile(&insertedFileID, "/testuser/file3.c4gh", "testuser") + fileID, err := db.RegisterFile(&insertedFileID, "/inbox", "/testuser/file3.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.UpdateFileEventLog(fileID, "uploaded", "testuser", "{}", "{}") @@ -73,7 +73,7 @@ func (suite *DatabaseTests) TestUpdateFileEventLog() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/file4.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/file4.c4gh", "testuser") assert.Nil(suite.T(), err, "failed to register file in database") // Attempt to mark a file that doesn't exist as uploaded @@ -98,7 +98,7 @@ func (suite *DatabaseTests) TestStoreHeader() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestStoreHeader.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestStoreHeader.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, fileID) @@ -116,7 +116,7 @@ func (suite *DatabaseTests) TestRotateHeaderKey() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // Register a new key and a new file - fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.addKeyHash("someKeyHash", "this is a test key") assert.NoError(suite.T(), err, "failed to register key in database") @@ -164,7 +164,7 @@ func (suite *DatabaseTests) TestSetArchived() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestSetArchived.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestSetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/Test/SetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), -1, fmt.Sprintf("%x", sha256.New())} @@ -192,7 +192,7 @@ func (suite *DatabaseTests) TestSetArchived() { }, } { suite.T().Run(step.name, func(t *testing.T) { - err := db.setArchived(step.location, step.fileInfo, step.fileID) + err := db.SetArchived(step.location, step.fileInfo, step.fileID) if step.expectedError != "" { assert.ErrorContains(suite.T(), err, step.expectedError) } else { @@ -208,7 +208,7 @@ func (suite *DatabaseTests) TestGetFileStatus() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestGetFileStatus.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetFileStatus.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.UpdateFileEventLog(fileID, "downloaded", "testuser", "{}", "{}") @@ -226,7 +226,7 @@ func (suite *DatabaseTests) TestGetHeader() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestGetHeader.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetHeader.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.StoreHeader([]byte{15, 45, 20, 40, 48}, fileID) @@ -244,7 +244,7 @@ func (suite *DatabaseTests) TestBackupHeader() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) defer db.Close() - fileID, err := db.RegisterFile(nil, "/testuser/TestBackupHeader.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestBackupHeader.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") testKeyHash := "test-key-hash-123" @@ -271,7 +271,7 @@ func (suite *DatabaseTests) TestSetVerified() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestSetVerified.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestSetVerified.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/testuser/TestSetVerified.c4gh", fmt.Sprintf("%x", sha256.New()), 948, fmt.Sprintf("%x", sha256.New())} @@ -289,12 +289,12 @@ func (suite *DatabaseTests) TestGetArchived() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/testuser/TestGetArchived.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchivedWithLocation("/archive", fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -313,11 +313,11 @@ func (suite *DatabaseTests) TestSetAccessionID() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestSetAccessionID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -333,11 +333,11 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestCheckAccessionIDExists.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestCheckAccessionIDExists.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -361,11 +361,11 @@ func (suite *DatabaseTests) TestGetAccessionID() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestSetAccessionID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -385,11 +385,11 @@ func (suite *DatabaseTests) TestGetAccessionID_wrongFileID() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestSetAccessionID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -416,7 +416,7 @@ func (suite *DatabaseTests) TestGetFileInfo() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestGetFileInfo.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetFileInfo.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") encSha := sha256.New() @@ -429,7 +429,7 @@ func (suite *DatabaseTests) TestGetFileInfo() { fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/tmp/TestGetFileInfo.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -450,7 +450,7 @@ func (suite *DatabaseTests) TestMapFilesToDataset() { accessions := []string{} for i := 1; i < 12; i++ { - fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestMapFilesToDataset-%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", fmt.Sprintf("/testuser/TestMapFilesToDataset-%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("acession-%d", i), fileID) @@ -489,7 +489,7 @@ func (suite *DatabaseTests) TestGetInboxPath() { accessions := []string{} for i := 0; i < 5; i++ { - fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("acession-00%d", i), fileID) @@ -513,7 +513,7 @@ func (suite *DatabaseTests) TestUpdateDatasetEvent() { accessions := []string{} for i := 0; i < 5; i++ { - fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", fmt.Sprintf("/testuser/TestGetInboxPath-00%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("acession-00%d", i), fileID) @@ -547,7 +547,7 @@ func (suite *DatabaseTests) TestGetHeaderForStableID() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestGetHeaderForStableID.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetHeaderForStableID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.StoreHeader([]byte("HEADER"), fileID) @@ -569,13 +569,13 @@ func (suite *DatabaseTests) TestGetSyncData() { assert.NoError(suite.T(), err, "got %v when creating new connection", err) // register a file in the database - fileID, err := db.RegisterFile(nil, "/testuser/TestGetGetSyncData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetGetSyncData.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/tmp/TestGetGetSyncData.c4gh", checksum, 999, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = db.SetVerified(fileInfo, fileID) @@ -600,7 +600,7 @@ func (suite *DatabaseTests) TestCheckIfDatasetExists() { accessions := []string{} for i := 0; i <= 3; i++ { - fileID, err := db.RegisterFile(nil, fmt.Sprintf("/testuser/TestCheckIfDatasetExists-%d.c4gh", i), "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", fmt.Sprintf("/testuser/TestCheckIfDatasetExists-%d.c4gh", i), "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetAccessionID(fmt.Sprintf("accession-%d", i), fileID) @@ -633,13 +633,13 @@ func (suite *DatabaseTests) TestGetArchivePath() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/testuser/TestGetArchivePath-001.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetArchivePath-001.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New()) corrID := uuid.New().String() fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1234, corrID, checksum, 999, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchivedWithLocation("/archive", fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = db.SetAccessionID("acession-0001", fileID) @@ -665,7 +665,7 @@ func (suite *DatabaseTests) TestGetUserFiles() { sub = "submission_b" } - fileID, err := db.RegisterFile(nil, fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", testUser, sub, i), testUser) + fileID, err := db.RegisterFile(nil, "/inbox", fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", testUser, sub, i), testUser) assert.NoError(suite.T(), err, "failed to register file in database") err = db.UpdateFileEventLog(fileID, "uploaded", testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") @@ -701,7 +701,7 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { filePath := "/testuser/file10.c4gh" user := "testuser" - fileID, err := db.RegisterFile(nil, filePath, user) + fileID, err := db.RegisterFile(nil, "/inbox", filePath, user) if err != nil { suite.FailNow("failed to register file in database") } @@ -711,7 +711,7 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, fileID, checksum, 999, fmt.Sprintf("%x", sha256.New())} - if err := db.SetArchived(fileInfo, fileID); err != nil { + if err := db.SetArchived("/archive", fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as archived") } if err := db.UpdateFileEventLog(fileID, "archived", user, "{}", "{}"); err != nil { @@ -721,7 +721,7 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) } - fileID2, err := db.RegisterFile(nil, filePath, user) + fileID2, err := db.RegisterFile(nil, "/inbox", filePath, user) assert.NoError(suite.T(), err, "failed to register file in database") if err := db.UpdateFileEventLog(fileID2, "uploaded", user, "{}", "{}"); err != nil { suite.FailNow("failed to update satus of file in database") @@ -739,7 +739,7 @@ func (suite *DatabaseTests) TestListActiveUsers() { for _, user := range testUsers { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i) - fileID, err := db.RegisterFile(nil, filePath, user) + fileID, err := db.RegisterFile(nil, "/inbox", filePath, user) if err != nil { suite.FailNow("Failed to register file") } @@ -750,7 +750,7 @@ func (suite *DatabaseTests) TestListActiveUsers() { checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999, fmt.Sprintf("%x", sha256.New())} - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -794,7 +794,7 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", "User-Q", i) - fileID, err := db.RegisterFile(nil, filePath, "User-Q") + fileID, err := db.RegisterFile(nil, "/inbox", filePath, "User-Q") if err != nil { suite.FailNow("Failed to register file") } @@ -812,7 +812,7 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { 999, fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -949,7 +949,7 @@ func (suite *DatabaseTests) TestSetKeyHash() { keyDescription := "this is a test key" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") // Test that the key hash can be set in the files table @@ -973,7 +973,7 @@ func (suite *DatabaseTests) TestSetKeyHash_wrongHash() { keyDescription := "this is a test hash" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile(nil, "/testuser/file2.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/file2.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") // Ensure failure if a non existing hash is used @@ -992,7 +992,7 @@ func (suite *DatabaseTests) TestGetKeyHash() { keyDescription := "this is a test key" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetKeyHash(keyHex, fileID) assert.NoError(suite.T(), err, "failed to set key hash in database") @@ -1013,7 +1013,7 @@ func (suite *DatabaseTests) TestGetKeyHash_wrongFileID() { keyDescription := "this is a test key" err = db.addKeyHash(keyHex, keyDescription) assert.NoError(suite.T(), err, "failed to register key in database") - fileID, err := db.RegisterFile(nil, "/testuser/file1.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/file1.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") err = db.SetKeyHash(keyHex, fileID) assert.NoError(suite.T(), err, "failed to set key hash in database") @@ -1073,7 +1073,7 @@ func (suite *DatabaseTests) TestListDatasets() { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", "User-Q", i) - fileID, err := db.RegisterFile(nil, filePath, "User-Q") + fileID, err := db.RegisterFile(nil, "/inbox", filePath, "User-Q") if err != nil { suite.FailNow("Failed to register file") } @@ -1091,7 +1091,7 @@ func (suite *DatabaseTests) TestListDatasets() { 999, fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -1153,7 +1153,7 @@ func (suite *DatabaseTests) TestListUserDatasets() { user := "User-Q" for i := 0; i < 6; i++ { filePath := fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i) - fileID, err := db.RegisterFile(nil, filePath, user) + fileID, err := db.RegisterFile(nil, "/inbox", filePath, user) if err != nil { suite.FailNow("Failed to register file") } @@ -1171,7 +1171,7 @@ func (suite *DatabaseTests) TestListUserDatasets() { 999, fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -1204,7 +1204,7 @@ func (suite *DatabaseTests) TestListUserDatasets() { suite.FailNow("failed to update dataset event") } - fileID, err := db.RegisterFile(nil, "filePath", "user") + fileID, err := db.RegisterFile(nil, "/inbox", "filePath", "user") if err != nil { suite.FailNow("Failed to register file") } @@ -1292,7 +1292,7 @@ func (suite *DatabaseTests) TestGetReVerificationData() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile(nil, "/testuser/TestGetReVerificationData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetReVerificationData.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1310,7 +1310,7 @@ func (suite *DatabaseTests) TestGetReVerificationData() { } fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - if err = db.SetArchived(fileInfo, fileID); err != nil { + if err = db.SetArchived("/archive", fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } if err = db.SetVerified(fileInfo, fileID); err != nil { @@ -1332,7 +1332,7 @@ func (suite *DatabaseTests) TestGetReVerificationDataFromFileID() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile(nil, "/testuser/TestGetReVerificationData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetReVerificationData.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1350,7 +1350,7 @@ func (suite *DatabaseTests) TestGetReVerificationDataFromFileID() { } fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - if err = db.SetArchived(fileInfo, fileID); err != nil { + if err = db.SetArchived("/archive", fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } if err = db.SetVerified(fileInfo, fileID); err != nil { @@ -1368,7 +1368,7 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile(nil, "/testuser/TestGetReVerificationData.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetReVerificationData.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1387,7 +1387,7 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - if err = db.SetArchived(fileInfo, fileID); err != nil { + if err = db.SetArchived("/archive", fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } if err = db.SetVerified(fileInfo, fileID); err != nil { @@ -1409,7 +1409,7 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) - fileID, err := db.RegisterFile(nil, "/testuser/TestGetDecryptedChecksum.c4gh", "testuser") + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestGetDecryptedChecksum.c4gh", "testuser") if err != nil { suite.FailNow("failed to register file in database") } @@ -1428,7 +1428,7 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetDecryptedChecksum.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - if err = db.SetArchived(fileInfo, fileID); err != nil { + if err = db.SetArchived("/archive", fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } if err = db.SetVerified(fileInfo, fileID); err != nil { @@ -1449,7 +1449,7 @@ func (suite *DatabaseTests) TestGetDatasetFiles() { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetDsatasetFiles-00%d.c4gh", "User-Q", i) - fileID, err := db.RegisterFile(nil, filePath, "User-Q") + fileID, err := db.RegisterFile(nil, "/inbox", filePath, "User-Q") if err != nil { suite.FailNow("Failed to register file") } @@ -1467,7 +1467,7 @@ func (suite *DatabaseTests) TestGetDatasetFiles() { 999, fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -1506,7 +1506,7 @@ func (suite *DatabaseTests) TestGetDatasetFileIDs() { for i := 0; i < testCases; i++ { filePath := fmt.Sprintf("/%v/TestGetDatasetFileIDs-00%d.c4gh", "User-Q", i) - fileID, err := db.RegisterFile(nil, filePath, "User-Q") + fileID, err := db.RegisterFile(nil, "/inbox", filePath, "User-Q") if err != nil { suite.FailNow("Failed to register file") } @@ -1525,7 +1525,7 @@ func (suite *DatabaseTests) TestGetDatasetFileIDs() { 999, fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID) + err = db.SetArchived("/archive", fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -1566,7 +1566,7 @@ func (suite *DatabaseTests) TestGetSubmissionPathAndLocation() { user := "UserX" filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) - fileID, err := db.RegisterFileWithLocation(nil, "/inbox", filePath, user) + fileID, err := db.RegisterFile(nil, "/inbox", filePath, user) if err != nil { suite.FailNow("Failed to register file") } @@ -1592,7 +1592,7 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { user := "UserX" filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) - fileID, err := db.RegisterFile(nil, filePath, user) + fileID, err := db.RegisterFile(nil, "/inbox", filePath, user) if err != nil { suite.FailNow("Failed to register file") } @@ -1631,7 +1631,7 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUI_Found() { // Register a file to get a valid UUID filePath := "/dummy_user.org/Dummy_folder/dummyfile.c4gh" user := "dummy@user.org" - fileID, err := db.RegisterFile(nil, filePath, user) + fileID, err := db.RegisterFile(nil, "/inbox", filePath, user) if err != nil { suite.FailNow("failed to register file in database") } @@ -1667,7 +1667,7 @@ func (suite *DatabaseTests) TestSetSubmissionFileSize() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "failed to create new connection") - fileID, err := db.RegisterFile(nil, "/test.file", "user") + fileID, err := db.RegisterFile(nil, "/inbox", "/test.file", "user") if err != nil { suite.FailNow("failed to register file", err) } @@ -1773,12 +1773,12 @@ func (suite *DatabaseTests) TestGetSizeAndObjectCountOfLocation() { assert.NoError(t, err) for fileID, size := range test.filesToRegister { - _, err := db.RegisterFileWithLocation(&fileID, "/inbox", "/"+fileID, "user") + _, err := db.RegisterFile(&fileID, "/inbox", "/"+fileID, "user") assert.NoError(t, err) assert.NoError(t, db.setSubmissionFileSize(fileID, size)) } for fileID, size := range test.filesToArchive { - assert.NoError(t, db.SetArchivedWithLocation("/archive", FileInfo{ + assert.NoError(t, db.SetArchived("/archive", FileInfo{ ArchiveChecksum: "123", Size: size, Path: "/test.file3", @@ -1808,12 +1808,12 @@ func (suite *DatabaseTests) TestCancelFile() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "failed to create new connection") - fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/test.file", "user") + fileID, err := db.RegisterFile(nil, "/inbox", "/test.file", "user") if err != nil { suite.FailNow("failed to register file", err) } - assert.NoError(suite.T(), db.SetArchivedWithLocation("/archive", FileInfo{ + assert.NoError(suite.T(), db.SetArchived("/archive", FileInfo{ ArchiveChecksum: "123", Size: 500, Path: "/test.file3", @@ -1834,7 +1834,7 @@ func (suite *DatabaseTests) TestIsFileInDataset_No() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "failed to create new connection") - fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/test.file", "user") + fileID, err := db.RegisterFile(nil, "/inbox", "/test.file", "user") if err != nil { suite.FailNow("failed to register file", err) } @@ -1848,11 +1848,11 @@ func (suite *DatabaseTests) TestIsFileInDataset_Yes() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "failed to create new connection") - fileID, err := db.RegisterFileWithLocation(nil, "/inbox", "/test.file", "user") + fileID, err := db.RegisterFile(nil, "/inbox", "/test.file", "user") if err != nil { suite.FailNow("failed to register file", err) } - assert.NoError(suite.T(), db.SetArchivedWithLocation("/archive", FileInfo{ + assert.NoError(suite.T(), db.SetArchived("/archive", FileInfo{ ArchiveChecksum: "123", Size: 500, Path: "/test.file3", From d60572a3a3083bebe19efa15c2ff3d348bfe58bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Feb 2026 09:42:28 +0100 Subject: [PATCH 257/530] fix: ensure broker defer close does nil checks, and ensure ingest sends correct message if it failes to be validated against schema --- sda/cmd/api/api.go | 15 +++++++++++---- sda/cmd/finalize/finalize.go | 15 +++++++++++---- sda/cmd/ingest/ingest.go | 6 +++--- 3 files changed, 25 insertions(+), 11 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index e15805a15a..76f71cad79 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -83,11 +83,18 @@ func run() error { return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } defer func() { - if err := Conf.API.MQ.Channel.Close(); err != nil { - log.Errorf("failed to close mq broker channel due to: %v", err) + if Conf.API.MQ == nil { + return + } + if Conf.API.MQ.Channel != nil { + if err := Conf.API.MQ.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } } - if err := Conf.API.MQ.Connection.Close(); err != nil { - log.Errorf("failed to close mq broker connection due to: %v", err) + if Conf.API.MQ.Connection != nil { + if err := Conf.API.MQ.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } } }() diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 8e08c877ea..b5bac11bfd 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -59,11 +59,18 @@ func run() error { return fmt.Errorf("failed to initialize mq broker, due to: %v", err) } defer func() { - if err := mqBroker.Channel.Close(); err != nil { - log.Errorf("failed to close mq broker channel due to: %v", err) + if mqBroker == nil { + return + } + if mqBroker.Channel != nil { + if err := mqBroker.Channel.Close(); err != nil { + log.Errorf("failed to close mq broker channel due to: %v", err) + } } - if err := mqBroker.Connection.Close(); err != nil { - log.Errorf("failed to close mq broker connection due to: %v", err) + if mqBroker.Connection != nil { + if err := mqBroker.Connection.Close(); err != nil { + log.Errorf("failed to close mq broker connection due to: %v", err) + } } }() diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 4d25b06be2..99b55b23e9 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -159,7 +159,7 @@ func (app *Ingest) handleMessage(ctx context.Context, delivered amqp.Delivery) { ctx, cancel := context.WithCancel(ctx) defer cancel() log.Debugf("received a message (correlation-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - message := schema.IngestionTrigger{} + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.MQ.Conf.SchemasPath), delivered.Body) if err != nil { log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) @@ -167,7 +167,7 @@ func (app *Ingest) handleMessage(ctx context.Context, delivered amqp.Delivery) { infoErrorMessage := broker.InfoError{ Error: "Message validation failed", Reason: err.Error(), - OriginalMessage: message, + OriginalMessage: delivered, } body, _ := json.Marshal(infoErrorMessage) @@ -180,7 +180,7 @@ func (app *Ingest) handleMessage(ctx context.Context, delivered amqp.Delivery) { return } - + message := schema.IngestionTrigger{} // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) log.Infof("Received work (correlation-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) From 7790b3f362bd43103453bbfdef15456ada8ce030 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 16 Jan 2026 09:33:35 +0100 Subject: [PATCH 258/530] feat(integration tests): update config to set storage/v2 config, and bucket naming for multiple storage impl --- .../integration/scripts/charts/values.yaml | 25 +++--- .github/integration/sda/config.yaml | 77 ++++++++----------- .../tests/sda/30_backup-finalize_test.sh | 2 +- .../tests/sda/65_api_admin_rotate_key_test.sh | 2 +- .../tests/sda/70_rotate_key_test.sh | 2 +- 5 files changed, 52 insertions(+), 56 deletions(-) diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index c831981ec9..8d3700a11a 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -36,6 +36,22 @@ global: s3Port: 9000 s3ReadyPath: "/minio/health/ready" existingClaim: backup-pvc + inbox: + storageType: s3 + s3AccessKey: PLACEHOLDER_VALUE + s3SecretKey: PLACEHOLDER_VALUE + s3Url: http://minio.minio + s3Port: 9000 + s3Bucket: "inbox" + s3ReadyPath: "/minio/health/ready" + existingClaim: inbox-pvc + s3inbox: + s3AccessKey: PLACEHOLDER_VALUE + s3SecretKey: PLACEHOLDER_VALUE + s3Url: http://minio.minio + s3Port: 9000 + s3Bucket: "inbox" + s3ReadyPath: "/minio/health/ready" auth: jwtSecret: jwk jwtAlg: ES256 @@ -80,15 +96,6 @@ global: jwkPath: "/jwks" id: DfCieZLuBU secret: DfCieZLuBU - inbox: - storageType: s3 - s3AccessKey: PLACEHOLDER_VALUE - s3SecretKey: PLACEHOLDER_VALUE - s3Url: http://minio.minio - s3Port: 9000 - s3Bucket: "inbox" - s3ReadyPath: "/minio/health/ready" - existingClaim: inbox-pvc reencrypt: host: pipeline-sda-svc-reencrypt port: 50051 diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 9768feacb9..968a4eada8 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -3,15 +3,40 @@ log: level: "debug" api: rbacFile: /rbac.json -archive: - type: s3 - url: "http://s3" - port: 9000 - readypath: "/minio/health/ready" - accessKey: "access" - secretKey: "secretKey" - bucket: "archive" + +storage: + archive: + s3: + endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + bucket_prefix: "archive" + region: "us-east-1" + disable_https: true + backup: + s3: + endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + bucket_prefix: "backup" + region: "us-east-1" + disable_https: true + inbox: + s3: + endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + bucket_prefix: "inbox" + region: "us-east-1" + disable_https: true + +s3inbox: + endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + bucket: "inbox" region: "us-east-1" + ready_path: "/minio/health/ready" grpc: host: "reencrypt" @@ -33,33 +58,6 @@ auth: resignJwt: s3Inbox: "http://inbox:8000" -backup: - type: s3 - url: "http://s3" - port: 9000 - readypath: "/minio/health/ready" - accessKey: "access" - secretKey: "secretKey" - bucket: "backup" - region: "us-east-1" -inbox: - type: s3 - url: "http://s3" - port: 9000 - readypath: "/minio/health/ready" - accessKey: "access" - secretKey: "secretKey" - bucket: "inbox" - region: "us-east-1" - -s3inbox: - endpoint: "http://s3:9000" - access_key: "access" - secret_key: "secretKey" - bucket: "inbox" - region: "us-east-1" - ready_path: "/minio/health/ready" - broker: host: "rabbitmq" port: "5672" @@ -107,15 +105,6 @@ sync: password: "pass" user: "user" centerPrefix: "SYNC" - destination: - type: "s3" - url: "http://s3" - port: 9000 - readypath: "/minio/health/ready" - accessKey: "access" - secretKey: "secretKey" - bucket: "sync" - region: "us-east-1" remote: host: "http://sync-api" port: "8080" diff --git a/.github/integration/tests/sda/30_backup-finalize_test.sh b/.github/integration/tests/sda/30_backup-finalize_test.sh index 6ad7828261..78379130b0 100644 --- a/.github/integration/tests/sda/30_backup-finalize_test.sh +++ b/.github/integration/tests/sda/30_backup-finalize_test.sh @@ -95,7 +95,7 @@ EOD # check DB for archive file names for file in NA12878.bam.c4gh NA12878.bai.c4gh NA12878_20k_b37.bam.c4gh NA12878_20k_b37.bai.c4gh; do archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = '$file';") - size=$(s3cmd -c direct ls s3://backup/"$archiveName" | tr -s ' ' | cut -d ' ' -f 3) + size=$(s3cmd -c direct ls s3://backup1/"$archiveName" | tr -s ' ' | cut -d ' ' -f 3) if [ "$size" -eq 0 ]; then echo "Failed to get size of $file from backup site" exit 1 diff --git a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh index 300fa96ada..90d922d0f3 100755 --- a/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh +++ b/.github/integration/tests/sda/65_api_admin_rotate_key_test.sh @@ -257,7 +257,7 @@ for i in 1 2; do # get archive file archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where id='$fileID';") - s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive/"$archivePath" --force + s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive1/"$archivePath" --force # concatenate and decrypt cat "admin_rotate_testfile${i}_rotated.c4gh" "$archivePath" > tmp_file && mv tmp_file "admin_rotate_testfile${i}_rotated.c4gh" diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh index 32bd916ec0..b007c633f4 100644 --- a/.github/integration/tests/sda/70_rotate_key_test.sh +++ b/.github/integration/tests/sda/70_rotate_key_test.sh @@ -185,7 +185,7 @@ psql -U postgres -h postgres -d sda -At -c "select header from sda.files where i # get archive file archivePath=$(psql -U postgres -h postgres -d sda -At -c "select archive_file_path from sda.files where id='$fileID';") -s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive/"$archivePath" --force +s3cmd --access_key=access --secret_key=secretKey --host=minio:9000 --no-ssl --host-bucket=minio:9000 get s3://archive1/"$archivePath" --force # concatenate and decrypt cat testfile1_rotated.c4gh "$archivePath" > tmp_file && mv tmp_file testfile1_rotated.c4gh From e8ac5b1f0f0ed90c5d8eb9280a5ed43df041bd2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 19 Jan 2026 15:46:21 +0100 Subject: [PATCH 259/530] feat(integration tests): update integration tests to check that we rollover to next archive bucket when first reached its quota --- .github/integration/sda/config.yaml | 4 ++++ .../integration/tests/sda/20_ingest-verify_test.sh | 12 ++++++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 968a4eada8..c26bc8d6e2 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -13,6 +13,8 @@ storage: bucket_prefix: "archive" region: "us-east-1" disable_https: true + max_buckets: 10 + max_objects: 3 backup: s3: endpoint: "http://s3:9000" @@ -38,6 +40,8 @@ s3inbox: region: "us-east-1" ready_path: "/minio/health/ready" +location_broker.cache_ttl: 0 + grpc: host: "reencrypt" port: 50051 diff --git a/.github/integration/tests/sda/20_ingest-verify_test.sh b/.github/integration/tests/sda/20_ingest-verify_test.sh index f500d68781..43fb5d424e 100644 --- a/.github/integration/tests/sda/20_ingest-verify_test.sh +++ b/.github/integration/tests/sda/20_ingest-verify_test.sh @@ -80,4 +80,16 @@ if [ "$key_hashes" -eq 0 ]; then exit 1 fi +num_in_archive_1="$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(id) FROM sda.files WHERE archive_location LIKE '%archive1'")" +if [ "$num_in_archive_1" -ne 3 ]; then + echo "::error::Unexpected amount of files in archive bucket 1: $num_in_archive_1, expected 3" + exit 1 +fi + +num_in_archive_2="$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(id) FROM sda.files WHERE archive_location LIKE '%archive2'")" +if [ "$num_in_archive_2" -ne 1 ]; then + echo "::error::Unexpected amount of files in archive bucket 2: : $num_in_archive_2, expected 1" + exit 1 +fi + echo "ingestion and verification test completed successfully" \ No newline at end of file From c777095fc93b9d2f1bf7283eba615bc70aa9d431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 27 Jan 2026 15:57:10 +0100 Subject: [PATCH 260/530] feat(integration tests): update integration tests to check that cancel file will delete file from archive, backup and unset archive data from db --- .../integration/tests/sda/21_cancel_test.sh | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/integration/tests/sda/21_cancel_test.sh b/.github/integration/tests/sda/21_cancel_test.sh index f67dffe08f..f916a7b2b1 100644 --- a/.github/integration/tests/sda/21_cancel_test.sh +++ b/.github/integration/tests/sda/21_cancel_test.sh @@ -60,6 +60,47 @@ if [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_e exit 1 fi + +# check database to verify file archive location and path has been unset +if [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT 1 FROM sda.files WHERE id = '$CORRID' AND archive_file_path = '' AND archive_location IS NULL")" != "1" ]; then + echo "canceling file failed" + exit 1 +fi + +cat >/shared/direct < Date: Fri, 16 Jan 2026 15:47:26 +0100 Subject: [PATCH 261/530] feat(charts): update sda-svc chart to support and populate the storage/v2 config and the updated s3inbox config --- charts/sda-svc/README.md | 631 +++++++++--------- charts/sda-svc/templates/api-secrets.yaml | 28 +- charts/sda-svc/templates/doa-deploy.yaml | 2 +- .../sda-svc/templates/download-secrets.yaml | 38 +- .../sda-svc/templates/finalize-secrets.yaml | 82 ++- charts/sda-svc/templates/ingest-deploy.yaml | 2 +- charts/sda-svc/templates/ingest-secrets.yaml | 74 +- charts/sda-svc/templates/mapper-deploy.yaml | 2 +- charts/sda-svc/templates/mapper-secrets.yaml | 32 +- .../sda-svc/templates/s3-inbox-secrets.yaml | 26 +- charts/sda-svc/templates/sync-secrets.yaml | 57 +- charts/sda-svc/templates/verify-secrets.yaml | 37 +- charts/sda-svc/values.yaml | 49 +- 13 files changed, 543 insertions(+), 517 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 75133cc1ee..5a93ef7802 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -8,13 +8,16 @@ Source repositories: ## Installing the Chart Edit the values.yaml file and specify the relevant parts of the `global` section. -If no shared credentials for the broker and database are used, the credentials for each service shuld be set in the `credentials` section. +If no shared credentials for the broker and database are used, the credentials for each service shuld be set in the +`credentials` section. -While it is possible to deploy this chart with the crypt4gh keys included in the values file as base64 encoded strings, it is advisable to create the secret containing the crypt4gh keys manually. +While it is possible to deploy this chart with the crypt4gh keys included in the values file as base64 encoded strings, +it is advisable to create the secret containing the crypt4gh keys manually. ## Upgrading an existing Release to a new version -A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing +manual actions. ### To 3.0.0 @@ -26,322 +29,336 @@ This version adds jobs that perform the following actions: When upgrading from a V1 release *All these jobs* should be run. -Unless the same queries are being executed manually by a database administrator a `Basic authentication Secret` containing the credentials to perform the upgrade needs to be created. +Unless the same queries are being executed manually by a database administrator a `Basic authentication Secret` +containing the credentials to perform the upgrade needs to be created. ### Configuration The following table lists the configurable parameters of the `sda-svc` chart and their default values. -Parameter | Description | Default ---------- | ----------- | ------- -`image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` -`image.tag` | Tag version to deploy | `chart appVersion` -`image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` -`global.secretsPath` | Path where the sensitive files can be found | `/.secrets` -`global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` -`global.tlsPath` | This path will be a subpath to the secretsPath | `tls` -`global.jwtPath` | This path will be a subpath to the secretsPath | `jwt` -`global.confFile` | Name of config file, used when secrets are handled by hasicorp vault | `config.yaml` -`global.confFilePath` | This path will be a subpath to the secretsPath | `""` -`global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all` -`global.schemaType` | Standalone version requires a custom JOSN schema, available options are `federated` or `isolated`, unset defaults to federated. | `""` -`global.ingress.deploy` | | `false` -`global.ingress.hostName.auth` | | `""` -`global.ingress.hostName.doa` | | `""` -`global.ingress.hostName.s3Inbox` | | `""` -`global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` -`global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` -`global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` -`global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` -`global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` -`global.ingress.annotations` | extra annotations shared by all ingress definitions | `""` -`global.ingress.ingressClassName` | Class of ingress controller to use | `""` -`global.log.format` | Log format for all services, JSON or TEXT. | `json` -`global.log.level` | Log level for all services. | `info` -`global.networkPolicy.create` | Use network isolation. | `false` -`global.networkPolicy.brokerNamespace` | Namespace where the broker is deployed. | `""` -`global.networkPolicy.databaseNamespace` | Namespace where the database is deployed. | `""` -`global.networkPolicy.externalNamespace` | Namespace where the external components are deployed. | `""` -`global.networkPolicy.internalNamespace` | Namespace where the internal components are deployed. | `""` -`global.networkPolicy.ingressNamespace` | Namespace where the ingress is deployed. | `""` -`global.networkPolicy.ingressLabelMatch` | Match to use to allow connections from ingress controller pod. | `""` -`global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` -`global.podAnnotations` | Annotations applied to pods of all services. | `{}` -`global.pkiService` | If an external PKI infrastructure is used set this to true. | `false` -`global.rbacEnabled` | Use role based access control. | `true` -`global.vaultSecrets` | If Hashicorp Vault is used for secrets management | `false` -`global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. | `s3` -`global.archive.s3Url` | URL to S3 archive instance. | `""` -`global.archive.s3Bucket` | S3 archive bucket. | `""` -`global.archive.s3Region` | S3 archive region. | `us-east-1` -`global.archive.s3ChunkSize` | S3 chunk size in MB. | `15` -`global.archive.s3AccessKey` | Access key to S3 archive . | `null` -`global.archive.s3SecretKey` | Secret key to S3 archive. | `null` -`global.archive.s3CaFile` | CA certificate to use if the S3 archive is internal. | `null` -`global.archive.s3Port` | Port that the S3 S3 archive is available on. | `443` -`global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive` -`global.archive.nfsServer` | URL or IP address to a NFS server. | `""` -`global.archive.nfsPath` | Path on the NFS server for the archive. | `""` -`global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. | `null` -`global.backupArchive.s3Url` | URL to S3 backup archive instance. | `""` -`global.backupArchive.s3Bucket` | S3 backup archive bucket. | `""` -`global.backupArchive.s3Region` | S3 backup archive region. | `us-east-1` -`global.backupArchive.s3ChunkSize` | S3 chunk size in MB. | `15` -`global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` -`global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` -`global.backupArchive.s3CaFile` | CA certificate to use if the S3 backup archive is internal. | `null` -`global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `443` -`global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup` -`global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""` -`global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. | `""` -`global.api.jwtPubKeyName` | Public key used to verify the JWT. | `` -`global.api.jwtSecret` | The name of the secret holding the JWT public key | `` -`global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) | `` -`global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type | `"ES256"` -`global.auth.jwtKey` | Private key used to sign the JWT. | `""` -`global.auth.jwtPub` | Public key ues to verify the JWT. | `""` -`global.auth.jwtTTL` | TTL of the resigned token (hours). | `168` -`global.auth.resignJWT` | Resign the LS-AAI JWTs. | `true` -`global.auth.useTLS` | Run a TLS secured server. | `true` -`global.auth.corsOrigins` | Domain name allowed for cross-domain requests. | `""` -`global.auth.corsMethods` | Allowed cross-domain request methods. | `""` -`global.auth.corsCreds` | Include credentials in cross-domain requests. | `false` -`global.broker.host` | Domain name or IP address to the message broker. | `""` -`global.broker.exchange` | Exchange to publish messages to. | `""` -`global.broker.port` | Port for the message broker. | `5671` -`global.broker.verifyPeer` | Use Client/Server verification. | `true` -`global.broker.vhost` | Virtual host to connect to. | `/` -`global.broker.password` | Shared password to the message broker. | `/` -`global.broker.username` | Shared user to the message broker. | `/` -`global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` -`global.cega.host` | Full URI to the EGA user authentication service. | `""` -`global.cega.user` | Username for the EGA user authentication service. | `""` -`global.cega.password` | Password for the EGA user authentication service. | `""` -`global.c4gh.privateKeys` | List of Private C4GH keys. | `` -`global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. | `""` -`global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. | `""` -`global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""` -`global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""` -`global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""` -`global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` -`global.db.admin.passKey` | Key in the secret that holds the password. | `""` -`global.db.admin.userkey` | Key in the secret that holds the username. | `""` -`global.db.host` | Hostname for the database. | `""` -`global.db.name` | Database to connect to. | `lega` -`global.db.passIngest` | Password used for `data in` services. | `""` -`global.db.passOutgest` | Password used for `data out` services. | `""` -`global.db.port` | Port that the database is listening on. | `5432` -`global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. | `verify-full` -`global.doa.enabled` | Deploy the DOA service | `false` -`global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env` -`global.doa.serviceport` | Port that the DOA service is accessible on | `443` -`global.doa.outbox.enabled` | Enable Outbox functionality of Data Out API | `false` -`global.doa.outbox.queue` | MQ queue name for files/datasets export requests | `""` -`global.doa.outbox.type` | Outbox type can be either S3 or POSIX | `""` -`global.doa.outbox.path` | Posix outbox location with placeholder for the username | `""` -`global.doa.outbox.s3Url` | Outbox S3 URL | `""` -`global.doa.outbox.s3Port` | Outbox S3 port | `443` -`global.doa.outbox.s3Region` | Outbox S3 region | `""` -`global.doa.outbox.s3Bucket` | Outbox S3 bucket | `""` -`global.doa.outbox.s3CaFile` | Outbox S3 CA certificate to use | `null` -`global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null` -`global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null` -`global.download.enabled` | Deploy the download service | `true` -`global.download.serveDecrypted.c4ghKeyFile` | Transient private C4GH key | `""` -`global.download.serveDecrypted.secretName` | Secret holding the transient private C4GH key and its passphrase | `""` -`global.download.sessionExpiration` | Session key expiration time in seconds | `28800` -`global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` -`global.download.trusted.configFile` | Name of ISS config file | `iss.json` -`global.download.trusted.iss` | Array of trusted OIDC endpoints | `` -`global.download.trusted.iss[iss]` | URI to the OIDC service | `https://proxy.aai.lifescience-ri.eu` -`global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://proxy.aai.lifescience-ri.eu/OIDC/jwks` -`global.oidc.provider` | URL to the OIDc service. | `"https://proxy.aai.lifescience-ri.eu"` -`global.oidc.jwkPath` | Public key path on the OIDC host. | `/OIDC/jwks` -`global.oidc.id` | User ID to the OIDC host. | `` -`global.oidc.secret` | User credentials to the OIDC host. | `` -`global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` -`global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. | `posix` -`global.inbox.path` | Path to the mounted `posix` volume. | `/inbox` -`global.inbox.user` | Path to the mounted `posix` volume. | `lega` -`global.inbox.nfsServer` | URL or IP address to a NFS server. | `""` -`global.inbox.nfsPath` | Path on the NFS server for the inbox. | `""` -`global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""` -`global.inbox.s3Url` | URL to S3 inbox instance. | `""` -`global.inbox.s3Port` | Port that the S3 inbox is available on. | `443` -`global.inbox.s3Bucket` | S3 inbox bucket. | `""` -`global.inbox.s3Region` | S3 inbox region. | `""` -`global.inbox.s3ChunkSize` | S3 chunk size in MB. | `15` -`global.inbox.s3AccessKey` | Access key to S3 inbox . | `null` -`global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null` -`global.inbox.s3CaFile` | CA certificate to use if the S3 inbox is internal. | `null` -`global.inbox.s3ReadyPath` | Endpoint to verify that the inbox is respondig. | `""` -`global.sync.api.password` | Password for authenticating to the syncAPI server | `null` -`global.sync.api.user` | User for authenticating to the syncAPI server | `null` -`global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` -`global.sync.centerPrefix` | Prefix for locally generated datasets | `null` -`global.sync.destination.storageType` | Storage type for the sync destination, currently only supports S3 | `s3` -`global.sync.destination.accesskey` | Access key to S3 sync destination | `null` -`global.sync.destination.bucket` | sync destination bucket | `null` -`global.sync.destination.port` | Port that the S3 sync destination instance is available on | `443` -`global.sync.destination.secretkey` | Secret key to S3 sync destination | `null` -`global.sync.destination.url` | URL to S3 sync destination instance. | `null` -`global.sync.remote.host` | URL to the remote syncAPI host | `null` -`global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` -`global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null` -`global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null` -`global.tls.enabled` | Use TLS for all connections. | `true` -`global.tls.issuer` | Issuer for TLS certificate creation. | `""` -`global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` -`global.reencrypt.host` | gRPC host for reencryption | `""` -`global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` +| Parameter | Description | Default | +|----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------| +| `image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` | +| `image.tag` | Tag version to deploy | `chart appVersion` | +| `image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` | +| `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` | +| `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` | +| `global.tlsPath` | This path will be a subpath to the secretsPath | `tls` | +| `global.jwtPath` | This path will be a subpath to the secretsPath | `jwt` | +| `global.confFile` | Name of config file, used when secrets are handled by hasicorp vault | `config.yaml` | +| `global.confFilePath` | This path will be a subpath to the secretsPath | `""` | +| `global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all` | +| `global.schemaType` | Standalone version requires a custom JOSN schema, available options are `federated` or `isolated`, unset defaults to federated. | `""` | +| `global.ingress.deploy` | | `false` | +| `global.ingress.hostName.auth` | | `""` | +| `global.ingress.hostName.doa` | | `""` | +| `global.ingress.hostName.s3Inbox` | | `""` | +| `global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | +| `global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | +| `global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | +| `global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | +| `global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | +| `global.ingress.annotations` | extra annotations shared by all ingress definitions | `""` | +| `global.ingress.ingressClassName` | Class of ingress controller to use | `""` | +| `global.log.format` | Log format for all services, JSON or TEXT. | `json` | +| `global.log.level` | Log level for all services. | `info` | +| `global.networkPolicy.create` | Use network isolation. | `false` | +| `global.networkPolicy.brokerNamespace` | Namespace where the broker is deployed. | `""` | +| `global.networkPolicy.databaseNamespace` | Namespace where the database is deployed. | `""` | +| `global.networkPolicy.externalNamespace` | Namespace where the external components are deployed. | `""` | +| `global.networkPolicy.internalNamespace` | Namespace where the internal components are deployed. | `""` | +| `global.networkPolicy.ingressNamespace` | Namespace where the ingress is deployed. | `""` | +| `global.networkPolicy.ingressLabelMatch` | Match to use to allow connections from ingress controller pod. | `""` | +| `global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` | +| `global.podAnnotations` | Annotations applied to pods of all services. | `{}` | +| `global.pkiService` | If an external PKI infrastructure is used set this to true. | `false` | +| `global.rbacEnabled` | Use role based access control. | `true` | +| `global.vaultSecrets` | If Hashicorp Vault is used for secrets management | `false` | +| `global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. | `s3` | +| `global.archive.s3Url` | URL to S3 archive instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.archive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"archive"` | +| `global.archive.s3Region` | S3 archive region. | `us-east-1` | +| `global.archive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `50mb` | +| `global.archive.s3AccessKey` | Access key to S3 archive . | `null` | +| `global.archive.s3SecretKey` | Secret key to S3 archive. | `null` | +| `global.archive.s3CaFile` | CA certificate to use. | `null` | +| `global.archive.s3Port` | Port that the S3 S3 archive is available on. | `` | +| `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | +| `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | +| `global.archive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | +| `global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive` | +| `global.archive.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.archive.nfsPath` | Path on the NFS server for the archive. | `""` | +| `global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. | `null` | +| `global.backupArchive.s3Url` | URL to S3 backup archive instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.backupArchive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"backup"` | +| `global.backupArchive.s3Region` | S3 backup archive region. | `us-east-1` | +| `global.backupArchive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `15` | +| `global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` | +| `global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` | +| `global.backupArchive.s3CaFile` | CA certificate to use. | `null` | +| `global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `` | +| `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | +| `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | +| `global.backupArchive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | +| `global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup` | +| `global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. | `""` | +| `global.api.jwtPubKeyName` | Public key used to verify the JWT. | `` | +| `global.api.jwtSecret` | The name of the secret holding the JWT public key | `` | +| `global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) | `` | +| `global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type | `"ES256"` | +| `global.auth.jwtKey` | Private key used to sign the JWT. | `""` | +| `global.auth.jwtPub` | Public key ues to verify the JWT. | `""` | +| `global.auth.jwtTTL` | TTL of the resigned token (hours). | `168` | +| `global.auth.resignJWT` | Resign the LS-AAI JWTs. | `true` | +| `global.auth.useTLS` | Run a TLS secured server. | `true` | +| `global.auth.corsOrigins` | Domain name allowed for cross-domain requests. | `""` | +| `global.auth.corsMethods` | Allowed cross-domain request methods. | `""` | +| `global.auth.corsCreds` | Include credentials in cross-domain requests. | `false` | +| `global.broker.host` | Domain name or IP address to the message broker. | `""` | +| `global.broker.exchange` | Exchange to publish messages to. | `""` | +| `global.broker.port` | Port for the message broker. | `5671` | +| `global.broker.verifyPeer` | Use Client/Server verification. | `true` | +| `global.broker.vhost` | Virtual host to connect to. | `/` | +| `global.broker.password` | Shared password to the message broker. | `/` | +| `global.broker.username` | Shared user to the message broker. | `/` | +| `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` | +| `global.cega.host` | Full URI to the EGA user authentication service. | `""` | +| `global.cega.user` | Username for the EGA user authentication service. | `""` | +| `global.cega.password` | Password for the EGA user authentication service. | `""` | +| `global.c4gh.privateKeys` | List of Private C4GH keys. | `` | +| `global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. | `""` | +| `global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. | `""` | +| `global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""` | +| `global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""` | +| `global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""` | +| `global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` | +| `global.db.admin.passKey` | Key in the secret that holds the password. | `""` | +| `global.db.admin.userkey` | Key in the secret that holds the username. | `""` | +| `global.db.host` | Hostname for the database. | `""` | +| `global.db.name` | Database to connect to. | `lega` | +| `global.db.passIngest` | Password used for `data in` services. | `""` | +| `global.db.passOutgest` | Password used for `data out` services. | `""` | +| `global.db.port` | Port that the database is listening on. | `5432` | +| `global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. | `verify-full` | +| `global.doa.enabled` | Deploy the DOA service | `false` | +| `global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env` | +| `global.doa.serviceport` | Port that the DOA service is accessible on | `443` | +| `global.doa.outbox.enabled` | Enable Outbox functionality of Data Out API | `false` | +| `global.doa.outbox.queue` | MQ queue name for files/datasets export requests | `""` | +| `global.doa.outbox.type` | Outbox type can be either S3 or POSIX | `""` | +| `global.doa.outbox.path` | Posix outbox location with placeholder for the username | `""` | +| `global.doa.outbox.s3Url` | Outbox S3 URL | `""` | +| `global.doa.outbox.s3Port` | Outbox S3 port | `443` | +| `global.doa.outbox.s3Region` | Outbox S3 region | `""` | +| `global.doa.outbox.s3Bucket` | Outbox S3 bucket | `""` | +| `global.doa.outbox.s3CaFile` | Outbox S3 CA certificate to use | `null` | +| `global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null` | +| `global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null` | +| `global.download.enabled` | Deploy the download service | `true` | +| `global.download.serveDecrypted.c4ghKeyFile` | Transient private C4GH key | `""` | +| `global.download.serveDecrypted.secretName` | Secret holding the transient private C4GH key and its passphrase | `""` | +| `global.download.sessionExpiration` | Session key expiration time in seconds | `28800` | +| `global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` | +| `global.download.trusted.configFile` | Name of ISS config file | `iss.json` | +| `global.download.trusted.iss` | Array of trusted OIDC endpoints | `` | +| `global.download.trusted.iss[iss]` | URI to the OIDC service | `https://proxy.aai.lifescience-ri.eu` | +| `global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://proxy.aai.lifescience-ri.eu/OIDC/jwks` | +| `global.oidc.provider` | URL to the OIDc service. | `"https://proxy.aai.lifescience-ri.eu"` | +| `global.oidc.jwkPath` | Public key path on the OIDC host. | `/OIDC/jwks` | +| `global.oidc.id` | User ID to the OIDC host. | `` | +| `global.oidc.secret` | User credentials to the OIDC host. | `` | +| `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` | +| `global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. | `posix` | +| `global.inbox.volumePath` | Path to the mounted `posix` volume. | `/inbox` | +| `global.inbox.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.inbox.nfsPath` | Path on the NFS server for the inbox. | `""` | +| `global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""` | +| `global.inbox.s3Url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.inbox.s3Port` | Port that the S3 inbox is available on. | `` | +| `global.inbox.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | +| `global.inbox.s3Region` | S3 inbox region. | `"us-east-1"` | +| `global.inbox.s3AccessKey` | Access key to S3 inbox . | `null` | +| `global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null` | +| `global.inbox.s3CaFile` | CA certificate to use. | `null` | +| `global.s3inbox.url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.s3inbox.port` | Port that the S3 inbox is available on. | `` | +| `global.s3inbox.bucket` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | +| `global.s3inbox.accessKey` | Access key to S3 inbox . | `null` | +| `global.s3inbox.secretKey` | Secret key to S3 inbox. | `null` | +| `global.s3inbox.region` | S3 inbox region. | `"us-east-1"` | +| `global.s3inbox.s3CaFile` | CA certificate to use. | `null` | +| `global.s3inbox.readyPath` | Path to the s3 ready/healthy endpoint. | `null` | +| `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` | +| `global.sync.api.user` | User for authenticating to the syncAPI server | `null` | +| `global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` | +| `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` | +| `global.sync.destination.s3Url` | URL to S3 sync destination instance. | `null` | +| `global.sync.destination.s3Port` | Port that the S3 sync destination instance is available on | `` | +| `global.sync.destination.s3Accesskey` | Access key to S3 sync destination | `null` | +| `global.sync.destination.s3Secretkey` | Secret key to S3 sync destination | `null` | +| `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | +| `global.sync.destination.s3CaFile` | CA certificate to use. | `null` | +| `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | +| `global.sync.remote.host` | URL to the remote syncAPI host | `null` | +| `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | +| `global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null` | +| `global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null` | +| `global.tls.enabled` | Use TLS for all connections. | `true` | +| `global.tls.issuer` | Issuer for TLS certificate creation. | `""` | +| `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` | +| `global.reencrypt.host` | gRPC host for reencryption | `""` | +| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` | ### Credentials -If no shared credentials for the message broker and database are used these should be set in the `credentials` section of the values file. +If no shared credentials for the message broker and database are used these should be set in the `credentials` section +of the values file. -Parameter | Description | Default ---------- | ----------- | ------- -`credentials.api.dbUser` | Database user for api | `""` -`credentials.api.dbPassword` | Database password for api | `""` -`credentials.api.mqUser` | Broker user for api | `""` -`credentials.auth.dbUser` | Database user for auth | `""` -`credentials.auth.dbPassword` | Database password for auth | `""` -`credentials.api.mqPassword` | Broker password for api | `""` -`credentials.doa.dbUser` | Database user for doa | `""` -`credentials.doa.dbPassword` | Database password for doa | `""` -`credentials.download.dbUser` | Database user for download | `""` -`credentials.download.dbPassword` | Database password for download | `""` -`credentials.finalize.dbUser` | Database user for finalize | `""` -`credentials.finalize.dbPassword` | Database password for finalize | `""` -`credentials.finalize.mqUser` | Broker user for finalize | `""` -`credentials.finalize.mqPassword` | Broker password for finalize | `""` -`credentials.inbox.mqUser` | Broker user for inbox | `""` -`credentials.inbox.mqPassword` | Broker password for inbox | `""` -`credentials.ingest.dbUser` | Database user for ingest | `""` -`credentials.ingest.dbPassword` | Database password for ingest | `""` -`credentials.ingest.mqUser` | Broker user for ingest | `""` -`credentials.ingest.mqPassword` | Broker password for ingest | `""` -`credentials.intercept.mqUser` | Broker user for intercept | `""` -`credentials.intercept.mqPassword` | Broker password for intercept | `""` -`credentials.sync.dbUser` | Database user for sync | `""` -`credentials.sync.dbPassword` | Database password for sync | `""` -`credentials.sync.mqUser` | Broker user for sync | `""` -`credentials.sync.mqPassword` | Broker password for sync | `""` -`credentials.syncapi.mqUser` | Broker user for sync | `""` -`credentials.syncapi.mqPassword` | Broker password for sync | `""` -`credentials.test.dbUser` | Database user for test | `""` -`credentials.test.dbPassword` | Database password for test | `""` -`credentials.test.mqUser` | Broker user for test | `""` -`credentials.test.mqPassword` | Broker password for test | `""` -`credentials.verify.dbUser` | Database user for verify | `""` -`credentials.verify.dbPassword` | Database password for verify | `""` -`credentials.verify.mqUser` | Broker user for verify | `""` -`credentials.verify.mqPassword` | Broker password for verify | `""` + Parameter | Description | Default +------------------------------------|--------------------------------|--------- + `credentials.api.dbUser` | Database user for api | `""` + `credentials.api.dbPassword` | Database password for api | `""` + `credentials.api.mqUser` | Broker user for api | `""` + `credentials.auth.dbUser` | Database user for auth | `""` + `credentials.auth.dbPassword` | Database password for auth | `""` + `credentials.api.mqPassword` | Broker password for api | `""` + `credentials.doa.dbUser` | Database user for doa | `""` + `credentials.doa.dbPassword` | Database password for doa | `""` + `credentials.download.dbUser` | Database user for download | `""` + `credentials.download.dbPassword` | Database password for download | `""` + `credentials.finalize.dbUser` | Database user for finalize | `""` + `credentials.finalize.dbPassword` | Database password for finalize | `""` + `credentials.finalize.mqUser` | Broker user for finalize | `""` + `credentials.finalize.mqPassword` | Broker password for finalize | `""` + `credentials.inbox.mqUser` | Broker user for inbox | `""` + `credentials.inbox.mqPassword` | Broker password for inbox | `""` + `credentials.ingest.dbUser` | Database user for ingest | `""` + `credentials.ingest.dbPassword` | Database password for ingest | `""` + `credentials.ingest.mqUser` | Broker user for ingest | `""` + `credentials.ingest.mqPassword` | Broker password for ingest | `""` + `credentials.intercept.mqUser` | Broker user for intercept | `""` + `credentials.intercept.mqPassword` | Broker password for intercept | `""` + `credentials.sync.dbUser` | Database user for sync | `""` + `credentials.sync.dbPassword` | Database password for sync | `""` + `credentials.sync.mqUser` | Broker user for sync | `""` + `credentials.sync.mqPassword` | Broker password for sync | `""` + `credentials.syncapi.mqUser` | Broker user for sync | `""` + `credentials.syncapi.mqPassword` | Broker password for sync | `""` + `credentials.test.dbUser` | Database user for test | `""` + `credentials.test.dbPassword` | Database password for test | `""` + `credentials.test.mqUser` | Broker user for test | `""` + `credentials.test.mqPassword` | Broker password for test | `""` + `credentials.verify.dbUser` | Database user for verify | `""` + `credentials.verify.dbPassword` | Database password for verify | `""` + `credentials.verify.mqUser` | Broker user for verify | `""` + `credentials.verify.mqPassword` | Broker password for verify | `""` ### Pod settings -Parameter | Description | Default ---------- | ----------- | ------- -`api.replicaCount` | Desired number of replicas | `2` -`api.annotations` | Specific annotation for the auth pod | `{}` -`api.ingressAnnotations | Ingress annotations specific to the api service | `{}` -`api.livenessProbe` | Liveness definition for the api pod. | `{}` -`api.readinessProbe` | Rediness definition for the api pod. | `{}` -`api.resources.requests.memory` | Memory request for container. | `128Mi` -`api.resources.requests.cpu` | CPU request for container. | `100m` -`api.resources.limits.memory` | Memory limit for container. | `256Mi` -`api.resources.limits.cpu` | CPU limit for container. | `1000m` -`api.tls.secretName` | Secret holding the application TLS certificates | `` -`auth.replicaCount` | desired number of replicas | `2` -`auth.annotations` | Specific annotation for the auth pod | `{}` -`auth.ingressAnnotations | Ingress annotations specific to the auth service | `{}` -`auth.livenessProbe` | Liveness definition for the auth pod. | `{}` -`auth.readinessProbe` | Rediness definition for the auth pod. | `{}` -`auth.resources.requests.memory` | Memory request for container. | `128Mi` -`auth.resources.requests.cpu` | CPU request for container. | `100m` -`auth.resources.limits.memory` | Memory limit for container. | `256Mi` -`auth.resources.limits.cpu` | CPU limit for container. | 1000m` -`doa.replicaCount` | desired number of replicas | `2` -`doa.keystorePass` | keystore password | `changeit` -`doa.annotations` | Specific annotation for the doa pod | `{}` -`doa.ingressAnnotations | Ingress annotations specific to the doa service | `{}` -`doa.livenessProbe` | Liveness definition for the doa pod. | `{}` -`doa.readinessProbe` | Rediness definition for the doa pod. | `{}` -`doa.resources.requests.memory` | Memory request for dataedge container. | `128Mi` -`doa.resources.requests.cpu` | CPU request for dataedge container. | `100m` -`doa.resources.limits.memory` | Memory limit for dataedge container. | `1024Mi` -`doa.resources.limits.cpu` | CPU limit for dataedge container. | `2000m` -`download.replicaCount` | desired number of replicas | `2` -`download.keystorePass` | keystore password | `changeit` -`download.annotations` | Specific annotation for the dataedge pod | `{}` -`download.ingressAnnotations | Ingress annotations specific to the download service | `{}` -`download.livenessProbe` | Liveness definition for the download pod. | `{}` -`download.readinessProbe` | Rediness definition for the download pod. | `{}` -`download.resources.requests.memory` | Memory request for dataedge container. | `256Mi` -`download.resources.requests.cpu` | CPU request for dataedge container. | `100m` -`download.resources.limits.memory` | Memory limit for dataedge container. | `512Mi` -`download.resources.limits.cpu` | CPU limit for dataedge container. | `1000m` -`download.minClientVersion` | Minimal version of the `sda-cli` to accept | `v0.3.0` -`finalize.annotations` | Specific annotation for the finalize pod | `{}` -`finalize.resources.requests.memory` | Memory request for finalize container. | `128Mi` -`finalize.resources.requests.cpu` | CPU request for finalize container. | `100m` -`finalize.resources.limits.memory` | Memory limit for finalize container. | `256Mi` -`finalize.resources.limits.cpu` | CPU limit for finalize container. | `1000m` -`ingest.replicaCount` | desired number of ingest workers | `1` -`ingest.annotations` | Specific annotation for the ingest pod | `{}` -`ingest.resources.requests.memory` | Memory request for ingest container. | `128Mi` -`ingest.resources.requests.cpu` | CPU request for ingest container. | `100m` -`ingest.resources.limits.memory` | Memory limit for ingest container. | `512Mi` -`ingest.resources.limits.cpu` | CPU limit for ingest container. | `2000m` -`intercept.replicaCount` | desired number of intercept workers | `1` -`intercept.annotations` | Specific annotation for the intercept pod | `{}` -`intercept.resources.requests.memory` | Memory request for intercept container. | `32Mi` -`intercept.resources.requests.cpu` | CPU request for intercept container. | `100m` -`intercept.resources.limits.memory` | Memory limit for intercept container. | `128Mi` -`intercept.resources.limits.cpu` | CPU limit for intercept container. | `1000m` -`s3Inbox.replicaCount` | desired number of S3inbox containers | `2` -`s3Inbox.annotations` | Specific annotation for the S3inbox pod | `{}` -`s3Inbox.ingressAnnotations | Ingress annotations specific to the s3Inbox service | `{}` -`s3Inbox.livenessProbe` | Liveness definition for the s3Inbox pod. | `{}` -`s3Inbox.readinessProbe` | Rediness definition for the S3inbox pod. | `{}` -`s3Inbox.resources.requests.memory` | Memory request for s3Inbox container. | `128Mi` -`s3Inbox.resources.requests.cpu` | CPU request for s3Inbox container. | `100m` -`s3Inbox.resources.limits.memory` | Memory limit for s3Inbox container. | `1024Mi` -`s3Inbox.resources.limits.cpu` | CPU limit for s3Inbox container. | `1000m` -`sftpInbox.replicaCount` | desired number of sftp inbox containers | `2` -`sftpInbox.keystorePass` | sftp inbox keystore password | `changeit` -`sftpInbox.nodeHostname` | Node name if the sftp inbox needs to be deployed on a specific node | `""` -`sftpInbox.annotations` | Specific annotation for the sftp inbox pod | `{}` -`sftpInbox.livenessProbe` | Liveness definition for the sftpInbox pod. | `{}` -`sftpInbox.readinessProbe` | Rediness definition for the sftpInbox pod. | `{}` -`sftpInbox.resources.requests.memory` | Memory request for sftpInbox container. | `128Mi` -`sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. | `100m` -`sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. | `256Mi` -`sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. | `1000m` -`sync.annotations` | Specific annotation for the sync pod | `{}` -`sync.deploy` | Set to true if the sync service should be active | `false` -`sync.replicaCount` | desired number of sync containers | `1` -`sync.resources.requests.memory` | Memory request for sync container. | `128Mi` -`sync.resources.requests.cpu` | CPU request for sync container. | `100m` -`sync.resources.limits.memory` | Memory limit for sync container. | `512Mi` -`sync.resources.limits.cpu` | CPU limit for sync container. | `1000m` -`syncapi.replicaCount` | desired number of syncAPI containers | `1` -`syncapi.annotations` | Specific annotation for the syncAPI pod | `{}` -`syncapi.ingressAnnotations | Ingress annotations specific to the syncapi service | `{}` -`syncapi.readinessProbe` | Rediness definition for the syncAPI pod. | `{}` -`syncapi.resources.requests.memory` | Memory request for syncAPI container. | `64Mi` -`syncapi.resources.requests.cpu` | CPU request for syncAPI container. | `100m` -`syncapi.resources.limits.memory` | Memory limit for syncAPI container. | `256Mi` -`syncapi.resources.limits.cpu` | CPU limit for syncAPI container. | `1000m` -`verify.replicaCount` | desired number of verify containers | `1` -`verify.annotations` | Specific annotation for the verify pod | `{}` -`verify.resources.requests.memory` | Memory request for verify container. | `128Mi` -`verify.resources.requests.cpu` | CPU request for verify container. | `100m` -`verify.resources.limits.memory` | Memory limit for verify container. | `512Mi` -`verify.resources.limits.cpu` | CPU limit for verify container. | `2000m` -`releasetest.repository` | inbox container image repository | `neicnordic/sda-helm-test-support` -`releasetest.imageTag` | inbox container image version | `latest` -`releasetest.imagePullPolicy` | inbox container image pull policy | `Always` + Parameter | Description | Default +---------------------------------------|----------------------------------------------------------------------|------------------------------------ + `api.replicaCount` | Desired number of replicas | `2` + `api.annotations` | Specific annotation for the auth pod | `{}` + `api.ingressAnnotations | Ingress annotations specific to the api service | `{}` + `api.livenessProbe` | Liveness definition for the api pod. | `{}` + `api.readinessProbe` | Rediness definition for the api pod. | `{}` + `api.resources.requests.memory` | Memory request for container. | `128Mi` + `api.resources.requests.cpu` | CPU request for container. | `100m` + `api.resources.limits.memory` | Memory limit for container. | `256Mi` + `api.resources.limits.cpu` | CPU limit for container. | `1000m` + `api.tls.secretName` | Secret holding the application TLS certificates | `` + `auth.replicaCount` | desired number of replicas | `2` + `auth.annotations` | Specific annotation for the auth pod | `{}` + `auth.ingressAnnotations | Ingress annotations specific to the auth service | `{}` + `auth.livenessProbe` | Liveness definition for the auth pod. | `{}` + `auth.readinessProbe` | Rediness definition for the auth pod. | `{}` + `auth.resources.requests.memory` | Memory request for container. | `128Mi` + `auth.resources.requests.cpu` | CPU request for container. | `100m` + `auth.resources.limits.memory` | Memory limit for container. | `256Mi` + `auth.resources.limits.cpu` | CPU limit for container. | 1000m` + `doa.replicaCount` | desired number of replicas | `2` + `doa.keystorePass` | keystore password | `changeit` + `doa.annotations` | Specific annotation for the doa pod | `{}` + `doa.ingressAnnotations | Ingress annotations specific to the doa service | `{}` + `doa.livenessProbe` | Liveness definition for the doa pod. | `{}` + `doa.readinessProbe` | Rediness definition for the doa pod. | `{}` + `doa.resources.requests.memory` | Memory request for dataedge container. | `128Mi` + `doa.resources.requests.cpu` | CPU request for dataedge container. | `100m` + `doa.resources.limits.memory` | Memory limit for dataedge container. | `1024Mi` + `doa.resources.limits.cpu` | CPU limit for dataedge container. | `2000m` + `download.replicaCount` | desired number of replicas | `2` + `download.keystorePass` | keystore password | `changeit` + `download.annotations` | Specific annotation for the dataedge pod | `{}` + `download.ingressAnnotations | Ingress annotations specific to the download service | `{}` + `download.livenessProbe` | Liveness definition for the download pod. | `{}` + `download.readinessProbe` | Rediness definition for the download pod. | `{}` + `download.resources.requests.memory` | Memory request for dataedge container. | `256Mi` + `download.resources.requests.cpu` | CPU request for dataedge container. | `100m` + `download.resources.limits.memory` | Memory limit for dataedge container. | `512Mi` + `download.resources.limits.cpu` | CPU limit for dataedge container. | `1000m` + `download.minClientVersion` | Minimal version of the `sda-cli` to accept | `v0.3.0` + `finalize.annotations` | Specific annotation for the finalize pod | `{}` + `finalize.resources.requests.memory` | Memory request for finalize container. | `128Mi` + `finalize.resources.requests.cpu` | CPU request for finalize container. | `100m` + `finalize.resources.limits.memory` | Memory limit for finalize container. | `256Mi` + `finalize.resources.limits.cpu` | CPU limit for finalize container. | `1000m` + `ingest.replicaCount` | desired number of ingest workers | `1` + `ingest.annotations` | Specific annotation for the ingest pod | `{}` + `ingest.resources.requests.memory` | Memory request for ingest container. | `128Mi` + `ingest.resources.requests.cpu` | CPU request for ingest container. | `100m` + `ingest.resources.limits.memory` | Memory limit for ingest container. | `512Mi` + `ingest.resources.limits.cpu` | CPU limit for ingest container. | `2000m` + `intercept.replicaCount` | desired number of intercept workers | `1` + `intercept.annotations` | Specific annotation for the intercept pod | `{}` + `intercept.resources.requests.memory` | Memory request for intercept container. | `32Mi` + `intercept.resources.requests.cpu` | CPU request for intercept container. | `100m` + `intercept.resources.limits.memory` | Memory limit for intercept container. | `128Mi` + `intercept.resources.limits.cpu` | CPU limit for intercept container. | `1000m` + `s3Inbox.replicaCount` | desired number of S3inbox containers | `2` + `s3Inbox.annotations` | Specific annotation for the S3inbox pod | `{}` + `s3Inbox.ingressAnnotations | Ingress annotations specific to the s3Inbox service | `{}` + `s3Inbox.livenessProbe` | Liveness definition for the s3Inbox pod. | `{}` + `s3Inbox.readinessProbe` | Rediness definition for the S3inbox pod. | `{}` + `s3Inbox.resources.requests.memory` | Memory request for s3Inbox container. | `128Mi` + `s3Inbox.resources.requests.cpu` | CPU request for s3Inbox container. | `100m` + `s3Inbox.resources.limits.memory` | Memory limit for s3Inbox container. | `1024Mi` + `s3Inbox.resources.limits.cpu` | CPU limit for s3Inbox container. | `1000m` + `sftpInbox.replicaCount` | desired number of sftp inbox containers | `2` + `sftpInbox.keystorePass` | sftp inbox keystore password | `changeit` + `sftpInbox.nodeHostname` | Node name if the sftp inbox needs to be deployed on a specific node | `""` + `sftpInbox.annotations` | Specific annotation for the sftp inbox pod | `{}` + `sftpInbox.livenessProbe` | Liveness definition for the sftpInbox pod. | `{}` + `sftpInbox.readinessProbe` | Rediness definition for the sftpInbox pod. | `{}` + `sftpInbox.resources.requests.memory` | Memory request for sftpInbox container. | `128Mi` + `sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. | `100m` + `sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. | `256Mi` + `sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. | `1000m` + `sync.annotations` | Specific annotation for the sync pod | `{}` + `sync.deploy` | Set to true if the sync service should be active | `false` + `sync.replicaCount` | desired number of sync containers | `1` + `sync.resources.requests.memory` | Memory request for sync container. | `128Mi` + `sync.resources.requests.cpu` | CPU request for sync container. | `100m` + `sync.resources.limits.memory` | Memory limit for sync container. | `512Mi` + `sync.resources.limits.cpu` | CPU limit for sync container. | `1000m` + `syncapi.replicaCount` | desired number of syncAPI containers | `1` + `syncapi.annotations` | Specific annotation for the syncAPI pod | `{}` + `syncapi.ingressAnnotations | Ingress annotations specific to the syncapi service | `{}` + `syncapi.readinessProbe` | Rediness definition for the syncAPI pod. | `{}` + `syncapi.resources.requests.memory` | Memory request for syncAPI container. | `64Mi` + `syncapi.resources.requests.cpu` | CPU request for syncAPI container. | `100m` + `syncapi.resources.limits.memory` | Memory limit for syncAPI container. | `256Mi` + `syncapi.resources.limits.cpu` | CPU limit for syncAPI container. | `1000m` + `verify.replicaCount` | desired number of verify containers | `1` + `verify.annotations` | Specific annotation for the verify pod | `{}` + `verify.resources.requests.memory` | Memory request for verify container. | `128Mi` + `verify.resources.requests.cpu` | CPU request for verify container. | `100m` + `verify.resources.limits.memory` | Memory limit for verify container. | `512Mi` + `verify.resources.limits.cpu` | CPU limit for verify container. | `2000m` + `releasetest.repository` | inbox container image repository | `neicnordic/sda-helm-test-support` + `releasetest.imageTag` | inbox container image version | `latest` + `releasetest.imagePullPolicy` | inbox container image pull policy | `Always` ### Jobs diff --git a/charts/sda-svc/templates/api-secrets.yaml b/charts/sda-svc/templates/api-secrets.yaml index f29d39b30c..a367d84b93 100644 --- a/charts/sda-svc/templates/api-secrets.yaml +++ b/charts/sda-svc/templates/api-secrets.yaml @@ -55,23 +55,21 @@ stringData: {{- if .Values.global.reencrypt.port}} port: {{ .Values.global.reencrypt.port }} {{- end }} - inbox: - type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + storage: + inbox: {{- if eq "s3" .Values.global.inbox.storageType }} - accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} - bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} - {{- if .Values.global.inbox.s3Port }} - port: {{ .Values.global.inbox.s3Port }} - {{- end }} - readypath: {{ .Values.global.inbox.s3ReadyPath }} - secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} - url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + s3: + - endpoint: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }}{{- if .Values.global.inbox.s3Port }}:{{.Values.global.inbox.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + secret_key: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + bucket_prefix: {{ required "S3 inbox bucket prefix not set" .Values.global.inbox.s3BucketPrefix }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} {{- else }} - location: /inbox/ + posix: + - path: {{ .Values.global.inbox.volumePath }} {{- end }} log: format: {{ .Values.global.log.format }} diff --git a/charts/sda-svc/templates/doa-deploy.yaml b/charts/sda-svc/templates/doa-deploy.yaml index 6bc39ecbab..f59a546d4a 100644 --- a/charts/sda-svc/templates/doa-deploy.yaml +++ b/charts/sda-svc/templates/doa-deploy.yaml @@ -185,7 +185,7 @@ spec: - name: S3_SECURE value: "true" - name: S3_BUCKET - value: {{ .Values.global.archive.s3Bucket | quote}} + value: {{ .Values.global.archive.s3BucketPrefix | quote}} {{- if .Values.global.archive.s3CaFile }} - name: S3_ROOT_CERT_PATH value: "/etc/ssl/certs/java/ca.crt" diff --git a/charts/sda-svc/templates/download-secrets.yaml b/charts/sda-svc/templates/download-secrets.yaml index 7c97386fe5..87f3fc8b27 100644 --- a/charts/sda-svc/templates/download-secrets.yaml +++ b/charts/sda-svc/templates/download-secrets.yaml @@ -14,25 +14,25 @@ stringData: serverKey: {{ template "tlsPath" . }}/tls.key {{- end }} minimalcliversion: {{ default "v0.3.0" .Values.download.minClientVersion }} - archive: - type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} - {{- if eq "s3" .Values.global.archive.storageType }} - accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} - bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{- end }} - chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} - {{- if .Values.global.archive.s3Port }} - port: {{ .Values.global.archive.s3Port }} - {{- end }} - readypath: {{ .Values.global.archive.s3ReadyPath }} - secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} - url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- else }} - location: /archive/ - {{- end }} - {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} + storage: + archive: + {{- if eq "s3" .Values.global.archive.storageType }} + s3: + - endpoint: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}{{- if .Values.global.archive.s3Port }}:{{.Values.global.archive.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- if .Values.global.archive.s3ChunkSize}} + chunk_size: {{ .Values.global.archive.s3ChunkSize }} + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- else }} + posix: + - path: {{ .Values.global.archive.volumePath }} + {{- end }} c4gh: transientKeyPath: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }} transientKeyPassphrase: {{ .Values.global.download.serveDecrypted.passphrase }} diff --git a/charts/sda-svc/templates/finalize-secrets.yaml b/charts/sda-svc/templates/finalize-secrets.yaml index c8d92e3a2a..3ffeb85255 100644 --- a/charts/sda-svc/templates/finalize-secrets.yaml +++ b/charts/sda-svc/templates/finalize-secrets.yaml @@ -9,42 +9,52 @@ type: Opaque stringData: config.yaml: |- {{- if .Values.global.backupArchive.storageType }} - archive: - type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} - {{- if eq "s3" .Values.global.archive.storageType }} - accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} - bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} - {{- if .Values.global.archive.s3Port }} - port: {{ .Values.global.archive.s3Port }} - {{- end }} - readypath: {{ .Values.global.archive.s3ReadyPath }} - secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} - url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- else }} - location: /archive/ - {{- end }} - backup: - type: {{ .Values.global.backupArchive.storageType }} - {{- if eq "s3" .Values.global.backupArchive.storageType }} - accessKey: {{ required "Accesskey required for backup archive" .Values.global.backupArchive.s3AccessKey }} - bucket: {{ required "S3 backup archive bucket not set" .Values.global.backupArchive.s3Bucket }} - {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.backupArchive.s3ChunkSize }} - {{- if .Values.global.backupArchive.s3Port }} - port: {{ .Values.global.backupArchive.s3Port }} - {{- end }} - readypath: {{ .Values.global.backupArchive.s3ReadyPath }} - secretKey: {{ required "Secretkey required for backup archive" .Values.global.backupArchive.s3SecretKey }} - url: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }} - {{- else }} - location: /backup/ - {{- end }} + storage: + archive: + {{- if eq "s3" .Values.global.archive.storageType }} + s3: + - endpoint: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}{{- if .Values.global.archive.s3Port }}:{{.Values.global.archive.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- if .Values.global.archive.s3ChunkSize}} + chunk_size: {{ .Values.global.archive.s3ChunkSize }} + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- else }} + posix: + - path: {{ .Values.global.archive.volumePath }} + {{- end }} + backup: + {{- if eq "s3" .Values.global.backupArchive.storageType }} + s3: + - endpoint: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }}{{- if .Values.global.backupArchive.s3Port }}:{{.Values.global.backupArchive.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for backup archive" .Values.global.backupArchive.s3AccessKey }} + secret_key: {{ required "Secretkey required for backup archive" .Values.global.backupArchive.s3SecretKey }} + bucket_prefix: {{ required "S3 backup archive bucket not set" .Values.global.backupArchive.s3BucketPrefix }} + {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- if .Values.global.backupArchive.s3ChunkSize}} + chunk_size: {{ .Values.global.backupArchive.s3ChunkSize }} + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- if .Values.global.backupArchive.maxBuckets }} + max_buckets: {{ .Values.global.backupArchive.maxBuckets }} + {{- end }} + {{- else }} + posix: + - path: {{ .Values.global.backupArchive.volumePath }} + {{- end }} + {{- if .Values.global.backupArchive.maxObjects }} + max_objects: {{ .Values.global.backupArchive.maxObjects }} + {{- end }} + {{- if .Values.global.backupArchive.maxSize }} + max_size: {{ .Values.global.backupArchive.maxSize }} + {{- end }} {{- end }} broker: {{- if .Values.global.tls.enabled }} diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index 644cec717b..11fbcdc18f 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -93,7 +93,7 @@ spec: {{- end }} {{- if eq "posix" .Values.global.inbox.storageType }} - name: inbox - mountPath: {{ .Values.global.inbox.path | quote }} + mountPath: {{ .Values.global.inbox.volumePath | quote }} {{- end }} enableServiceLinks: false volumes: diff --git a/charts/sda-svc/templates/ingest-secrets.yaml b/charts/sda-svc/templates/ingest-secrets.yaml index d08426f88b..af241794bc 100644 --- a/charts/sda-svc/templates/ingest-secrets.yaml +++ b/charts/sda-svc/templates/ingest-secrets.yaml @@ -8,24 +8,46 @@ metadata: type: Opaque stringData: config.yaml: |- - archive: - type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} - {{- if eq "s3" .Values.global.archive.storageType }} - accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} - bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} - {{- if .Values.global.archive.s3Port }} - port: {{ .Values.global.archive.s3Port }} + storage: + archive: + {{- if eq "s3" .Values.global.archive.storageType }} + s3: + - endpoint: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}{{- if .Values.global.archive.s3Port }}:{{.Values.global.archive.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + bucket_prefix: {{ required "S3 archive bucket prefix not set" .Values.global.archive.s3BucketPrefix }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- if .Values.global.archive.maxBuckets }} + max_buckets: {{ .Values.global.archive.maxBuckets }} + {{- end }} + {{- else }} + posix: + - path: {{ .Values.global.archive.volumePath }} + {{- end }} + {{- if .Values.global.archive.maxObjects }} + max_objects: {{ .Values.global.archive.maxObjects }} + {{- end }} + {{- if .Values.global.archive.maxSize }} + max_size: {{ .Values.global.archive.maxSize }} + {{- end }} + inbox: + {{- if eq "s3" .Values.global.inbox.storageType }} + s3: + - endpoint: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }}{{- if .Values.global.inbox.s3Port }}:{{.Values.global.inbox.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + secret_key: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + bucket_prefix: {{ required "S3 inbox bucket prefix not set" .Values.global.inbox.s3BucketPrefix }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- else }} + posix: + - path: {{ .Values.global.inbox.volumePath }} {{- end }} - readypath: {{ .Values.global.archive.s3ReadyPath }} - secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} - url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- else }} - location: /archive/ - {{- end }} broker: {{- if .Values.global.tls.enabled }} caCert: {{ template "tlsPath" . }}/ca.crt @@ -66,24 +88,6 @@ stringData: port: {{ .Values.global.db.port }} sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} user: {{ required "DB user is required" (include "dbUserIngest" .) }} - inbox: - type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} - {{- if eq "s3" .Values.global.inbox.storageType }} - accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} - bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} - {{- if .Values.global.inbox.s3Port }} - port: {{ .Values.global.inbox.s3Port }} - {{- end }} - readypath: {{ .Values.global.inbox.s3ReadyPath }} - secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} - url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- else }} - location: /inbox/ - {{- end }} log: format: {{ .Values.global.log.format }} level: {{ .Values.global.log.level }} diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml index 9a9fa5427f..d9aac0c9a3 100644 --- a/charts/sda-svc/templates/mapper-deploy.yaml +++ b/charts/sda-svc/templates/mapper-deploy.yaml @@ -69,7 +69,7 @@ spec: {{- end }} {{- if eq "posix" .Values.global.inbox.storageType }} - name: inbox - mountPath: {{ .Values.global.inbox.path | quote }} + mountPath: {{ .Values.global.inbox.volumePath | quote }} {{- end }} enableServiceLinks: false volumes: diff --git a/charts/sda-svc/templates/mapper-secrets.yaml b/charts/sda-svc/templates/mapper-secrets.yaml index c66ed0dc15..b9cda932d2 100644 --- a/charts/sda-svc/templates/mapper-secrets.yaml +++ b/charts/sda-svc/templates/mapper-secrets.yaml @@ -43,24 +43,22 @@ stringData: port: {{ .Values.global.db.port }} sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} user: {{ required "DB user is required" (include "dbUserMapper" .) }} - inbox: - type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} - {{- if eq "s3" .Values.global.inbox.storageType }} - accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} - bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} - {{- if .Values.global.inbox.s3Port }} - port: {{ .Values.global.inbox.s3Port }} + storage: + inbox: + {{- if eq "s3" .Values.global.inbox.storageType }} + s3: + - endpoint: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }}{{- if .Values.global.inbox.s3Port }}:{{.Values.global.inbox.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + secret_key: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + bucket_prefix: {{ required "S3 inbox bucket prefix not set" .Values.global.inbox.s3BucketPrefix }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- else }} + posix: + - path: {{ .Values.global.inbox.volumePath }} {{- end }} - readypath: {{ .Values.global.inbox.s3ReadyPath }} - secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} - url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- else }} - location: /inbox/ - {{- end }} log: format: {{ .Values.global.log.format }} level: {{ .Values.global.log.level }} diff --git a/charts/sda-svc/templates/s3-inbox-secrets.yaml b/charts/sda-svc/templates/s3-inbox-secrets.yaml index 340a6a7f86..ecc3db753e 100644 --- a/charts/sda-svc/templates/s3-inbox-secrets.yaml +++ b/charts/sda-svc/templates/s3-inbox-secrets.yaml @@ -43,24 +43,16 @@ stringData: port: {{ .Values.global.db.port }} sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} user: {{ required "DB user is required" (include "dbUserInbox" .) }} - inbox: - type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} - {{- if eq "s3" .Values.global.inbox.storageType }} - accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} - bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} - {{- if .Values.global.inbox.s3Port }} - port: {{ .Values.global.inbox.s3Port }} + s3inbox: + endpoint: {{ required "S3 archive URL missing" .Values.global.s3Inbox.url }}{{- if .Values.global.s3Inbox.port }}:{{.Values.global.s3Inbox.port}}{{- end }} + access_key: {{ required "Accesskey required for archive" .Values.global.s3Inbox.accessKey }} + secret_key: {{ required "Secretkey required for archive" .Values.global.s3Inbox.secretKey }} + bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.s3Inbox.bucket }} + {{- if and .Values.global.s3Inbox.caFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} - readypath: {{ .Values.global.inbox.s3ReadyPath }} - secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} - url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- else }} - location: /inbox/ - {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + ready_path: {{ .Values.global.s3Inbox.readyPath }} log: format: {{ .Values.global.log.format }} level: {{ .Values.global.log.level }} diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index d58a4c0a83..159ada78e8 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -10,24 +10,32 @@ metadata: type: Opaque stringData: config.yaml: |- - archive: - type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} - {{- if eq "s3" .Values.global.archive.storageType }} - accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} - bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} - {{- if .Values.global.archive.s3Port }} - port: {{ .Values.global.archive.s3Port }} - {{- end }} - readypath: {{ .Values.global.archive.s3ReadyPath }} - secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} - url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- else }} - location: /archive/ - {{- end }} + storage: + archive: + {{- if eq "s3" .Values.global.archive.storageType }} + s3: + - endpoint: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}{{- if .Values.global.archive.s3Port }}:{{.Values.global.archive.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- if .Values.global.archive.s3ChunkSize}} + chunk_size: {{ .Values.global.archive.s3ChunkSize }} + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- else }} + posix: + - path: {{ .Values.global.archive.volumePath }} + {{- end }} + sync: + s3: + - endpoint: {{ required "S3 sync URL missing" .Values.global.sync.destination.s3Url }}{{- if .Values.global.sync.destination.s3Port }}:{{.Values.global.sync.destination.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for sync" .Values.global.sync.destination.s3AccessKey }} + secret_key: {{ required "Secretkey required for sync" .Values.global.sync.destination.s3SecretKey }} + bucket_prefix: {{ required "S3 sync bucket not set" .Values.global.sync.destination.s3BucketPrefix }} + disable_https: {{ not .Values.global.tls.enabled }} broker: {{- if .Values.global.tls.enabled }} caCert: {{ template "tlsPath" . }}/ca.crt @@ -74,19 +82,6 @@ stringData: type: {{ default "isolated" .Values.global.schemaType }} sync: centerPrefix: {{ .Values.global.sync.centerPrefix }} - destination: - type: s3 - url: {{ required "S3 sync archive URL missing" .Values.global.sync.destination.url }} - {{- if not (empty .Values.global.sync.destination.port) }} - port: {{ .Values.global.sync.destination.port }} - {{- end }} - {{- if not (empty .Values.global.sync.destination.readypath) }} - readypath: {{ .Values.global.sync.destination.readypath }} - {{- end }} - accessKey: {{ required "Accesskey required for sync destination" .Values.global.sync.destination.accessKey }} - secretKey: {{ required "SecretKey required for sync destination" .Values.global.sync.destination.secretKey }} - bucket: {{ required "S3 sync destination bucket missing" .Values.global.sync.destination.bucket }} - region: {{ default "us-east-1" .Values.global.sync.destination.region }} remote: host: {{ required "remote sync API host is required" .Values.global.sync.remote.host }} password: {{ required "remote sync API password is required" .Values.global.sync.remote.password }} diff --git a/charts/sda-svc/templates/verify-secrets.yaml b/charts/sda-svc/templates/verify-secrets.yaml index 7fde154e2d..7b95978b2d 100644 --- a/charts/sda-svc/templates/verify-secrets.yaml +++ b/charts/sda-svc/templates/verify-secrets.yaml @@ -9,24 +9,25 @@ metadata: type: Opaque stringData: config.yaml: |- - archive: - type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} - {{- if eq "s3" .Values.global.archive.storageType }} - accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} - bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - caCert: {{ template "tlsPath" . }}/ca.crt - {{ end }} - chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} - {{- if .Values.global.archive.s3Port }} - port: {{ .Values.global.archive.s3Port }} - {{- end }} - readypath: {{ .Values.global.archive.s3ReadyPath }} - secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} - url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- else }} - location: /archive/ - {{- end }} + storage: + archive: + {{- if eq "s3" .Values.global.archive.storageType }} + s3: + - endpoint: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}{{- if .Values.global.archive.s3Port }}:{{.Values.global.archive.s3Port}}{{- end }} + access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- if .Values.global.archive.s3ChunkSize}} + chunk_size: {{ .Values.global.archive.s3ChunkSize }} + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- else }} + posix: + - path: {{ .Values.global.archive.volumePath }} + {{- end }} broker: {{- if .Values.global.tls.enabled }} caCert: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 4128e55910..6d74fef502 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -110,35 +110,40 @@ global: rbacFileSecret: "" archive: storageType: "" # s3 or posix + maxObjects: null + maxSize: "" # The six lines below is only used with S3 backend s3Url: "" - s3Bucket: "" + s3BucketPrefix: "archive" s3Region: "us-east-1" - s3ChunkSize: 15 # Chunk size in MB > 5 + s3ChunkSize: "" s3AccessKey: null s3SecretKey: null s3CaFile: null s3Port: 443 + maxBuckets: 1 # The lines below is only used with POSIX backend volumePath: "/archive" nfsServer: "" nfsPath: "" backupArchive: - storageType: "" # s3 or posix + storageType: "" # s3 or posix, leave empty for no backup + maxObjects: null + maxSize: "" # The six lines below is only used with S3 backend s3Url: "" - s3Bucket: "" + s3BucketPrefix: "backup" s3Region: "us-east-1" - s3ChunkSize: 15 # Chunk size in MB > 5 + s3ChunkSize: "" s3AccessKey: null s3SecretKey: null s3Port: 443 + maxBuckets: 1 # The lines below is only used with POSIX backend volumePath: "/backup" nfsServer: "" nfsPath: "" - copyHeader: false auth: # @param jwtSecret, name of the secret holding the jwt signing key @@ -253,22 +258,30 @@ global: id: "" secret: "" + s3Inbox: + url: "" + port: 443 + bucket: "inbox" + accessKey: null + secretKey: null + caFile: null + region: "us-east-1" + readyPath: "" + inbox: - servicePort: 2222 storageType: "" # These lines below are only used with posix - path: "/inbox" - user: "lega" + volumePath: "/inbox" nfsServer: "" nfsPath: "" existingClaim: "" # These lines below are only used with s3 s3Url: "" - s3Bucket: "inbox" + s3BucketPrefix: "inbox" s3AccessKey: null s3SecretKey: null s3Port: 443 - s3ReadyPath: "" + s3CaFile: null s3Region: "" reencrypt: @@ -283,14 +296,12 @@ global: brokerQueue: "" centerPrefix: "" destination: - storageType: "s3" - url: "" - port: 443 - readypath: "" - accessKey: "" - secretKey: "" - bucket: "" - region: "" + s3Url: "" + s3Port: 443 + s3AccessKey: "" + s3SecretKey: "" + s3BucketPrefix: "sync" + s3Region: "us-east-1" remote: host: "" port: "" From 610e1d2a96362d980c662f4e203d8d724d6197e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Feb 2026 09:55:53 +0100 Subject: [PATCH 262/530] feat(ingest): revert change to checkS3Bucket just doing HeadBucket, move back to using CreateBucket --- sda/cmd/s3inbox/s3inbox.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 5df0868b43..f8b20bc0b3 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -141,7 +141,7 @@ func run() error { } func checkS3Bucket(ctx context.Context, s3Client *s3.Client, bucket string) error { - _, err := s3Client.HeadBucket(ctx, &s3.HeadBucketInput{Bucket: &bucket}) + _, err := s3Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: &bucket}) if err != nil { var apiErr smithy.APIError if errors.As(err, &apiErr) { From 021d644665a333b06027e399ebc803311ddaa43e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Feb 2026 15:01:17 +0100 Subject: [PATCH 263/530] fix(ingest): fix error check for backupWriter failing to initialize --- sda/cmd/ingest/ingest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 99b55b23e9..034f518c2c 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -110,7 +110,7 @@ func run() error { } backupWriter, err := storage.NewWriter(ctx, "backup", storageLocationBroker) - if err != nil && errors.Is(err, storageerrors.ErrorNoValidWriter) { + if err != nil && !errors.Is(err, storageerrors.ErrorNoValidWriter) { return fmt.Errorf("failed to initialize backup writer due to: %v", err) } if backupWriter != nil { From 348d5e8bb4b4f03dfeaa2b7e1d9c9e5c05657220 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Feb 2026 15:05:27 +0100 Subject: [PATCH 264/530] fix(config): separate mapper required configs from ingest, finalize, verify, as mapper does not require broker.routingkey --- sda/internal/config/config.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 75b7332d5d..049e1b0ef6 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -274,7 +274,7 @@ func NewConfig(app string) (*Config, error) { if viper.GetBool("auth.resignJwt") { requiredConfVars = append(requiredConfVars, []string{"auth.jwt.issuer", "auth.jwt.privateKey", "auth.jwt.signatureAlg", "auth.jwt.tokenTTL"}...) } - case "ingest", "finalize", "mapper", "verify": + case "ingest", "finalize", "verify": requiredConfVars = []string{ "broker.host", "broker.port", @@ -288,7 +288,20 @@ func NewConfig(app string) (*Config, error) { "db.password", "db.database", } - + case "mapper": + // Mapper does not require broker.routingkey thus we remove it + requiredConfVars = []string{ + "broker.host", + "broker.port", + "broker.user", + "broker.password", + "broker.queue", + "db.host", + "db.port", + "db.user", + "db.password", + "db.database", + } case "intercept": requiredConfVars = []string{ "broker.host", From 140750ecb62faf76949c0499afc18803d409e01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Feb 2026 15:20:31 +0100 Subject: [PATCH 265/530] feat(charts+integration tests): fix chart s3 config missing region, and get ca cert file from tlsPath template(instead of s3CaCert file config), update dependencies.sh to be able to deploy minio with tls enabled, add make command for deploying k3d dependencies with s3 tls, update chart tests to use s3 tls --- .../scripts/charts/dependencies.sh | 71 ++++++++++++++----- .../scripts/charts/dependencies.yaml | 1 + .../integration/scripts/charts/values.yaml | 37 +++++----- .github/workflows/build_pr_container.yaml | 2 +- Makefile | 4 +- charts/sda-svc/README.md | 19 ++--- charts/sda-svc/templates/api-secrets.yaml | 3 +- .../sda-svc/templates/download-secrets.yaml | 4 +- .../sda-svc/templates/finalize-secrets.yaml | 6 +- charts/sda-svc/templates/ingest-secrets.yaml | 6 +- charts/sda-svc/templates/mapper-secrets.yaml | 3 +- .../sda-svc/templates/s3-inbox-secrets.yaml | 7 +- charts/sda-svc/templates/sync-secrets.yaml | 7 +- charts/sda-svc/templates/verify-secrets.yaml | 3 +- charts/sda-svc/values.yaml | 4 -- 15 files changed, 110 insertions(+), 67 deletions(-) diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index a56a68f5d2..5093823554 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -77,15 +77,58 @@ helm install --namespace default nfs-ganesha nfs-ganesha-server-and-external-pro kubectl create namespace minio kubectl apply -f .github/integration/scripts/charts/dependencies.yaml + +values_file=".github/integration/scripts/charts/values.yaml" +if [ "$1" == "local" ]; then + values_file=/tmp/values.yaml + cp .github/integration/scripts/charts/values.yaml /tmp/values.yaml +fi + if [ "$2" == "s3" ]; then - ## S3 storage backend - MINIO_ACCESS="$(random-string)" - export MINIO_ACCESS - MINIO_SECRET="$(random-string)" - export MINIO_SECRET - helm install minio minio/minio \ - --namespace minio \ - --set rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi + if [ "$3" = true ] ; then + # Sleep to give cert issuer time to issue certs so we can create an secret with format minio expects + sleep 5 + + kubectl -n minio create secret generic minio-tls \ + --from-file=public.crt=<(kubectl -n minio get secret minio-cert -o jsonpath='{.data.tls\.crt}' | base64 -d) \ + --from-file=private.key=<(kubectl -n minio get secret minio-cert -o jsonpath='{.data.tls\.key}' | base64 -d) + + ## S3 storage backend + MINIO_ACCESS="$(random-string)" + export MINIO_ACCESS + MINIO_SECRET="$(random-string)" + export MINIO_SECRET + helm install minio minio/minio \ + --namespace minio \ + --set tls.enabled=true,tls.certSecret=minio-tls,rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi + + yq -i ' +.global.archive.s3Url = "https://minio.minio" | +.global.backupArchive.s3Url = "https://minio.minio" | +.global.inbox.s3Url = "https://minio.minio" | +.global.s3Inbox.url = "https://minio.minio" | +.global.sync.destination.s3Url = "https://minio.minio" +' "$values_file" + + else + ## S3 storage backend + MINIO_ACCESS="$(random-string)" + export MINIO_ACCESS + MINIO_SECRET="$(random-string)" + export MINIO_SECRET + helm install minio minio/minio \ + --namespace minio \ + --set rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi + + yq -i ' +.global.archive.s3Url = "http://minio.minio" | +.global.backupArchive.s3Url = "http://minio.minio" | +.global.inbox.s3Url = "http://minio.minio" | +.global.s3Inbox.url = "http://minio.minio" | +.global.sync.destination.s3Url = "http://minio.minio" +' "$values_file" + + fi fi PGPASSWORD="$(random-string)" @@ -97,12 +140,6 @@ export MQPASSWORD TEST_TOKEN="$(bash .github/integration/scripts/sign_jwt.sh ES256 "$dir/jwt.key")" export TEST_TOKEN -values_file=".github/integration/scripts/charts/values.yaml" -if [ "$1" == "local" ]; then - values_file=/tmp/values.yaml - cp .github/integration/scripts/charts/values.yaml /tmp/values.yaml -fi - ## update values file with all credentials if [ "$2" == "federated" ]; then yq -i '.global.schemaType = "federated"' "$values_file" @@ -118,8 +155,10 @@ yq -i ' .global.db.password = strenv(PGPASSWORD) | .global.inbox.s3AccessKey = strenv(MINIO_ACCESS) | .global.inbox.s3SecretKey = strenv(MINIO_SECRET) | -.global.sync.destination.accessKey = strenv(MINIO_ACCESS) | -.global.sync.destination.secretKey = strenv(MINIO_SECRET) | +.global.s3Inbox.accessKey = strenv(MINIO_ACCESS) | +.global.s3Inbox.secretKey = strenv(MINIO_SECRET) | +.global.sync.destination.s3AccessKey = strenv(MINIO_ACCESS) | +.global.sync.destination.s3SecretKey = strenv(MINIO_SECRET) | .releasetest.secrets.accessToken = strenv(TEST_TOKEN) ' "$values_file" diff --git a/.github/integration/scripts/charts/dependencies.yaml b/.github/integration/scripts/charts/dependencies.yaml index aa60fdea25..47197ac5cc 100644 --- a/.github/integration/scripts/charts/dependencies.yaml +++ b/.github/integration/scripts/charts/dependencies.yaml @@ -50,6 +50,7 @@ spec: dnsNames: - localhost - minio + - minio.minio - minio.minio.svc - minio.minio.svc.cluster.local ipAddresses: diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index 8d3700a11a..21cc2c42a9 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -22,36 +22,33 @@ global: storageType: s3 s3AccessKey: PLACEHOLDER_VALUE s3SecretKey: PLACEHOLDER_VALUE - s3Url: "http://minio.minio" + s3Url: PLACEHOLDER_VALUE s3Bucket: "archive" s3Port: 9000 - s3ReadyPath: "/minio/health/ready" existingClaim: archive-pvc backupArchive: storageType: "s3" s3AccessKey: PLACEHOLDER_VALUE s3SecretKey: PLACEHOLDER_VALUE - s3Url: "http://minio.minio" + s3Url: PLACEHOLDER_VALUE s3Bucket: "backup" s3Port: 9000 - s3ReadyPath: "/minio/health/ready" existingClaim: backup-pvc inbox: storageType: s3 s3AccessKey: PLACEHOLDER_VALUE s3SecretKey: PLACEHOLDER_VALUE - s3Url: http://minio.minio + s3Url: PLACEHOLDER_VALUE s3Port: 9000 s3Bucket: "inbox" - s3ReadyPath: "/minio/health/ready" existingClaim: inbox-pvc - s3inbox: - s3AccessKey: PLACEHOLDER_VALUE - s3SecretKey: PLACEHOLDER_VALUE - s3Url: http://minio.minio - s3Port: 9000 - s3Bucket: "inbox" - s3ReadyPath: "/minio/health/ready" + s3Inbox: + accessKey: PLACEHOLDER_VALUE + secretKey: PLACEHOLDER_VALUE + url: PLACEHOLDER_VALUE + port: 9000 + bucket: "inbox" + readyPath: "/minio/health/ready" auth: jwtSecret: jwk jwtAlg: ES256 @@ -106,14 +103,12 @@ global: brokerQueue: "mapping_stream" centerPrefix: "SYNC" destination: - storageType: "s3" - url: "http://minio.minio" - port: 9000 - readypath: "/minio/health/ready" - accessKey: PLACEHOLDER_VALUE - secretKey: PLACEHOLDER_VALUE - bucket: "sync" - region: "us-east-1" + s3Url: PLACEHOLDER_VALUE + s3Port: 9000 + s3AccessKey: PLACEHOLDER_VALUE + s3SecretKey: PLACEHOLDER_VALUE + s3BucketPrefix: "sync" + s3Region: "us-east-1" remote: host: "http://remote-sync" port: "8080" diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 3358a07fe0..ac9d29c1f0 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -350,7 +350,7 @@ jobs: shell: bash - name: Deploy external services - run: bash .github/integration/scripts/charts/dependencies.sh ${{matrix.deployment}} ${{matrix.storage}} + run: bash .github/integration/scripts/charts/dependencies.sh ${{matrix.deployment}} ${{matrix.storage}} ${{matrix.tls}} shell: bash - name: Deploy DB diff --git a/Makefile b/Makefile index b9754c7c8e..f6406f0405 100644 --- a/Makefile +++ b/Makefile @@ -209,7 +209,9 @@ k3d-deploy-dependencies-federated: k3d-deploy-dependencies-isolated: @bash .github/integration/scripts/charts/dependencies.sh local isolated k3d-deploy-dependencies-s3: - @bash .github/integration/scripts/charts/dependencies.sh local s3 + @bash .github/integration/scripts/charts/dependencies.sh local s3 false +k3d-deploy-dependencies-s3-tls: + @bash .github/integration/scripts/charts/dependencies.sh local s3 true k3d-import-images: build-all @bash .github/integration/scripts/charts/import_local_images.sh k3s-default k3d-deploy-postgres: diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 5a93ef7802..878c8bbbfd 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -81,7 +81,6 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.archive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `50mb` | | `global.archive.s3AccessKey` | Access key to S3 archive . | `null` | | `global.archive.s3SecretKey` | Secret key to S3 archive. | `null` | -| `global.archive.s3CaFile` | CA certificate to use. | `null` | | `global.archive.s3Port` | Port that the S3 S3 archive is available on. | `` | | `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | | `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | @@ -96,7 +95,6 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.backupArchive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `15` | | `global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` | | `global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` | -| `global.backupArchive.s3CaFile` | CA certificate to use. | `null` | | `global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `` | | `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | | `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | @@ -181,15 +179,13 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.inbox.s3Region` | S3 inbox region. | `"us-east-1"` | | `global.inbox.s3AccessKey` | Access key to S3 inbox . | `null` | | `global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null` | -| `global.inbox.s3CaFile` | CA certificate to use. | `null` | -| `global.s3inbox.url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.s3inbox.port` | Port that the S3 inbox is available on. | `` | -| `global.s3inbox.bucket` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | -| `global.s3inbox.accessKey` | Access key to S3 inbox . | `null` | -| `global.s3inbox.secretKey` | Secret key to S3 inbox. | `null` | -| `global.s3inbox.region` | S3 inbox region. | `"us-east-1"` | -| `global.s3inbox.s3CaFile` | CA certificate to use. | `null` | -| `global.s3inbox.readyPath` | Path to the s3 ready/healthy endpoint. | `null` | +| `global.s3Inbox.url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.s3Inbox.port` | Port that the S3 inbox is available on. | `` | +| `global.s3Inbox.bucket` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | +| `global.s3Inbox.accessKey` | Access key to S3 inbox . | `null` | +| `global.s3Inbox.secretKey` | Secret key to S3 inbox. | `null` | +| `global.s3Inbox.region` | S3 inbox region. | `"us-east-1"` | +| `global.s3Inbox.readyPath` | Path to the s3 ready/healthy endpoint. | `null` | | `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` | | `global.sync.api.user` | User for authenticating to the syncAPI server | `null` | | `global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` | @@ -199,7 +195,6 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.sync.destination.s3Accesskey` | Access key to S3 sync destination | `null` | | `global.sync.destination.s3Secretkey` | Secret key to S3 sync destination | `null` | | `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | -| `global.sync.destination.s3CaFile` | CA certificate to use. | `null` | | `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | | `global.sync.remote.host` | URL to the remote syncAPI host | `null` | | `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | diff --git a/charts/sda-svc/templates/api-secrets.yaml b/charts/sda-svc/templates/api-secrets.yaml index a367d84b93..7b980d8ef3 100644 --- a/charts/sda-svc/templates/api-secrets.yaml +++ b/charts/sda-svc/templates/api-secrets.yaml @@ -63,7 +63,8 @@ stringData: access_key: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} secret_key: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} bucket_prefix: {{ required "S3 inbox bucket prefix not set" .Values.global.inbox.s3BucketPrefix }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.inbox.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} disable_https: {{ not .Values.global.tls.enabled }} diff --git a/charts/sda-svc/templates/download-secrets.yaml b/charts/sda-svc/templates/download-secrets.yaml index 87f3fc8b27..fa5bbcb42e 100644 --- a/charts/sda-svc/templates/download-secrets.yaml +++ b/charts/sda-svc/templates/download-secrets.yaml @@ -22,7 +22,8 @@ stringData: access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.archive.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} {{- if .Values.global.archive.s3ChunkSize}} @@ -33,6 +34,7 @@ stringData: posix: - path: {{ .Values.global.archive.volumePath }} {{- end }} + {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} c4gh: transientKeyPath: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }} transientKeyPassphrase: {{ .Values.global.download.serveDecrypted.passphrase }} diff --git a/charts/sda-svc/templates/finalize-secrets.yaml b/charts/sda-svc/templates/finalize-secrets.yaml index 3ffeb85255..0a362d5132 100644 --- a/charts/sda-svc/templates/finalize-secrets.yaml +++ b/charts/sda-svc/templates/finalize-secrets.yaml @@ -17,7 +17,8 @@ stringData: access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.archive.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} {{- if .Values.global.archive.s3ChunkSize}} @@ -35,7 +36,8 @@ stringData: access_key: {{ required "Accesskey required for backup archive" .Values.global.backupArchive.s3AccessKey }} secret_key: {{ required "Secretkey required for backup archive" .Values.global.backupArchive.s3SecretKey }} bucket_prefix: {{ required "S3 backup archive bucket not set" .Values.global.backupArchive.s3BucketPrefix }} - {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.backupArchive.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} {{- if .Values.global.backupArchive.s3ChunkSize}} diff --git a/charts/sda-svc/templates/ingest-secrets.yaml b/charts/sda-svc/templates/ingest-secrets.yaml index af241794bc..381c974d84 100644 --- a/charts/sda-svc/templates/ingest-secrets.yaml +++ b/charts/sda-svc/templates/ingest-secrets.yaml @@ -16,7 +16,8 @@ stringData: access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} bucket_prefix: {{ required "S3 archive bucket prefix not set" .Values.global.archive.s3BucketPrefix }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.archive.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} disable_https: {{ not .Values.global.tls.enabled }} @@ -40,7 +41,8 @@ stringData: access_key: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} secret_key: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} bucket_prefix: {{ required "S3 inbox bucket prefix not set" .Values.global.inbox.s3BucketPrefix }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.inbox.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} disable_https: {{ not .Values.global.tls.enabled }} diff --git a/charts/sda-svc/templates/mapper-secrets.yaml b/charts/sda-svc/templates/mapper-secrets.yaml index b9cda932d2..e86e85e271 100644 --- a/charts/sda-svc/templates/mapper-secrets.yaml +++ b/charts/sda-svc/templates/mapper-secrets.yaml @@ -51,7 +51,8 @@ stringData: access_key: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} secret_key: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} bucket_prefix: {{ required "S3 inbox bucket prefix not set" .Values.global.inbox.s3BucketPrefix }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.inbox.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} disable_https: {{ not .Values.global.tls.enabled }} diff --git a/charts/sda-svc/templates/s3-inbox-secrets.yaml b/charts/sda-svc/templates/s3-inbox-secrets.yaml index ecc3db753e..510a7e6eaf 100644 --- a/charts/sda-svc/templates/s3-inbox-secrets.yaml +++ b/charts/sda-svc/templates/s3-inbox-secrets.yaml @@ -44,11 +44,12 @@ stringData: sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} user: {{ required "DB user is required" (include "dbUserInbox" .) }} s3inbox: - endpoint: {{ required "S3 archive URL missing" .Values.global.s3Inbox.url }}{{- if .Values.global.s3Inbox.port }}:{{.Values.global.s3Inbox.port}}{{- end }} + endpoint: {{ required "S3 s3inbox URL missing" .Values.global.s3Inbox.url }}{{- if .Values.global.s3Inbox.port }}:{{.Values.global.s3Inbox.port}}{{- end }} access_key: {{ required "Accesskey required for archive" .Values.global.s3Inbox.accessKey }} secret_key: {{ required "Secretkey required for archive" .Values.global.s3Inbox.secretKey }} - bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.s3Inbox.bucket }} - {{- if and .Values.global.s3Inbox.caFile .Values.global.tls.enabled }} + bucket: {{ required "S3 archive bucket not set" .Values.global.s3Inbox.bucket }} + region: {{ default "us-east-1" .Values.global.s3Inbox.region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} disable_https: {{ not .Values.global.tls.enabled }} diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index 159ada78e8..e72f037215 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -18,7 +18,8 @@ stringData: access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.archive.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} {{- if .Values.global.archive.s3ChunkSize}} @@ -35,7 +36,11 @@ stringData: access_key: {{ required "Accesskey required for sync" .Values.global.sync.destination.s3AccessKey }} secret_key: {{ required "Secretkey required for sync" .Values.global.sync.destination.s3SecretKey }} bucket_prefix: {{ required "S3 sync bucket not set" .Values.global.sync.destination.s3BucketPrefix }} + region: {{ default "us-east-1" .Values.global.sync.destination.s3Region }} disable_https: {{ not .Values.global.tls.enabled }} + {{- if .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} broker: {{- if .Values.global.tls.enabled }} caCert: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/templates/verify-secrets.yaml b/charts/sda-svc/templates/verify-secrets.yaml index 7b95978b2d..f4e08aea4c 100644 --- a/charts/sda-svc/templates/verify-secrets.yaml +++ b/charts/sda-svc/templates/verify-secrets.yaml @@ -17,7 +17,8 @@ stringData: access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + region: {{ default "us-east-1" .Values.global.archive.s3Region }} + {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} {{- if .Values.global.archive.s3ChunkSize}} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 6d74fef502..1f97851b3e 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -119,7 +119,6 @@ global: s3ChunkSize: "" s3AccessKey: null s3SecretKey: null - s3CaFile: null s3Port: 443 maxBuckets: 1 # The lines below is only used with POSIX backend @@ -227,7 +226,6 @@ global: s3Port: 443 s3Region: "" s3Bucket: "" - s3CaFile: null s3AccessKey: null s3SecretKey: null servicePort: 443 @@ -264,7 +262,6 @@ global: bucket: "inbox" accessKey: null secretKey: null - caFile: null region: "us-east-1" readyPath: "" @@ -281,7 +278,6 @@ global: s3AccessKey: null s3SecretKey: null s3Port: 443 - s3CaFile: null s3Region: "" reencrypt: From b7281733ce63c2e027f012d8bfe72146edefb9af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Feb 2026 17:02:15 +0100 Subject: [PATCH 266/530] feat(integration tests): set sync storage in config.yaml, use sync1 bucket instead of sync when confirming files were synced, fix error log when accession fails --- .github/integration/sda/config.yaml | 14 +++++++++++--- .github/integration/tests/sda/45_sync_test.sh | 2 +- .github/integration/tests/sda/60_api_admin_test.sh | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index c26bc8d6e2..90d7f221f6 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -7,7 +7,7 @@ api: storage: archive: s3: - endpoint: "http://s3:9000" + - endpoint: "http://s3:9000" access_key: "access" secret_key: "secretKey" bucket_prefix: "archive" @@ -17,7 +17,7 @@ storage: max_objects: 3 backup: s3: - endpoint: "http://s3:9000" + - endpoint: "http://s3:9000" access_key: "access" secret_key: "secretKey" bucket_prefix: "backup" @@ -25,12 +25,20 @@ storage: disable_https: true inbox: s3: - endpoint: "http://s3:9000" + - endpoint: "http://s3:9000" access_key: "access" secret_key: "secretKey" bucket_prefix: "inbox" region: "us-east-1" disable_https: true + sync: + s3: + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + bucket_prefix: "sync" + region: "us-east-1" + disable_https: true s3inbox: endpoint: "http://s3:9000" diff --git a/.github/integration/tests/sda/45_sync_test.sh b/.github/integration/tests/sda/45_sync_test.sh index 276fbfd3c2..41c66f0d61 100644 --- a/.github/integration/tests/sda/45_sync_test.sh +++ b/.github/integration/tests/sda/45_sync_test.sh @@ -11,7 +11,7 @@ fi # check bucket for synced files for file in NA12878.bai NA12878_20k_b37.bai; do RETRY_TIMES=0 - until [ "$(s3cmd -c direct ls s3://sync/"$file")" != "" ]; do + until [ "$(s3cmd -c direct ls s3://sync1/"$file")" != "" ]; do RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then echo "::error::Time out while waiting for files to be synced" diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index d4340c05e5..90d489d4e2 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -159,7 +159,7 @@ jq -c -n \ resp_accession_payload="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d "$accession_payload" "http://api:8080/file/accession")" if [ "$resp_accession_payload" != "200" ]; then - echo "Error when requesting to ingesting file, expected 200 got: $resp_accession_payload" + echo "Error when requesting to set accession id to file, expected 200 got: $resp_accession_payload" exit 1 fi From 502fc3470beb5ad4d6daf03afa29e47e7e937ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Feb 2026 17:34:50 +0100 Subject: [PATCH 267/530] feat(integration tests): wait for file event log verified before setting accession id in 60_api_admin_test.sh --- .github/integration/tests/sda/60_api_admin_test.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 90d489d4e2..398933c8c9 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -137,11 +137,11 @@ fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/te # wait for the fail to get the correct status RETRY_TIMES=0 -until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_events e where e.title in (select event from sda.file_event_log where file_id = '$fileid' order by started_at desc limit 1);")" -gt 30 ]; do - echo "waiting for ingest to complete" +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.file_event_log WHERE file_id='$fileid' order by started_at desc limit 1;")" = "verified" ]; do + echo "waiting for ingest and verification to complete for file: $fileid" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then - echo "::error::Time out while waiting for ingest to complete" + echo "::error::Time out while waiting for file: $fileid ingest and verification to complete" exit 1 fi sleep 2 From 4173baa3d7e4a8b07f2c541f3ec11434c5c7ff5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 16 Feb 2026 09:54:38 +0100 Subject: [PATCH 268/530] feat(storage-v2): update posix reader to validate that the requested file path combined with location is within the location --- .../storage/v2/posix/reader/get_file_size.go | 20 ++++++++++++++++++- .../v2/posix/reader/new_file_reader.go | 18 ++++++++++++++++- .../storage/v2/posix/reader/get_file_size.go | 20 ++++++++++++++++++- .../v2/posix/reader/new_file_reader.go | 18 ++++++++++++++++- 4 files changed, 72 insertions(+), 4 deletions(-) diff --git a/sda-download/internal/storage/v2/posix/reader/get_file_size.go b/sda-download/internal/storage/v2/posix/reader/get_file_size.go index 4634c34530..1d430a9db0 100644 --- a/sda-download/internal/storage/v2/posix/reader/get_file_size.go +++ b/sda-download/internal/storage/v2/posix/reader/get_file_size.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" ) @@ -24,7 +25,24 @@ func (reader *Reader) GetFileSize(_ context.Context, location, filePath string) return 0, storageerrors.ErrorNoEndpointConfiguredForLocation } - stat, err := os.Stat(filepath.Join(location, filePath)) + basePath, err := filepath.Abs(location) + if err != nil { + return 0, fmt.Errorf("failed to resolve base path for location %s: %w", location, err) + } + + fullFilePath, err := filepath.Abs(filepath.Join(basePath, filePath)) + if err != nil { + return 0, fmt.Errorf("failed to resolve file path for %s at location %s: %w", filePath, location, err) + } + + // Ensure the resolved path is within the configured base path to prevent path traversal. + baseWithSep := basePath + string(os.PathSeparator) + fullWithSep := fullFilePath + string(os.PathSeparator) + if !strings.HasPrefix(fullWithSep, baseWithSep) { + return 0, storageerrors.ErrorFileNotFoundInLocation + } + + stat, err := os.Stat(fullFilePath) if err != nil { if errors.Is(err, os.ErrNotExist) { return 0, storageerrors.ErrorFileNotFoundInLocation diff --git a/sda-download/internal/storage/v2/posix/reader/new_file_reader.go b/sda-download/internal/storage/v2/posix/reader/new_file_reader.go index 10bb49f71a..0defee47fa 100644 --- a/sda-download/internal/storage/v2/posix/reader/new_file_reader.go +++ b/sda-download/internal/storage/v2/posix/reader/new_file_reader.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" ) @@ -25,7 +26,22 @@ func (reader *Reader) NewFileReader(_ context.Context, location, filePath string return nil, storageerrors.ErrorNoEndpointConfiguredForLocation } - fullFilePath := filepath.Join(location, filePath) + basePath, err := filepath.Abs(location) + if err != nil { + return nil, fmt.Errorf("failed to resolve base path for location %s: %w", location, err) + } + + fullFilePath, err := filepath.Abs(filepath.Join(basePath, filePath)) + if err != nil { + return nil, fmt.Errorf("failed to resolve file path for %s at location %s: %w", filePath, location, err) + } + + // Ensure the resolved path is within the configured base path to prevent path traversal. + baseWithSep := basePath + string(os.PathSeparator) + fullWithSep := fullFilePath + string(os.PathSeparator) + if !strings.HasPrefix(fullWithSep, baseWithSep) { + return nil, storageerrors.ErrorFileNotFoundInLocation + } if _, err := os.Stat(fullFilePath); errors.Is(err, os.ErrNotExist) { return nil, storageerrors.ErrorFileNotFoundInLocation diff --git a/sda/internal/storage/v2/posix/reader/get_file_size.go b/sda/internal/storage/v2/posix/reader/get_file_size.go index ff9e8a2b52..480761af81 100644 --- a/sda/internal/storage/v2/posix/reader/get_file_size.go +++ b/sda/internal/storage/v2/posix/reader/get_file_size.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" ) @@ -24,7 +25,24 @@ func (reader *Reader) GetFileSize(_ context.Context, location, filePath string) return 0, storageerrors.ErrorNoEndpointConfiguredForLocation } - stat, err := os.Stat(filepath.Join(location, filePath)) + basePath, err := filepath.Abs(location) + if err != nil { + return 0, fmt.Errorf("failed to resolve base path for location %s: %w", location, err) + } + + fullFilePath, err := filepath.Abs(filepath.Join(basePath, filePath)) + if err != nil { + return 0, fmt.Errorf("failed to resolve file path for %s at location %s: %w", filePath, location, err) + } + + // Ensure the resolved path is within the configured base path to prevent path traversal. + baseWithSep := basePath + string(os.PathSeparator) + fullWithSep := fullFilePath + string(os.PathSeparator) + if !strings.HasPrefix(fullWithSep, baseWithSep) { + return 0, storageerrors.ErrorFileNotFoundInLocation + } + + stat, err := os.Stat(fullFilePath) if err != nil { if errors.Is(err, os.ErrNotExist) { return 0, storageerrors.ErrorFileNotFoundInLocation diff --git a/sda/internal/storage/v2/posix/reader/new_file_reader.go b/sda/internal/storage/v2/posix/reader/new_file_reader.go index bd7792d675..f5bdd22d0a 100644 --- a/sda/internal/storage/v2/posix/reader/new_file_reader.go +++ b/sda/internal/storage/v2/posix/reader/new_file_reader.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "strings" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" ) @@ -25,7 +26,22 @@ func (reader *Reader) NewFileReader(_ context.Context, location, filePath string return nil, storageerrors.ErrorNoEndpointConfiguredForLocation } - fullFilePath := filepath.Join(location, filePath) + basePath, err := filepath.Abs(location) + if err != nil { + return nil, fmt.Errorf("failed to resolve base path for location %s: %w", location, err) + } + + fullFilePath, err := filepath.Abs(filepath.Join(basePath, filePath)) + if err != nil { + return nil, fmt.Errorf("failed to resolve file path for %s at location %s: %w", filePath, location, err) + } + + // Ensure the resolved path is within the configured base path to prevent path traversal. + baseWithSep := basePath + string(os.PathSeparator) + fullWithSep := fullFilePath + string(os.PathSeparator) + if !strings.HasPrefix(fullWithSep, baseWithSep) { + return nil, storageerrors.ErrorFileNotFoundInLocation + } if _, err := os.Stat(fullFilePath); errors.Is(err, os.ErrNotExist) { return nil, storageerrors.ErrorFileNotFoundInLocation From 0ee0554864a32ea87903c2c636a96c59c6b7f026 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 16 Feb 2026 14:36:53 +0100 Subject: [PATCH 269/530] feat(storage): remove now deprecated and unused storage v1 and its related configuration stuff in config package --- sda/internal/config/config.go | 76 ---- sda/internal/storage/storage.go | 512 --------------------------- sda/internal/storage/storage_test.go | 470 ------------------------ 3 files changed, 1058 deletions(-) delete mode 100644 sda/internal/storage/storage.go delete mode 100644 sda/internal/storage/storage_test.go diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 049e1b0ef6..c2f43f77bd 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -11,19 +11,12 @@ import ( "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" - "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" "google.golang.org/grpc/credentials" ) -const ( - POSIX = "posix" - S3 = "s3" - SFTP = "sftp" -) - var requiredConfVars []string // ServerConfig stores general server information @@ -37,7 +30,6 @@ type ServerConfig struct { // Config is a parent object for all the different configuration parts type Config struct { - Archive storage.Conf Broker broker.MQConf Database database.DBConf Server ServerConfig @@ -111,7 +103,6 @@ type APIConf struct { Session SessionConfig DB *database.SDAdb MQ *broker.AMQPBroker - INBOX storage.Backend Grpc Grpc AuditLogger *log.Logger } @@ -647,17 +638,6 @@ func (c *Config) apiDefaults() { viper.SetDefault("api.audit", true) } -// configArchive provides configuration for the archive storage -func (c *Config) configArchive() { - if viper.GetString("archive.type") == S3 { - c.Archive.Type = S3 - c.Archive.S3 = configS3Storage("archive") - } else { - c.Archive.Type = POSIX - c.Archive.Posix.Location = viper.GetString("archive.location") - } -} - // configBroker provides configuration for the message broker func (c *Config) configBroker() error { // Setup broker @@ -908,62 +888,6 @@ func (c *Config) configSchemas() { } } -// configS3Storage populates and returns a S3Conf from the -// configuration -func configS3Storage(prefix string) storage.S3Conf { - s3 := storage.S3Conf{} - // All these are required - s3.URL = viper.GetString(prefix + ".url") - s3.AccessKey = viper.GetString(prefix + ".accesskey") - s3.SecretKey = viper.GetString(prefix + ".secretkey") - s3.Bucket = viper.GetString(prefix + ".bucket") - - // Defaults (move to viper?) - - s3.Region = "us-east-1" - s3.NonExistRetryTime = 2 * time.Minute - - if viper.IsSet(prefix + ".port") { - s3.Port = viper.GetInt(prefix + ".port") - } - - if viper.IsSet(prefix + ".region") { - s3.Region = viper.GetString(prefix + ".region") - } - - if viper.IsSet(prefix + ".readypath") { - s3.Readypath = viper.GetString(prefix + ".readypath") - } - - if viper.IsSet(prefix + ".chunksize") { - s3.Chunksize = viper.GetInt(prefix+".chunksize") * 1024 * 1024 - } - - if viper.IsSet(prefix + ".cacert") { - s3.CAcert = viper.GetString(prefix + ".cacert") - } - - return s3 -} - -// configSFTP populates and returns a sftpConf with sftp backend configuration -func configSFTP(prefix string) storage.SftpConf { - sftpConf := storage.SftpConf{} - if viper.IsSet(prefix + ".sftp.hostKey") { - sftpConf.HostKey = viper.GetString(prefix + ".sftp.hostKey") - } else { - sftpConf.HostKey = "" - } - // All these are required - sftpConf.Host = viper.GetString(prefix + ".sftp.host") - sftpConf.Port = viper.GetString(prefix + ".sftp.port") - sftpConf.UserName = viper.GetString(prefix + ".sftp.userName") - sftpConf.PemKeyPath = viper.GetString(prefix + ".sftp.pemKeyPath") - sftpConf.PemKeyPass = viper.GetString(prefix + ".sftp.pemKeyPass") - - return sftpConf -} - // configNotify provides configuration for the backup storage func (c *Config) configSMTP() { c.Notify = SMTPConf{} diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go deleted file mode 100644 index cddaee0135..0000000000 --- a/sda/internal/storage/storage.go +++ /dev/null @@ -1,512 +0,0 @@ -// Package storage provides interface for storage areas, e.g. s3 or POSIX file system. -package storage - -import ( - "context" - "crypto/tls" - "crypto/x509" - "encoding/base64" - "errors" - "fmt" - "io" - "net" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" - "github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager" - "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/aws/smithy-go" - "github.com/pkg/sftp" - "golang.org/x/crypto/ssh" - - log "github.com/sirupsen/logrus" -) - -// Backend defines methods to be implemented by PosixBackend, S3Backend and sftpBackend -type Backend interface { - GetFileSize(filePath string, expectDelay bool) (int64, error) - RemoveFile(filePath string) error - NewFileReader(filePath string) (io.ReadCloser, error) - NewFileWriter(filePath string) (io.WriteCloser, error) -} - -// Conf is a wrapper for the storage config -type Conf struct { - Type string - S3 S3Conf - Posix posixConf - SFTP SftpConf -} - -type posixBackend struct { - FileReader io.Reader - FileWriter io.Writer - Location string -} - -type posixConf struct { - Location string -} - -// NewBackend initiates a storage backend -func NewBackend(conf Conf) (Backend, error) { - switch conf.Type { - case "s3": - return newS3Backend(conf.S3) - case "sftp": - return newSftpBackend(conf.SFTP) - default: - return newPosixBackend(conf.Posix) - } -} - -func newPosixBackend(conf posixConf) (*posixBackend, error) { - fileInfo, err := os.Stat(conf.Location) - - if err != nil { - return nil, err - } - - if !fileInfo.IsDir() { - return nil, fmt.Errorf("%s is not a directory", conf.Location) - } - - return &posixBackend{Location: conf.Location}, nil -} - -// NewFileReader returns an io.Reader instance -func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { - if pb == nil { - return nil, errors.New("invalid posixBackend") - } - - file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) - if err != nil { - log.Error(err) - - return nil, err - } - - return file, nil -} - -// NewFileWriter returns an io.Writer instance -func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { - if pb == nil { - return nil, errors.New("invalid posixBackend") - } - - file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) - if err != nil { - log.Error(err) - - return nil, err - } - - return file, nil -} - -// GetFileSize returns the size of the file -func (pb *posixBackend) GetFileSize(filePath string, _ bool) (int64, error) { - if pb == nil { - return 0, errors.New("invalid posixBackend") - } - - stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) - if err != nil { - log.Error(err) - - return 0, err - } - - return stat.Size(), nil -} - -// RemoveFile removes a file from a given path -func (pb *posixBackend) RemoveFile(filePath string) error { - if pb == nil { - return errors.New("invalid posixBackend") - } - - err := os.Remove(filepath.Join(filepath.Clean(pb.Location), filePath)) - if err != nil { - log.Error(err) - - return err - } - - return nil -} - -type s3Backend struct { - Client *s3.Client - Uploader *transfermanager.Client - Bucket string - Conf *S3Conf -} - -// S3Conf stores information about the S3 storage backend -type S3Conf struct { - URL string - Port int - AccessKey string // #nosec G117 -- Export needed to access configuration atm - SecretKey string - Bucket string - Region string - UploadConcurrency int - Chunksize int - CAcert string - NonExistRetryTime time.Duration - Readypath string -} - -func newS3Backend(conf S3Conf) (*s3Backend, error) { - s3Client, err := NewS3Client(conf) - if err != nil { - return nil, err - } - - sb := &s3Backend{ - Bucket: conf.Bucket, - Client: s3Client, - Conf: &conf, - Uploader: transfermanager.New(s3Client, func(u *transfermanager.Options) { - u.PartSizeBytes = int64(conf.Chunksize) - u.Concurrency = conf.UploadConcurrency - }), - } - - err = CheckS3Bucket(conf.Bucket, s3Client) - if err != nil { - return sb, err - } - - return sb, nil -} -func NewS3Client(conf S3Conf) (*s3.Client, error) { - s3cfg, err := config.LoadDefaultConfig( - context.TODO(), - config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.SecretKey, "")), - config.WithHTTPClient(&http.Client{Transport: transportConfigS3(conf)}), - ) - if err != nil { - return nil, err - } - - endpoint := conf.URL - if conf.Port != 0 { - endpoint = fmt.Sprintf("%s:%d", conf.URL, conf.Port) - } - - s3Client := s3.NewFromConfig( - s3cfg, - func(o *s3.Options) { - o.BaseEndpoint = aws.String(endpoint) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(conf.URL, "http:") - o.Region = conf.Region - o.UsePathStyle = true - o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired - o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired - }, - ) - - return s3Client, nil -} -func CheckS3Bucket(bucket string, s3Client *s3.Client) error { - _, err := s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: &bucket}) - if err != nil { - var apiErr smithy.APIError - if errors.As(err, &apiErr) { - var bae *types.BucketAlreadyExists - var baoby *types.BucketAlreadyOwnedByYou - if errors.As(err, &bae) || errors.As(err, &baoby) { - return nil - } - - return fmt.Errorf("unexpected issue while creating bucket: %s", err.Error()) - } - - return fmt.Errorf("verifying bucket failed, check S3 configuration: %s", err.Error()) - } - - return nil -} - -// NewFileReader returns an io.Reader instance -func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { - r, err := sb.Client.GetObject(context.TODO(), &s3.GetObjectInput{ - Bucket: &sb.Bucket, - Key: &filePath, - }) - - retryTime := 2 * time.Minute - if sb.Conf != nil { - retryTime = sb.Conf.NonExistRetryTime - } - - start := time.Now() - for err != nil && time.Since(start) < retryTime { - if strings.Contains(err.Error(), "NoSuchKey:") { - return nil, err - } - time.Sleep(1 * time.Second) - r, err = sb.Client.GetObject(context.TODO(), &s3.GetObjectInput{ - Bucket: &sb.Bucket, - Key: &filePath, - }) - } - - if err != nil { - return nil, err - } - - return r.Body, nil -} - -// NewFileWriter uploads the contents of an io.Reader to a S3 bucket -func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { - reader, writer := io.Pipe() - go func() { - _, err := sb.Uploader.UploadObject(context.TODO(), &transfermanager.UploadObjectInput{ - Body: reader, - Bucket: &sb.Bucket, - Key: &filePath, - ContentEncoding: aws.String("application/octet-stream"), - }) - - if err != nil { - _ = reader.CloseWithError(err) - } - }() - - return writer, nil -} - -// GetFileSize returns the size of a specific object -func (sb *s3Backend) GetFileSize(filePath string, expectDelay bool) (int64, error) { - r, err := sb.Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ - Bucket: &sb.Bucket, - Key: &filePath, - }) - - start := time.Now() - - retryTime := 2 * time.Minute - if sb.Conf != nil { - retryTime = sb.Conf.NonExistRetryTime - } - - // Retry on error up to five minutes to allow for - // "slow writes' or s3 eventual consistency - for err != nil && time.Since(start) < retryTime { - if !expectDelay && (strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:")) { - return 0, err - } - time.Sleep(1 * time.Second) - - r, err = sb.Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ - Bucket: &sb.Bucket, - Key: &filePath, - }) - } - - if err != nil { - return 0, err - } - - return *r.ContentLength, nil -} - -// RemoveFile removes an object from a bucket -func (sb *s3Backend) RemoveFile(filePath string) error { - _, err := sb.Client.DeleteObject(context.TODO(), &s3.DeleteObjectInput{ - Bucket: &sb.Bucket, - Key: &filePath, - }) - if err != nil { - return err - } - - return nil -} - -// transportConfigS3 is a helper method to setup TLS for the S3 client. -func transportConfigS3(conf S3Conf) http.RoundTripper { - cfg := new(tls.Config) - - // Read system CAs - systemCAs, err := x509.SystemCertPool() - if err != nil { - log.Errorf("failed to read system CAs: %v, using an empty pool as base", err) - systemCAs = x509.NewCertPool() - } - - cfg.RootCAs = systemCAs - - if conf.CAcert != "" { - cacert, e := os.ReadFile(conf.CAcert) // #nosec this file comes from our config - if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main - } - if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { - log.Debug("no certs appended, using system certs only") - } - } - - return &http.Transport{TLSClientConfig: cfg, ForceAttemptHTTP2: true} -} - -type sftpBackend struct { - Connection *ssh.Client - Client *sftp.Client - Conf *SftpConf -} - -// sftpConf stores information about the sftp storage backend -type SftpConf struct { - Host string - Port string - UserName string - PemKeyPath string - PemKeyPass string - HostKey string -} - -func newSftpBackend(conf SftpConf) (*sftpBackend, error) { - // read in and parse pem key - key, err := os.ReadFile(conf.PemKeyPath) - if err != nil { - return nil, fmt.Errorf("failed to read from key file, %v", err) - } - - var signer ssh.Signer - if conf.PemKeyPass == "" { - signer, err = ssh.ParsePrivateKey(key) - } else { - signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(conf.PemKeyPass)) - } - if err != nil { - return nil, fmt.Errorf("failed to parse private key, %v", err) - } - - // connect - conn, err := ssh.Dial("tcp", conf.Host+":"+conf.Port, - &ssh.ClientConfig{ - User: conf.UserName, - Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: TrustedHostKeyCallback(conf.HostKey), - }, - ) - if err != nil { - return nil, fmt.Errorf("failed to start ssh connection, %v", err) - } - - // create new SFTP client - client, err := sftp.NewClient(conn) - if err != nil { - return nil, fmt.Errorf("failed to start sftp client, %v", err) - } - - sfb := &sftpBackend{ - Connection: conn, - Client: client, - Conf: &conf, - } - - _, err = client.ReadDir("./") - - if err != nil { - return nil, fmt.Errorf("failed to list files with sftp, %v", err) - } - - return sfb, nil -} - -// NewFileWriter returns an io.Writer instance for the sftp remote -func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { - if sfb == nil { - return nil, errors.New("invalid sftpBackend") - } - // Make remote directories - parent := filepath.Dir(filePath) - err := sfb.Client.MkdirAll(parent) - if err != nil { - return nil, fmt.Errorf("failed to create dir with sftp, %v", err) - } - - file, err := sfb.Client.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_RDWR) - if err != nil { - return nil, fmt.Errorf("failed to create file with sftp, %v", err) - } - - return file, nil -} - -// GetFileSize returns the size of the file -func (sfb *sftpBackend) GetFileSize(filePath string, _ bool) (int64, error) { - if sfb == nil { - return 0, errors.New("invalid sftpBackend") - } - - stat, err := sfb.Client.Lstat(filePath) - if err != nil { - return 0, fmt.Errorf("failed to get file size with sftp, %v", err) - } - - return stat.Size(), nil -} - -// NewFileReader returns an io.Reader instance -func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { - if sfb == nil { - return nil, errors.New("invalid sftpBackend") - } - - file, err := sfb.Client.Open(filePath) - if err != nil { - return nil, fmt.Errorf("failed to open file with sftp, %v", err) - } - - return file, nil -} - -// RemoveFile removes a file or an empty directory. -func (sfb *sftpBackend) RemoveFile(filePath string) error { - if sfb == nil { - return errors.New("invalid sftpBackend") - } - - err := sfb.Client.Remove(filePath) - if err != nil { - return fmt.Errorf("failed to remove file with sftp, %v", err) - } - - return nil -} - -func TrustedHostKeyCallback(key string) ssh.HostKeyCallback { - if key == "" { - return func(_ string, _ net.Addr, k ssh.PublicKey) error { - keyString := k.Type() + " " + base64.StdEncoding.EncodeToString(k.Marshal()) - log.Warningf("host key verification is not in effect (Fix by adding trustedKey: %q)", keyString) - - return nil - } - } - - return func(_ string, _ net.Addr, k ssh.PublicKey) error { - keyString := k.Type() + " " + base64.StdEncoding.EncodeToString(k.Marshal()) - if ks := keyString; key != ks { - return fmt.Errorf("host key verification expected %q but got %q", key, ks) - } - - return nil - } -} diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go deleted file mode 100644 index c87c5d50d8..0000000000 --- a/sda/internal/storage/storage_test.go +++ /dev/null @@ -1,470 +0,0 @@ -package storage - -import ( - "bytes" - "fmt" - "io" - "net/http" - "os" - "strconv" - "testing" - "time" - - "github.com/neicnordic/sensitive-data-archive/internal/helper" - "github.com/ory/dockertest/v3" - "github.com/ory/dockertest/v3/docker" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/suite" - - log "github.com/sirupsen/logrus" -) - -type StorageTestSuite struct { - suite.Suite -} - -var testConf = Conf{} -var sshPath string -var s3Port, sftpPort int -var writeData = []byte("this is a test") - -const posixType = "posix" -const s3Type = "s3" -const sftpType = "sftp" - -func TestMain(m *testing.M) { - sshPath, _ = os.MkdirTemp("", "ssh") - if err := helper.CreateSSHKey(sshPath); err != nil { - log.Panicf("Failed to create SSH keys, reason: %v", err.Error()) - } - - // uses a sensible default on windows (tcp/http) and linux/osx (socket) - pool, err := dockertest.NewPool("") - if err != nil { - log.Panicf("Could not construct pool: %s", err) - } - - // uses pool to try to connect to Docker - err = pool.Client.Ping() - if err != nil { - log.Panicf("Could not connect to Docker: %s", err) - } - - // pulls an image, creates a container based on it and runs it - sftp, err := pool.RunWithOptions(&dockertest.RunOptions{ - Name: "sftp", - Repository: "atmoz/sftp", - Tag: "latest", - Cmd: []string{"user:test:1001::share"}, - Mounts: []string{ - fmt.Sprintf("%s/id_rsa.pub:/home/user/.ssh/keys/id_rsa.pub", sshPath), - fmt.Sprintf("%s/id_rsa:/etc/ssh/ssh_host_rsa_key", sshPath), - }, - }, func(config *docker.HostConfig) { - // set AutoRemove to true so that stopped container goes away by itself - config.AutoRemove = true - config.RestartPolicy = docker.RestartPolicy{ - Name: "no", - } - }) - if err != nil { - log.Panicf("Could not start resource: %s", err) - } - - // sftpHostAndPort := sftp.GetHostPort("22/tcp") - sftpPort, _ = strconv.Atoi(sftp.GetPort("22/tcp")) - - // pulls an image, creates a container based on it and runs it - minio, err := pool.RunWithOptions(&dockertest.RunOptions{ - Name: "s3", - Repository: "minio/minio", - Tag: "RELEASE.2023-05-18T00-05-36Z", - Cmd: []string{"server", "/data", "--console-address", ":9001"}, - Env: []string{ - "MINIO_ROOT_USER=access", - "MINIO_ROOT_PASSWORD=secretKey", - "MINIO_SERVER_URL=http://127.0.0.1:9000", - }, - }, func(config *docker.HostConfig) { - // set AutoRemove to true so that stopped container goes away by itself - config.AutoRemove = true - config.RestartPolicy = docker.RestartPolicy{ - Name: "no", - } - }) - if err != nil { - log.Panicf("Could not start resource: %s", err) - } - - s3HostAndPort := minio.GetHostPort("9000/tcp") - s3Port, _ = strconv.Atoi(minio.GetPort("9000/tcp")) - - client := http.Client{Timeout: 5 * time.Second} - req, err := http.NewRequest(http.MethodGet, "http://"+s3HostAndPort+"/minio/health/live", http.NoBody) - if err != nil { - log.Panic(err) - } - - // exponential backoff-retry, because the application in the container might not be ready to accept connections yet - if err := pool.Retry(func() error { - res, err := client.Do(req) // #nosec G704 -- request controlled by unit test - if err != nil { - return err - } - _ = res.Body.Close() - - return nil - }); err != nil { - if err := pool.Purge(minio); err != nil { - log.Panicf("Could not purge resource: %s", err) - } - log.Panicf("Could not connect to minio: %s", err) - } - - code := m.Run() - - log.Println("tests completed") - if err := pool.Purge(minio); err != nil { - log.Panicf("Could not purge resource: %s", err) - } - if err := pool.Purge(sftp); err != nil { - log.Panicf("Could not purge resource: %s", err) - } - - _ = os.RemoveAll(sshPath) - - os.Exit(code) -} - -func TestStorageTestSuite(t *testing.T) { - suite.Run(t, new(StorageTestSuite)) -} - -func (ts *StorageTestSuite) SetupTest() { - testS3Conf := S3Conf{ - "http://127.0.0.1", - s3Port, - "access", - "secretKey", - "bucket", - "region", - 10, - 5 * 1024 * 1024, - "", - 2 * time.Second, - "", - } - - testSftpConf := SftpConf{ - "localhost", - strconv.Itoa(sftpPort), - "user", - fmt.Sprintf("%s/id_rsa", sshPath), - "password", - "", - } - - testPosixConf := posixConf{ - os.TempDir(), - } - - testConf = Conf{posixType, testS3Conf, testPosixConf, testSftpConf} -} - -func (ts *StorageTestSuite) TestNewBackend() { - testConf.Type = posixType - p, err := NewBackend(testConf) - assert.NoError(ts.T(), err, "Backend posix failed") - assert.IsType(ts.T(), p, &posixBackend{}, "Wrong type from NewBackend with posix") - - var buf bytes.Buffer - log.SetOutput(&buf) - - testConf.Type = sftpType - sf, err := NewBackend(testConf) - assert.NoError(ts.T(), err, "Backend sftp failed") - assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") - assert.IsType(ts.T(), sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") - buf.Reset() - - testConf.Type = s3Type - s, err := NewBackend(testConf) - assert.NoError(ts.T(), err, "Backend s3 failed") - assert.IsType(ts.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") - - // test some extra ssl handling - testConf.S3.CAcert = "/dev/null" - s, err = NewBackend(testConf) - assert.NoError(ts.T(), err, "Backend s3 failed") - assert.IsType(ts.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") -} - -func (ts *StorageTestSuite) TestNewS3Client() { - c, err := NewS3Client(testConf.S3) - assert.NoError(ts.T(), err) - assert.NotNil(ts.T(), c) -} - -func (ts *StorageTestSuite) TestCheckS3Bucket() { - s3, err := newS3Backend(testConf.S3) - assert.NoError(ts.T(), err) - assert.NoError(ts.T(), CheckS3Bucket(testConf.S3.Bucket, s3.Client)) - - testConf.S3.URL = "file://tmp/" - bad, err := newS3Backend(testConf.S3) - assert.Error(ts.T(), err) - err = CheckS3Bucket(testConf.S3.Bucket, bad.Client) - assert.Error(ts.T(), err) -} - -func (ts *StorageTestSuite) TestPosixBackend() { - posixPath, _ := os.MkdirTemp("", "posix") - defer os.RemoveAll(posixPath) - testConf.Type = posixType - testConf.Posix = posixConf{posixPath} - backend, err := NewBackend(testConf) - assert.Nil(ts.T(), err, "POSIX backend failed unexpectedly") - - log.SetOutput(os.Stdout) - - writer, err := backend.NewFileWriter("testFile") - assert.NotNil(ts.T(), writer, "Got a nil reader for writer from posix") - assert.NoError(ts.T(), err, "posix NewFileWriter failed when it shouldn't") - - written, err := writer.Write(writeData) - assert.NoError(ts.T(), err, "Failure when writing to posix writer") - assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") - _ = writer.Close() - - reader, err := backend.NewFileReader("testFile") - assert.Nil(ts.T(), err, "posix NewFileReader failed when it should work") - assert.NotNil(ts.T(), reader, "Reader that should be usable is nosuite.T(), bailing out") - - var buf bytes.Buffer - log.SetOutput(&buf) - writer, err = backend.NewFileWriter("posix/Not/Creatable") - assert.Nil(ts.T(), writer, "Got a non-nil reader for writer from posix") - assert.Error(ts.T(), err, "posix NewFileWriter worked when it shouldn't") - assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") - buf.Reset() - log.SetOutput(os.Stdout) - - var readBackBuffer [4096]byte - readBack, err := reader.Read(readBackBuffer[0:4096]) - - assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") - assert.Nil(ts.T(), err, "unexpected error when reading back data") - - size, err := backend.GetFileSize("testFile", false) - assert.Nil(ts.T(), err, "posix NewFileReader failed when it should work") - assert.NotNil(ts.T(), size, "Got a nil size for posix") - - err = backend.RemoveFile("testFile") - assert.Nil(ts.T(), err, "posix RemoveFile failed when it should work") - - log.SetOutput(&buf) - reader, err = backend.NewFileReader("posixDoesNotExist") - assert.Error(ts.T(), err, "posix NewFileReader worked when it should not") - assert.Nil(ts.T(), reader, "Got a non-nil reader for posix") - assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") - - buf.Reset() - _, err = backend.GetFileSize("posixDoesNotExist", false) - assert.Error(ts.T(), err, "posix GetFileSize worked when it should not") - assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") - - log.SetOutput(os.Stdout) - - testConf.Posix.Location = "/thisdoesnotexist" - backEnd, err := NewBackend(testConf) - assert.NotNil(ts.T(), err, "Backend worked when it should not") - assert.Nil(ts.T(), backEnd, "Got a backend when expected not to") - - testConf.Posix.Location = "/etc/passwd" - - backEnd, err = NewBackend(testConf) - assert.NotNil(ts.T(), err, "Backend worked when it should not") - assert.Nil(ts.T(), backEnd, "Got a backend when expected not to") - - var dummyBackend *posixBackend - failReader, err := dummyBackend.NewFileReader("/") - assert.NotNil(ts.T(), err, "NewFileReader worked when it should not") - assert.Nil(ts.T(), failReader, "Got a Reader when expected not to") - - failWriter, err := dummyBackend.NewFileWriter("/") - assert.NotNil(ts.T(), err, "NewFileWriter worked when it should not") - assert.Nil(ts.T(), failWriter, "Got a Writer when expected not to") - - _, err = dummyBackend.GetFileSize("/", false) - assert.NotNil(ts.T(), err, "GetFileSize worked when it should not") - - err = dummyBackend.RemoveFile("/") - assert.NotNil(ts.T(), err, "RemoveFile worked when it should not") -} - -func (ts *StorageTestSuite) TestS3Backend() { - testConf.Type = s3Type - s3back, err := NewBackend(testConf) - assert.NoError(ts.T(), err, "Backend failed") - - writer, err := s3back.NewFileWriter("s3Creatable") - assert.NotNil(ts.T(), writer, "Got a nil reader for writer from s3") - assert.Nil(ts.T(), err, "s3 NewFileWriter failed when it shouldn't") - - written, err := writer.Write(writeData) - assert.Nil(ts.T(), err, "Failure when writing to s3 writer") - assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") - _ = writer.Close() - // sleep to allow the write to complete, otherwise the next step will fail due to timing issues. - time.Sleep(1 * time.Second) - - reader, err := s3back.NewFileReader("s3Creatable") - assert.NoError(ts.T(), err, "s3 NewFileReader failed when it should work") - assert.NotNil(ts.T(), reader, "Reader that should be usable is not, bailing out") - - size, err := s3back.GetFileSize("s3Creatable", false) - assert.Nil(ts.T(), err, "s3 GetFileSize failed when it should work") - assert.NotNil(ts.T(), size, "Got a nil size for s3") - assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") - - // make sure expectDelay=true works - // delete file and make sure the file size can not be retrieved anymore - err = s3back.RemoveFile("s3Creatable") - assert.Nil(ts.T(), err, "s3 RemoveFile failed when it should work") - _, err = s3back.GetFileSize("s3Creatable", true) - assert.NotNil(ts.T(), err, "s3 GetFileSize worked when it should not") - assert.Error(ts.T(), err) - // rewrite file, do not wait before retrieving file size - writer, err = s3back.NewFileWriter("s3Creatable") - assert.Nil(ts.T(), err, "s3 NewFileWriter failed when it shouldn't") - written, err = writer.Write(writeData) - assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") - assert.Nil(ts.T(), err, "Failure when writing to s3 writer") - _ = writer.Close() - size, err = s3back.GetFileSize("s3Creatable", true) - assert.Nil(ts.T(), err, "s3 GetFileSize with expected delay failed when it should work") - assert.NotNil(ts.T(), size, "Got a nil size for s3") - assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") - - var readBackBuffer [4096]byte - readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") - if err != nil && err != io.EOF { - assert.Nil(ts.T(), err, "unexpected error when reading back data") - } - - err = s3back.RemoveFile("s3Creatable") - assert.Nil(ts.T(), err, "s3 RemoveFile failed when it should work") - - _, err = s3back.GetFileSize("s3DoesNotExist", false) - assert.NotNil(ts.T(), err, "s3 GetFileSize worked when it should not") - assert.Error(ts.T(), err) - - reader, err = s3back.NewFileReader("s3DoesNotExist") - assert.NotNil(ts.T(), err, "s3 NewFileReader worked when it should not") - assert.Error(ts.T(), err) - assert.Nil(ts.T(), reader, "Got a non-nil reader for s3") - - testConf.S3.URL = "file://tmp/" - _, err = NewBackend(testConf) - assert.Error(ts.T(), err, "Backend worked when it should not") -} - -func (ts *StorageTestSuite) TestSftpBackend() { - var buf bytes.Buffer - log.SetOutput(&buf) - testConf.Type = sftpType - sftpBack, err := NewBackend(testConf) - assert.NoError(ts.T(), err, "Backend failed") - buf.Reset() - - var sftpDoesNotExist = "nonexistent/file" - var sftpCreatable = "/share/file/exists" - - writer, err := sftpBack.NewFileWriter(sftpCreatable) - assert.NotNil(ts.T(), writer, "Got a nil reader for writer from sftp") - assert.Nil(ts.T(), err, "sftp NewFileWriter failed when it shouldn't") - - written, err := writer.Write(writeData) - assert.Nil(ts.T(), err, "Failure when writing to sftp writer") - assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") - _ = writer.Close() - - reader, err := sftpBack.NewFileReader(sftpCreatable) - assert.Nil(ts.T(), err, "sftp NewFileReader failed when it should work") - assert.NotNil(ts.T(), reader, "Reader that should be usable is not, bailing out") - - size, err := sftpBack.GetFileSize(sftpCreatable, false) - assert.Nil(ts.T(), err, "sftp GetFileSize failed when it should work") - assert.NotNil(ts.T(), size, "Got a nil size for sftp") - assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") - - err = sftpBack.RemoveFile(sftpCreatable) - assert.Nil(ts.T(), err, "sftp RemoveFile failed when it should work") - - err = sftpBack.RemoveFile(sftpDoesNotExist) - assert.ErrorContains(ts.T(), err, "file does not exist") - - var readBackBuffer [4096]byte - readBack, err := reader.Read(readBackBuffer[0:4096]) - - assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") - - if err != nil && err != io.EOF { - assert.Nil(ts.T(), err, "unexpected error when reading back data") - } - - _, err = sftpBack.GetFileSize(sftpDoesNotExist, false) - assert.EqualError(ts.T(), err, "failed to get file size with sftp, file does not exist") - reader, err = sftpBack.NewFileReader(sftpDoesNotExist) - assert.EqualError(ts.T(), err, "failed to open file with sftp, file does not exist") - assert.Nil(ts.T(), reader, "Got a non-nil reader for sftp") - - // wrong host key - testConf.SFTP.HostKey = "wronghostkey" - _, err = NewBackend(testConf) - assert.ErrorContains(ts.T(), err, "failed to start ssh connection, ssh: handshake failed: host key verification expected") - - // wrong key password - testConf.SFTP.PemKeyPass = "wrongkey" - _, err = NewBackend(testConf) - assert.EqualError(ts.T(), err, "failed to parse private key, x509: decryption password incorrect") - - // missing key password - testConf.SFTP.PemKeyPass = "" - _, err = NewBackend(testConf) - assert.EqualError(ts.T(), err, "failed to parse private key, ssh: this private key is passphrase protected") - - // wrong key - testConf.SFTP.PemKeyPath = "nonexistentkey" - _, err = NewBackend(testConf) - assert.EqualError(ts.T(), err, "failed to read from key file, open nonexistentkey: no such file or directory") - - f, _ := os.CreateTemp(sshPath, "dummy") - testConf.SFTP.PemKeyPath = f.Name() - _, err = NewBackend(testConf) - assert.EqualError(ts.T(), err, "failed to parse private key, ssh: no key found") - - testConf.SFTP.Host = "nonexistenthost" - _, err = NewBackend(testConf) - assert.NotNil(ts.T(), err, "Backend worked when it should not") - - var dummyBackend *sftpBackend - failReader, err := dummyBackend.NewFileReader("/") - assert.NotNil(ts.T(), err, "NewFileReader worked when it should not") - assert.Nil(ts.T(), failReader, "Got a Reader when expected not to") - - failWriter, err := dummyBackend.NewFileWriter("/") - assert.NotNil(ts.T(), err, "NewFileWriter worked when it should not") - assert.Nil(ts.T(), failWriter, "Got a Writer when expected not to") - - _, err = dummyBackend.GetFileSize("/", false) - assert.NotNil(ts.T(), err, "GetFileSize worked when it should not") - - err = dummyBackend.RemoveFile("/") - assert.NotNil(ts.T(), err, "RemoveFile worked when it should not") - assert.EqualError(ts.T(), err, "invalid sftpBackend") -} From 1bb69be7c70fe419531a014c11208a93085305e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Feb 2026 13:47:08 +0100 Subject: [PATCH 270/530] feat(storage-v2): update location broker to support RegisterSizeAndCountFinderFunc which allows the registration of size and object count finder functions such that we can power the location broker from other sources than the DB --- .../v2/locationbroker/location_broker.go | 77 ++++++++++++++++--- .../v2/locationbroker/location_broker_test.go | 48 ++++++++++-- .../storage/v2/posix/writer/config.go | 6 +- .../storage/v2/posix/writer/write_file.go | 2 +- .../storage/v2/posix/writer/writer.go | 39 +++++++++- .../storage/v2/posix/writer/writer_test.go | 8 +- sda/internal/storage/v2/s3/writer/config.go | 6 +- .../storage/v2/s3/writer/remove_file.go | 2 +- .../storage/v2/s3/writer/write_file.go | 4 +- sda/internal/storage/v2/s3/writer/writer.go | 53 ++++++++++++- .../storage/v2/s3/writer/writer_test.go | 45 ++++++----- 11 files changed, 239 insertions(+), 51 deletions(-) diff --git a/sda/internal/storage/v2/locationbroker/location_broker.go b/sda/internal/storage/v2/locationbroker/location_broker.go index 1a70efd60c..c2c25b294e 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker.go +++ b/sda/internal/storage/v2/locationbroker/location_broker.go @@ -3,6 +3,7 @@ package locationbroker import ( "context" "errors" + "fmt" "sync" "time" @@ -12,18 +13,29 @@ import ( // LocationBroker is responsible for being able to serve the count of objects and the current accumulated size of all objects in a location type LocationBroker interface { // GetObjectCount returns the current amount of objects in a location - GetObjectCount(ctx context.Context, location string) (uint64, error) + GetObjectCount(ctx context.Context, backendName, location string) (uint64, error) // GetSize returns the accumulated size(in bytes) of all objects in a location - GetSize(ctx context.Context, location string) (uint64, error) + GetSize(ctx context.Context, backendName, location string) (uint64, error) + // RegisterSizeAndCountFinderFunc registers a function that returns the size and object count for a backend and location which matches + // the locationMatcher, + // for which we can not be supported by the database, i.e any other backends than "inbox", "archive", and "backup" + RegisterSizeAndCountFinderFunc(backend string, locationMatcher func(location string) bool, sizeAndCountFunc func(ctx context.Context, location string) (uint64, uint64, error)) } type locationBroker struct { - checkedLocations map[string]*locationEntry - config *config - db database.Database + checkedLocations map[string]*locationEntry + config *config + db database.Database + sizeAndCountFinders []*sizeAndCountFinder sync.Mutex } +type sizeAndCountFinder struct { + backend string + locationMatcher func(location string) bool + sizeAndCountFinderFunc func(ctx context.Context, location string) (uint64, uint64, error) +} + type locationEntry struct { lastChecked time.Time objectCount uint64 @@ -48,7 +60,14 @@ func NewLocationBroker(db database.Database, options ...func(*config)) (Location }, nil } -func (l *locationBroker) GetObjectCount(ctx context.Context, location string) (uint64, error) { +func (l *locationBroker) RegisterSizeAndCountFinderFunc(backend string, locationMatcher func(location string) bool, sizeAndCountFunc func(ctx context.Context, location string) (uint64, uint64, error)) { + l.sizeAndCountFinders = append(l.sizeAndCountFinders, &sizeAndCountFinder{ + backend: backend, + locationMatcher: locationMatcher, + sizeAndCountFinderFunc: sizeAndCountFunc, + }) +} +func (l *locationBroker) GetObjectCount(ctx context.Context, backendName, location string) (uint64, error) { l.Lock() defer l.Unlock() @@ -57,7 +76,27 @@ func (l *locationBroker) GetObjectCount(ctx context.Context, location string) (u return loc.objectCount, nil } - size, objectCount, err := l.db.GetSizeAndObjectCountOfLocation(ctx, location) + var size, objectCount uint64 + var err error + switch backendName { + case "inbox", "archive", "backup": + size, objectCount, err = l.db.GetSizeAndObjectCountOfLocation(ctx, location) + default: + var sizeAndCountFinderFunc func(ctx context.Context, location string) (uint64, uint64, error) + for _, scf := range l.sizeAndCountFinders { + if scf.backend != backendName || !scf.locationMatcher(location) { + continue + } + if sizeAndCountFinderFunc != nil { + return 0, fmt.Errorf("multiple size and count finder func matching location: %s, backend: %s", location, backendName) + } + sizeAndCountFinderFunc = scf.sizeAndCountFinderFunc + } + if sizeAndCountFinderFunc == nil { + return 0, fmt.Errorf("no size and count finder function defined for location: %s, backed %s", location, backendName) + } + size, objectCount, err = sizeAndCountFinderFunc(ctx, location) + } if err != nil { return 0, err } @@ -71,7 +110,7 @@ func (l *locationBroker) GetObjectCount(ctx context.Context, location string) (u return objectCount, nil } -func (l *locationBroker) GetSize(ctx context.Context, location string) (uint64, error) { +func (l *locationBroker) GetSize(ctx context.Context, backendName, location string) (uint64, error) { l.Lock() defer l.Unlock() @@ -80,7 +119,27 @@ func (l *locationBroker) GetSize(ctx context.Context, location string) (uint64, return loc.size, nil } - size, objectCount, err := l.db.GetSizeAndObjectCountOfLocation(ctx, location) + var size, objectCount uint64 + var err error + switch backendName { + case "inbox", "archive", "backup": + size, objectCount, err = l.db.GetSizeAndObjectCountOfLocation(ctx, location) + default: + var sizeAndCountFinderFunc func(ctx context.Context, location string) (uint64, uint64, error) + for _, scf := range l.sizeAndCountFinders { + if scf.backend != backendName || !scf.locationMatcher(location) { + continue + } + if sizeAndCountFinderFunc != nil { + return 0, fmt.Errorf("multiple size and count finder func matching location: %s, backend: %s", location, backendName) + } + sizeAndCountFinderFunc = scf.sizeAndCountFinderFunc + } + if sizeAndCountFinderFunc == nil { + return 0, fmt.Errorf("no size and count finder function defined for location: %s, backed %s", location, backendName) + } + size, objectCount, err = sizeAndCountFinderFunc(ctx, location) + } if err != nil { return 0, err } diff --git a/sda/internal/storage/v2/locationbroker/location_broker_test.go b/sda/internal/storage/v2/locationbroker/location_broker_test.go index fd554bdd8f..fadc907d61 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker_test.go +++ b/sda/internal/storage/v2/locationbroker/location_broker_test.go @@ -62,7 +62,7 @@ func (ts *LocationBrokerTestSuite) TestGetSize() { ts.FailNow(err.Error()) } - size, err := lb.GetSize(context.TODO(), "mock_location") + size, err := lb.GetSize(context.TODO(), "inbox", "mock_location") ts.NoError(err) ts.Equal(uint64(123), size) } @@ -76,7 +76,7 @@ func (ts *LocationBrokerTestSuite) TestGetObjectCount() { ts.FailNow(err.Error()) } - count, err := lb.GetObjectCount(context.TODO(), "mock_location") + count, err := lb.GetObjectCount(context.TODO(), "inbox", "mock_location") ts.NoError(err) ts.Equal(uint64(321), count) } @@ -90,11 +90,11 @@ func (ts *LocationBrokerTestSuite) TestGetObjectCount_WithCache() { ts.FailNow(err.Error()) } - countFromDB, err := lb.GetObjectCount(context.TODO(), "mock_location") + countFromDB, err := lb.GetObjectCount(context.TODO(), "inbox", "mock_location") ts.NoError(err) ts.Equal(uint64(321), countFromDB) - countFromCache, err := lb.GetObjectCount(context.TODO(), "mock_location") + countFromCache, err := lb.GetObjectCount(context.TODO(), "inbox", "mock_location") ts.NoError(err) ts.Equal(countFromDB, countFromCache) mockDb.AssertNumberOfCalls(ts.T(), "GetSizeAndObjectCountOfLocation", 1) @@ -109,12 +109,48 @@ func (ts *LocationBrokerTestSuite) TestGetSize_WithCache() { ts.FailNow(err.Error()) } - sizeFromDB, err := lb.GetObjectCount(context.TODO(), "mock_location") + sizeFromDB, err := lb.GetObjectCount(context.TODO(), "inbox", "mock_location") ts.NoError(err) ts.Equal(uint64(321), sizeFromDB) - sizeFromCache, err := lb.GetObjectCount(context.TODO(), "mock_location") + sizeFromCache, err := lb.GetObjectCount(context.TODO(), "inbox", "mock_location") ts.NoError(err) ts.Equal(sizeFromDB, sizeFromCache) mockDb.AssertNumberOfCalls(ts.T(), "GetSizeAndObjectCountOfLocation", 1) } + +func (ts *LocationBrokerTestSuite) TestGetSize_WithDefaultFinderFunc() { + mockDb := &mockDatabase{} + + lb, err := NewLocationBroker(mockDb) + if err != nil { + ts.FailNow(err.Error()) + } + lb.RegisterSizeAndCountFinderFunc("sync", func(_ string) bool { + return true + }, func(_ context.Context, _ string) (uint64, uint64, error) { + return uint64(789), uint64(987), nil + }) + + size, err := lb.GetSize(context.TODO(), "sync", "mock_location") + ts.NoError(err) + ts.Equal(uint64(789), size) +} + +func (ts *LocationBrokerTestSuite) TestGetObjectCount_WithDefaultFinderFunc() { + mockDb := &mockDatabase{} + + lb, err := NewLocationBroker(mockDb) + if err != nil { + ts.FailNow(err.Error()) + } + lb.RegisterSizeAndCountFinderFunc("sync", func(_ string) bool { + return true + }, func(_ context.Context, _ string) (uint64, uint64, error) { + return uint64(789), uint64(987), nil + }) + + size, err := lb.GetObjectCount(context.TODO(), "sync", "mock_location") + ts.NoError(err) + ts.Equal(uint64(987), size) +} diff --git a/sda/internal/storage/v2/posix/writer/config.go b/sda/internal/storage/v2/posix/writer/config.go index 0ec6c38fc3..57d1bb23a4 100644 --- a/sda/internal/storage/v2/posix/writer/config.go +++ b/sda/internal/storage/v2/posix/writer/config.go @@ -52,8 +52,8 @@ func loadConfig(backendName string) ([]*endpointConfig, error) { return endpointConf, nil } -func (endpointConf *endpointConfig) isUsable(ctx context.Context, locationBroker locationbroker.LocationBroker) (bool, error) { - count, err := locationBroker.GetObjectCount(ctx, endpointConf.Path) +func (endpointConf *endpointConfig) isUsable(ctx context.Context, backendName string, locationBroker locationbroker.LocationBroker) (bool, error) { + count, err := locationBroker.GetObjectCount(ctx, backendName, endpointConf.Path) if err != nil { return false, err } @@ -61,7 +61,7 @@ func (endpointConf *endpointConfig) isUsable(ctx context.Context, locationBroker return false, nil } - size, err := locationBroker.GetSize(ctx, endpointConf.Path) + size, err := locationBroker.GetSize(ctx, backendName, endpointConf.Path) if err != nil { return false, err } diff --git a/sda/internal/storage/v2/posix/writer/write_file.go b/sda/internal/storage/v2/posix/writer/write_file.go index 2bbec37d02..3e1ba6f860 100644 --- a/sda/internal/storage/v2/posix/writer/write_file.go +++ b/sda/internal/storage/v2/posix/writer/write_file.go @@ -23,7 +23,7 @@ func (writer *Writer) WriteFile(ctx context.Context, filePath string, fileConten return "", storageerrors.ErrorNoValidLocations } - usable, err := writer.activeEndpoints[0].isUsable(ctx, writer.locationBroker) + usable, err := writer.activeEndpoints[0].isUsable(ctx, writer.backendName, writer.locationBroker) if err != nil { return "", fmt.Errorf("failed to check if location: %s is usable: %v", writer.activeEndpoints[0].Path, err) } diff --git a/sda/internal/storage/v2/posix/writer/writer.go b/sda/internal/storage/v2/posix/writer/writer.go index 1f320ef27a..386cefb521 100644 --- a/sda/internal/storage/v2/posix/writer/writer.go +++ b/sda/internal/storage/v2/posix/writer/writer.go @@ -4,7 +4,10 @@ import ( "context" "errors" "fmt" + "io/fs" "os" + "path/filepath" + "strings" "sync" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" @@ -13,6 +16,7 @@ import ( ) type Writer struct { + backendName string configuredEndpoints []*endpointConfig activeEndpoints []*endpointConfig locationBroker locationbroker.LocationBroker @@ -31,8 +35,12 @@ func NewWriter(ctx context.Context, backendName string, locationBroker locationb } writer := &Writer{ + backendName: backendName, locationBroker: locationBroker, } + writer.locationBroker.RegisterSizeAndCountFinderFunc(backendName, func(location string) bool { + return !strings.HasPrefix(location, "/") + }, findSizeAndObjectCountInDir) // Verify locations for _, endpointConf := range endPoints { @@ -48,7 +56,7 @@ func NewWriter(ctx context.Context, backendName string, locationBroker locationb writer.configuredEndpoints = append(writer.configuredEndpoints, endpointConf) - usable, err := endpointConf.isUsable(ctx, writer.locationBroker) + usable, err := endpointConf.isUsable(ctx, backendName, writer.locationBroker) if err != nil { return nil, err } @@ -67,3 +75,32 @@ func NewWriter(ctx context.Context, backendName string, locationBroker locationb return writer, nil } + +// findSizeAndObjectCountInDir find the total size and total amount of objects in an directory if we do not store +// this information in the database +func findSizeAndObjectCountInDir(_ context.Context, dir string) (uint64, uint64, error) { + totalObjects := uint64(0) + totalSize := uint64(0) + + if err := filepath.Walk(dir, func(_ string, info fs.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + + fileSize := info.Size() + if fileSize < 0 { + return fmt.Errorf("file: %s has negative size", info.Name()) + } + totalSize += uint64(fileSize) + totalObjects++ + + return nil + }); err != nil { + return 0, 0, err + } + + return totalSize, totalObjects, nil +} diff --git a/sda/internal/storage/v2/posix/writer/writer_test.go b/sda/internal/storage/v2/posix/writer/writer_test.go index a01218b9e5..f8653eda68 100644 --- a/sda/internal/storage/v2/posix/writer/writer_test.go +++ b/sda/internal/storage/v2/posix/writer/writer_test.go @@ -32,7 +32,7 @@ type mockLocationBroker struct { mock.Mock } -func (m *mockLocationBroker) GetObjectCount(_ context.Context, location string) (uint64, error) { +func (m *mockLocationBroker) GetObjectCount(_ context.Context, _, location string) (uint64, error) { args := m.Called(location) count := args.Int(0) if count < 0 { @@ -42,7 +42,7 @@ func (m *mockLocationBroker) GetObjectCount(_ context.Context, location string) return uint64(count), args.Error(1) } -func (m *mockLocationBroker) GetSize(_ context.Context, location string) (uint64, error) { +func (m *mockLocationBroker) GetSize(_ context.Context, _, location string) (uint64, error) { args := m.Called(location) size := args.Int(0) if size < 0 { @@ -51,6 +51,9 @@ func (m *mockLocationBroker) GetSize(_ context.Context, location string) (uint64 //nolint:gosec // disable G115 return uint64(size), args.Error(1) } +func (m *mockLocationBroker) RegisterSizeAndCountFinderFunc(_ string, _ func(string) bool, _ func(context.Context, string) (uint64, uint64, error)) { + _ = m.Called() +} func TestWriterTestSuite(t *testing.T) { suite.Run(t, new(WriterTestSuite)) @@ -88,6 +91,7 @@ storage: ts.locationBrokerMock.On("GetSize", ts.dir1).Return(0, nil).Once() ts.locationBrokerMock.On("GetObjectCount", ts.dir2).Return(0, nil).Once() ts.locationBrokerMock.On("GetSize", ts.dir2).Return(0, nil).Once() + ts.locationBrokerMock.On("RegisterSizeAndCountFinderFunc").Return().Once() ts.writer, err = NewWriter(context.TODO(), "test", ts.locationBrokerMock) if err != nil { ts.FailNow(err.Error()) diff --git a/sda/internal/storage/v2/s3/writer/config.go b/sda/internal/storage/v2/s3/writer/config.go index d0ef2cef12..40e483d0c2 100644 --- a/sda/internal/storage/v2/s3/writer/config.go +++ b/sda/internal/storage/v2/s3/writer/config.go @@ -180,7 +180,7 @@ func (endpointConf *endpointConfig) transportConfigS3() (http.RoundTripper, erro }, nil } -func (endpointConf *endpointConfig) findActiveBucket(ctx context.Context, locationBroker locationbroker.LocationBroker) (string, error) { +func (endpointConf *endpointConfig) findActiveBucket(ctx context.Context, backendName string, locationBroker locationbroker.LocationBroker) (string, error) { client, err := endpointConf.getS3Client(ctx) if err != nil { return "", fmt.Errorf("failed to create S3 client to endpoint: %s, due to %v", endpointConf.Endpoint, err) @@ -213,7 +213,7 @@ func (endpointConf *endpointConfig) findActiveBucket(ctx context.Context, locati // find first bucket with available object count and size for _, bucket := range bucketsWithPrefix { loc := endpointConf.Endpoint + "/" + bucket - count, err := locationBroker.GetObjectCount(ctx, loc) + count, err := locationBroker.GetObjectCount(ctx, backendName, loc) if err != nil { return "", fmt.Errorf("failed to get object count of location %s, due to %v", loc, err) } @@ -221,7 +221,7 @@ func (endpointConf *endpointConfig) findActiveBucket(ctx context.Context, locati continue } - size, err := locationBroker.GetSize(ctx, loc) + size, err := locationBroker.GetSize(ctx, backendName, loc) if err != nil { return "", fmt.Errorf("failed to get size of location %s, due to %v", loc, err) } diff --git a/sda/internal/storage/v2/s3/writer/remove_file.go b/sda/internal/storage/v2/s3/writer/remove_file.go index f7ac63b2e6..77646857d8 100644 --- a/sda/internal/storage/v2/s3/writer/remove_file.go +++ b/sda/internal/storage/v2/s3/writer/remove_file.go @@ -15,7 +15,7 @@ func (writer *Writer) RemoveFile(ctx context.Context, location, filePath string) return err } - client, err := writer.getS3ClientForEndpoint(ctx, endpoint) + client, err := getS3ClientForEndpoint(ctx, writer.configuredEndpoints, endpoint) if err != nil { return err } diff --git a/sda/internal/storage/v2/s3/writer/write_file.go b/sda/internal/storage/v2/s3/writer/write_file.go index c0c94fa1cf..bbe06eebf0 100644 --- a/sda/internal/storage/v2/s3/writer/write_file.go +++ b/sda/internal/storage/v2/s3/writer/write_file.go @@ -14,7 +14,7 @@ import ( func (writer *Writer) WriteFile(ctx context.Context, filePath string, fileContent io.Reader) (string, error) { // Find endpoint / bucket that is to be used for writing writer.Lock() - activeBucket, err := writer.activeEndpoint.findActiveBucket(ctx, writer.locationBroker) + activeBucket, err := writer.activeEndpoint.findActiveBucket(ctx, writer.backendName, writer.locationBroker) if err != nil && !errors.Is(err, storageerrors.ErrorNoFreeBucket) { writer.Unlock() @@ -28,7 +28,7 @@ func (writer *Writer) WriteFile(ctx context.Context, filePath string, fileConten continue } - activeBucket, err = endpointConf.findActiveBucket(ctx, writer.locationBroker) + activeBucket, err = endpointConf.findActiveBucket(ctx, writer.backendName, writer.locationBroker) if err != nil { if errors.Is(err, storageerrors.ErrorNoFreeBucket) { continue diff --git a/sda/internal/storage/v2/s3/writer/writer.go b/sda/internal/storage/v2/s3/writer/writer.go index 2c80b97d00..37dc33c5ad 100644 --- a/sda/internal/storage/v2/s3/writer/writer.go +++ b/sda/internal/storage/v2/s3/writer/writer.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" @@ -16,6 +17,7 @@ import ( ) type Writer struct { + backendName string configuredEndpoints []*endpointConfig activeEndpoint *endpointConfig @@ -36,12 +38,16 @@ func NewWriter(ctx context.Context, backendName string, locationBroker locationb } writer := &Writer{ + backendName: backendName, locationBroker: locationBroker, } + writer.locationBroker.RegisterSizeAndCountFinderFunc(backendName, func(location string) bool { + return !strings.HasPrefix(location, "/") + }, findSizeAndObjectCountOfLocation(endPointConf)) // Verify endpointConfig connections for _, e := range endPointConf { - _, err := e.findActiveBucket(ctx, writer.locationBroker) + _, err := e.findActiveBucket(ctx, backendName, writer.locationBroker) if err != nil { if errors.Is(err, storageerrors.ErrorNoFreeBucket) { log.Warningf("s3: %s has no available bucket", e.Endpoint) @@ -65,8 +71,8 @@ func NewWriter(ctx context.Context, backendName string, locationBroker locationb return writer, nil } -func (writer *Writer) getS3ClientForEndpoint(ctx context.Context, endpoint string) (*s3.Client, error) { - for _, e := range writer.configuredEndpoints { +func getS3ClientForEndpoint(ctx context.Context, configuredEndpoints []*endpointConfig, endpoint string) (*s3.Client, error) { + for _, e := range configuredEndpoints { if e.Endpoint != endpoint { continue } @@ -100,3 +106,44 @@ func parseLocation(location string) (string, string, error) { return endpoint, bucketName, nil } + +// findSizeAndObjectCountOfLocation find the total size and total amount of objects in a s3 bucket if we do not store +// this information in the database +func findSizeAndObjectCountOfLocation(configuredEndpoints []*endpointConfig) func(ctx context.Context, location string) (uint64, uint64, error) { + return func(ctx context.Context, location string) (uint64, uint64, error) { + endpoint, bucket, err := parseLocation(location) + if err != nil { + return 0, 0, err + } + + s3Client, err := getS3ClientForEndpoint(ctx, configuredEndpoints, endpoint) + if err != nil { + return 0, 0, err + } + + var totalSize uint64 + var totalObjects uint64 + + // Create paginator + paginator := s3.NewListObjectsV2Paginator(s3Client, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + }) + + // Iterate through all pages + for paginator.HasMorePages() { + page, err := paginator.NextPage(ctx) + if err != nil { + return 0, 0, err + } + + for _, obj := range page.Contents { + totalObjects++ + if obj.Size != nil && *obj.Size > 0 { + totalSize += uint64(*obj.Size) + } + } + } + + return totalSize, totalObjects, nil + } +} diff --git a/sda/internal/storage/v2/s3/writer/writer_test.go b/sda/internal/storage/v2/s3/writer/writer_test.go index 1078616307..aad6404ac3 100644 --- a/sda/internal/storage/v2/s3/writer/writer_test.go +++ b/sda/internal/storage/v2/s3/writer/writer_test.go @@ -31,6 +31,30 @@ type WriterTestSuite struct { type mockLocationBroker struct { mock.Mock } + +func (m *mockLocationBroker) GetObjectCount(_ context.Context, _, location string) (uint64, error) { + args := m.Called(location) + count := args.Int(0) + if count < 0 { + count = 0 + } + //nolint:gosec // disable G115 + return uint64(count), args.Error(1) +} + +func (m *mockLocationBroker) GetSize(_ context.Context, _, location string) (uint64, error) { + args := m.Called(location) + size := args.Int(0) + if size < 0 { + size = 0 + } + //nolint:gosec // disable G115 + return uint64(size), args.Error(1) +} +func (m *mockLocationBroker) RegisterSizeAndCountFinderFunc(_ string, _ func(string) bool, _ func(context.Context, string) (uint64, uint64, error)) { + _ = m.Called() +} + type mockS3 struct { server *httptest.Server buckets map[string]map[string]string // "bucket name" -> "file name" -> "content" @@ -118,26 +142,6 @@ func (m *mockS3) CreateBucket(w http.ResponseWriter, req *http.Request) { w.WriteHeader(http.StatusOK) } -func (m *mockLocationBroker) GetObjectCount(_ context.Context, location string) (uint64, error) { - args := m.Called(location) - count := args.Int(0) - if count < 0 { - count = 0 - } - //nolint:gosec // disable G115 - return uint64(count), args.Error(1) -} - -func (m *mockLocationBroker) GetSize(_ context.Context, location string) (uint64, error) { - args := m.Called(location) - size := args.Int(0) - if size < 0 { - size = 0 - } - //nolint:gosec // disable G115 - return uint64(size), args.Error(1) -} - func TestReaderTestSuite(t *testing.T) { suite.Run(t, new(WriterTestSuite)) } @@ -193,6 +197,7 @@ func (ts *WriterTestSuite) SetupTest() { ts.locationBrokerMock = &mockLocationBroker{} var err error + ts.locationBrokerMock.On("RegisterSizeAndCountFinderFunc").Return().Once() ts.writer, err = NewWriter(context.TODO(), "test", ts.locationBrokerMock) if err != nil { ts.FailNow(err.Error()) From a87808d7f87c7cd70b89b70b6edc908f038cf936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 16 Feb 2026 10:47:56 +0100 Subject: [PATCH 271/530] feat(integration tests): set max_objects on sync storage to ensure it uses the s3 registered sizeAndCountFinderFunc --- .github/integration/sda/config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 90d7f221f6..74a8818c55 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -39,6 +39,8 @@ storage: bucket_prefix: "sync" region: "us-east-1" disable_https: true + max_buckets: 10 + max_objects: 10 s3inbox: endpoint: "http://s3:9000" From b6056143d8ed40026abf66da14ed2e7711c28167 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 16 Feb 2026 10:48:49 +0100 Subject: [PATCH 272/530] feat(storage-v2): fix posix writer sizeAndCountFinderFunc registration location matcher func --- sda/internal/storage/v2/posix/writer/writer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/storage/v2/posix/writer/writer.go b/sda/internal/storage/v2/posix/writer/writer.go index 386cefb521..85dbf0b6cb 100644 --- a/sda/internal/storage/v2/posix/writer/writer.go +++ b/sda/internal/storage/v2/posix/writer/writer.go @@ -39,7 +39,7 @@ func NewWriter(ctx context.Context, backendName string, locationBroker locationb locationBroker: locationBroker, } writer.locationBroker.RegisterSizeAndCountFinderFunc(backendName, func(location string) bool { - return !strings.HasPrefix(location, "/") + return strings.HasPrefix(location, "/") }, findSizeAndObjectCountInDir) // Verify locations From f76db9fbf0c1cf03ad7599c7a07685e49408c042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 16 Feb 2026 15:30:03 +0100 Subject: [PATCH 273/530] feat(storage-v2): add unit test that uses the s3 and posix default size and object count finder funcs --- .../storage/v2/posix/writer/writer_test.go | 66 +++++++++++++ .../storage/v2/s3/writer/writer_test.go | 97 +++++++++++++++++++ 2 files changed, 163 insertions(+) diff --git a/sda/internal/storage/v2/posix/writer/writer_test.go b/sda/internal/storage/v2/posix/writer/writer_test.go index f8653eda68..db51d84860 100644 --- a/sda/internal/storage/v2/posix/writer/writer_test.go +++ b/sda/internal/storage/v2/posix/writer/writer_test.go @@ -12,6 +12,8 @@ import ( "strings" "testing" + "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" "github.com/spf13/viper" "github.com/stretchr/testify/mock" @@ -315,3 +317,67 @@ func (ts *WriterTestSuite) TestRemoveFile_LocationNotConfigured() { err := ts.writer.RemoveFile(context.TODO(), "/tmp/no_access_here", "test_file_1.txt") ts.EqualError(err, storageerrors.ErrorNoEndpointConfiguredForLocation.Error()) } + +func (ts *WriterTestSuite) TestWriteFile_NoMockLocationBroker_FirstDirFull() { + content := "test file 2" + lb, err := locationbroker.NewLocationBroker(¬ImplementedDatabase{}, locationbroker.CacheTTL(0)) + ts.NoError(err) + + writer, err := NewWriter(context.TODO(), "test", lb) + if err != nil { + ts.FailNow(err.Error()) + } + + for i := 0; i < 10; i++ { + if err := os.WriteFile(filepath.Join(ts.dir1, fmt.Sprintf("file-%d.txt", i)), []byte(fmt.Sprintf("file content %d", i)), 0600); err != nil { + ts.FailNow(err.Error()) + } + } + + contentReader := bytes.NewReader([]byte(content)) + location, err := writer.WriteFile(context.TODO(), "test_file_1.txt", contentReader) + if err != nil { + ts.FailNow(err.Error()) + } + + readContent, err := os.ReadFile(filepath.Join(ts.dir2, "test_file_1.txt")) + ts.NoError(err) + ts.Equal(content, string(readContent)) + ts.Equal(ts.dir2, location) + // Ensure location/tmp folder empty + tmpFiles, err := os.ReadDir(filepath.Join(ts.dir2, "tmp")) + ts.NoError(err) + ts.Len(tmpFiles, 0) +} + +type notImplementedDatabase struct { +} + +func (m *notImplementedDatabase) GetSizeAndObjectCountOfLocation(_ context.Context, _ string) (uint64, uint64, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) GetUploadedSubmissionFilePathAndLocation(_ context.Context, _, _ string) (string, string, error) { + panic("function not expected to be called in unit tests") +} +func (m *notImplementedDatabase) GetArchiveLocation(_ string) (string, error) { + panic("function not expected to be called in unit tests") +} +func (m *notImplementedDatabase) GetArchivePathAndLocation(_ string) (string, string, error) { + panic("function not expected to be called in unit tests") +} +func (m *notImplementedDatabase) GetMappingData(_ string) (*database.MappingData, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) GetSubmissionLocation(_ context.Context, _ string) (string, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) IsFileInDataset(_ context.Context, _ string) (bool, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) CancelFile(_ context.Context, _, _ string) error { + panic("function not expected to be called in unit tests") +} diff --git a/sda/internal/storage/v2/s3/writer/writer_test.go b/sda/internal/storage/v2/s3/writer/writer_test.go index aad6404ac3..0b1a6e3dcd 100644 --- a/sda/internal/storage/v2/s3/writer/writer_test.go +++ b/sda/internal/storage/v2/s3/writer/writer_test.go @@ -12,6 +12,8 @@ import ( "strings" "testing" + "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/locationbroker" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" "github.com/spf13/viper" "github.com/stretchr/testify/mock" @@ -70,10 +72,40 @@ func (m *mockS3) handler(w http.ResponseWriter, req *http.Request) { m.CreateBucket(w, req) case req.Method == "DELETE": m.Delete(w, req) + case strings.HasSuffix(req.RequestURI, "list-type=2"): + m.ListFiles(w, req) default: w.WriteHeader(http.StatusNotImplemented) } } +func (m *mockS3) ListFiles(w http.ResponseWriter, req *http.Request) { + bucket := strings.Split(strings.Split(req.RequestURI, "/")[1], "?")[0] + + var b strings.Builder + _, _ = b.WriteString(` + + + false`) + + for buk, files := range m.buckets { + if buk != bucket { + continue + } + + for fileName, fileContent := range files { + _, _ = b.WriteString(fmt.Sprintf(` + + %d + %s + `, len(fileContent), fileName)) + } + } + + _, _ = b.WriteString(` + +`) + _, _ = w.Write([]byte(b.String())) +} func (m *mockS3) ListBuckets(w http.ResponseWriter) { w.WriteHeader(http.StatusOK) @@ -298,3 +330,68 @@ func (ts *WriterTestSuite) TestRemoveFile_InvalidLocation() { err := ts.writer.RemoveFile(context.TODO(), "http://different_s3_url/bucket", "file_to_be_removed") ts.EqualError(err, storageerrors.ErrorNoEndpointConfiguredForLocation.Error()) } + +func (ts *WriterTestSuite) TestWriteFile_NoMockLocationBroker_FirstBucketFull() { + content := "test file 2" + lb, err := locationbroker.NewLocationBroker(¬ImplementedDatabase{}, locationbroker.CacheTTL(0)) + ts.NoError(err) + + ts.s3Mock1.buckets["bucket_in_1-1"] = map[string]string{ + "test_file_1.txt": content, + "test_file_2.txt": content, + "test_file_3.txt": content, + "test_file_4.txt": content, + "test_file_5.txt": content, + "test_file_6.txt": content, + "test_file_7.txt": content, + "test_file_8.txt": content, + "test_file_9.txt": content, + "test_file_10.txt": content, + } + + writer, err := NewWriter(context.TODO(), "test", lb) + if err != nil { + ts.FailNow(err.Error()) + } + + contentReader := bytes.NewReader([]byte(content)) + location, err := writer.WriteFile(context.TODO(), "test_file_1.txt", contentReader) + if err != nil { + ts.FailNow(err.Error()) + } + + ts.Equal(ts.s3Mock1.buckets["bucket_in_1-2"]["test_file_1.txt"], content) + ts.Equal(fmt.Sprintf("%s/bucket_in_1-2", ts.s3Mock1.server.URL), location) +} + +type notImplementedDatabase struct { +} + +func (m *notImplementedDatabase) GetSizeAndObjectCountOfLocation(_ context.Context, _ string) (uint64, uint64, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) GetUploadedSubmissionFilePathAndLocation(_ context.Context, _, _ string) (string, string, error) { + panic("function not expected to be called in unit tests") +} +func (m *notImplementedDatabase) GetArchiveLocation(_ string) (string, error) { + panic("function not expected to be called in unit tests") +} +func (m *notImplementedDatabase) GetArchivePathAndLocation(_ string) (string, string, error) { + panic("function not expected to be called in unit tests") +} +func (m *notImplementedDatabase) GetMappingData(_ string) (*database.MappingData, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) GetSubmissionLocation(_ context.Context, _ string) (string, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) IsFileInDataset(_ context.Context, _ string) (bool, error) { + panic("function not expected to be called in unit tests") +} + +func (m *notImplementedDatabase) CancelFile(_ context.Context, _, _ string) error { + panic("function not expected to be called in unit tests") +} From 71db604dc5c769c6a4f56157968e9b04d7b04838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 17 Feb 2026 08:57:31 +0100 Subject: [PATCH 274/530] feat(storage-v2): remove redundant comments --- sda/internal/storage/v2/s3/writer/writer.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/sda/internal/storage/v2/s3/writer/writer.go b/sda/internal/storage/v2/s3/writer/writer.go index 37dc33c5ad..18840a86ab 100644 --- a/sda/internal/storage/v2/s3/writer/writer.go +++ b/sda/internal/storage/v2/s3/writer/writer.go @@ -124,12 +124,10 @@ func findSizeAndObjectCountOfLocation(configuredEndpoints []*endpointConfig) fun var totalSize uint64 var totalObjects uint64 - // Create paginator paginator := s3.NewListObjectsV2Paginator(s3Client, &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), }) - // Iterate through all pages for paginator.HasMorePages() { page, err := paginator.NextPage(ctx) if err != nil { From 74451b5ac4aadda8ec05d23c2ad84b6a767f99aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 18 Feb 2026 10:25:13 +0100 Subject: [PATCH 275/530] feat(storage-v2): update s3 config to require scheme in endpoint --- sda/internal/storage/v2/s3/reader/config.go | 4 ++++ .../storage/v2/s3/reader/reader_test.go | 20 ++++++++++++++++--- sda/internal/storage/v2/s3/writer/config.go | 5 +++++ 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/sda/internal/storage/v2/s3/reader/config.go b/sda/internal/storage/v2/s3/reader/config.go index 0bc206bd5a..a49be56f1d 100644 --- a/sda/internal/storage/v2/s3/reader/config.go +++ b/sda/internal/storage/v2/s3/reader/config.go @@ -65,6 +65,10 @@ func loadConfig(backendName string) ([]*endpointConfig, error) { default: } + if !strings.HasPrefix(e.Endpoint, "https:") && !strings.HasPrefix(e.Endpoint, "http:") { + return nil, errors.New("unsupported or no scheme in endpoint") + } + e.chunkSizeBytes = 50 * 1024 * 1024 if e.ChunkSize != "" { s, err := datasize.ParseString(e.ChunkSize) diff --git a/sda/internal/storage/v2/s3/reader/reader_test.go b/sda/internal/storage/v2/s3/reader/reader_test.go index d228b4e8ed..eb49450c20 100644 --- a/sda/internal/storage/v2/s3/reader/reader_test.go +++ b/sda/internal/storage/v2/s3/reader/reader_test.go @@ -289,12 +289,26 @@ storage: `, expectedErrorMsg: "missing required parameter: endpoint", }, { - testName: "MissingAccessKey", + testName: "MissingEndpointScheme", config: ` storage: config_test: s3: - endpoint: 123 + access_key: access_key1 + secret_key: secret_key1 + disable_https: true + region: us-east-1 + chunk_size: 1mb +`, + expectedErrorMsg: "unsupported or no scheme in endpoint", + }, { + testName: "MissingAccessKey", + config: ` +storage: + config_test: + s3: + - endpoint: http://123 secret_key: secret_key1 disable_https: true region: us-east-1 @@ -307,7 +321,7 @@ storage: storage: config_test: s3: - - endpoint: 123 + - endpoint: http://123 access_key: access_key1 disable_https: true region: us-east-1 @@ -320,7 +334,7 @@ storage: storage: config_test: s3: - - endpoint: 123 + - endpoint: http://123 access_key: access_key1 secret_key: secret_key1 disable_https: true diff --git a/sda/internal/storage/v2/s3/writer/config.go b/sda/internal/storage/v2/s3/writer/config.go index 40e483d0c2..39275fcc4b 100644 --- a/sda/internal/storage/v2/s3/writer/config.go +++ b/sda/internal/storage/v2/s3/writer/config.go @@ -78,6 +78,11 @@ func loadConfig(backendName string) ([]*endpointConfig, error) { return nil, errors.New("https scheme in endpoint when HTTPS is disabled") default: } + + if !strings.HasPrefix(e.Endpoint, "https:") && !strings.HasPrefix(e.Endpoint, "http:") { + return nil, errors.New("unsupported or no scheme in endpoint") + } + if e.ChunkSize != "" { byteSize, err := datasize.ParseString(e.ChunkSize) if err != nil { From 7bc6eceb8271f6f3d6595bb8201f8932e51918c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 19 Feb 2026 10:33:04 +0100 Subject: [PATCH 276/530] fix: linter issues, add suppression of gosec and add reasoning, fix QF1012 in s3 writer test --- sda-download/internal/storage/v2/s3/reader/config.go | 2 +- .../internal/storage/v2/s3/reader/reader_test.go | 12 ++++++------ sda/cmd/s3inbox/s3inbox.go | 2 +- sda/internal/config/config.go | 2 +- sda/internal/storage/v2/s3/writer/writer.go | 2 +- sda/internal/storage/v2/s3/writer/writer_test.go | 9 +++++---- 6 files changed, 15 insertions(+), 14 deletions(-) diff --git a/sda-download/internal/storage/v2/s3/reader/config.go b/sda-download/internal/storage/v2/s3/reader/config.go index d80e0ed8c3..0bc206bd5a 100644 --- a/sda-download/internal/storage/v2/s3/reader/config.go +++ b/sda-download/internal/storage/v2/s3/reader/config.go @@ -21,7 +21,7 @@ import ( ) type endpointConfig struct { - AccessKey string `mapstructure:"access_key"` + AccessKey string `mapstructure:"access_key"` // #nosec G117 -- needs to be exported for unmarshalling CACert string `mapstructure:"ca_cert"` ChunkSize string `mapstructure:"chunk_size"` chunkSizeBytes uint64 diff --git a/sda-download/internal/storage/v2/s3/reader/reader_test.go b/sda-download/internal/storage/v2/s3/reader/reader_test.go index cf726923ed..9660bacea9 100644 --- a/sda-download/internal/storage/v2/s3/reader/reader_test.go +++ b/sda-download/internal/storage/v2/s3/reader/reader_test.go @@ -93,7 +93,7 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test case strings.HasPrefix(req.RequestURI, "/mock_s3_1_bucket_2"): if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file2.txt") { @@ -140,10 +140,10 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test default: w.WriteHeader(http.StatusInternalServerError) - _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test } })) @@ -213,7 +213,7 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test case strings.HasPrefix(req.RequestURI, "/mock_s3_2_bucket_2"): if req.Method == "GET" && strings.HasSuffix(req.RequestURI, "GetObject") && strings.Contains(req.RequestURI, "file4.txt") { w.WriteHeader(http.StatusOK) @@ -230,10 +230,10 @@ func (ts *ReaderTestSuite) SetupSuite() { } w.WriteHeader(http.StatusNotFound) - _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "no file found at: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test default: w.WriteHeader(http.StatusInternalServerError) - _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) + _, _ = fmt.Fprintf(w, "unexpected path called: %s", req.RequestURI) // #nosec G705 -- request controlled by unit test } })) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index f8b20bc0b3..81487383ce 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -171,7 +171,7 @@ func configTLS(c config.S3InboxConf) (*tls.Config, error) { cfg.RootCAs = systemCAs if c.CaCert != "" { - caCert, e := os.ReadFile(c.CaCert) + caCert, e := os.ReadFile(c.CaCert) // #nosec G703 -- file path controlled by configuration if e != nil { return nil, fmt.Errorf("failed to append %q to RootCAs: %v", c.CaCert, e) } diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index c2f43f77bd..c795ad00fa 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -86,7 +86,7 @@ type SyncAPIConf struct { type S3InboxConf struct { Endpoint string `mapstructure:"endpoint"` - AccessKey string `mapstructure:"access_key"` + AccessKey string `mapstructure:"access_key"` // #nosec G117 -- Export needed for configuration access SecretKey string `mapstructure:"secret_key"` Bucket string `mapstructure:"bucket"` Region string `mapstructure:"region"` diff --git a/sda/internal/storage/v2/s3/writer/writer.go b/sda/internal/storage/v2/s3/writer/writer.go index 18840a86ab..8018db3dae 100644 --- a/sda/internal/storage/v2/s3/writer/writer.go +++ b/sda/internal/storage/v2/s3/writer/writer.go @@ -137,7 +137,7 @@ func findSizeAndObjectCountOfLocation(configuredEndpoints []*endpointConfig) fun for _, obj := range page.Contents { totalObjects++ if obj.Size != nil && *obj.Size > 0 { - totalSize += uint64(*obj.Size) + totalSize += uint64(*obj.Size) // #nosec G115 -- *obj.Size has been checked to be bigger than 0 } } } diff --git a/sda/internal/storage/v2/s3/writer/writer_test.go b/sda/internal/storage/v2/s3/writer/writer_test.go index 0b1a6e3dcd..eeb5ad96e4 100644 --- a/sda/internal/storage/v2/s3/writer/writer_test.go +++ b/sda/internal/storage/v2/s3/writer/writer_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" @@ -93,11 +94,11 @@ func (m *mockS3) ListFiles(w http.ResponseWriter, req *http.Request) { } for fileName, fileContent := range files { - _, _ = b.WriteString(fmt.Sprintf(` + _, _ = b.WriteString(` - %d - %s - `, len(fileContent), fileName)) + ` + strconv.Itoa(len(fileContent)) + ` + ` + fileName + ` + `) } } From 38da649e52de1d9f913d944405083f66410fd216 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 09:22:25 +0100 Subject: [PATCH 277/530] fix(verify): add missing return after db error on GetArchiveLocation --- sda/cmd/verify/verify.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 8c0fc73cd4..75b6a30be0 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -215,6 +215,8 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } + + return } if archiveLocation == "" { log.Errorf("archive location for file: %s, not known in database", message.FileID) From 75ec23db5959f9ec4fd8a54fdefb664c1e31f399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 09:30:26 +0100 Subject: [PATCH 278/530] fix(s3inbox): fix checkS3Bucket checking correct errors incase bucket already exists, etc --- sda/cmd/s3inbox/s3inbox.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 81487383ce..6f37bfc02a 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -13,6 +13,7 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" "github.com/gorilla/mux" @@ -145,11 +146,13 @@ func checkS3Bucket(ctx context.Context, s3Client *s3.Client, bucket string) erro if err != nil { var apiErr smithy.APIError if errors.As(err, &apiErr) { - if apiErr.ErrorCode() == "NotFound" { - return fmt.Errorf("bucket: %s does not exist at the configured s3 endpoint", bucket) + var bae *types.BucketAlreadyExists + var baoby *types.BucketAlreadyOwnedByYou + if errors.As(err, &bae) || errors.As(err, &baoby) { + return nil } - return fmt.Errorf("unexpected issue while checking that bucket: %s exists, due to %v", bucket, err) + return fmt.Errorf("unexpected issue while creating bucket: %s", err.Error()) } return fmt.Errorf("verifying bucket failed, check S3 configuration: %v", err) From 375a15de13f0db6d61b585db545d37eaa82074ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 09:50:00 +0100 Subject: [PATCH 279/530] fix(ingest): fix error log when RegisterFile fails if requested file to ingest does not exists in db --- sda/cmd/ingest/ingest.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 034f518c2c..45635ae0f7 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -321,14 +321,14 @@ func (app *Ingest) ingestFile(ctx context.Context, fileID string, message schema switch status { case "": - // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox + // Catch all for implementations inbox uploading that does not register the file in the DB, e.g. for those not using S3inbox or sftpInbox // Since we dont have the submission location in storage, we need to look through all configured storage locations. var findFileErr, registerErr error submissionLocation, findFileErr = app.InboxReader.FindFile(ctx, message.FilePath) // Register file even if FindFile didnt succeed with submissionLocation == "", as we will add an error file event log to it in that case fileID, registerErr = app.DB.RegisterFile(&fileID, submissionLocation, message.FilePath, message.User) if registerErr != nil { - log.Errorf("failed to register file, fileID: %s, reason: (%s)", fileID, err.Error()) + log.Errorf("failed to register file, fileID: %s, reason: (%s)", fileID, registerErr.Error()) return "nack" } From ae1b6c222e32c58d2c34f7e39474987ef1999ac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 10:51:16 +0100 Subject: [PATCH 280/530] feat(s3inbox): update to create one s3 client for the whole lifetime of the proxy instead of one s3 client per request --- sda/cmd/s3inbox/healthchecks_test.go | 32 +++++--- sda/cmd/s3inbox/proxy.go | 78 ++++++------------ sda/cmd/s3inbox/proxy_test.go | 116 ++++++++++++++------------- sda/cmd/s3inbox/s3inbox.go | 36 ++++++++- sda/cmd/s3inbox/s3inbox_test.go | 3 +- 5 files changed, 140 insertions(+), 125 deletions(-) diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index 738394951e..9afcfee2c5 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -1,11 +1,13 @@ package main import ( + "context" "crypto/tls" "net/http" "net/http/httptest" "testing" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" @@ -16,12 +18,13 @@ import ( type HealthcheckTestSuite struct { suite.Suite - mockS3Conf config.S3InboxConf // fakeserver - DBConf database.DBConf - fakeServer *FakeServer - MQConf broker.MQConf - messenger *broker.AMQPBroker - database *database.SDAdb + mockS3Conf config.S3InboxConf // fakeserver + s3ClientToMock *s3.Client + DBConf database.DBConf + fakeServer *FakeServer + MQConf broker.MQConf + messenger *broker.AMQPBroker + database *database.SDAdb } func TestHealthTestSuite(t *testing.T) { @@ -39,6 +42,11 @@ func (ts *HealthcheckTestSuite) SetupTest() { Bucket: "buckbuck", Region: "us-east-1", } + var err error + ts.s3ClientToMock, err = newS3Client(context.TODO(), ts.mockS3Conf) + if err != nil { + ts.FailNow(err.Error()) + } // Create a configuration for the fake MQ ts.MQConf = broker.MQConf{ @@ -74,7 +82,7 @@ func (ts *HealthcheckTestSuite) TearDownTest() { } func (ts *HealthcheckTestSuite) TestHttpsGetCheck() { - p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, ts.s3ClientToMock, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) url, _ := p.getS3ReadyPath() assert.NoError(ts.T(), p.httpsGetCheck(url)) @@ -82,7 +90,7 @@ func (ts *HealthcheckTestSuite) TestHttpsGetCheck() { } func (ts *HealthcheckTestSuite) TestS3URL() { - p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, ts.s3ClientToMock, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) _, err := p.getS3ReadyPath() assert.NoError(ts.T(), err) @@ -98,7 +106,7 @@ func (ts *HealthcheckTestSuite) TestHealthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, ts.s3ClientToMock, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) @@ -112,7 +120,7 @@ func (ts *HealthcheckTestSuite) TestClosedDBHealthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, ts.s3ClientToMock, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Check that 200 is reported w := httptest.NewRecorder() @@ -135,7 +143,7 @@ func (ts *HealthcheckTestSuite) TestNoS3Healthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, ts.s3ClientToMock, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // S3 unavailable, check that 503 is reported w := httptest.NewRecorder() @@ -151,7 +159,7 @@ func (ts *HealthcheckTestSuite) TestNoMQHealthchecks() { db, _ := database.NewSDAdb(ts.DBConf) messenger, err := broker.NewMQ(ts.MQConf) assert.NoError(ts.T(), err) - p := NewProxy(ts.mockS3Conf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) + p := NewProxy(ts.mockS3Conf, ts.s3ClientToMock, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Messenger unavailable, check that 503 is reported p.messenger.Conf.Port = 123456 diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index c4d5a40647..8b7156cdeb 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -16,8 +16,6 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" - s3config "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/minio/minio-go/v6/pkg/signer" @@ -37,6 +35,7 @@ type uniqueFileID struct { // Proxy represents the toplevel object in this application type Proxy struct { s3Conf config.S3InboxConf + s3Client *s3.Client auth userauth.Authenticator messenger *broker.AMQPBroker database *database.SDAdb @@ -83,11 +82,19 @@ const ( ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. -func NewProxy(s3conf config.S3InboxConf, auth userauth.Authenticator, messenger *broker.AMQPBroker, db *database.SDAdb, tlsConf *tls.Config) *Proxy { +func NewProxy(s3conf config.S3InboxConf, s3Client *s3.Client, auth userauth.Authenticator, messenger *broker.AMQPBroker, db *database.SDAdb, tlsConf *tls.Config) *Proxy { tr := &http.Transport{TLSClientConfig: tlsConf} client := &http.Client{Transport: tr, Timeout: 30 * time.Second} - return &Proxy{s3conf, auth, messenger, db, client, make(map[uniqueFileID]string)} + return &Proxy{ + s3Conf: s3conf, + s3Client: s3Client, + auth: auth, + messenger: messenger, + database: db, + client: client, + fileIDs: make(map[uniqueFileID]string), + } } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -170,13 +177,6 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw filePath: filePath, } - s3Client, err := newS3Client(r.Context(), p.s3Conf) - if err != nil { - reportError(http.StatusInternalServerError, fmt.Sprintf("failed to connect to s3, reason: %v", err), w) - - return - } - location := p.s3Conf.Endpoint + "/" + p.s3Conf.Bucket // if this is an upload request if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { @@ -192,7 +192,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // check if the file already exists, in that case send an overwrite message, // so that the FEGA portal is informed that a new version of the file exists. - err = p.sendMessageOnOverwrite(r, s3Client, p.fileIDs[fileIdentifier], rawFilepath, token) + err = p.sendMessageOnOverwrite(r, p.fileIDs[fileIdentifier], rawFilepath, token) if err != nil { p.internalServerError(w, r, err.Error()) @@ -212,7 +212,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") - message, err := p.CreateMessageFromRequest(r, token, s3Client, anonymizedFilepath) + message, err := p.CreateMessageFromRequest(r, token, anonymizedFilepath) if err != nil { p.internalServerError(w, r, err.Error()) @@ -245,7 +245,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw log.Debugf("resuming work on file with fileId: %v", p.fileIDs[fileIdentifier]) } - if err := p.storeObjectSizeInDB(r.Context(), s3Client, rawFilepath, p.fileIDs[fileIdentifier]); err != nil { + if err := p.storeObjectSizeInDB(r.Context(), rawFilepath, p.fileIDs[fileIdentifier]); err != nil { log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) p.internalServerError(w, r, "storeObjectSizeInDB failed") @@ -509,12 +509,12 @@ func (p *Proxy) detectRequestType(r *http.Request) S3RequestType { // CreateMessageFromRequest is a function that can take a http request and // figure out the correct rabbitmq message to send from it. -func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, s3Client *s3.Client, anonymizedFilepath string) (Event, error) { +func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anonymizedFilepath string) (Event, error) { event := Event{} checksum := Checksum{} var err error - checksum.Value, event.Filesize, err = p.requestInfo(r.Context(), s3Client, r.URL.Path) + checksum.Value, event.Filesize, err = p.requestInfo(r.Context(), r.URL.Path) if err != nil { return event, fmt.Errorf("could not get checksum information: %s", err) } @@ -534,7 +534,7 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, s3Cl // RequestInfo is a function that makes a request to the S3 and collects // the etag and size information for the uploaded document -func (p *Proxy) requestInfo(ctx context.Context, s3Client *s3.Client, fullPath string) (string, int64, error) { +func (p *Proxy) requestInfo(ctx context.Context, fullPath string) (string, int64, error) { filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) input := &s3.HeadObjectInput{ @@ -542,7 +542,7 @@ func (p *Proxy) requestInfo(ctx context.Context, s3Client *s3.Client, fullPath s Key: aws.String(filePath), } - result, err := s3Client.HeadObject(ctx, input) + result, err := p.s3Client.HeadObject(ctx, input) if err != nil { log.Debug(err.Error()) @@ -556,42 +556,12 @@ func (p *Proxy) requestInfo(ctx context.Context, s3Client *s3.Client, fullPath s return strings.Trim(*result.ETag, "\""), *result.ContentLength, nil } -func newS3Client(ctx context.Context, conf config.S3InboxConf) (*s3.Client, error) { - tlsConfig, err := configTLS(conf) - if err != nil { - return nil, err - } - - s3cfg, err := s3config.LoadDefaultConfig( - ctx, - s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.SecretKey, "")), - s3config.WithHTTPClient(&http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig, ForceAttemptHTTP2: true}}), - ) - if err != nil { - return nil, err - } - - s3Client := s3.NewFromConfig( - s3cfg, - func(o *s3.Options) { - o.BaseEndpoint = aws.String(conf.Endpoint) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(conf.Endpoint, "http:") - o.Region = conf.Region - o.UsePathStyle = true - o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired - o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired - }, - ) - - return s3Client, nil -} - // checkFileExists makes a request to the S3 to check whether the file already // is uploaded. Returns a bool indicating whether the file was found. -func (p *Proxy) checkFileExists(ctx context.Context, s3Client *s3.Client, fullPath string) (bool, error) { +func (p *Proxy) checkFileExists(ctx context.Context, fullPath string) (bool, error) { filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) - result, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{ + result, err := p.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(filePath), }) @@ -605,8 +575,8 @@ func (p *Proxy) checkFileExists(ctx context.Context, s3Client *s3.Client, fullPa return result != nil, err } -func (p *Proxy) sendMessageOnOverwrite(r *http.Request, s3Client *s3.Client, fileID, rawFilepath string, token jwt.Token) error { - exist, err := p.checkFileExists(r.Context(), s3Client, r.URL.Path) +func (p *Proxy) sendMessageOnOverwrite(r *http.Request, fileID, rawFilepath string, token jwt.Token) error { + exist, err := p.checkFileExists(r.Context(), r.URL.Path) if err != nil { return err } @@ -681,8 +651,8 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { } } -func (p *Proxy) storeObjectSizeInDB(ctx context.Context, s3Client *s3.Client, path, fileID string) error { - o, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{ +func (p *Proxy) storeObjectSizeInDB(ctx context.Context, path, fileID string) error { + o, err := p.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(path), }) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index e519a11999..6ba0d851cb 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -34,8 +34,11 @@ import ( type ProxyTests struct { suite.Suite - S3Fakeconf config.S3InboxConf // fakeserver - S3conf config.S3InboxConf // actual s3 container + s3Fakeconf config.S3InboxConf // fakeserver + s3ClientToFake *s3.Client + s3Conf config.S3InboxConf // actual s3 container + s3Client *s3.Client + DBConf database.DBConf fakeServer *FakeServer MQConf broker.MQConf @@ -54,21 +57,30 @@ func (s *ProxyTests) SetupTest() { s.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - s.S3Fakeconf = config.S3InboxConf{ + s.s3Fakeconf = config.S3InboxConf{ Endpoint: "http://127.0.0.1:9024", AccessKey: "someAccess", SecretKey: "someSecret", Bucket: "buckbuck", Region: "us-east-1", } + var err error + s.s3ClientToFake, err = newS3Client(context.TODO(), s.s3Fakeconf) + if err != nil { + s.FailNow(err.Error()) + } - s.S3conf = config.S3InboxConf{ + s.s3Conf = config.S3InboxConf{ Endpoint: fmt.Sprintf("http://127.0.0.1:%d", s3Port), AccessKey: "access", SecretKey: "secretKey", Bucket: "buckbuck", Region: "us-east-1", } + s.s3Client, err = newS3Client(context.TODO(), s.s3Conf) + if err != nil { + s.FailNow(err.Error()) + } // Create a configuration for the fake MQ s.MQConf = broker.MQConf{ @@ -123,27 +135,27 @@ func (s *ProxyTests) SetupTest() { s.T().FailNow() } - s3cfg, err := s3config.LoadDefaultConfig(context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) + s3cfg, err := s3config.LoadDefaultConfig(context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.s3Conf.AccessKey, s.s3Conf.SecretKey, ""))) if err != nil { s.FailNow("bad") } s3Client := s3.NewFromConfig( s3cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String(s.S3conf.Endpoint) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.Endpoint, "http:") - o.Region = s.S3conf.Region + o.BaseEndpoint = aws.String(s.s3Conf.Endpoint) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.s3Conf.Endpoint, "http:") + o.Region = s.s3Conf.Region o.UsePathStyle = true }, ) - _, _ = s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(s.S3conf.Bucket)}) + _, _ = s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(s.s3Conf.Bucket)}) if err != nil { _, _ = fmt.Println(err.Error()) } output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ Body: strings.NewReader("This is a test"), - Bucket: aws.String(s.S3conf.Bucket), + Bucket: aws.String(s.s3Conf.Bucket), Key: aws.String("/dummy/file"), ContentEncoding: aws.String("application/octet-stream"), }) @@ -227,12 +239,12 @@ func (m *MockMessenger) SendMessage(uuid string, body []byte) error { // nolint:bodyclose func (s *ProxyTests) TestServeHTTP_disallowed() { - proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) + proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() - log.Warnf("using proxy at %s", s.S3Fakeconf.Endpoint) + log.Warnf("using proxy at %s", s.s3Fakeconf.Endpoint) // Remove bucket disallowed r.Method = "DELETE" r.URL, _ = url.Parse("/asdf/") @@ -281,7 +293,7 @@ func (s *ProxyTests) TestServeHTTP_disallowed() { assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Not authorized user get 401 response - proxy = NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) + proxy = NewProxy(s.s3Fakeconf, s.s3ClientToFake, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) w = httptest.NewRecorder() r.Method = "GET" r.URL, _ = url.Parse("/username/file") @@ -298,7 +310,7 @@ func (s *ProxyTests) TestServeHTTPS3Unresponsive() { Bucket: "buckbuck", Region: "us-east-1", } - proxy := NewProxy(s3conf, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) + proxy := NewProxy(s3conf, s.s3Client, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() @@ -315,7 +327,7 @@ func (s *ProxyTests) TestServeHTTP_MQConnectionClosed() { messenger, err := broker.NewMQ(s.MQConf) assert.NoError(s.T(), err) db, _ := database.NewSDAdb(s.DBConf) - proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) + proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the mq connection will be restored when needed proxy.messenger.Connection.Close() @@ -334,7 +346,7 @@ func (s *ProxyTests) TestServeHTTP_MQChannelClosed() { messenger, err := broker.NewMQ(s.MQConf) assert.NoError(s.T(), err) db, _ := database.NewSDAdb(s.DBConf) - proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) + proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the mq channel will be restored when needed proxy.messenger.Channel.Close() @@ -353,7 +365,7 @@ func (s *ProxyTests) TestServeHTTP_MQ_Unavailable() { messenger, err := broker.NewMQ(s.MQConf) assert.NoError(s.T(), err) db, _ := database.NewSDAdb(s.DBConf) - proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) + proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the correct status code is returned when mq connection can't be created proxy.messenger.Conf.Port = 123456 @@ -372,7 +384,7 @@ func (s *ProxyTests) TestServeHTTP_allowed() { messenger, err := broker.NewMQ(s.MQConf) assert.NoError(s.T(), err) db, _ := database.NewSDAdb(s.DBConf) - proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) + proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // List files works r, err := http.NewRequest("GET", "/dummy/file", nil) @@ -481,12 +493,10 @@ func (s *ProxyTests) TestMessageFormatting() { assert.NoError(s.T(), claims.Set("sub", user)) // start proxy that denies everything - proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) + proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "1234"} - s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) - assert.NoError(s.T(), err) - msg, err := proxy.CreateMessageFromRequest(r, claims, s3Client, "new_file.txt") + msg, err := proxy.CreateMessageFromRequest(r, claims, "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) @@ -502,7 +512,7 @@ func (s *ProxyTests) TestMessageFormatting() { // Test single shot upload r.Method = "PUT" - msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), s3Client, "new_file.txt") + msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), "upload", msg.Operation) @@ -517,7 +527,7 @@ func (s *ProxyTests) TestDatabaseConnection() { assert.NoError(s.T(), err) defer messenger.Connection.Close() // Start proxy that allows everything - proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) + proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // PUT a file into the system filename := "/dummy/db-test-file" @@ -544,7 +554,7 @@ func (s *ProxyTests) TestDatabaseConnection() { err = db.DB.QueryRow(query, anonymFilename).Scan(&fileID, &location) assert.Nil(s.T(), err, "Failed to query database") assert.NotNil(s.T(), fileID, "File not found in database") - assert.Equal(s.T(), fmt.Sprintf("%s/%s", s.S3Fakeconf.Endpoint, s.S3Fakeconf.Bucket), location) + assert.Equal(s.T(), fmt.Sprintf("%s/%s", s.s3Fakeconf.Endpoint, s.s3Fakeconf.Bucket), location) // Check that the "registered" status is in the database for this file for _, status := range []string{"registered", "uploaded"} { @@ -581,10 +591,8 @@ func (s *ProxyTests) TestCheckFileExists() { messenger, err := broker.NewMQ(s.MQConf) assert.NoError(s.T(), err) defer messenger.Connection.Close() - proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) - s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) - assert.NoError(s.T(), err) - res, err := proxy.checkFileExists(context.TODO(), s3Client, "/dummy/file") + proxy := NewProxy(s.s3Conf, s.s3Client, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) + res, err := proxy.checkFileExists(context.TODO(), "/dummy/file") assert.True(s.T(), res) assert.Nil(s.T(), err) } @@ -598,10 +606,8 @@ func (s *ProxyTests) TestCheckFileExists_nonExistingFile() { messenger, err := broker.NewMQ(s.MQConf) assert.NoError(s.T(), err) defer messenger.Connection.Close() - proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) - s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) - assert.NoError(s.T(), err) - res, err := proxy.checkFileExists(context.TODO(), s3Client, "nonexistingfilepath") + proxy := NewProxy(s.s3Conf, s.s3Client, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + res, err := proxy.checkFileExists(context.TODO(), "nonexistingfilepath") assert.False(s.T(), res) assert.Nil(s.T(), err) } @@ -617,21 +623,21 @@ func (s *ProxyTests) TestCheckFileExists_unresponsive() { defer messenger.Connection.Close() // Unaccessible S3 (wrong port) - proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + proxy := NewProxy(s.s3Conf, s.s3Client, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.s3Conf.Endpoint = "http://127.0.0.1:1111" - s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + proxy.s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) assert.NoError(s.T(), err) - res, err := proxy.checkFileExists(context.TODO(), s3Client, "nonexistingfilepath") + res, err := proxy.checkFileExists(context.TODO(), "nonexistingfilepath") assert.False(s.T(), res) assert.NotNil(s.T(), err) assert.Contains(s.T(), err.Error(), "S3: HeadObject") // Bad access key gives 403 - proxy.s3Conf.Endpoint = s.S3conf.Endpoint + proxy.s3Conf.Endpoint = s.s3Conf.Endpoint proxy.s3Conf.AccessKey = "invaild" - s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) + proxy.s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) assert.NoError(s.T(), err) - res, err = proxy.checkFileExists(context.TODO(), s3Client, "nonexistingfilepath") + res, err = proxy.checkFileExists(context.TODO(), "nonexistingfilepath") assert.False(s.T(), res) assert.NotNil(s.T(), err) assert.Contains(s.T(), err.Error(), "StatusCode: 403") @@ -645,17 +651,15 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { assert.NoError(s.T(), err) defer mq.Connection.Close() - proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + proxy := NewProxy(s.s3Conf, s.s3Client, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.database = db - s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) - assert.NoError(s.T(), err) fileID, err := db.RegisterFile(nil, "/inbox", "/dummy/file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) db.Close() - assert.NoError(s.T(), proxy.storeObjectSizeInDB(context.TODO(), s3Client, "/dummy/file", fileID)) + assert.NoError(s.T(), proxy.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) } func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { @@ -666,7 +670,7 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { assert.NoError(s.T(), err) defer mq.Connection.Close() - proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + proxy := NewProxy(s.s3Conf, s.s3Client, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.database = db fileID, err := db.RegisterFile(nil, "/inbox", "/dummy/file", "test-user") assert.NoError(s.T(), err) @@ -674,15 +678,15 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { // Detect autentication failure proxy.s3Conf.AccessKey = "badKey" - s3Client, err := newS3Client(context.TODO(), proxy.s3Conf) + proxy.s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) assert.NoError(s.T(), err) - assert.Error(s.T(), proxy.storeObjectSizeInDB(context.TODO(), s3Client, "/dummy/file", fileID)) + assert.Error(s.T(), proxy.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) // Detect unresponsive backend service proxy.s3Conf.Endpoint = "http://127.0.0.1:1234" - s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) + proxy.s3Client, err = newS3Client(context.TODO(), proxy.s3Conf) assert.NoError(s.T(), err) - assert.Error(s.T(), proxy.storeObjectSizeInDB(context.TODO(), s3Client, "/dummy/file", fileID)) + assert.Error(s.T(), proxy.storeObjectSizeInDB(context.TODO(), "/dummy/file", fileID)) } // This test is intended to try to catch some issues we sometimes see when a query to the S3 backend @@ -696,37 +700,37 @@ func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { assert.NoError(s.T(), err) defer mq.Connection.Close() - p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + p := NewProxy(s.s3Conf, s.s3Client, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db fileID, err := db.RegisterFile(nil, "/inbox", "/test/new_file", "test-user") assert.NoError(s.T(), err) assert.NotNil(s.T(), fileID) - s3cfg, err := s3config.LoadDefaultConfig(context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) + s3cfg, err := s3config.LoadDefaultConfig(context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.s3Conf.AccessKey, s.s3Conf.SecretKey, ""))) if err != nil { - s.FailNow("bad") + s.FailNow(err.Error()) } s3Client := s3.NewFromConfig( s3cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String(s.S3conf.Endpoint) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.Endpoint, "http:") - o.Region = s.S3conf.Region + o.BaseEndpoint = aws.String(s.s3Conf.Endpoint) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.s3Conf.Endpoint, "http:") + o.Region = s.s3Conf.Region o.UsePathStyle = true }, ) output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ Body: strings.NewReader(strings.Repeat("A", 10*1024*1024)), - Bucket: aws.String(s.S3conf.Bucket), + Bucket: aws.String(s.s3Conf.Bucket), Key: aws.String("/test/new_file"), ContentEncoding: aws.String("application/octet-stream"), }) assert.NoError(s.T(), err) assert.NotNil(s.T(), output, output) - assert.NoError(s.T(), p.storeObjectSizeInDB(context.TODO(), s3Client, "/test/new_file", fileID)) + assert.NoError(s.T(), p.storeObjectSizeInDB(context.TODO(), "/test/new_file", fileID)) const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" var objectSize int64 diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 6f37bfc02a..4f2099c789 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -9,9 +9,13 @@ import ( "net/http" "os" "os/signal" + "strings" "syscall" "time" + "github.com/aws/aws-sdk-go-v2/aws" + s3config "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/aws/smithy-go" @@ -96,7 +100,7 @@ func run() error { } } router := mux.NewRouter() - proxy := NewProxy(conf.S3Inbox, auth, mqBroker, sdaDB, tlsProxy) + proxy := NewProxy(conf.S3Inbox, s3Client, auth, mqBroker, sdaDB, tlsProxy) router.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") router.HandleFunc("/health", proxy.CheckHealth) router.PathPrefix("/").Handler(proxy) @@ -185,3 +189,33 @@ func configTLS(c config.S3InboxConf) (*tls.Config, error) { return cfg, nil } + +func newS3Client(ctx context.Context, conf config.S3InboxConf) (*s3.Client, error) { + tlsConfig, err := configTLS(conf) + if err != nil { + return nil, err + } + + s3cfg, err := s3config.LoadDefaultConfig( + ctx, + s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(conf.AccessKey, conf.SecretKey, "")), + s3config.WithHTTPClient(&http.Client{Transport: &http.Transport{TLSClientConfig: tlsConfig, ForceAttemptHTTP2: true}}), + ) + if err != nil { + return nil, err + } + + s3Client := s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(conf.Endpoint) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(conf.Endpoint, "http:") + o.Region = conf.Region + o.UsePathStyle = true + o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired + o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired + }, + ) + + return s3Client, nil +} diff --git a/sda/cmd/s3inbox/s3inbox_test.go b/sda/cmd/s3inbox/s3inbox_test.go index d8f426d325..4ea9f0b251 100644 --- a/sda/cmd/s3inbox/s3inbox_test.go +++ b/sda/cmd/s3inbox/s3inbox_test.go @@ -18,7 +18,7 @@ import ( "github.com/ory/dockertest/v3/docker" ) -var DBport, MQport, OIDCport, s3Port int +var DBport, MQport, s3Port int func TestMain(m *testing.M) { if _, err := os.Stat("/.dockerenv"); err == nil { @@ -203,7 +203,6 @@ func TestMain(m *testing.M) { log.Fatalf("Could not start resource: %s", err) } - OIDCport, _ = strconv.Atoi(oidc.GetPort("8080/tcp")) oidcHostAndPort := oidc.GetHostPort("8080/tcp") client = http.Client{Timeout: 5 * time.Second} From 3f2232fc5b84610782957a62ef76db70a2291377 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 10:52:30 +0100 Subject: [PATCH 281/530] fix(finalize): fix correct err var in error log, fix stray ) in error log when io.Copy failes --- sda/cmd/finalize/finalize.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index b5bac11bfd..66f75c3ac6 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -216,7 +216,7 @@ func handleMessage(ctx context.Context, delivered amqp.Delivery) { body, _ := json.Marshal(fileError) // Send the message to an error queue so it can be analyzed. - if e := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, "error", body); e != nil { + if err := mqBroker.SendMessage(fileID, mqBroker.Conf.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: %v", err) } @@ -311,7 +311,7 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { }() if copiedSize, err := io.Copy(contentWriter, file); err != nil { - _ = contentWriter.CloseWithError(fmt.Errorf("failed to copy file, reason: %v)", err)) + _ = contentWriter.CloseWithError(fmt.Errorf("failed to copy file, reason: %v", err)) } else if copiedSize != archiveData.FileSize { _ = contentWriter.CloseWithError(errors.New("copied size does not match file size")) } From 9a662ff778f810d479164ab2104bd4518f1b4ce9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 10:54:34 +0100 Subject: [PATCH 282/530] fix(storage-v2): fix typo backed -> backend --- sda/internal/storage/v2/locationbroker/location_broker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/internal/storage/v2/locationbroker/location_broker.go b/sda/internal/storage/v2/locationbroker/location_broker.go index c2c25b294e..7d98b75534 100644 --- a/sda/internal/storage/v2/locationbroker/location_broker.go +++ b/sda/internal/storage/v2/locationbroker/location_broker.go @@ -93,7 +93,7 @@ func (l *locationBroker) GetObjectCount(ctx context.Context, backendName, locati sizeAndCountFinderFunc = scf.sizeAndCountFinderFunc } if sizeAndCountFinderFunc == nil { - return 0, fmt.Errorf("no size and count finder function defined for location: %s, backed %s", location, backendName) + return 0, fmt.Errorf("no size and count finder function defined for location: %s, backend %s", location, backendName) } size, objectCount, err = sizeAndCountFinderFunc(ctx, location) } @@ -136,7 +136,7 @@ func (l *locationBroker) GetSize(ctx context.Context, backendName, location stri sizeAndCountFinderFunc = scf.sizeAndCountFinderFunc } if sizeAndCountFinderFunc == nil { - return 0, fmt.Errorf("no size and count finder function defined for location: %s, backed %s", location, backendName) + return 0, fmt.Errorf("no size and count finder function defined for location: %s, backend %s", location, backendName) } size, objectCount, err = sizeAndCountFinderFunc(ctx, location) } From b5347e2c7b8d9d7fc80f914a63e4292c01315496 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 10:55:26 +0100 Subject: [PATCH 283/530] fix(storage-v2): fix typo witin -> within --- .../internal/storage/v2/s3/reader/new_file_read_seeker.go | 2 +- sda/internal/storage/v2/s3/reader/new_file_read_seeker.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go index e6202c3659..4d160cc41e 100644 --- a/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda-download/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -386,7 +386,7 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { } if r.currentOffset < contentRange.start || r.currentOffset > contentRange.end { - return 0, fmt.Errorf("unexpected content range %v - expected %v to be witin the content range", *object.ContentRange, r.currentOffset) + return 0, fmt.Errorf("unexpected content range %v - expected %v to be within the content range", *object.ContentRange, r.currentOffset) } b := bytes.Buffer{} diff --git a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go index e6202c3659..4d160cc41e 100644 --- a/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go +++ b/sda/internal/storage/v2/s3/reader/new_file_read_seeker.go @@ -386,7 +386,7 @@ func (r *s3SeekableReader) Read(dst []byte) (n int, err error) { } if r.currentOffset < contentRange.start || r.currentOffset > contentRange.end { - return 0, fmt.Errorf("unexpected content range %v - expected %v to be witin the content range", *object.ContentRange, r.currentOffset) + return 0, fmt.Errorf("unexpected content range %v - expected %v to be within the content range", *object.ContentRange, r.currentOffset) } b := bytes.Buffer{} From be5f165697dbdf58cb2d7730ecadfcab60cb3ef7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 10:56:33 +0100 Subject: [PATCH 284/530] fix(s3inbox): fix S3INBOX_SECRET_KEY docs in s3inbox.md --- sda/cmd/s3inbox/s3inbox.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/s3inbox.md b/sda/cmd/s3inbox/s3inbox.md index 9b7d1335e6..887530fb3c 100644 --- a/sda/cmd/s3inbox/s3inbox.md +++ b/sda/cmd/s3inbox/s3inbox.md @@ -86,7 +86,7 @@ These settings control how verify connects to the RabbitMQ message broker. - `S3INBOX_ENDPOINT`: Endpoint to the S3 service including the port (e.g., https://s3.example.com:9000) - `S3INBOX_ACCESS_KEY`: The S3 access key (username equivalent), used to authenticate to S3 [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) -- `S3INBOX_SECRET_KEY`: The S3 access key (username equivalent), used to authenticate to S3 +- `S3INBOX_SECRET_KEY`: The S3 secret key (password equivalent), used to authenticate to S3 [more info at AWS](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) - `S3INBOX_BUCKET`: The S3 bucket to be used for storage of uploaded files when forwarding upload requests - `S3INBOX_REGION`: S3 region From dad25921a8cfca77a59825c2a520ee4781bb9fa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 11:00:27 +0100 Subject: [PATCH 285/530] feat(storage-v2): add protection against path traversal in posix reader find file as well --- .../storage/v2/posix/reader/find_file.go | 20 ++++++++++++++++++- .../storage/v2/posix/reader/find_file.go | 20 ++++++++++++++++++- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/sda-download/internal/storage/v2/posix/reader/find_file.go b/sda-download/internal/storage/v2/posix/reader/find_file.go index 9ab6454c0a..2a0bc3257e 100644 --- a/sda-download/internal/storage/v2/posix/reader/find_file.go +++ b/sda-download/internal/storage/v2/posix/reader/find_file.go @@ -7,13 +7,31 @@ import ( "io/fs" "os" "path/filepath" + "strings" "github.com/neicnordic/sda-download/internal/storage/v2/storageerrors" ) func (reader *Reader) FindFile(_ context.Context, filePath string) (string, error) { for _, endpointConf := range reader.configuredEndpoints { - _, err := os.Stat(filepath.Join(endpointConf.Path, filePath)) + basePath, err := filepath.Abs(endpointConf.Path) + if err != nil { + return "", fmt.Errorf("failed to resolve base path for location %s: %w", endpointConf.Path, err) + } + + fullFilePath, err := filepath.Abs(filepath.Join(basePath, filePath)) + if err != nil { + return "", fmt.Errorf("failed to resolve file path for %s at location %s: %w", filePath, endpointConf.Path, err) + } + + // Ensure the resolved path is within the configured base path to prevent path traversal. + baseWithSep := basePath + string(os.PathSeparator) + fullWithSep := fullFilePath + string(os.PathSeparator) + if !strings.HasPrefix(fullWithSep, baseWithSep) { + return "", storageerrors.ErrorFileNotFoundInLocation + } + + _, err = os.Stat(fullFilePath) if err != nil { if errors.Is(err, fs.ErrNotExist) { continue diff --git a/sda/internal/storage/v2/posix/reader/find_file.go b/sda/internal/storage/v2/posix/reader/find_file.go index 7f41186066..e550469e38 100644 --- a/sda/internal/storage/v2/posix/reader/find_file.go +++ b/sda/internal/storage/v2/posix/reader/find_file.go @@ -7,13 +7,31 @@ import ( "io/fs" "os" "path/filepath" + "strings" "github.com/neicnordic/sensitive-data-archive/internal/storage/v2/storageerrors" ) func (reader *Reader) FindFile(_ context.Context, filePath string) (string, error) { for _, endpointConf := range reader.configuredEndpoints { - _, err := os.Stat(filepath.Join(endpointConf.Path, filePath)) + basePath, err := filepath.Abs(endpointConf.Path) + if err != nil { + return "", fmt.Errorf("failed to resolve base path for location %s: %w", endpointConf.Path, err) + } + + fullFilePath, err := filepath.Abs(filepath.Join(basePath, filePath)) + if err != nil { + return "", fmt.Errorf("failed to resolve file path for %s at location %s: %w", filePath, endpointConf.Path, err) + } + + // Ensure the resolved path is within the configured base path to prevent path traversal. + baseWithSep := basePath + string(os.PathSeparator) + fullWithSep := fullFilePath + string(os.PathSeparator) + if !strings.HasPrefix(fullWithSep, baseWithSep) { + return "", storageerrors.ErrorFileNotFoundInLocation + } + + _, err = os.Stat(fullFilePath) if err != nil { if errors.Is(err, fs.ErrNotExist) { continue From 72cf4a20461dc8457a3d31f8b38eb02cc184e723 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 11:02:51 +0100 Subject: [PATCH 286/530] fix(charts): fix typo in s3inbox secrets saying archive instead of inbox --- charts/sda-svc/templates/s3-inbox-secrets.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/sda-svc/templates/s3-inbox-secrets.yaml b/charts/sda-svc/templates/s3-inbox-secrets.yaml index 510a7e6eaf..0a8daa25e9 100644 --- a/charts/sda-svc/templates/s3-inbox-secrets.yaml +++ b/charts/sda-svc/templates/s3-inbox-secrets.yaml @@ -45,9 +45,9 @@ stringData: user: {{ required "DB user is required" (include "dbUserInbox" .) }} s3inbox: endpoint: {{ required "S3 s3inbox URL missing" .Values.global.s3Inbox.url }}{{- if .Values.global.s3Inbox.port }}:{{.Values.global.s3Inbox.port}}{{- end }} - access_key: {{ required "Accesskey required for archive" .Values.global.s3Inbox.accessKey }} - secret_key: {{ required "Secretkey required for archive" .Values.global.s3Inbox.secretKey }} - bucket: {{ required "S3 archive bucket not set" .Values.global.s3Inbox.bucket }} + access_key: {{ required "Accesskey required for inbox" .Values.global.s3Inbox.accessKey }} + secret_key: {{ required "Secretkey required for inbox" .Values.global.s3Inbox.secretKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.s3Inbox.bucket }} region: {{ default "us-east-1" .Values.global.s3Inbox.region }} {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt From bed84da2d368dcffcb98a5312481290888b00b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Feb 2026 15:28:56 +0100 Subject: [PATCH 287/530] feat(charts): update sda-svc and add job to verify that locations have been populated as helm pre install hook, update db admin configuration to be able to take username and password directly if not given through secret --- .../scripts/charts/dependencies.sh | 1 + .../integration/scripts/charts/values.yaml | 3 + charts/sda-svc/README.md | 2 + .../job-ensure-file-locations-populated.yaml | 76 +++++++++++++++++++ charts/sda-svc/values.yaml | 8 +- 5 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 charts/sda-svc/templates/job-ensure-file-locations-populated.yaml diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index 5093823554..363ed51437 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -153,6 +153,7 @@ yq -i ' .global.broker.password = strenv(MQPASSWORD) | .global.c4gh.privateKeys[0].passphrase = strenv(C4GHPASSPHRASE) | .global.db.password = strenv(PGPASSWORD) | +.global.db.admin.password = strenv(PGPASSWORD) | .global.inbox.s3AccessKey = strenv(MINIO_ACCESS) | .global.inbox.s3SecretKey = strenv(MINIO_SECRET) | .global.s3Inbox.accessKey = strenv(MINIO_ACCESS) | diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index 21cc2c42a9..2b28de1cd2 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -78,6 +78,9 @@ global: host: "postgres-sda-db" user: "postgres" password: PLACEHOLDER_VALUE + admin: + username: "postgres" + password: PLACEHOLDER_VALUE doa: enabled: false download: diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 878c8bbbfd..9bfa304ab5 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -134,6 +134,8 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` | | `global.db.admin.passKey` | Key in the secret that holds the password. | `""` | | `global.db.admin.userkey` | Key in the secret that holds the username. | `""` | +| `global.db.admin.username` | If secretName is not set will default to this to populate PGUSER used in migration jobs | `""` | +| `global.db.admin.password` | If secretName is not set will default to this to populate PGPASSWORD used in migration jobs | `""` | | `global.db.host` | Hostname for the database. | `""` | | `global.db.name` | Database to connect to. | `lega` | | `global.db.passIngest` | Password used for `data in` services. | `""` | diff --git a/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml new file mode 100644 index 0000000000..b6aecee78f --- /dev/null +++ b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml @@ -0,0 +1,76 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sda.fullname" . }}-ensure-file-locations-populated + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed +spec: + template: + metadata: + name: {{ template "sda.fullname" . }}-ensure-file-locations-populated + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + spec: + containers: + - name: {{ template "sda.fullname" . }}-ensure-file-locations-populated + command: + - "/bin/sh" + - "-c" + - | + pg_isready + psql -At -c "SELECT count(id) FROM sda.files WHERE submission_location IS NULL OR (archive_location IS NULL AND archive_file_path != '')" | grep -q 0 && exit 0 || (echo "There exists rows with unset submission or archive location, can not proceed with deployment" && exit 1) + env: + - name: PGDATABASE + value: {{ .Values.global.db.name }} + - name: PGHOST + value: {{ .Values.global.db.host }} + - name: PGPASSWORD + {{- if .Values.global.db.admin.secretName }} + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- else }} + value: {{ .Values.global.db.admin.password }} + {{- end }} + {{- if .Values.global.db.port }} + - name: PGPORT + value: {{ .Values.global.db.port | quote }} + {{- end }} + - name: PGUSER + {{- if .Values.global.db.admin.secretName }} + valueFrom: + secretKeyRef: + name: {{ .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + {{- else }} + value: {{ .Values.global.db.admin.username }} + {{- end }} + image: {{ .Values.jobs.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + restartPolicy: Never + securityContext: + runAsUser: 72 + runAsGroup: 72 + fsGroup: 72 + backoffLimit: 0 diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 1f97851b3e..bda2e417e8 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -202,9 +202,11 @@ global: admin: # These credentials need at least dbOwner permissions # in order to perform schema migrations. - secretName: "" - passKey: "" - userKey: "" + secretName: "" # Name of k8s secret which contains the username and password + passKey: "" # Name of field in k8s secret which contains the postgres password + userKey: "" # Name of field in k8s secret which contains the postgres username + username: "postgres" # If secretName is not set will default to this to populate PGUSER used in migration jobs + password: "" # If secretName is not set will default to this to populate PGPASSWORD used in migration jobs host: "" name: "sda" user: "" From e0fb08cd64ccbc3cc1d11cd90db91ef84a026ec9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 24 Feb 2026 10:47:49 +0100 Subject: [PATCH 288/530] feat(charts): update sda-svc jobs to support pg tls connection --- charts/sda-svc/README.md | 13 ++++--- charts/sda-svc/templates/job-certificate.yaml | 37 +++++++++++++++++++ .../job-ensure-file-locations-populated.yaml | 30 +++++++++++++++ .../job-perform-db-optimization.yaml | 28 ++++++++++++++ .../templates/job-remove-user-name.yaml | 30 +++++++++++++++ .../templates/job-set-missing-keyhash.yaml | 28 ++++++++++++++ .../templates/job-upgrade-from-v1.yaml | 28 ++++++++++++++ charts/sda-svc/values.yaml | 4 +- 8 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 charts/sda-svc/templates/job-certificate.yaml diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 9bfa304ab5..1e68e675d0 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -359,8 +359,11 @@ of the values file. ### Jobs -`jobs.image` | Container image used for running the DB migration jobs | `postgres:15.4-alpine` -`jobs.setKeyHash` | Populate the key_hash table after migration from V1 | `false` -`jobs.upgradeFromV1` | Upgrade database schema from a version 1 release. | `false` -`jobs.removeUserName` | remove username from file path. | `false` -`jobs.removeCorrID` | remove correlation ID column from file event log table. | `false` +| Parameter | Description | Default | +|-----------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------| +| `jobs.image` | Container image used for running the DB migration jobs | `postgres:15.4-alpine` | +| `jobs.setKeyHash` | Populate the key_hash table after migration from V1 | `false` | +| `jobs.upgradeFromV1` | Upgrade database schema from a version 1 release. | `false` | +| `jobs.removeUserName` | remove username from file path. | `false` | +| `jobs.removeCorrID` | remove correlation ID column from file event log table. | `false` | +| `jobs.tls.secretName` | Secret holding the application TLS certificates, required if `global.tls.enabled` is true, and `global.tls.clusterIssuer` or `global.tls.issuer` are not set | `` | diff --git a/charts/sda-svc/templates/job-certificate.yaml b/charts/sda-svc/templates/job-certificate.yaml new file mode 100644 index 0000000000..c613ba5f87 --- /dev/null +++ b/charts/sda-svc/templates/job-certificate.yaml @@ -0,0 +1,37 @@ +{{- if .Values.global.tls.enabled }} +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-job-certs + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-1" + helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-job-certs + + duration: 2160h # 90d + + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer ) }} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml index b6aecee78f..0bc07bd394 100644 --- a/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml +++ b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml @@ -59,6 +59,18 @@ spec: {{- else }} value: {{ .Values.global.db.admin.username }} {{- end }} + {{- if .Values.global.tls.enabled }} + - name: PGSSLROOTCERT + value: {{ template "tlsPath" . }}/ca.crt + {{- if eq "verify-full" .Values.global.db.sslMode }} + - name: PGSSLKEY + value: {{ template "tlsPath" . }}/tls.key + - name: PGSSLCERT + value: {{ template "tlsPath" . }}/tls.crt + {{- end }} + {{- end }} + - name: PGSSLMODE + value: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} image: {{ .Values.jobs.image }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: @@ -68,9 +80,27 @@ spec: runAsNonRoot: true seccompProfile: type: "RuntimeDefault" + {{- if .Values.global.tls.enabled }} + volumeMounts: + - name: tls + mountPath: {{ template "tlsPath" . }} + {{- end }} restartPolicy: Never securityContext: runAsUser: 72 runAsGroup: 72 fsGroup: 72 + {{- if .Values.global.tls.enabled }} + volumes: + - name: tls + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + secret: + defaultMode: 0640 + secretName: {{ template "sda.fullname" . }}-job-certs + {{- else }} + secret: + defaultMode: 0640 + secretName: {{ required "An certificate issuer or a TLS secret name is required for the jobs" .Values.jobs.tls.secretName }} + {{- end }} + {{- end }} backoffLimit: 0 diff --git a/charts/sda-svc/templates/job-perform-db-optimization.yaml b/charts/sda-svc/templates/job-perform-db-optimization.yaml index e5d7b4ffb9..940d030be5 100644 --- a/charts/sda-svc/templates/job-perform-db-optimization.yaml +++ b/charts/sda-svc/templates/job-perform-db-optimization.yaml @@ -103,6 +103,18 @@ spec: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} key: {{ default "username" .Values.global.db.admin.userKey }} + {{- if .Values.global.tls.enabled }} + - name: PGSSLROOTCERT + value: {{ template "tlsPath" . }}/ca.crt + {{- if eq "verify-full" .Values.global.db.sslMode }} + - name: PGSSLKEY + value: {{ template "tlsPath" . }}/tls.key + - name: PGSSLCERT + value: {{ template "tlsPath" . }}/tls.crt + {{- end }} + {{- end }} + - name: PGSSLMODE + value: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} image: {{ .Values.jobs.image }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: @@ -115,6 +127,10 @@ spec: volumeMounts: - name: sql mountPath: /migratedb + {{- if .Values.global.tls.enabled }} + - name: tls + mountPath: {{ template "tlsPath" . }} + {{- end }} restartPolicy: Never securityContext: runAsUser: 72 @@ -124,4 +140,16 @@ spec: - name: sql configMap: name: optimize-db + {{- if .Values.global.tls.enabled }} + - name: tls + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + secret: + defaultMode: 0640 + secretName: {{ template "sda.fullname" . }}-job-certs + {{- else }} + secret: + defaultMode: 0640 + secretName: {{ required "An certificate issuer or a TLS secret name is required for the jobs" .Values.jobs.tls.secretName }} + {{- end }} + {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/job-remove-user-name.yaml b/charts/sda-svc/templates/job-remove-user-name.yaml index 650b959e45..d5497958b1 100644 --- a/charts/sda-svc/templates/job-remove-user-name.yaml +++ b/charts/sda-svc/templates/job-remove-user-name.yaml @@ -51,6 +51,18 @@ spec: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} key: {{ default "username" .Values.global.db.admin.userKey }} + {{- if .Values.global.tls.enabled }} + - name: PGSSLROOTCERT + value: {{ template "tlsPath" . }}/ca.crt + {{- if eq "verify-full" .Values.global.db.sslMode }} + - name: PGSSLKEY + value: {{ template "tlsPath" . }}/tls.key + - name: PGSSLCERT + value: {{ template "tlsPath" . }}/tls.crt + {{- end }} + {{- end }} + - name: PGSSLMODE + value: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} image: {{ .Values.jobs.image }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: @@ -60,9 +72,27 @@ spec: runAsNonRoot: true seccompProfile: type: "RuntimeDefault" + {{- if .Values.global.tls.enabled }} + volumeMounts: + - name: tls + mountPath: {{ template "tlsPath" . }} + {{- end }} restartPolicy: Never securityContext: runAsUser: 72 runAsGroup: 72 fsGroup: 72 + {{- if .Values.global.tls.enabled }} + volumes: + - name: tls + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + secret: + defaultMode: 0640 + secretName: {{ template "sda.fullname" . }}-job-certs + {{- else }} + secret: + defaultMode: 0640 + secretName: {{ required "An certificate issuer or a TLS secret name is required for the jobs" .Values.jobs.tls.secretName }} + {{- end }} + {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/job-set-missing-keyhash.yaml b/charts/sda-svc/templates/job-set-missing-keyhash.yaml index 25addc599a..095d9b0e8b 100644 --- a/charts/sda-svc/templates/job-set-missing-keyhash.yaml +++ b/charts/sda-svc/templates/job-set-missing-keyhash.yaml @@ -68,6 +68,18 @@ spec: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} key: {{ default "username" .Values.global.db.admin.userKey }} + {{- if .Values.global.tls.enabled }} + - name: PGSSLROOTCERT + value: {{ template "tlsPath" . }}/ca.crt + {{- if eq "verify-full" .Values.global.db.sslMode }} + - name: PGSSLKEY + value: {{ template "tlsPath" . }}/tls.key + - name: PGSSLCERT + value: {{ template "tlsPath" . }}/tls.crt + {{- end }} + {{- end }} + - name: PGSSLMODE + value: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} image: {{ .Values.jobs.image }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: @@ -80,6 +92,10 @@ spec: volumeMounts: - name: sql mountPath: /migratedb + {{- if .Values.global.tls.enabled }} + - name: tls + mountPath: {{ template "tlsPath" . }} + {{- end }} restartPolicy: Never securityContext: runAsUser: 72 @@ -89,4 +105,16 @@ spec: - name: sql configMap: name: set-key-hash + {{- if .Values.global.tls.enabled }} + - name: tls + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + secret: + defaultMode: 0640 + secretName: {{ template "sda.fullname" . }}-job-certs + {{- else }} + secret: + defaultMode: 0640 + secretName: {{ required "An certificate issuer or a TLS secret name is required for the jobs" .Values.jobs.tls.secretName }} + {{- end }} + {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/job-upgrade-from-v1.yaml b/charts/sda-svc/templates/job-upgrade-from-v1.yaml index e6f20b7a78..592b40195d 100644 --- a/charts/sda-svc/templates/job-upgrade-from-v1.yaml +++ b/charts/sda-svc/templates/job-upgrade-from-v1.yaml @@ -91,6 +91,18 @@ spec: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} key: {{ default "username" .Values.global.db.admin.userKey }} + {{- if .Values.global.tls.enabled }} + - name: PGSSLROOTCERT + value: {{ template "tlsPath" . }}/ca.crt + {{- if eq "verify-full" .Values.global.db.sslMode }} + - name: PGSSLKEY + value: {{ template "tlsPath" . }}/tls.key + - name: PGSSLCERT + value: {{ template "tlsPath" . }}/tls.crt + {{- end }} + {{- end }} + - name: PGSSLMODE + value: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} image: {{ .Values.jobs.image }} imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: @@ -103,6 +115,10 @@ spec: volumeMounts: - name: sql mountPath: /migratedb + {{- if .Values.global.tls.enabled }} + - name: tls + mountPath: {{ template "tlsPath" . }} + {{- end }} restartPolicy: Never securityContext: runAsUser: 72 @@ -112,4 +128,16 @@ spec: - name: sql configMap: name: upgrade-from-v1 + {{- if .Values.global.tls.enabled }} + - name: tls + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + secret: + defaultMode: 0640 + secretName: {{ template "sda.fullname" . }}-job-certs + {{- else }} + secret: + defaultMode: 0640 + secretName: {{ required "An certificate issuer or a TLS secret name is required for the jobs" .Values.jobs.tls.secretName }} + {{- end }} + {{- end }} {{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index bda2e417e8..93409540ea 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -752,4 +752,6 @@ jobs: removeUserName: false setKeyHash: false upgradeFromV1: false - removeCorrID: false \ No newline at end of file + removeCorrID: false + tls: + secretName: "" From f8b058e318620eb3b8ff8f9327189d130aacbb37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 24 Feb 2026 10:49:06 +0100 Subject: [PATCH 289/530] feat(charts): increment sda-svc version 3.1.4 -> 3.2.0 --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 7f04b6dfea..2889d5159d 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.1.4 +version: 3.2.0 appVersion: v3.1.25 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From 4333f089e3314f4497f8c36818d47295f61b9c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Mar 2026 11:11:19 +0100 Subject: [PATCH 290/530] feat(download): ensure file reader from storage is closed --- sda-download/api/sda/sda.go | 8 ++++++-- sda-download/api/sda/sda_test.go | 9 ++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index e59c814bef..f85fcca6ca 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -326,19 +326,23 @@ func Download(c *gin.Context) { } // Get archive file handle - var file io.Reader + var file io.ReadCloser if wholeFile { file, err = ArchiveReader.NewFileReader(c, fileDetails.ArchiveLocation, fileDetails.ArchivePath) } else { file, err = ArchiveReader.NewFileReadSeeker(c, fileDetails.ArchiveLocation, fileDetails.ArchivePath) } - if err != nil { log.Errorf("could not find archive file %s, %s", fileDetails.ArchivePath, err) c.String(http.StatusInternalServerError, "archive error") return } + defer func() { + if err := file.Close(); err != nil { + log.Warnf("failed to close file reader to: %s, location: %s, error: %v", fileDetails.ArchivePath, fileDetails.ArchiveLocation, err) + } + }() c.Header("Content-Type", "application/octet-stream") if c.GetBool("S3") { diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go index 6a7f7bc2c3..0e50240f54 100644 --- a/sda-download/api/sda/sda_test.go +++ b/sda-download/api/sda/sda_test.go @@ -1409,12 +1409,19 @@ func (m *mockBackend) NewFileReader(_ context.Context, _, _ string) (io.ReadClos return io.NopCloser(bytes.NewReader(m.file)), nil } -func (m *mockBackend) NewFileReadSeeker(ctx context.Context, location, filePath string) (io.ReadSeekCloser, error) { +func (m *mockBackend) NewFileReadSeeker(_ context.Context, _, _ string) (io.ReadSeekCloser, error) { return mockReedSeekCloser{ ReadSeeker: io.ReadSeeker(bytes.NewReader(m.file)), + Closer: &mockCloser{}, }, nil } +type mockCloser struct{} + +func (mc *mockCloser) Close() error { + return nil +} + func (m *mockBackend) FindFile(ctx context.Context, filePath string) (string, error) { panic("not expected to be used by unit tests") } From d3d0629800fa54b0faef8258fb8a89666ab4b450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Mar 2026 11:20:34 +0100 Subject: [PATCH 291/530] feat(charts): fix job-ensure-file-locations-populated job, add set -e for job to fail if pg isnt ready, and fix grep check for 0 rows --- .../sda-svc/templates/job-ensure-file-locations-populated.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml index 0bc07bd394..f61756e40d 100644 --- a/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml +++ b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml @@ -30,8 +30,9 @@ spec: - "/bin/sh" - "-c" - | + set -e pg_isready - psql -At -c "SELECT count(id) FROM sda.files WHERE submission_location IS NULL OR (archive_location IS NULL AND archive_file_path != '')" | grep -q 0 && exit 0 || (echo "There exists rows with unset submission or archive location, can not proceed with deployment" && exit 1) + psql -At -c "SELECT count(id) FROM sda.files WHERE submission_location IS NULL OR (archive_location IS NULL AND archive_file_path != '')" | grep -qx '0' && exit 0 || (echo "There exists rows with unset submission or archive location, can not proceed with deployment" && exit 1) env: - name: PGDATABASE value: {{ .Values.global.db.name }} From 1016110f2fad8a6cef0ba93437e0d1f5c215c9da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Mar 2026 11:34:31 +0100 Subject: [PATCH 292/530] feat(charts): update previous jobs to support inline pg credentials as well as getting from secret --- .../templates/job-perform-db-optimization.yaml | 10 +++++++++- charts/sda-svc/templates/job-remove-user-name.yaml | 10 +++++++++- .../sda-svc/templates/job-set-missing-keyhash.yaml | 12 ++++++++++-- charts/sda-svc/templates/job-upgrade-from-v1.yaml | 12 ++++++++++-- 4 files changed, 38 insertions(+), 6 deletions(-) diff --git a/charts/sda-svc/templates/job-perform-db-optimization.yaml b/charts/sda-svc/templates/job-perform-db-optimization.yaml index 940d030be5..f14b6d3c92 100644 --- a/charts/sda-svc/templates/job-perform-db-optimization.yaml +++ b/charts/sda-svc/templates/job-perform-db-optimization.yaml @@ -90,19 +90,27 @@ spec: - name: PGHOST value: {{ .Values.global.db.host }} - name: PGPASSWORD + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} key: {{ default "password" .Values.global.db.admin.passKey }} + {{- else }} + value: {{ .Values.global.db.admin.password }} + {{- end }} {{- if .Values.global.db.port }} - name: PGPORT value: {{ .Values.global.db.port | quote }} {{- end }} - name: PGUSER + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: - name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + name: {{ .Values.global.db.admin.secretName }} key: {{ default "username" .Values.global.db.admin.userKey }} + {{- else }} + value: {{ .Values.global.db.admin.username }} + {{- end }} {{- if .Values.global.tls.enabled }} - name: PGSSLROOTCERT value: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/templates/job-remove-user-name.yaml b/charts/sda-svc/templates/job-remove-user-name.yaml index d5497958b1..b568cc829f 100644 --- a/charts/sda-svc/templates/job-remove-user-name.yaml +++ b/charts/sda-svc/templates/job-remove-user-name.yaml @@ -38,19 +38,27 @@ spec: - name: PGHOST value: {{ .Values.global.db.host }} - name: PGPASSWORD + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} key: {{ default "password" .Values.global.db.admin.passKey }} + {{- else }} + value: {{ .Values.global.db.admin.password }} + {{- end }} {{- if .Values.global.db.port }} - name: PGPORT value: {{ .Values.global.db.port | quote }} {{- end }} - name: PGUSER + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: - name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + name: {{ .Values.global.db.admin.secretName }} key: {{ default "username" .Values.global.db.admin.userKey }} + {{- else }} + value: {{ .Values.global.db.admin.username }} + {{- end }} {{- if .Values.global.tls.enabled }} - name: PGSSLROOTCERT value: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/templates/job-set-missing-keyhash.yaml b/charts/sda-svc/templates/job-set-missing-keyhash.yaml index 095d9b0e8b..faedd92d64 100644 --- a/charts/sda-svc/templates/job-set-missing-keyhash.yaml +++ b/charts/sda-svc/templates/job-set-missing-keyhash.yaml @@ -55,19 +55,27 @@ spec: - name: PGHOST value: {{ .Values.global.db.host }} - name: PGPASSWORD + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} key: {{ default "password" .Values.global.db.admin.passKey }} + {{- else }} + value: {{ .Values.global.db.admin.password }} + {{- end }} {{- if .Values.global.db.port }} - name: PGPORT value: {{ .Values.global.db.port | quote }} {{- end }} - name: PGUSER + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: - name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} - key: {{ default "username" .Values.global.db.admin.userKey }} + name: {{ .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + {{- else }} + value: {{ .Values.global.db.admin.username }} + {{- end }} {{- if .Values.global.tls.enabled }} - name: PGSSLROOTCERT value: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/templates/job-upgrade-from-v1.yaml b/charts/sda-svc/templates/job-upgrade-from-v1.yaml index 592b40195d..953c63921d 100644 --- a/charts/sda-svc/templates/job-upgrade-from-v1.yaml +++ b/charts/sda-svc/templates/job-upgrade-from-v1.yaml @@ -78,19 +78,27 @@ spec: - name: PGHOST value: {{ .Values.global.db.host }} - name: PGPASSWORD + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} - key: {{ default "password" .Values.global.db.admin.passKey }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- else }} + value: {{ .Values.global.db.admin.password }} + {{- end }} {{- if .Values.global.db.port }} - name: PGPORT value: {{ .Values.global.db.port | quote }} {{- end }} - name: PGUSER + {{- if .Values.global.db.admin.secretName }} valueFrom: secretKeyRef: - name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + name: {{ .Values.global.db.admin.secretName }} key: {{ default "username" .Values.global.db.admin.userKey }} + {{- else }} + value: {{ .Values.global.db.admin.username }} + {{- end }} {{- if .Values.global.tls.enabled }} - name: PGSSLROOTCERT value: {{ template "tlsPath" . }}/ca.crt From 2d3a7f68debda006509f10af93caa113a294d9f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Mar 2026 11:36:56 +0100 Subject: [PATCH 293/530] feat(charts): update helm hook weight of job certificate to exists before all jobs, and ensure file locations populated job to run after all currently existing jobs --- charts/sda-svc/templates/job-certificate.yaml | 2 +- .../sda-svc/templates/job-ensure-file-locations-populated.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/sda-svc/templates/job-certificate.yaml b/charts/sda-svc/templates/job-certificate.yaml index c613ba5f87..da36ccd6c6 100644 --- a/charts/sda-svc/templates/job-certificate.yaml +++ b/charts/sda-svc/templates/job-certificate.yaml @@ -8,7 +8,7 @@ metadata: # This is what defines this resource as a hook. # Without this line, the job is considered part of the release. helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "-1" + helm.sh/hook-weight: "-10" helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed spec: # Secret names are always required. diff --git a/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml index f61756e40d..cd723f1816 100644 --- a/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml +++ b/charts/sda-svc/templates/job-ensure-file-locations-populated.yaml @@ -12,7 +12,7 @@ metadata: # This is what defines this resource as a hook. # Without this line, the job is considered part of the release. helm.sh/hook: pre-install,pre-upgrade - helm.sh/hook-weight: "0" + helm.sh/hook-weight: "2" helm.sh/hook-delete-policy: before-hook-creation,hook-succeeded,hook-failed spec: template: From 222bca49f7af8b96d58ccfead2d70c53f44167f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Mar 2026 11:44:03 +0100 Subject: [PATCH 294/530] fix(s3inbox): fix checksum type to MD5 after change for it to be the aws etag instead of the sha256 hashed etag --- sda/cmd/s3inbox/proxy.go | 2 +- sda/cmd/s3inbox/proxy_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 8b7156cdeb..1ba4f8a4ee 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -524,7 +524,7 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anon event.Filepath = anonymizedFilepath event.Username = claims.Subject() - checksum.Type = "sha256" + checksum.Type = "md5" event.Checksum = []any{checksum} privateClaims := claims.PrivateClaims() log.Info("user ", event.Username, " with pilot ", privateClaims["pilot"], " uploaded file ", event.Filepath, " with checksum ", checksum.Value, " at ", time.Now()) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 6ba0d851cb..eb4139d78d 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -507,7 +507,7 @@ func (s *ProxyTests) TestMessageFormatting() { c, _ := json.Marshal(msg.Checksum[0]) checksum := Checksum{} _ = json.Unmarshal(c, &checksum) - assert.Equal(s.T(), "sha256", checksum.Type) + assert.Equal(s.T(), "md5", checksum.Type) assert.Equal(s.T(), "0a44282bd39178db9680f24813c41aec-1", checksum.Value) // Test single shot upload From 6249a80e824530947ff98d0b3a044cca309c3b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 3 Mar 2026 14:55:34 +0100 Subject: [PATCH 295/530] fix(charts): add hook weights to the config map used by the sda-svc jobs with one lower than job to ensure they are applied before the job --- charts/sda-svc/templates/job-perform-db-optimization.yaml | 4 ++++ charts/sda-svc/templates/job-set-missing-keyhash.yaml | 6 ++++++ charts/sda-svc/templates/job-upgrade-from-v1.yaml | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/charts/sda-svc/templates/job-perform-db-optimization.yaml b/charts/sda-svc/templates/job-perform-db-optimization.yaml index f14b6d3c92..809de6a892 100644 --- a/charts/sda-svc/templates/job-perform-db-optimization.yaml +++ b/charts/sda-svc/templates/job-perform-db-optimization.yaml @@ -3,6 +3,10 @@ apiVersion: v1 kind: ConfigMap metadata: name: optimize-db + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" # Hook weight is just one lower than job using config map to ensure it's applied before job + helm.sh/hook-delete-policy: hook-succeeded data: optimize-db.sql: | DO diff --git a/charts/sda-svc/templates/job-set-missing-keyhash.yaml b/charts/sda-svc/templates/job-set-missing-keyhash.yaml index faedd92d64..9163adffb8 100644 --- a/charts/sda-svc/templates/job-set-missing-keyhash.yaml +++ b/charts/sda-svc/templates/job-set-missing-keyhash.yaml @@ -3,6 +3,12 @@ apiVersion: v1 kind: ConfigMap metadata: name: set-key-hash + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "-5" # Hook weight is just one lower than job using config map to ensure it's applied before job + helm.sh/hook-delete-policy: hook-succeeded data: set-key-hash.sql: | DO diff --git a/charts/sda-svc/templates/job-upgrade-from-v1.yaml b/charts/sda-svc/templates/job-upgrade-from-v1.yaml index 953c63921d..f90a65f938 100644 --- a/charts/sda-svc/templates/job-upgrade-from-v1.yaml +++ b/charts/sda-svc/templates/job-upgrade-from-v1.yaml @@ -3,6 +3,12 @@ apiVersion: v1 kind: ConfigMap metadata: name: upgrade-from-v1 + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "-6" # Hook weight is just one lower than job using config map to ensure it's applied before job + helm.sh/hook-delete-policy: hook-succeeded data: upgrade-from-v1.sql: | DO From d3aefcc337af0b76c11c8f5abcb7fe7cb83f129b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 5 Mar 2026 04:03:35 +0000 Subject: [PATCH 296/530] build(deps): bump github.com/docker/cli in /sda Bumps [github.com/docker/cli](https://github.com/docker/cli) from 27.4.1+incompatible to 29.2.0+incompatible. - [Commits](https://github.com/docker/cli/compare/v27.4.1...v29.2.0) --- updated-dependencies: - dependency-name: github.com/docker/cli dependency-version: 29.2.0+incompatible dependency-type: indirect ... Signed-off-by: dependabot[bot] --- sda/go.mod | 30 ++++++++++++++-------- sda/go.sum | 73 ++++++++++++++++++++++++------------------------------ 2 files changed, 52 insertions(+), 51 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 8d15c9f77b..adfbfa5854 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -27,7 +27,6 @@ require ( github.com/ory/dockertest v3.3.5+incompatible github.com/ory/dockertest/v3 v3.12.0 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.10 github.com/rabbitmq/amqp091-go v1.10.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.4 @@ -42,7 +41,7 @@ require ( require ( dario.cat/mergo v1.0.1 // indirect filippo.io/edwards25519 v1.2.0 // indirect - github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect github.com/BurntSushi/toml v1.6.0 // indirect github.com/CloudyKit/fastprinter v0.0.0-20251202014920-1725d2651bd4 // indirect github.com/CloudyKit/jet/v6 v6.3.1 // indirect @@ -73,28 +72,33 @@ require ( github.com/casbin/govaluate v1.10.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/containerd/continuity v0.4.5 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 // indirect - github.com/docker/cli v27.4.1+incompatible // indirect - github.com/docker/docker v28.0.0+incompatible // indirect - github.com/docker/go-connections v0.5.0 // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/cli v29.2.0+incompatible // indirect + github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/structs v1.1.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.4 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.30.1 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.19.2 // indirect - github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/gomarkdown/markdown v0.0.0-20260217112301-37c66b85d6ab // indirect @@ -111,7 +115,6 @@ require ( github.com/kataras/tunnel v0.0.4 // indirect github.com/klauspost/compress v1.18.4 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect - github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.4 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -124,18 +127,19 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/moby/api v1.53.0 // indirect + github.com/moby/moby/client v0.2.2 // indirect github.com/moby/sys/user v0.3.0 // indirect - github.com/moby/term v0.5.0 // indirect + github.com/moby/term v0.5.2 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/opencontainers/runc v1.2.8 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/quic-go/qpack v0.6.0 // indirect github.com/quic-go/quic-go v0.59.0 // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect @@ -157,6 +161,11 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yosssi/ace v0.0.5 // indirect go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/otel/trace v1.39.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect @@ -166,7 +175,6 @@ require ( golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/ini.v1 v1.67.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect gotest.tools v2.2.0+incompatible // indirect ) diff --git a/sda/go.sum b/sda/go.sum index bbc2ef47b8..4dad7eb741 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -2,8 +2,8 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s= dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= -github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk= github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= @@ -87,17 +87,20 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= -github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= -github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -106,12 +109,12 @@ github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFn github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= -github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= -github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= -github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= -github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM= +github.com/docker/cli v29.2.0+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pMmjSD94= +github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -119,6 +122,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw= github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= @@ -135,6 +140,7 @@ github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFS github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -157,8 +163,6 @@ github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= -github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= -github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= @@ -214,16 +218,12 @@ github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA= github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= -github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= -github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -267,10 +267,14 @@ github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/moby/api v1.53.0 h1:PihqG1ncw4W+8mZs69jlwGXdaYBeb5brF6BL7mPIS/w= +github.com/moby/moby/api v1.53.0/go.mod h1:8mb+ReTlisw4pS6BRzCMts5M49W5M7bKt1cJy/YbAqc= +github.com/moby/moby/client v0.2.2 h1:Pt4hRMCAIlyjL3cr8M5TrXCwKzguebPAc2do2ur7dEM= +github.com/moby/moby/client v0.2.2/go.mod h1:2EkIPVNCqR05CMIzL1mfA07t0HvVUUOl85pasRz/GmQ= github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= -github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= -github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= github.com/mocktools/go-smtp-mock v1.10.0 h1:glrRmjNqASyy+jf1IJ2nCWgEbJScD3Amf2IGcXgdEVg= github.com/mocktools/go-smtp-mock v1.10.0/go.mod h1:mmvlBVX6MTOBHtROX+tor9YZF5JENN8d8wrToD1vvg4= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -286,8 +290,8 @@ github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 h1:9bCMuD3Tc github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25/go.mod h1:eDjgYHYDJbPLBLsyZ6qRaugP0mX8vePOhZ5id1fdzJw= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= -github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= github.com/opencontainers/runc v1.2.8 h1:RnEICeDReapbZ5lZEgHvj7E9Q3Eex9toYmaGBsbvU5Q= github.com/opencontainers/runc v1.2.8/go.mod h1:cC0YkmZcuvr+rtBZ6T7NBoVbMGNAdLa/21vIElJDOzI= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= @@ -298,8 +302,6 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0 github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= -github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -309,8 +311,8 @@ github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SA github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= @@ -389,14 +391,14 @@ github.com/yudai/gojsondiff v1.0.0 h1:27cbfqXLVEJ1o8I6v3y9lg8Ydm53EKqHXAOMxEGlCO github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3IfnEUduWvb9is428/nNb5L3U01M= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF3xaE= go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= @@ -418,15 +420,12 @@ golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -435,8 +434,6 @@ golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -447,8 +444,6 @@ golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -456,7 +451,6 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -492,14 +486,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= @@ -523,7 +514,9 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= -gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= moul.io/http2curl/v2 v2.3.0 h1:9r3JfDzWPcbIklMOs2TnIFzDYvfAZvjeavG6EzP7jYs= moul.io/http2curl/v2 v2.3.0/go.mod h1:RW4hyBjTWSYDOxapodpNEtX0g5Eb16sxklBqmd2RHcE= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= From 169bed8f7b2dd7371332b14080ba2adc7ab201cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 27 Feb 2026 13:09:12 +0100 Subject: [PATCH 297/530] feat(github workflows): update workflow and jobs - Rename file names to be more clear in what stage the workflow runs and what they do - Update image building - Build the 7 images by matrix - Populate additional org.opencontainers.image labels with correct values - Do not retag images from PR when merged, rebuild images from commit in main which was tagged - Update sda "release" process - Tag main when PR is merged - Tag in main triggers the actual building of images, and github release creation - sda now creates a github release with automatic release notes and references to images built for that release - Scan all images during PR, run trivy, upload sarif result, and add report as PR comment if PR changes affect image - Added trivy report markdown template - Update trivy sarif report to include which image to make it more clear from which image a vulnerability in Github Code Scanning originates from - merge the functionality.yaml jobs into the pr image building and integration test workflow (noted TODOs to not rebuild the images for the stfp-inbox and sda-download tests) - But given the upcoming download v2 the download(v1) will be deprecated and replaced with new integration tests - And stfp-inbox currently have no dedicated maintainer so might become deprecated and these tests removed - Update CodeQL workflow to not trigger by cronjob and instead run when sda is merged to have the main result available - Remove disabled "Delete old unused container images" job, replaced by image cleaning when a PR is closed --- .github/trivy_report_template_markdown.tpl | 18 + ...int-markdown.yml => adr_lint_markdown.yml} | 10 +- .github/workflows/build_pr_container.yaml | 411 ------------------ .github/workflows/functionality.yml | 92 ---- .github/workflows/ghcr-actions.yml | 29 -- .../workflows/pr_build_and_test_images.yml | 379 ++++++++++++++++ .github/workflows/pr_closed_clean_images.yml | 16 + .../{code-linter.yaml => pr_code_linter.yaml} | 3 +- .../workflows/{test.yml => pr_code_tests.yml} | 8 +- ...ql-analysis.yml => pr_codeql_analysis.yml} | 5 - .../pr_merge_increment_version_tag_sda.yml | 32 ++ .github/workflows/publish_container.yml | 141 ------ ...{publish_charts.yml => release_charts.yml} | 0 .github/workflows/release_sda.yml | 173 ++++++++ ..._sda-admin.yaml => release_sda_admin.yaml} | 0 .github/workflows/scan-images.yml | 71 --- 16 files changed, 626 insertions(+), 762 deletions(-) create mode 100644 .github/trivy_report_template_markdown.tpl rename .github/workflows/{lint-markdown.yml => adr_lint_markdown.yml} (66%) delete mode 100644 .github/workflows/build_pr_container.yaml delete mode 100644 .github/workflows/functionality.yml delete mode 100644 .github/workflows/ghcr-actions.yml create mode 100644 .github/workflows/pr_build_and_test_images.yml create mode 100644 .github/workflows/pr_closed_clean_images.yml rename .github/workflows/{code-linter.yaml => pr_code_linter.yaml} (98%) rename .github/workflows/{test.yml => pr_code_tests.yml} (96%) rename .github/workflows/{codeql-analysis.yml => pr_codeql_analysis.yml} (94%) create mode 100644 .github/workflows/pr_merge_increment_version_tag_sda.yml delete mode 100644 .github/workflows/publish_container.yml rename .github/workflows/{publish_charts.yml => release_charts.yml} (100%) create mode 100644 .github/workflows/release_sda.yml rename .github/workflows/{release_sda-admin.yaml => release_sda_admin.yaml} (100%) delete mode 100644 .github/workflows/scan-images.yml diff --git a/.github/trivy_report_template_markdown.tpl b/.github/trivy_report_template_markdown.tpl new file mode 100644 index 0000000000..0236b4effd --- /dev/null +++ b/.github/trivy_report_template_markdown.tpl @@ -0,0 +1,18 @@ +{{- if . }} +{{- range . }} +## Target `{{ escapeXML .Target }}` +{{- if (eq (len .Vulnerabilities) 0) }} +### No Vulnerabilities found +{{- else }} +### Vulnerabilities ({{ len .Vulnerabilities }}) +| Package | ID | Severity | Installed Version | Fixed Version | Title | +| -------- | ---- | -------- | ---------------- | ------------ | ---- | + {{- range .Vulnerabilities }} +| `{{ escapeXML .PkgName }}` | [{{ escapeXML .VulnerabilityID }}]({{ escapeXML .PrimaryURL }}) | {{ escapeXML .Severity }} | {{ escapeXML .InstalledVersion }} | {{ escapeXML .FixedVersion }} | {{ escapeXML .Title }} | + {{- end }} + +{{- end }} +{{- end }} +{{- else }} +## Trivy Returned Empty Report +{{- end }} \ No newline at end of file diff --git a/.github/workflows/lint-markdown.yml b/.github/workflows/adr_lint_markdown.yml similarity index 66% rename from .github/workflows/lint-markdown.yml rename to .github/workflows/adr_lint_markdown.yml index a2f1ba7420..8e04b1eb9c 100644 --- a/.github/workflows/lint-markdown.yml +++ b/.github/workflows/adr_lint_markdown.yml @@ -5,16 +5,8 @@ on: pull_request: paths: - "docs/decisions/**" - - ".github/workflows/lint-markdown.yml" + - ".github/workflows/adr_lint_markdown.yml" - ".markdownlint.yml" - push: - branches: - - main - paths: - - "docs/decisions/**" - - ".github/workflows/lint-markdown.yml" - - ".markdownlint.yml" - workflow_dispatch: jobs: markdownlint: diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml deleted file mode 100644 index ac9d29c1f0..0000000000 --- a/.github/workflows/build_pr_container.yaml +++ /dev/null @@ -1,411 +0,0 @@ -name: Build PR container - -on: - pull_request: - paths-ignore: - - ".github/integration/**" - - ".github/workflows/**" - - ".gitignore" - - "**/*.md" - - ".github/dependabot.yaml" - - "Makefile" - - "sda-admin/**" - -env: - PR_NUMBER: ${{ github.event.number }} - -jobs: - build_go_images: - name: Build PR image (golang) - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Log in to the Github Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build container for sda-download - uses: docker/build-push-action@v6 - with: - context: ./sda-download - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-download - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-download - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - - name: Build container for sda-validator-orchestrator - uses: docker/build-push-action@v6 - with: - context: ./sda-validator/orchestrator - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-validator-orchestrator - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-validator-orchestrator - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - - name: Build container for sensitive-data-archive - uses: docker/build-push-action@v6 - with: - context: ./sda - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }} - ghcr.io/${{ github.repository }}:PR${{ github.event.number }} - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - build_server_images: - name: Build PR image (servers) - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Log in to the Github Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build container for postgres - uses: docker/build-push-action@v6 - with: - context: ./postgresql - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-postgres - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-postgres - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - - name: Build container for rabbitmq - uses: docker/build-push-action@v6 - with: - context: ./rabbitmq - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-rabbitmq - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-rabbitmq - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.34.2 - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true - with: - image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-postgres - format: "sarif" - hide-progress: true - ignore-unfixed: true - output: 'postgres-results.sarif' - severity: "CRITICAL,HIGH" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: 'postgres-results.sarif' - category: postgres - - - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.34.2 - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true - with: - image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-rabbitmq - format: "sarif" - hide-progress: true - ignore-unfixed: true - output: 'rabbitmq-results.sarif' - severity: "CRITICAL,HIGH" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: 'rabbitmq-results.sarif' - category: rabbitmq - - build_java_images: - name: Build PR image (java) - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write - steps: - - name: Checkout code - uses: actions/checkout@v6 - - - name: Log in to the Github Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build container for sda-sftp-inbox - uses: docker/build-push-action@v6 - with: - context: ./sda-sftp-inbox - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-sftp-inbox - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-sftp-inbox - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.34.2 - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true - with: - image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-sftp-inbox - format: "sarif" - hide-progress: true - ignore-unfixed: true - output: 'inbox-results.sarif' - severity: "CRITICAL,HIGH" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: 'inbox-results.sarif' - category: sftp-inbox - - - name: Build container for sda-doa - uses: docker/build-push-action@v6 - with: - context: ./sda-doa - push: true - tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-doa - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-doa - labels: | - org.opencontainers.image.source=${{ github.event.repository.clone_url }} - org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') - org.opencontainers.image.revision=${{ github.sha }} - - - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.34.2 - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true - with: - image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-doa - format: "sarif" - hide-progress: true - ignore-unfixed: true - output: 'inbox-results.sarif' - severity: "CRITICAL,HIGH" - - rabbitmq: - needs: - - build_go_images - - build_server_images - name: rabbitmq-federation-test - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Test rabbitmq federation - run: docker compose -f .github/integration/rabbitmq-federation.yml run federation_test - - postgres: - needs: - - build_server_images - name: postgres-test-suite - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Test postgres - run: docker compose -f .github/integration/postgres.yml run tests - - sda: - needs: - - build_go_images - - build_server_images - runs-on: ubuntu-latest - strategy: - matrix: - storage: ["posix", "s3"] - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Test sensitive-data-archive - run: docker compose -f .github/integration/sda-${{matrix.storage}}-integration.yml run integration_test - - sda-sync: - needs: - - build_go_images - - build_server_images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 - id: changes - with: - filters: | - sync: - - "sda/cmd/sync/*" - sync-api: - - "sda/cmd/syncapi/*" - - name: Test sda-sync - run: docker compose -f .github/integration/sda-sync-integration.yml run integration_test - if: steps.changes.outputs.sync == 'true' || steps.changes.outputs.sync-api == 'true' - - sda-validator-orchestrator: - needs: - - build_go_images - - build_server_images - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Test sda-validator-orchestrator - run: | - echo 'kernel.apparmor_restrict_unprivileged_userns = 0' | sudo tee /etc/sysctl.d/20-apparmor-donotrestrict.conf - sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - docker compose -f .github/integration/sda-validator-orchestrator-integration.yml run integration_test - - sda-doa: - needs: - - build_java_images - - build_server_images - runs-on: ubuntu-latest - strategy: - matrix: - storage: [s3, posix] - steps: - - name: Check out code - uses: actions/checkout@v6 - - - name: Test sda-doa for ${{ matrix.storage }} storage - run: docker compose -f .github/integration/sda-doa-${{ matrix.storage }}-outbox.yml run integration_test - - chart: - needs: - - build_go_images - - build_server_images - - build_java_images - runs-on: ubuntu-latest - strategy: - matrix: - deployment: ["federated", "isolated"] - version: ["1.32", "1.33"] - tls: ["true", "false"] - storage: ["posix", "s3"] - exclude: - - version: "1.32" - tls: "false" - storage: "posix" - - version: "1.33" - tls: "false" - storage: "posix" - steps: - - name: Checkout - uses: actions/checkout@v6 - - - name: Install Helm - uses: azure/setup-helm@v4 - - - name: Initialise k3d - id: initK3D - run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} - shell: bash - - name: debug - if: steps.initK3D.outcome == 'failure' - run: k3d version list k3s | grep ${{matrix.version}} - shell: bash - - - name: Deploy external services - run: bash .github/integration/scripts/charts/dependencies.sh ${{matrix.deployment}} ${{matrix.storage}} ${{matrix.tls}} - shell: bash - - - name: Deploy DB - id: deployDB - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db ${{ github.event.number }} ${{matrix.tls}} - - name: debug - if: failure() && steps.deployDB.outcome == 'failure' - run: | - kubectl describe pod postgres-sda-db-0 - sleep 1 - kubectl logs postgres-sda-db-0 - - - name: Deploy MQ - id: deployMQ - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} ${{matrix.tls}} ${{matrix.deployment}} - shell: bash - - name: debug - if: failure() && steps.deployMQ.outcome == 'failure' - run: | - kubectl describe pod broker-sda-mq-0 - sleep 1 - kubectl logs broker-sda-mq-0 - - - name: Deploy pipeline - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} ${{matrix.tls}} ${{matrix.storage}} ${{matrix.deployment}} - shell: bash - - - name: Check deployment - run: | - sleep 30 - for n in api auth download finalize inbox ingest mapper reencrypt sync syncapi verify; do - if [ ${{matrix.storage}} == "posix" ] && [ "$n" == "auth" ] || [ "$n" == "sync" ] || [ "$n" == "syncapi" ]; then - continue - fi - if [ ! $(kubectl get pods -l role="$n" -o=jsonpath='{.items[*].status.containerStatuses[0].ready}' | grep true) ]; then - echo "$n is not ready after 30s, exiting" - exit 1 - fi - done - - - name: test - if: always() - run: | - kubectl get pods - sleep 1 - for svc in api auth finalize inbox ingest mapper reencrypt sync syncapi verify; do - echo "## describe $svc" && kubectl describe pod -l role="$svc" - sleep 1 - echo "## logs $svc" && kubectl logs -l role="$svc" - sleep 1 - done - if [ ${{matrix.deployment}} == "federated" ]; then - echo "## describe intercept" && kubectl describe pod -l role=intercept - sleep 1 - echo "## logs intercept" && kubectl logs -l role=intercept - sleep 1 - fi - shell: bash \ No newline at end of file diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml deleted file mode 100644 index 739d7af572..0000000000 --- a/.github/workflows/functionality.yml +++ /dev/null @@ -1,92 +0,0 @@ -name: Functionality tests - -on: - merge_group: - pull_request: - -jobs: - check_changes: - outputs: - sda-download: ${{ steps.changes.outputs.sda-download }} - sftp-inbox: ${{ steps.changes.outputs.sftp-inbox }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 - id: changes - with: - filters: | - sda-download: - - 'sda-download/**' - sftp-inbox: - - 'sda-sftp-inbox/**' - - sda-download: - needs: check_changes - if: needs.check_changes.outputs.sda-download == 'true' - name: sda-download-integration-${{ matrix.storagetype }} - runs-on: ubuntu-latest - env: - STORAGETYPE: ${{ matrix.storagetype }} - - strategy: - matrix: - storagetype: [s3, posix, s3notls] - fail-fast: false - steps: - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Check out code into the Go module directory - uses: actions/checkout@v6 - - - name: Run setup scripts - run: | - cd sda-download - ls -1 .github/integration/setup/{common,${{ matrix.storagetype }}}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do - echo "Executing setup script $runscript"; - bash -x "$runscript"; - done - - - name: Run tests - run: | - cd sda-download - ls -1 .github/integration/tests/{common,${{ matrix.storagetype }}}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do - echo "Executing test script $runscript"; - bash -x "$runscript"; - done - - sftp-inbox: - needs: check_changes - if: needs.check_changes.outputs.sftp-inbox == 'true' - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v6 - - - name: Build image - run: | - cd sda-sftp-inbox - docker build -t neicnordic/sda-inbox-sftp:latest . - - - name: Start Services - run: | - cd sda-sftp-inbox/dev_utils - bash ./make_certs.sh - docker compose up certfixer - docker compose up -d - sleep 20 - - - name: Run test for sftp ssh connection - run: | - cd sda-sftp-inbox - chmod 700 src/test/resources/id_ed25519 - sh ./dev_utils/sftp-testing-ssh.sh - - - name: Run test for sftp with password - run: | - cd sda-sftp-inbox - sudo apt-get install expect -y - sh ./dev_utils/sftp-testing-pass.sh \ No newline at end of file diff --git a/.github/workflows/ghcr-actions.yml b/.github/workflows/ghcr-actions.yml deleted file mode 100644 index 3a3d304310..0000000000 --- a/.github/workflows/ghcr-actions.yml +++ /dev/null @@ -1,29 +0,0 @@ -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: ghcr.io - -on: - schedule: - - cron: "21 21 * * *" - workflow_dispatch: - -jobs: - clean-ghcr: - name: Delete old unused container images - runs-on: ubuntu-latest - permissions: - packages: write - steps: - - name: Delete 'PR' containers older than one month - uses: snok/container-retention-policy@v3.0.1 - with: - account: ${{ github.repository_owner }} - image-names: sensitive-data-archive - image-tags: "!v*" - cut-off: One month ago UTC - keep-n-most-recent: 1 - token: ${{ secrets.GHCR_TOKEN }} - timestamp-to-use: updated_at diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml new file mode 100644 index 0000000000..fd8020900a --- /dev/null +++ b/.github/workflows/pr_build_and_test_images.yml @@ -0,0 +1,379 @@ +name: "PR " +on: + pull_request: + paths: + - "charts/**" # We want to run integration tests if charts are changed, given the chart integration tests + - "sda/**" + - "sda-download/**" + - "sda-validator/orchestrator/**" + - "sda-doa/**" + - "postgresql/**" + - "rabbitmq/**" +env: + PR_NUMBER: ${{ github.event.number }} # Needed by the triggered integration tests to pull image for this PR + +jobs: + build_images: + name: Build ${{ matrix.name }} Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + pull-requests: write + security-events: write + strategy: + matrix: + include: + - name: "SDA Services" + path: "sda" + image-tag-suffix: "" + + - name: "Validator Orchestrator" + path: "sda-validator/orchestrator" + image-tag-suffix: "-validator-orchestrator" + + - name: "Download" + path: "sda-download" + image-tag-suffix: "-download" + + - name: "Doa" + path: "sda-doa" + image-tag-suffix: "-doa" + + - name: "PostgresSQL" + path: "postgresql" + image-tag-suffix: "-postgres" + + - name: "Rabbitmq" + path: "rabbitmq" + image-tag-suffix: "-rabbitmq" + + - name: "STFP inbox" + path: "sda-sftp-inbox" + image-tag-suffix: "-sftp-inbox" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Log in to the Github Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build ${{ matrix.name }} container + uses: docker/build-push-action@v6 + with: + context: ./${{ matrix.path }} + push: true + tags: | + ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + labels: | # As this is a PR we set the version by the git sha + org.opencontainers.image.title=${{ github.repository }} + org.opencontainers.image.description=Sensitive Data Archive - ${{ matrix.name }} + org.opencontainers.image.authors=NeIC System Developers + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}$/{{ matrix.path }} + org.opencontainers.image.created=${{ github.event.pull_request.updated_at }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ github.sha }} + + - name: Run Trivy vulnerability scanner on ${{ matrix.name }} sarif + uses: aquasecurity/trivy-action@0.34.2 + env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: results.sarif + severity: "CRITICAL,HIGH" + + # We do this step such that the report in the Github code scanning is more clear for which image a vulnerability originates from + - name: Append image info to sarif file + run: | + jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri |= "ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }}/" + .' \ + results.sarif > annotated_results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: annotated_results.sarif + category: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + + # Only run trivy scan with markdown output and add PR comment if PR changes the relevant file + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + changed: + - '${{ matrix.path }}/**' + + - name: Run Trivy vulnerability scanner on ${{ matrix.name }} markdown output + if: steps.changes.outputs.changed == 'true' + uses: aquasecurity/trivy-action@0.34.2 + env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + format: template + template: '@.github/trivy_report_template_markdown.tpl' + output: trivy-results.md + + - name: Generate PR comment + if: steps.changes.outputs.changed == 'true' + run: | + echo "## 🔍 Trivy Scan - ${{ matrix.name }} 🔍" > trivy-comment.md + cat trivy-results.md >> trivy-comment.md + + - name: Comment PR + if: steps.changes.outputs.changed == 'true' + uses: marocchino/sticky-pull-request-comment@v2 + with: + header: ${{ matrix.name }} Trivy results + path: trivy-comment.md + + rabbitmq_test_federated: + needs: + - build_images + name: Rabbitmq federated tests + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Test rabbitmq federation + run: docker compose -f .github/integration/rabbitmq-federation.yml run federation_test + + postgres_test: + needs: + - build_images + name: Postgres tests + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Test postgres + run: docker compose -f .github/integration/postgres.yml run tests + + sda_integration_test: + name: sda ${{ matrix.storage }} integration tests + needs: + - build_images + runs-on: ubuntu-latest + strategy: + matrix: + storage: ["posix", "s3"] + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Test sensitive-data-archive ${{ matrix.storage }} storage + run: docker compose -f .github/integration/sda-${{matrix.storage}}-integration.yml run integration_test + + sda-sync_integration_test: + name: sda-sync integration tests + needs: + - build_images + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v6 + - name: Test sda-sync + run: docker compose -f .github/integration/sda-sync-integration.yml run integration_test + + sda-validator-orchestrator_integration_test: + name: sda-validator-orchestrator integration tests + needs: + - build_images + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Test sda-validator-orchestrator + # Given sda-validator-orchestrator needs to run apptainer we need to modify some apparmor restrictions to allow it + run: | + echo 'kernel.apparmor_restrict_unprivileged_userns = 0' | sudo tee /etc/sysctl.d/20-apparmor-donotrestrict.conf + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + docker compose -f .github/integration/sda-validator-orchestrator-integration.yml run integration_test + + + sda-download_integration_test: # TODO this tests is rebuilding all images, shouldnt be needed + needs: + - build_images + name: sda-download-integration ${{ matrix.storagetype }} integration tests + runs-on: ubuntu-latest + env: + STORAGETYPE: ${{ matrix.storagetype }} + + strategy: + matrix: + storagetype: [s3, posix, s3notls] + fail-fast: false + steps: + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Check out code into the Go module directory + uses: actions/checkout@v6 + + - name: Run setup scripts + run: | + cd sda-download + ls -1 .github/integration/setup/{common,${{ matrix.storagetype }}}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do + echo "Executing setup script $runscript"; + bash -x "$runscript"; + done + + - name: Run tests + run: | + cd sda-download + ls -1 .github/integration/tests/{common,${{ matrix.storagetype }}}/*.sh 2>/dev/null | sort -t/ -k5 -n | while read -r runscript; do + echo "Executing test script $runscript"; + bash -x "$runscript"; + done + + sda-doa_integration_test: + name: sda-doa ${{ matrix.storage }} integration tests + needs: + - build_images + runs-on: ubuntu-latest + strategy: + matrix: + storage: [s3, posix] + steps: + - name: Check out code + uses: actions/checkout@v6 + + - name: Test sda-doa ${{ matrix.storage }} storage + run: docker compose -f .github/integration/sda-doa-${{ matrix.storage }}-outbox.yml run integration_test + + sftp-inbox_integration_test: # TODO this tests is rebuilding stfp-inbox image, shouldnt be needed + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + + - name: Build image + run: | + cd sda-sftp-inbox + docker build -t neicnordic/sda-inbox-sftp:latest . + + - name: Start Services + run: | + cd sda-sftp-inbox/dev_utils + bash ./make_certs.sh + docker compose up certfixer + docker compose up -d + sleep 20 + + - name: Run test for sftp ssh connection + run: | + cd sda-sftp-inbox + chmod 700 src/test/resources/id_ed25519 + sh ./dev_utils/sftp-testing-ssh.sh + + - name: Run test for sftp with password + run: | + cd sda-sftp-inbox + sudo apt-get install expect -y + sh ./dev_utils/sftp-testing-pass.sh + + chart_tests: + needs: + - build_images + runs-on: ubuntu-latest + strategy: + matrix: + deployment: ["federated", "isolated"] + version: ["1.32", "1.33"] + tls: ["true", "false"] + storage: ["posix", "s3"] + exclude: + - version: "1.32" + tls: "false" + storage: "posix" + - version: "1.33" + tls: "false" + storage: "posix" + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Initialise k3d + id: initK3D + run: bash .github/integration/scripts/charts/k3d.sh ${{matrix.version}} + shell: bash + - name: debug + if: steps.initK3D.outcome == 'failure' + run: k3d version list k3s | grep ${{matrix.version}} + shell: bash + + - name: Deploy external services + run: bash .github/integration/scripts/charts/dependencies.sh ${{matrix.deployment}} ${{matrix.storage}} ${{matrix.tls}} + shell: bash + + - name: Deploy DB + id: deployDB + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-db ${{ github.event.number }} ${{matrix.tls}} + - name: debug + if: failure() && steps.deployDB.outcome == 'failure' + run: | + kubectl describe pod postgres-sda-db-0 + sleep 1 + kubectl logs postgres-sda-db-0 + + - name: Deploy MQ + id: deployMQ + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} ${{matrix.tls}} ${{matrix.deployment}} + shell: bash + - name: debug + if: failure() && steps.deployMQ.outcome == 'failure' + run: | + kubectl describe pod broker-sda-mq-0 + sleep 1 + kubectl logs broker-sda-mq-0 + + - name: Deploy pipeline + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} ${{matrix.tls}} ${{matrix.storage}} ${{matrix.deployment}} + shell: bash + + - name: Check deployment + run: | + sleep 30 + for n in api auth download finalize inbox ingest mapper reencrypt sync syncapi verify; do + if [ ${{matrix.storage}} == "posix" ] && [ "$n" == "auth" ] || [ "$n" == "sync" ] || [ "$n" == "syncapi" ]; then + continue + fi + if [ ! $(kubectl get pods -l role="$n" -o=jsonpath='{.items[*].status.containerStatuses[0].ready}' | grep true) ]; then + echo "$n is not ready after 30s, exiting" + exit 1 + fi + done + + - name: test + if: always() + run: | + kubectl get pods + sleep 1 + for svc in api auth finalize inbox ingest mapper reencrypt sync syncapi verify; do + echo "## describe $svc" && kubectl describe pod -l role="$svc" + sleep 1 + echo "## logs $svc" && kubectl logs -l role="$svc" + sleep 1 + done + if [ ${{matrix.deployment}} == "federated" ]; then + echo "## describe intercept" && kubectl describe pod -l role=intercept + sleep 1 + echo "## logs intercept" && kubectl logs -l role=intercept + sleep 1 + fi + shell: bash \ No newline at end of file diff --git a/.github/workflows/pr_closed_clean_images.yml b/.github/workflows/pr_closed_clean_images.yml new file mode 100644 index 0000000000..18ce1563da --- /dev/null +++ b/.github/workflows/pr_closed_clean_images.yml @@ -0,0 +1,16 @@ +name: clean image registry +on: + merge_group: + pull_request: # We run this in all PRs even if PR didnt build images just to be safe + types: [ closed ] + +jobs: + clean_pr_images_from_registry: + name: Cleanup PR docker images from Github Container registry + runs-on: ubuntu-latest + steps: + - name: Delete pull request docker images + uses: jenskeiner/ghcr-container-repository-cleanup-action@v1 + with: + include-tags: ^PR${{ github.event.number }}($|[-.].*) + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/pr_code_linter.yaml similarity index 98% rename from .github/workflows/code-linter.yaml rename to .github/workflows/pr_code_linter.yaml index 4a9258e300..ea6e21c28f 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/pr_code_linter.yaml @@ -1,7 +1,6 @@ -name: Multilinters +name: Linter on: - merge_group: pull_request: paths: - '**.go' diff --git a/.github/workflows/test.yml b/.github/workflows/pr_code_tests.yml similarity index 96% rename from .github/workflows/test.yml rename to .github/workflows/pr_code_tests.yml index d619625b7b..a5d2236aec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/pr_code_tests.yml @@ -1,8 +1,12 @@ name: Code tests on: - push: - branches-ignore: [main] + pull_request: + paths: + - "sda/**" + - "sda-download/**" + - "sda-validator/orchestrator/**" + - "sda-admin/**" env: go-version: '1.25' diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/pr_codeql_analysis.yml similarity index 94% rename from .github/workflows/codeql-analysis.yml rename to .github/workflows/pr_codeql_analysis.yml index 059043c67a..9323beaf4a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/pr_codeql_analysis.yml @@ -1,17 +1,12 @@ name: "CodeQL" on: - push: - branches: - - main pull_request: branches: - main paths: - '**.go' - '**.java' - schedule: - - cron: '0 15 * * 4' jobs: analyze: diff --git a/.github/workflows/pr_merge_increment_version_tag_sda.yml b/.github/workflows/pr_merge_increment_version_tag_sda.yml new file mode 100644 index 0000000000..a73fb4917c --- /dev/null +++ b/.github/workflows/pr_merge_increment_version_tag_sda.yml @@ -0,0 +1,32 @@ +name: version tag SDA in main +on: + merge_group: + pull_request: + branches: + - main + paths: + - "sda/**" + - "sda-download/**" + - "sda-validator/orchestrator/**" + - "sda-doa/**" + - "postgresql/**" + - "rabbitmq/**" + types: [ closed ] + +jobs: + tag_release: + if: github.event.pull_request.merged == true + name: tag sda version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: '0' + - name: Bump version and push tag + id: bump_tag + uses: anothrNick/github-tag-action@1.75.0 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_PREFIX: v + DEFAULT_BUMP: patch + TAG_CONTEXT: branch diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml deleted file mode 100644 index 4a188d02ef..0000000000 --- a/.github/workflows/publish_container.yml +++ /dev/null @@ -1,141 +0,0 @@ -name: Publish version tag -on: - merge_group: - pull_request: - branches: - - main - paths-ignore: - - ".github/**" - - ".gitignore" - - "**/*.md" - - "charts/**" - - "Makefile" - - "sda-admin/**" - - "tools/*" - types: [ closed ] - -jobs: - tag_release: - if: github.event.pull_request.merged == true - name: tag version - outputs: - part: ${{ steps.bump_tag.outputs.part }} - tag: ${{ steps.bump_tag.outputs.tag }} - new_tag: ${{ steps.bump_tag.outputs.new_tag }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - fetch-depth: '0' - - name: Bump version and push tag - id: bump_tag - uses: anothrNick/github-tag-action@1.75.0 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG_PREFIX: v - DEFAULT_BUMP: patch - TAG_CONTEXT: branch - - push_to_registry: - needs: tag_release - if: needs.tag_release.outputs.tag != '' - name: Push Docker image to Github Container registry - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - steps: - - name: Check out the repo - uses: actions/checkout@v6 - - - name: Log in to the Github Container registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Pull PR containers - run : | - for t in -download -postgres -rabbitmq -sftp-inbox -doa -validator-orchestrator; do - docker pull ghcr.io/${{ github.repository }}:PR${{ github.event.number }}$t - done - docker pull ghcr.io/${{ github.repository }}:PR${{ github.event.number }} - - - name: Retag PR image for download - run: | - docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-download ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-download - docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-download - - - name: Retag PR image for postgres - run: | - docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-postgres ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-postgres - docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-postgres - - - name: Retag PR image for rabbitmq - run: | - docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-rabbitmq ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-rabbitmq - docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-rabbitmq - - - name: Retag PR image for sftp-inbox - run: | - docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-sftp-inbox ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-sftp-inbox - docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-sftp-inbox - - - name: Retag PR image for sensitive-data-archive - run: | - docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }} ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }} - docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }} - - - name: Retag PR image for sda-validator-orchestrator - run: | - docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-validator-orchestrator ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-validator-orchestrator - docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-validator-orchestrator - - - name: Retag PR image for doa - run: | - docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-doa ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-doa - docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-doa - - bump_chart_version: - needs: [tag_release, push_to_registry] - if: ${{ needs.tag_release.outputs.tag != '' && needs.push_to_registry.result == 'success' }} - permissions: - contents: write - pull-requests: write - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v6 - with: - fetch-depth: '0' - - - name: Install PyBump - run: | - python3 -m pip install pybump - shell: bash - - - name: Automated Version Bump - id: version-bump - run: | - for name in sda-db sda-mq sda-svc; do - pybump set --file charts/"$name"/Chart.yaml --set-version ${{ needs.tag_release.outputs.tag }} --app-version - pybump bump --file charts/"$name"/Chart.yaml --level patch - done - shell: bash - - - name: Create Pull Request - uses: peter-evans/create-pull-request@v8 - with: - token: ${{ secrets.GHCR_TOKEN }} - commit-message: Bump chart version - author: neicnordic - base: main - branch: bump - delete-branch: true - title: '[charts] Bump version' - body: | - Bump the appVersion to: ${{ needs.tag_release.outputs.tag }} - labels: | - automated pr - team-reviewers: neicnordic/sensitive-data-development-collaboration diff --git a/.github/workflows/publish_charts.yml b/.github/workflows/release_charts.yml similarity index 100% rename from .github/workflows/publish_charts.yml rename to .github/workflows/release_charts.yml diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml new file mode 100644 index 0000000000..deb189759d --- /dev/null +++ b/.github/workflows/release_sda.yml @@ -0,0 +1,173 @@ +name: Build versioned images +on: + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +env: + VERSION_TAG: ${{ github.ref_name }} + +jobs: + + build_images: + name: Build ${{ matrix.name }} Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + security-events: write + strategy: + matrix: + include: + - name: "SDA Services" + path: "sda" + image-tag-suffix: "" + + - name: "Validator Orchestrator" + path: "sda-validator/orchestrator" + image-tag-suffix: "-validator-orchestrator" + + - name: "Download" + path: "sda-download" + image-tag-suffix: "-download" + + - name: "Doa" + path: "sda-doa" + image-tag-suffix: "-doa" + + - name: "PostgresSQL" + path: "postgresql" + image-tag-suffix: "-postgres" + + - name: "Rabbitmq" + path: "rabbitmq" + image-tag-suffix: "-rabbitmq" + + - name: "STFP inbox" + path: "sda-sftp-inbox" + image-tag-suffix: "-sftp-inbox" + + steps: + - name: Checkout code + uses: actions/checkout@v6 + + - name: Log in to the Github Container registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build ${{ matrix.name }} container + uses: docker/build-push-action@v6 + with: + context: ./${{ matrix.path }} + push: true + tags: | + ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} + labels: | + org.opencontainers.image.title=${{ github.repository }} + org.opencontainers.image.description=Sensitive Data Archive - ${{ matrix.name }} + org.opencontainers.image.authors=NeIC System Developers + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}/tree/${{ env.VERSION_TAG }}/${{ matrix.path }} + org.opencontainers.image.created=${{ github.event.head_commit.timestamp }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ env.VERSION_TAG }} + + + - name: Run Trivy vulnerability scanner on ${{ matrix.name }} + uses: aquasecurity/trivy-action@0.34.2 + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: results.sarif + severity: "CRITICAL,HIGH" + + # We do this step such that the report in the Github code scanning is more clear for which image a vulnerability originates from + - name: Append image info to sarif file + run: | + jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri |= "ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }}/" + .' \ + results.sarif > annotated_results.sarif + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v4 + with: + sarif_file: annotated_results.sarif + category: ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} + + code_ql: + name: Run CodeQL + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: [ 'java', 'go' ] + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: ${{ matrix.language }} + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 21 + + - name: Build sda-doa + if: ${{ matrix.language == 'java' }} + working-directory: ./sda-doa + run: mvn clean install -DskipTests + + - name: Set up JDK 19 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 19 + + - name: Build sda-sftp-inbox + if: ${{ matrix.language == 'java' }} + working-directory: ./sda-sftp-inbox + run: mvn clean install -DskipTests + + - name: Autobuild + if: ${{ matrix.language == 'go' }} + uses: github/codeql-action/autobuild@v4 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: "/language:${{matrix.language}}" + + + create_release: + needs: + - build_images + name: Create SDA release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + - name: Release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: "true" + tag_name: "${{ env.VERSION }}" + body: | + # Docker images: + - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }} + - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-download + - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-doa + - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-validator-orchestrator + - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-stfp-inbox + - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-postgres + - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-rabbitmq \ No newline at end of file diff --git a/.github/workflows/release_sda-admin.yaml b/.github/workflows/release_sda_admin.yaml similarity index 100% rename from .github/workflows/release_sda-admin.yaml rename to .github/workflows/release_sda_admin.yaml diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml deleted file mode 100644 index 80c2fd6a84..0000000000 --- a/.github/workflows/scan-images.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: Scan Images -on: - pull_request: - branches: - - main - paths-ignore: - - ".github/**" - - ".gitignore" - - "**/*.md" - - "charts/**" - - "Makefile" - - "sda-admin/**" - - "tools/*" - types: [ closed ] - -jobs: - scan-all: - if: github.event.pull_request.merged == true - name: Scan ${{ matrix.image-name }} Docker Image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write - strategy: - matrix: - image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] - steps: - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.34.2 - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true - with: - image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-${{ matrix.image-name }} - format: "sarif" - hide-progress: true - ignore-unfixed: true - output: '${{ matrix.image-name }}-results.sarif' - severity: "CRITICAL,HIGH" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: '${{ matrix.image-name }}-results.sarif' - category: ${{ matrix.image-name }} - scan-sda: - if: github.event.pull_request.merged == true - name: Scan SDA Docker image - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - security-events: write - steps: - - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.34.2 - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true - with: - image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }} - format: "sarif" - hide-progress: true - ignore-unfixed: true - output: 'sda-results.sarif' - severity: "CRITICAL,HIGH" - - name: Upload Trivy scan results to GitHub Security tab - uses: github/codeql-action/upload-sarif@v4 - with: - sarif_file: 'sda-results.sarif' - category: sda \ No newline at end of file From 96a918e159f5409393708273fa051ef0b49cfef8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 27 Feb 2026 13:25:40 +0100 Subject: [PATCH 298/530] feat(docker): Update docker files - Remove the org.label-schema labels as they are deprecated [see](http://label-schema.org/rc1/) in favour of the [OpenContainers Image Spec](https://specs.opencontainers.org/image-spec/annotations/?v=v1.0.1) which are now populated by the github actions workflow when images are built - Update postgres source image postgres:15.4-alpine3.17 -> postgres:15.16-alpine3.23 to resolve CVEs from alpine3.17 - Update SDA docker file to remove the "debug" stage - validator orchestrator - update grpc-health-probe:v0.4.42 -> grpc-health-probe:v0.4.45, - apptainer v1.4.3 -> v1.4.5 --- postgresql/Dockerfile | 8 +------- rabbitmq/Dockerfile | 5 ----- sda-download/Dockerfile | 6 ------ sda-validator/orchestrator/Dockerfile | 12 +++--------- sda-validator/orchestrator/README.md | 2 +- sda/Dockerfile | 24 ------------------------ 6 files changed, 5 insertions(+), 52 deletions(-) diff --git a/postgresql/Dockerfile b/postgresql/Dockerfile index 6e5917c897..ea936fe9ba 100644 --- a/postgresql/Dockerfile +++ b/postgresql/Dockerfile @@ -1,14 +1,8 @@ -FROM postgres:15.4-alpine3.17 +FROM postgres:15.16-alpine3.23 ARG BUILD_DATE ARG SOURCE_COMMIT -LABEL org.opencontainers.image.authors="NeIC System Developers" -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive" -LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT - COPY initdb.d /docker-entrypoint-initdb.d COPY migratedb.d /migratedb.d COPY --chmod=755 entrypoint.sh /usr/local/bin/entrypoint.sh diff --git a/rabbitmq/Dockerfile b/rabbitmq/Dockerfile index 4a2aa194c3..9077fe73ed 100644 --- a/rabbitmq/Dockerfile +++ b/rabbitmq/Dockerfile @@ -3,11 +3,6 @@ FROM rabbitmq:3.12.13-management-alpine ARG BUILD_DATE ARG SOURCE_COMMIT -LABEL org.opencontainers.image.authors="NeIC System Developers" -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sda" -LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT ENV RABBITMQ_CONFIG_FILE=/var/lib/rabbitmq/rabbitmq.conf ENV RABBITMQ_ADVANCED_CONFIG_FILE=/var/lib/rabbitmq/advanced.config diff --git a/sda-download/Dockerfile b/sda-download/Dockerfile index f050ab04c6..43d0c77306 100644 --- a/sda-download/Dockerfile +++ b/sda-download/Dockerfile @@ -13,12 +13,6 @@ FROM scratch ARG BUILD_DATE ARG SOURCE_COMMIT -LABEL maintainer="NeIC System Developers" -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sda-download" -LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT - COPY --from=builder /go/passwd /etc/passwd COPY --from=builder /go/sda-* /usr/bin/ COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/sda-validator/orchestrator/Dockerfile b/sda-validator/orchestrator/Dockerfile index 68383cd87d..a08486168e 100644 --- a/sda-validator/orchestrator/Dockerfile +++ b/sda-validator/orchestrator/Dockerfile @@ -3,8 +3,8 @@ FROM --platform=linux/amd64 debian:trixie-slim AS apptainer_builder RUN apt update \ && apt install -y wget \ && cd /tmp \ - && wget https://github.com/apptainer/apptainer/releases/download/v1.4.3/apptainer_1.4.3-trixie+_amd64.deb \ - && apt install -y ./apptainer_1.4.3-trixie+_amd64.deb libsubid5 squashfs-tools + && wget https://github.com/apptainer/apptainer/releases/download/v1.4.5/apptainer_1.4.5-trixie+_amd64.deb \ + && apt install -y ./apptainer_1.4.5-trixie+_amd64.deb libsubid5 squashfs-tools COPY ./apptainer.conf /etc/apptainer/apptainer.conf @@ -21,14 +21,8 @@ FROM --platform=linux/amd64 gcr.io/distroless/static-debian12 ARG BUILD_DATE ARG SOURCE_COMMIT -LABEL maintainer="NeIC System Developers" -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=$BUILD_DATE -LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive" -LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT - # Add health probe binary such that it can be used for docker, etc -COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.42 /ko-app/grpc-health-probe /bin/grpc_health_probe +COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.45 /ko-app/grpc-health-probe /bin/grpc_health_probe COPY --from=app_builder /go/sda-validator-orchestrator /usr/bin/ COPY --from=app_builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt diff --git a/sda-validator/orchestrator/README.md b/sda-validator/orchestrator/README.md index faca7f430c..07bb7de084 100644 --- a/sda-validator/orchestrator/README.md +++ b/sda-validator/orchestrator/README.md @@ -229,7 +229,7 @@ rm -rf api/openapi_interface/go/ ## Build and deploy The [Dockerfile](Dockerfile) will produce an image containing the sda-validator-orchestrator and only its dependant -binaries by utilising the [distroless image](gcr.io/distroless/static-debian12. +binaries by utilising the [distroless image](gcr.io/distroless/static-debian12). The main dependant binary is [Apptainer](#apptainer) which is described in its section. diff --git a/sda/Dockerfile b/sda/Dockerfile index 2a0f437fad..eac27add89 100644 --- a/sda/Dockerfile +++ b/sda/Dockerfile @@ -5,32 +5,8 @@ COPY . . SHELL ["bash", "-c"] RUN set -ex; for p in cmd/*; do go build -buildvcs=false -o "${p/cmd\//sda-}" "./$p"; done -FROM debian:trixie-slim AS Debug - -ARG SOURCE_COMMIT - -LABEL org.opencontainers.image.authors="NeIC System Developers" -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=now() -LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive" -LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT - -COPY --from=Build /go/sda-* /usr/local/bin/ -COPY --from=Build /go/cmd/auth/frontend /frontend -COPY --from=Build /go/schemas /schemas - -USER 65534 - FROM gcr.io/distroless/static-debian13 -ARG SOURCE_COMMIT - -LABEL org.opencontainers.image.authors="NeIC System Developers" -LABEL org.label-schema.schema-version="1.0" -LABEL org.label-schema.build-date=now() -LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive" -LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT - COPY --from=Build /go/sda-* /usr/local/bin/ COPY --from=Build /go/cmd/auth/frontend /frontend COPY --from=Build /go/schemas /schemas From 80ae896273126e213bd0d5cdc3e2ebcc7cdb6527 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 27 Feb 2026 14:06:00 +0100 Subject: [PATCH 299/530] feat(dependabot): add validator orchestrator, docker and gomod ecosystem to dependabot --- .github/dependabot.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 4c843fe89e..22cec02004 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -31,6 +31,12 @@ updates: interval: weekly open-pull-requests-limit: 10 + - package-ecosystem: docker + directory: "/sda-validator-orchestrator" + schedule: + interval: weekly + open-pull-requests-limit: 10 + - package-ecosystem: docker directory: "/sda-sftp-inbox" schedule: @@ -61,6 +67,17 @@ updates: schedule: interval: weekly + - package-ecosystem: gomod + directory: "/sda-validator/orchestrator" + groups: + all-modules: + patterns: + - "*" + open-pull-requests-limit: 10 + + schedule: + interval: weekly + # Each subfolder needs to be checked separately - package-ecosystem: maven directory: "/sda-sftp-inbox" From 053d2722481a661f331e3b326e50325b27756386 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 2 Mar 2026 09:02:14 +0100 Subject: [PATCH 300/530] feat(github workflow): update clean image registry workflow to use gh api to delete images --- .github/workflows/pr_closed_clean_images.yml | 74 ++++++++++++++++++-- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pr_closed_clean_images.yml b/.github/workflows/pr_closed_clean_images.yml index 18ce1563da..d7d7975f1b 100644 --- a/.github/workflows/pr_closed_clean_images.yml +++ b/.github/workflows/pr_closed_clean_images.yml @@ -1,6 +1,7 @@ name: clean image registry on: - merge_group: + schedule: + - cron: '0 8 * * *' # Run daily at 08:00 UTC pull_request: # We run this in all PRs even if PR didnt build images just to be safe types: [ closed ] @@ -8,9 +9,70 @@ jobs: clean_pr_images_from_registry: name: Cleanup PR docker images from Github Container registry runs-on: ubuntu-latest + permissions: + packages: write + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: - - name: Delete pull request docker images - uses: jenskeiner/ghcr-container-repository-cleanup-action@v1 - with: - include-tags: ^PR${{ github.event.number }}($|[-.].*) - token: ${{ secrets.GITHUB_TOKEN }} + - name: Get all container versions + run: | + gh api -H "Accept: application/vnd.github+json" /orgs/neicnordic/packages/container/sensitive-data-archive/versions --paginate > all_container_versions + + # Get containers built tagged by the PR number if the container does not have a semantic version tag(which should not be possible by the workflow but check is there for safety) + - name: Filter container versions from PR + if: ${{ github.event.number != '' }} + run: | + # (use jq to find the container versions which have do not have semantic version tag) + PR_TAG_REGEX="^PR${{ github.event.number }}(-[A-Za-z0-9_-]+)?$" + cat all_container_versions | jq -r --arg tag "$PR_TAG_REGEX" --arg sem_ver '^v?[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$' ' + .[] | + select(.metadata.container.tags != null) | + select([.metadata.container.tags[] | test($tag)] | any) | + select( + [ .metadata.container.tags[] | test($sem_ver) ] | any | not + ) | + "\(.id) \(.updated_at) \(.metadata.container.tags | join(","))" + ' > filtered_containers_versions + + # Get containers which has tags but no semantic version tag, which was last updated 30 days ago + - name: Filter container versions from cronjob, which has tags but no semantic version tags and is older than 30 days + if: ${{ github.event.number == '' }} + run: | + cat all_container_versions | jq -r --argjson now "$(date +%s)" --arg sem_ver '^v?[0-9]+\.[0-9]+\.[0-9]+([.-][0-9A-Za-z.-]+)?$' ' + .[] | + # parse updated_at timestamp + (.updated_at | sub("Z$"; "") | sub("\\s+$"; "") | strptime("%Y-%m-%dT%H:%M:%S") | mktime) as $updated_ts | + select( + # Has no semver tags + ([ .metadata.container.tags[] | test($sem_ver)] | any | not) + and + ((.metadata.container.tags // []) | length > 0) + and + # older than threshold + ((($now - $updated_ts) / 86400) > 30) + ) | + "\(.id) \(.updated_at) \(.metadata.container.tags | join(","))" + ' > filtered_containers_versions + + # Always delete all untagged container versions, as no point to keep them since no tags + - name: Filter all untagged container versions + run: | + cat all_container_versions | jq -r ' + .[] | + select((.metadata.container.tags // []) | length == 0) | + "\(.id) \(.updated_at) \(.metadata.container.tags | join(","))" + ' >> filtered_containers_versions + + - name: Delete container versions + run: | + while read -r id date tags; do + echo "Deleting id: $id, last updated at: $date (tags: $tags)" + + gh api \ + -X DELETE \ + -H "Accept: application/vnd.github+json" \ + /orgs/${{ github.repository_owner }}/packages/container/$(echo ${{ github.repository }} | cut -d "/" -f 2)/versions/$id + + done < filtered_containers_versions + + From e172dc82dbf9ca9be7033023e2f5b2e012b4242e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Mar 2026 13:14:37 +0100 Subject: [PATCH 301/530] fix(github actions): fix env.VERSION -> env.VERSION_TAG in sda release job --- .github/workflows/release_sda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml index deb189759d..8dfdad8bcc 100644 --- a/.github/workflows/release_sda.yml +++ b/.github/workflows/release_sda.yml @@ -161,7 +161,7 @@ jobs: uses: softprops/action-gh-release@v2 with: generate_release_notes: "true" - tag_name: "${{ env.VERSION }}" + tag_name: "${{ env.VERSION_TAG }}" body: | # Docker images: - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }} From 5ba6b546889ea670b6a6e7d3a98eb29f4b1a3d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Mar 2026 13:15:19 +0100 Subject: [PATCH 302/530] fix(github actions): add sda-stfp-inbox to pr_build_and_test_images --- .github/workflows/pr_build_and_test_images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index fd8020900a..02ce7c8cc2 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -7,6 +7,7 @@ on: - "sda-download/**" - "sda-validator/orchestrator/**" - "sda-doa/**" + - "sda-sftp-inbox/**" - "postgresql/**" - "rabbitmq/**" env: From bf7a0e197c462663531ebcb8b267248b53913328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Mar 2026 14:42:38 +0100 Subject: [PATCH 303/530] fix(github actions): fix unaligned indentation in pr_code_linter.yaml --- .github/workflows/pr_code_linter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_code_linter.yaml b/.github/workflows/pr_code_linter.yaml index ea6e21c28f..93dd8bb567 100644 --- a/.github/workflows/pr_code_linter.yaml +++ b/.github/workflows/pr_code_linter.yaml @@ -25,7 +25,7 @@ jobs: with: filters: | sda-download: - - 'sda-download/**' + - 'sda-download/**' sda-admin: - 'sda-admin/**' sda: From 08ce64e59327609e98cbbddcba7e95984c20a028 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:39:57 +0000 Subject: [PATCH 304/530] build(deps): bump docker/login-action from 3 to 4 Bumps [docker/login-action](https://github.com/docker/login-action) from 3 to 4. - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/v3...v4) --- updated-dependencies: - dependency-name: docker/login-action dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pr_build_and_test_images.yml | 2 +- .github/workflows/release_sda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index 02ce7c8cc2..c9fc68fff7 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -58,7 +58,7 @@ jobs: uses: actions/checkout@v6 - name: Log in to the Github Container registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml index 8dfdad8bcc..49ca6b86c7 100644 --- a/.github/workflows/release_sda.yml +++ b/.github/workflows/release_sda.yml @@ -52,7 +52,7 @@ jobs: uses: actions/checkout@v6 - name: Log in to the Github Container registry - uses: docker/login-action@v3 + uses: docker/login-action@v4 with: registry: ghcr.io username: ${{ github.actor }} From 9d1e40448f872ea9efecf9bc9da473aed9639271 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:40:06 +0000 Subject: [PATCH 305/530] build(deps): bump docker/build-push-action from 6 to 7 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 6 to 7. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v6...v7) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pr_build_and_test_images.yml | 2 +- .github/workflows/release_sda.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index 02ce7c8cc2..8423e4581c 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -65,7 +65,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build ${{ matrix.name }} container - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./${{ matrix.path }} push: true diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml index 8dfdad8bcc..20acb819c0 100644 --- a/.github/workflows/release_sda.yml +++ b/.github/workflows/release_sda.yml @@ -59,7 +59,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build ${{ matrix.name }} container - uses: docker/build-push-action@v6 + uses: docker/build-push-action@v7 with: context: ./${{ matrix.path }} push: true From 5e9f50889d732a74ac2de132781df621d7a379c5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:40:11 +0000 Subject: [PATCH 306/530] build(deps): bump the all-modules group Bumps the all-modules group in /sda-validator/orchestrator with 2 updates: [github.com/neicnordic/crypt4gh](https://github.com/neicnordic/crypt4gh) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/neicnordic/crypt4gh` from 1.14.2 to 1.15.0 - [Release notes](https://github.com/neicnordic/crypt4gh/releases) - [Commits](https://github.com/neicnordic/crypt4gh/compare/v1.14.2...v1.15.0) Updates `google.golang.org/grpc` from 1.79.1 to 1.79.2 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.1...v1.79.2) --- updated-dependencies: - dependency-name: github.com/neicnordic/crypt4gh dependency-version: 1.15.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.79.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-validator/orchestrator/go.mod | 4 ++-- sda-validator/orchestrator/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index bd016408f3..fa1f6e7549 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -9,14 +9,14 @@ require ( github.com/google/uuid v1.6.0 github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.11.2 - github.com/neicnordic/crypt4gh v1.14.2 + github.com/neicnordic/crypt4gh v1.15.0 github.com/rabbitmq/amqp091-go v1.10.0 github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - google.golang.org/grpc v1.79.1 + google.golang.org/grpc v1.79.2 ) require ( diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index 04a728461d..1b6cc6587c 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -99,8 +99,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.14.2 h1:iUYbA6djJWql5WUet7J9Mw+iQ9DYRGJWvNWC8WfuZBc= -github.com/neicnordic/crypt4gh v1.14.2/go.mod h1:+y61sviowARFg7hMiggWvXe0LQXPDcUDOdflddsU1l4= +github.com/neicnordic/crypt4gh v1.15.0 h1:as+O2Y2IwXAKJzSJX5RPE7PLTCtWkygdjIMk3N34YCo= +github.com/neicnordic/crypt4gh v1.15.0/go.mod h1:4FYcQUA0mtdEosuktXaWw8IID2VYL8lKKHbSKxoI4g4= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -194,8 +194,8 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= +google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From d3c8312beb9657d4b4a0d5f1137e20025888d4cc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:40:40 +0000 Subject: [PATCH 307/530] build(deps): bump the all-modules group in /sda-doa with 4 updates Bumps the all-modules group in /sda-doa with 4 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot), [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway), [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway) and [org.yaml:snakeyaml](https://bitbucket.org/snakeyaml/snakeyaml). Updates `org.springframework.boot:spring-boot-starter-parent` from 4.0.1 to 4.0.3 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v4.0.1...v4.0.3) Updates `no.elixir:crypt4gh` from 3.0.42 to 3.0.48 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.42...crypt4gh-3.0.48) Updates `no.elixir:clearinghouse` from 3.0.16 to 3.0.19 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.16...crypt4gh-3.0.19) Updates `org.yaml:snakeyaml` from 2.5 to 2.6 - [Commits](https://bitbucket.org/snakeyaml/snakeyaml/branches/compare/snakeyaml-2.6..snakeyaml-2.5) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 4.0.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.48 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.19 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: org.yaml:snakeyaml dependency-version: '2.6' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 183c0a0c4e..01c4f62839 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 4.0.1 + 4.0.3 no.uio.ifi @@ -109,7 +109,7 @@ no.elixir crypt4gh - 3.0.42 + 3.0.48 org.slf4j @@ -120,7 +120,7 @@ no.elixir clearinghouse - 3.0.16 + 3.0.19 org.slf4j @@ -131,7 +131,7 @@ org.yaml snakeyaml - 2.5 + 2.6 com.squareup.okhttp3 From 0f77e336adfe9f539afaa1dd5ab9cda0aa38085c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:45:03 +0000 Subject: [PATCH 308/530] build(deps): bump the all-modules group in /sda-download with 5 updates Bumps the all-modules group in /sda-download with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.2` | `1.41.3` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.10` | `1.32.11` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.19.10` | `1.19.11` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.96.2` | `1.96.4` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.79.1` | `1.79.2` | Updates `github.com/aws/aws-sdk-go-v2` from 1.41.2 to 1.41.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.2...v1.41.3) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.10 to 1.32.11 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.10...config/v1.32.11) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.10 to 1.19.11 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/iam/v1.19.10...credentials/v1.19.11) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.96.2 to 1.96.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.96.2...service/s3/v1.96.4) Updates `google.golang.org/grpc` from 1.79.1 to 1.79.2 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.1...v1.79.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.11 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.11 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.96.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.79.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 38 +++++++++++------------ sda-download/go.sum | 76 ++++++++++++++++++++++----------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 6af46fbcba..9636c0c080 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -5,10 +5,10 @@ go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/Masterminds/semver/v3 v3.4.0 - github.com/aws/aws-sdk-go-v2 v1.41.2 - github.com/aws/aws-sdk-go-v2/config v1.32.10 - github.com/aws/aws-sdk-go-v2/credentials v1.19.10 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 + github.com/aws/aws-sdk-go-v2 v1.41.3 + github.com/aws/aws-sdk-go-v2/config v1.32.11 + github.com/aws/aws-sdk-go-v2/credentials v1.19.11 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/dgraph-io/ristretto v1.0.0 @@ -23,26 +23,26 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.48.0 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa - google.golang.org/grpc v1.79.1 + google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 ) require ( filippo.io/edwards25519 v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 21b66a63ab..83f02fc0e4 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -4,42 +4,42 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= -github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= -github.com/aws/aws-sdk-go-v2/config v1.32.10 h1:9DMthfO6XWZYLfzZglAgW5Fyou2nRI5CuV44sTedKBI= -github.com/aws/aws-sdk-go-v2/config v1.32.10/go.mod h1:2rUIOnA2JaiqYmSKYmRJlcMWy6qTj1vuRFscppSBMcw= -github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= -github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= +github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA= +github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= +github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc= +github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= @@ -217,8 +217,8 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= +google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From af9b87495f7135a58101a6b367d88111a2c17738 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:46:32 +0000 Subject: [PATCH 309/530] build(deps): bump the all-modules group in /sda with 7 updates Bumps the all-modules group in /sda with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.2` | `1.41.3` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.10` | `1.32.11` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.19.10` | `1.19.11` | | [github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager](https://github.com/aws/aws-sdk-go-v2) | `0.1.6` | `0.1.8` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.96.2` | `1.96.4` | | [github.com/neicnordic/crypt4gh](https://github.com/neicnordic/crypt4gh) | `1.14.2` | `1.15.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.79.1` | `1.79.2` | Updates `github.com/aws/aws-sdk-go-v2` from 1.41.2 to 1.41.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.2...v1.41.3) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.10 to 1.32.11 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.10...config/v1.32.11) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.10 to 1.19.11 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/iam/v1.19.10...credentials/v1.19.11) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager` from 0.1.6 to 0.1.8 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/transfermanager/v0.1.6...feature/s3/transfermanager/v0.1.8) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.96.2 to 1.96.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.96.2...service/s3/v1.96.4) Updates `github.com/neicnordic/crypt4gh` from 1.14.2 to 1.15.0 - [Release notes](https://github.com/neicnordic/crypt4gh/releases) - [Commits](https://github.com/neicnordic/crypt4gh/compare/v1.14.2...v1.15.0) Updates `google.golang.org/grpc` from 1.79.1 to 1.79.2 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.1...v1.79.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.11 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.11 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager dependency-version: 0.1.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.96.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/neicnordic/crypt4gh dependency-version: 1.15.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.79.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 42 +++++++++++++-------------- sda/go.sum | 84 +++++++++++++++++++++++++++--------------------------- 2 files changed, 63 insertions(+), 63 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index adfbfa5854..a91f44ed5c 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.25.7 require ( - github.com/aws/aws-sdk-go-v2 v1.41.2 - github.com/aws/aws-sdk-go-v2/config v1.32.10 - github.com/aws/aws-sdk-go-v2/credentials v1.19.10 - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.6 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 + github.com/aws/aws-sdk-go-v2 v1.41.3 + github.com/aws/aws-sdk-go-v2/config v1.32.11 + github.com/aws/aws-sdk-go-v2/credentials v1.19.11 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.8 + github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 @@ -22,7 +22,7 @@ require ( github.com/lib/pq v1.11.2 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 - github.com/neicnordic/crypt4gh v1.14.2 + github.com/neicnordic/crypt4gh v1.15.0 github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 github.com/ory/dockertest v3.3.5+incompatible github.com/ory/dockertest/v3 v3.12.0 @@ -34,7 +34,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.48.0 golang.org/x/oauth2 v0.35.0 - google.golang.org/grpc v1.79.1 + google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 ) @@ -50,20 +50,20 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 // indirect github.com/andybalholm/brotli v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/sda/go.sum b/sda/go.sum index 4dad7eb741..cc1b3c9d5b 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -25,44 +25,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.2 h1:LuT2rzqNQsauaGkPK/7813XxcZ3o3yePY0Iy891T2ls= -github.com/aws/aws-sdk-go-v2 v1.41.2/go.mod h1:IvvlAZQXvTXznUPfRVfryiG1fbzE2NGK6m9u39YQ+S4= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 h1:zWFmPmgw4sveAYi1mRqG+E/g0461cJ5M4bJ8/nc6d3Q= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5/go.mod h1:nVUlMLVV8ycXSb7mSkcNu9e3v/1TJq2RTlrPwhYWr5c= -github.com/aws/aws-sdk-go-v2/config v1.32.10 h1:9DMthfO6XWZYLfzZglAgW5Fyou2nRI5CuV44sTedKBI= -github.com/aws/aws-sdk-go-v2/config v1.32.10/go.mod h1:2rUIOnA2JaiqYmSKYmRJlcMWy6qTj1vuRFscppSBMcw= -github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= -github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 h1:Ii4s+Sq3yDfaMLpjrJsqD6SmG/Wq/P5L/hw2qa78UAY= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18/go.mod h1:6x81qnY++ovptLE6nWQeWrpXxbnlIex+4H4eYYGcqfc= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.6 h1:AaBtONvA5gHSZHo2EgjXMQCvYdQWclo3MznTkQnQ8w0= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.6/go.mod h1:+gC6IbVU+BWeCB0W9MWNgOpKuTmggJ/ES54eyKA8zQM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18 h1:F43zk1vemYIqPAwhjTjYIz0irU2EY7sOb/F5eJ3HuyM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.18/go.mod h1:w1jdlZXrGKaJcNoL+Nnrj+k5wlpGXqnNrKoP22HvAug= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18 h1:xCeWVjj0ki0l3nruoyP2slHsGArMxeiiaoPN5QZH6YQ= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.18/go.mod h1:r/eLGuGCBw6l36ZRWiw6PaZwPXb6YOj+i/7MizNl5/k= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18 h1:eZioDaZGJ0tMM4gzmkNIO2aAoQd+je7Ug7TkvAzlmkU= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.18/go.mod h1:CCXwUKAJdoWr6/NcxZ+zsiPr6oH/Q5aTooRGYieAyj4= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5 h1:CeY9LUdur+Dxoeldqoun6y4WtJ3RQtzk0JMP2gfUay0= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.5/go.mod h1:AZLZf2fMaahW5s/wMRciu1sYbdsikT/UHwbUjOdEVTc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10 h1:fJvQ5mIBVfKtiyx0AHY6HeWcRX5LGANLpq8SVR+Uazs= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.10/go.mod h1:Kzm5e6OmNH8VMkgK9t+ry5jEih4Y8whqs+1hrkxim1I= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18 h1:LTRCYFlnnKFlKsyIQxKhJuDuA3ZkrDQMRYm6rXiHlLY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.18/go.mod h1:XhwkgGG6bHSd00nO/mexWTcTjgd6PjuvWQMqSn2UaEk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18 h1:/A/xDuZAVD2BpsS2fftFRo/NoEKQJ8YTnJDEHBy2Gtg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.18/go.mod h1:hWe9b4f+djUQGmyiGEeOnZv69dtMSgpDRIvNMvuvzvY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2 h1:M1A9AjcFwlxTLuf0Faj88L8Iqw0n/AJHjpZTQzMMsSc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.2/go.mod h1:KsdTV6Q9WKUZm2mNJnUFmIoXfZux91M3sr/a4REX8e0= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.6 h1:MzORe+J94I+hYu2a6XmV5yC9huoTv8NRcCrUNedDypQ= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.6/go.mod h1:hXzcHLARD7GeWnifd8j9RWqtfIgxj4/cAtIVIK7hg8g= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.11 h1:7oGD8KPfBOJGXiCoRKrrrQkbvCp8N++u36hrLMPey6o= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.11/go.mod h1:0DO9B5EUJQlIDif+XJRWCljZRKsAFKh3gpFz7UnDtOo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15 h1:edCcNp9eGIUDUCrzoCu1jWAXLGFIizeqkdkKgRlJwWc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.15/go.mod h1:lyRQKED9xWfgkYC/wmmYfv7iVIM68Z5OQ88ZdcV1QbU= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.7 h1:NITQpgo9A5NrDZ57uOWj+abvXSb83BbyggcUBVksN7c= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.7/go.mod h1:sks5UWBhEuWYDPdwlnRFn1w7xWdH29Jcpe+/PJQefEs= +github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA= +github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= +github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc= +github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.8 h1:fuVgGvKIwxwXeznHyzVYtHwUZNG+tAoLMyzmP7eBnvk= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.8/go.mod h1:qEZo3wYx35vB0xe9H7S5jXfl/kjhiw0sHnvDusOQ6qU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -283,8 +283,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.14.2 h1:iUYbA6djJWql5WUet7J9Mw+iQ9DYRGJWvNWC8WfuZBc= -github.com/neicnordic/crypt4gh v1.14.2/go.mod h1:+y61sviowARFg7hMiggWvXe0LQXPDcUDOdflddsU1l4= +github.com/neicnordic/crypt4gh v1.15.0 h1:as+O2Y2IwXAKJzSJX5RPE7PLTCtWkygdjIMk3N34YCo= +github.com/neicnordic/crypt4gh v1.15.0/go.mod h1:4FYcQUA0mtdEosuktXaWw8IID2VYL8lKKHbSKxoI4g4= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 h1:9bCMuD3TcnjeqjPT2gSlha4asp8NvgcFRYExCaikCxk= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25/go.mod h1:eDjgYHYDJbPLBLsyZ6qRaugP0mX8vePOhZ5id1fdzJw= @@ -496,8 +496,8 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.1 h1:zGhSi45ODB9/p3VAawt9a+O/MULLl9dpizzNNpq7flY= -google.golang.org/grpc v1.79.1/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= +google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From f502cb17254316002188676eb73d749e2988bdf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 5 Mar 2026 10:23:54 +0100 Subject: [PATCH 310/530] docs(adr): add ADR-0002 merge dependabot package managers --- .../0002-merge_dependabot_package_managers.md | 115 ++++++++++++++++++ 1 file changed, 115 insertions(+) create mode 100644 docs/decisions/0002-merge_dependabot_package_managers.md diff --git a/docs/decisions/0002-merge_dependabot_package_managers.md b/docs/decisions/0002-merge_dependabot_package_managers.md new file mode 100644 index 0000000000..dd7d1a3e2e --- /dev/null +++ b/docs/decisions/0002-merge_dependabot_package_managers.md @@ -0,0 +1,115 @@ +--- +status: accepted +date: 2026-03-05 +decision-makers: + - "@neicnordic/sensitive-data-development-collaboration" +--- + +# Merge Dependapot package managers + +## Context and Problem Statement + +There are quite a few (~22) [dependabot](https://github.com/dependabot) package managers, this results in a +high likelihood of it raising quite a lot of PRs weekly for dependency updates. Meaning each week the individual raised +PRs +runs their own integration tests, image building, etc, and needs to be manually reviewed, merged, etc. + +See [dependabot.yaml](../../.github/dependabot.yaml) for current configuration. + +## Decision Drivers + +* Reduce manual work needed to review / merge multiple PRs +* Reduce amount of PRs being raised each week and "clogging" up the PR tab + +## Considered Options + +1. Merge related package managers into one with multi-ecosystem (per release (ie main and release_v1 branches)) + * directories: "/sda", "/sda-download", "/sda-validator/orchestrator" + * package-ecosystems: docker, gomod + * directories: "/sda-sftp-inbox", "/sda-doa" + * package-ecosystems: docker, maven + * directories: "/rabbitmq" + * package-ecosystems: docker + * directories: "/postgresql" + * package-ecosystems: docker + * directories: "/" + * package-ecosystem: "github-actions" +2. Package manager per directory (per release (ie main and release_v1 branches)) + * directory: "/sda" + * package-ecosystem: docker, gomod + * directory: "/sda-download" + * package-ecosystem: docker, gomod + * directory: "/sda-validator/orchestrator" (not relevant for release_v1 branch ) + * package-ecosystem: docker, gomod + * directory: "/sda-sftp-inbox" + * package-ecosystem: docker, maven + * directory: "/sda-doa" + * package-ecosystem: docker, maven + * directory: "/rabbitmq" + * package-ecosystem: docker + * directory: "/postgresql" + * package-ecosystem: docker + * directory: "/" + * package-ecosystem: github-actions +3. Package manager per ecosystem (per release (ie main and release_v1 branches)) + * package-ecosystem: docker + * directories: "/sda", "/sda-download", "/sda-validator/orchestrator", "/sda-sftp-inbox", "/sda-doa", "/rabbitmq", "/postgresql" + * package-ecosystem: gomod + * directories: "/sda", "/sda-download", "/sda-validator/orchestrator" + * package-ecosystem: maven + * directories: "/sda-sftp-inbox", "/sda-doa" + * package-ecosystem: github-actions + * directories: "/" +4. All in one (per release (ie main and release_v1 branches)) + * directories: "/", "/sda", "/sda-download", "/sda-validator/orchestrator", "/sda-sftp-inbox", "/sda-doa", "/rabbitmq", "/postgresql" + * package-ecosystem: docker, gomod, maven, github-actions +5. Keep as is (separate package manager per directory & ecosystem) + +## Decision Outcome + +Chosen option: Option 3, with "/sda-sftp-inbox", and "/sda-doa" being seperated +because docker updates rarely occur, and when they do it is often a major version increment meaning additional changes +could need to take place for it to be successful. The "/sda-sftp-inbox", and "/sda-doa" are split since the +"/sda-sftp-inbox" is quite unmaintained, and often require manual intervention during dependency updates, which shouldnt +hinder the "/sda-doa". + +### Consequences + +* Good, because 10 instead of 24 potential PRs being opened weekly +* Good, because for most ecosystems the updates will be in the same PR +* Bad, because a dependency update which has an issue for one directory would hinder the PR until a manual intervention to exclude the problematic dependency update to enable the rest + +## Pros and Cons of the Options + +### Merge related package managers into one with multi-ecosystem (per release (ie main and release_v1 branches)) + +* Good, because 10 instead of 24 potential PRs being opened weekly +* Good, because the related directories would have similar(if not the same) dependency updates +* Bad, because a PR will affect multiple applications +* Bad, because a dependency update which has an issue for one directory would hinder the PR until a manual intervention to exclude the problematic dependency update to enable the rest + +### Package manager per directory (per release (ie main and release_v1 branches)) + +* Good, because 15 instead of 24 potential PRs being opened weekly +* Good, because the package manager will manage directories(which correlate to applications) separately +* Bad, because there will be similar PRs in multiple directories +* Bad, because a dependency update which has an issue for one ecosystem in a directory would hinder the PR until a manual intervention to exclude the problematic dependency update to enable the rest + +### Package manager per ecosystem (per release (ie main and release_v1 branches)) + +* Good, because 8 instead of 24 potential PRs being opened weekly +* Good, because all updates for an ecosystem is in the same PR +* Bad, because a PR will affect multiple applications maintained by different teams +* Bad, because a dependency update which has an issue for one directory would hinder the PR until a manual intervention to exclude the problematic dependency update to enable the rest + +### All in one (per release (ie main and release_v1 branches)) + +* Good, because 2 instead of 24 potential PRs being opened weekly +* Bad, because a PR will affect multiple applications maintained by different teams +* Bad, because a dependency update which has an issue for one directory would hinder the PR until a manual intervention to exclude the problematic dependency update to enable the rest + +### Keep as is (separate package manager per directory & ecosystem) + +* Good, because the dependabot PRs will be more confined per directory / ecosystem +* Bad, because a lot(~24) of potential PRs each week +* Bad, because a lot of similar PRs with the same updates in different directories From 6edce856cd047c994b644b5a55cc68e42659cadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Mar 2026 09:17:19 +0100 Subject: [PATCH 311/530] docs(adr): add ADR-0002 add mentioning of dependabot groups --- docs/decisions/0002-merge_dependabot_package_managers.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/decisions/0002-merge_dependabot_package_managers.md b/docs/decisions/0002-merge_dependabot_package_managers.md index dd7d1a3e2e..807ac56c15 100644 --- a/docs/decisions/0002-merge_dependabot_package_managers.md +++ b/docs/decisions/0002-merge_dependabot_package_managers.md @@ -73,6 +73,8 @@ could need to take place for it to be successful. The "/sda-sftp-inbox", and "/s "/sda-sftp-inbox" is quite unmaintained, and often require manual intervention during dependency updates, which shouldnt hinder the "/sda-doa". +To achieve this we can utilise [dependabot groups](https://docs.github.com/en/code-security/reference/supply-chain-security/dependabot-options-reference#groups--) + ### Consequences * Good, because 10 instead of 24 potential PRs being opened weekly From 598e70f220e9f3e268652438501072a75097554c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Mar 2026 09:49:10 +0100 Subject: [PATCH 312/530] fix(github actions): use secrets.GHCR_TOKEN when tagging sda releases such that release_sda will trigger on the created tag as github ignores events from secrets.GITHUB_TOKEN --- .github/workflows/pr_merge_increment_version_tag_sda.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_merge_increment_version_tag_sda.yml b/.github/workflows/pr_merge_increment_version_tag_sda.yml index a73fb4917c..5a756ad979 100644 --- a/.github/workflows/pr_merge_increment_version_tag_sda.yml +++ b/.github/workflows/pr_merge_increment_version_tag_sda.yml @@ -26,7 +26,7 @@ jobs: id: bump_tag uses: anothrNick/github-tag-action@1.75.0 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GHCR_TOKEN }} TAG_PREFIX: v DEFAULT_BUMP: patch - TAG_CONTEXT: branch + TAG_CONTEXT: branch \ No newline at end of file From 644ecc01afaafcb30d06a53b7c75c23d0cc0117b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:04:39 +0000 Subject: [PATCH 313/530] build(deps): bump aquasecurity/trivy-action from 0.34.2 to 0.35.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.34.2 to 0.35.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.34.2...0.35.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.35.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/pr_build_and_test_images.yml | 4 ++-- .github/workflows/release_sda.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index 63321b29c9..c17b53f070 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -81,7 +81,7 @@ jobs: org.opencontainers.image.version=${{ github.sha }} - name: Run Trivy vulnerability scanner on ${{ matrix.name }} sarif - uses: aquasecurity/trivy-action@0.34.2 + uses: aquasecurity/trivy-action@0.35.0 env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -115,7 +115,7 @@ jobs: - name: Run Trivy vulnerability scanner on ${{ matrix.name }} markdown output if: steps.changes.outputs.changed == 'true' - uses: aquasecurity/trivy-action@0.34.2 + uses: aquasecurity/trivy-action@0.35.0 env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml index e174b343ea..49346b57c5 100644 --- a/.github/workflows/release_sda.yml +++ b/.github/workflows/release_sda.yml @@ -76,7 +76,7 @@ jobs: - name: Run Trivy vulnerability scanner on ${{ matrix.name }} - uses: aquasecurity/trivy-action@0.34.2 + uses: aquasecurity/trivy-action@0.35.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From 752a22c7924aaa7a4e2f1385fa0a1160b963e5e7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:06:15 +0000 Subject: [PATCH 314/530] build(deps): bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) and [golang.org/x/crypto](https://github.com/golang/crypto). Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.96.4 to 1.97.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.96.4...service/s3/v1.97.0) Updates `golang.org/x/crypto` from 0.48.0 to 0.49.0 - [Commits](https://github.com/golang/crypto/compare/v0.48.0...v0.49.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.97.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.49.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 8 ++++---- sda-download/go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 9636c0c080..2a95c5e8d4 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.41.3 github.com/aws/aws-sdk-go-v2/config v1.32.11 github.com/aws/aws-sdk-go-v2/credentials v1.19.11 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/dgraph-io/ristretto v1.0.0 @@ -21,7 +21,7 @@ require ( github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.48.0 + golang.org/x/crypto v0.49.0 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 @@ -88,8 +88,8 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect golang.org/x/net v0.51.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 83f02fc0e4..0958bd3c5d 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -30,8 +30,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7su github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 h1:zyKY4OxzUImu+DigelJI9o49QQv8CjREs5E1CywjtIA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= @@ -200,19 +200,19 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= From f6ed46cea3f6932daa4cbdf97fb0deb8722cd0be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 12 Mar 2026 19:08:44 +0000 Subject: [PATCH 315/530] build(deps): bump the all-modules group in /sda with 3 updates Bumps the all-modules group in /sda with 3 updates: [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2), [golang.org/x/crypto](https://github.com/golang/crypto) and [golang.org/x/oauth2](https://github.com/golang/oauth2). Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.96.4 to 1.97.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.96.4...service/s3/v1.97.0) Updates `golang.org/x/crypto` from 0.48.0 to 0.49.0 - [Commits](https://github.com/golang/crypto/compare/v0.48.0...v0.49.0) Updates `golang.org/x/oauth2` from 0.35.0 to 0.36.0 - [Commits](https://github.com/golang/oauth2/compare/v0.35.0...v0.36.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.97.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.49.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-version: 0.36.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 10 +++++----- sda/go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index a91f44ed5c..42e874c1b9 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -7,7 +7,7 @@ require ( github.com/aws/aws-sdk-go-v2/config v1.32.11 github.com/aws/aws-sdk-go-v2/credentials v1.19.11 github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.8 - github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 @@ -32,8 +32,8 @@ require ( github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.48.0 - golang.org/x/oauth2 v0.35.0 + golang.org/x/crypto v0.49.0 + golang.org/x/oauth2 v0.36.0 google.golang.org/grpc v1.79.2 google.golang.org/protobuf v1.36.11 ) @@ -170,8 +170,8 @@ require ( golang.org/x/arch v0.24.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect golang.org/x/net v0.51.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/ini.v1 v1.67.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index cc1b3c9d5b..62db0b3b34 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -53,8 +53,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7su github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4 h1:4ExZyubQ6LQQVuF2Qp9OsfEvsTdAWh5Gfwf6PgIdLdk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.96.4/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 h1:zyKY4OxzUImu+DigelJI9o49QQv8CjREs5E1CywjtIA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= @@ -422,8 +422,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= @@ -441,8 +441,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= -golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= +golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -462,15 +462,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -478,8 +478,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From fd8040ff84a5d0fd33f1eaf7a835c2283bef93f7 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 09:52:35 +0100 Subject: [PATCH 316/530] docs(adr): fix ADR-0002 typo, naming convention, and missing index entry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix "Dependapot" → "Dependabot" typo in title heading - Rename file from underscores to dashes per NNNN-title-with-dashes.md convention - Add ADR-0002 to README.md index table - Add index table checklist item to PR template to prevent this gap recurring --- .github/PULL_REQUEST_TEMPLATE.md | 1 + ...ge_managers.md => 0002-merge-dependabot-package-managers.md} | 2 +- docs/decisions/README.md | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) rename docs/decisions/{0002-merge_dependabot_package_managers.md => 0002-merge-dependabot-package-managers.md} (99%) diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 3ff06bd75a..3fab15bee7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -9,5 +9,6 @@ This PR closes [issue number]. - [ ] This PR includes an architecturally significant decision (see `docs/decisions/README.md`) - [ ] A decision record has been added or updated in `docs/decisions/` + - [ ] The decision record is listed in the `docs/decisions/README.md` index table ## How to test diff --git a/docs/decisions/0002-merge_dependabot_package_managers.md b/docs/decisions/0002-merge-dependabot-package-managers.md similarity index 99% rename from docs/decisions/0002-merge_dependabot_package_managers.md rename to docs/decisions/0002-merge-dependabot-package-managers.md index 807ac56c15..6f5b888c5c 100644 --- a/docs/decisions/0002-merge_dependabot_package_managers.md +++ b/docs/decisions/0002-merge-dependabot-package-managers.md @@ -5,7 +5,7 @@ decision-makers: - "@neicnordic/sensitive-data-development-collaboration" --- -# Merge Dependapot package managers +# Merge Dependabot package managers ## Context and Problem Statement diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f3353c14c6..f96474007d 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -93,6 +93,7 @@ superseded-by: "0005-use-new-approach.md" | # | Decision | Status | | --- | --- | --- | | [0000](0000-use-markdown-architectural-decision-records.md) | Use Markdown Architectural Decision Records | accepted | +| [0002](0002-merge-dependabot-package-managers.md) | Merge Dependabot package managers | accepted | ## More information From 2ea59cf65004b290c19566b1ded079aee6e8b8f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Mar 2026 10:27:46 +0100 Subject: [PATCH 317/530] feat(dependabot): group dependabot package managers to group same ecosystems in same PRs (except sda-doa and sda-stfp-inbox) --- .github/dependabot.yaml | 189 +++++++--------------------------------- 1 file changed, 30 insertions(+), 159 deletions(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 22cec02004..e70f96c9eb 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -6,107 +6,46 @@ updates: interval: weekly open-pull-requests-limit: 10 -# Each subfolder needs to be checked separately + # Docker images across services - package-ecosystem: docker - directory: "/postgresql" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - - package-ecosystem: docker - directory: "/sda-download" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - - package-ecosystem: docker - directory: "/rabbitmq" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - - package-ecosystem: docker - directory: "/sda" - schedule: - interval: weekly + directories: + - "/postgresql" + - "/rabbitmq" + - "/sda-validator/orchestrator" + - "/sda" + - "/sda-download" + - "/sda-sftp-inbox" + - "/sda-doa" open-pull-requests-limit: 10 - - - package-ecosystem: docker - directory: "/sda-validator-orchestrator" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - - package-ecosystem: docker - directory: "/sda-sftp-inbox" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - -# Each subfolder needs to be checked separately - - package-ecosystem: gomod - directory: "/sda" groups: - all-modules: + docker: patterns: - "*" - open-pull-requests-limit: 10 - - schedule: - interval: weekly - package-ecosystem: gomod - directory: "/sda-download" - groups: - all-modules: - patterns: - - "*" + directories: + - "/sda-validator/orchestrator" + - "/sda" + - "/sda-download" open-pull-requests-limit: 10 - schedule: interval: weekly - - - package-ecosystem: gomod - directory: "/sda-validator/orchestrator" groups: - all-modules: + go: patterns: - "*" - open-pull-requests-limit: 10 - schedule: - interval: weekly - -# Each subfolder needs to be checked separately - package-ecosystem: maven directory: "/sda-sftp-inbox" - groups: - all-modules: - patterns: - - "*" open-pull-requests-limit: 10 - schedule: interval: weekly - package-ecosystem: maven directory: "/sda-doa" - groups: - all-modules: - patterns: - - "*" open-pull-requests-limit: 10 - - schedule: - interval: weekly - - - package-ecosystem: docker - directory: "/sda-doa" schedule: interval: weekly - open-pull-requests-limit: 10 - ## release v1 branch - package-ecosystem: "github-actions" @@ -115,80 +54,30 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - - commit-message: - prefix: "[release_v1]" -### Docker - - package-ecosystem: docker - target-branch: release_v1 - directory: "/postgresql" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - commit-message: - prefix: "[release_v1]" - - - package-ecosystem: docker - target-branch: release_v1 - directory: "/rabbitmq" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - commit-message: - prefix: "[release_v1]" - - - package-ecosystem: docker - target-branch: release_v1 - directory: "/sda" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - commit-message: - prefix: "[release_v1]" - - - package-ecosystem: docker - target-branch: release_v1 - directory: "/sda-doa" - schedule: - interval: weekly - open-pull-requests-limit: 10 - - commit-message: - prefix: "[release_v1]" - - - package-ecosystem: docker - target-branch: release_v1 - directory: "/sda-download" - schedule: - interval: weekly - open-pull-requests-limit: 10 - commit-message: prefix: "[release_v1]" - package-ecosystem: docker target-branch: release_v1 - directory: "/sda-sftp-inbox" - schedule: - interval: weekly + directories: + - "/postgresql" + - "/rabbitmq" + - "/sda" + - "/sda-download" + - "/sda-sftp-inbox" + - "/sda-doa" open-pull-requests-limit: 10 - + groups: + docker: + patterns: + - "*" commit-message: prefix: "[release_v1]" -### JAVA - package-ecosystem: maven target-branch: release_v1 directory: "/sda-doa" - groups: - all-modules: - patterns: - - "*" open-pull-requests-limit: 10 - schedule: interval: weekly commit-message: @@ -197,26 +86,7 @@ updates: - package-ecosystem: maven target-branch: release_v1 directory: "/sda-sftp-inbox" - groups: - all-modules: - patterns: - - "*" - open-pull-requests-limit: 10 - - schedule: - interval: weekly - commit-message: - prefix: "[release_v1]" -### GO - - package-ecosystem: gomod - target-branch: release_v1 - directory: "/sda-download" - groups: - all-modules: - patterns: - - "*" open-pull-requests-limit: 10 - schedule: interval: weekly commit-message: @@ -224,13 +94,14 @@ updates: - package-ecosystem: gomod target-branch: release_v1 - directory: "/sda" + directories: + - "/sda" + - "/sda-download" groups: - all-modules: + go: patterns: - "*" open-pull-requests-limit: 10 - schedule: interval: weekly commit-message: From 087babcf1e71cbcf85306684e220ca3ced8e2a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Mar 2026 10:37:13 +0100 Subject: [PATCH 318/530] feat(dependabot): add missing schedule in docker package-ecosystems, add groups to all for all dependencies within to be updated in same PR, and reorder settings to be aligned --- .github/dependabot.yaml | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index e70f96c9eb..85a68aed4e 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -2,9 +2,13 @@ version: 2 updates: - package-ecosystem: "github-actions" directory: "/" + open-pull-requests-limit: 10 schedule: interval: weekly - open-pull-requests-limit: 10 + groups: + github-actions: + patterns: + - "*" # Docker images across services - package-ecosystem: docker @@ -17,6 +21,8 @@ updates: - "/sda-sftp-inbox" - "/sda-doa" open-pull-requests-limit: 10 + schedule: + interval: weekly groups: docker: patterns: @@ -40,20 +46,32 @@ updates: open-pull-requests-limit: 10 schedule: interval: weekly + groups: + maven: + patterns: + - "*" - package-ecosystem: maven directory: "/sda-doa" open-pull-requests-limit: 10 schedule: interval: weekly + groups: + maven: + patterns: + - "*" ## release v1 branch - package-ecosystem: "github-actions" target-branch: release_v1 directory: "/" + open-pull-requests-limit: 10 schedule: interval: weekly - open-pull-requests-limit: 10 + groups: + github-actions: + patterns: + - "*" commit-message: prefix: "[release_v1]" @@ -67,6 +85,8 @@ updates: - "/sda-sftp-inbox" - "/sda-doa" open-pull-requests-limit: 10 + schedule: + interval: weekly groups: docker: patterns: @@ -80,6 +100,10 @@ updates: open-pull-requests-limit: 10 schedule: interval: weekly + groups: + maven: + patterns: + - "*" commit-message: prefix: "[release_v1]" @@ -89,6 +113,10 @@ updates: open-pull-requests-limit: 10 schedule: interval: weekly + groups: + maven: + patterns: + - "*" commit-message: prefix: "[release_v1]" @@ -97,12 +125,12 @@ updates: directories: - "/sda" - "/sda-download" + open-pull-requests-limit: 10 + schedule: + interval: weekly groups: go: patterns: - "*" - open-pull-requests-limit: 10 - schedule: - interval: weekly commit-message: - prefix: "[release_v1]" + prefix: "[release_v1]" \ No newline at end of file From e2746c04508571d6b4c48fe032905b9f60635f77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 13 Mar 2026 13:05:21 +0100 Subject: [PATCH 319/530] feat(dependabot): update open-pull-requests-limit for all to 1 --- .github/dependabot.yaml | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 85a68aed4e..1a74895fdd 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -2,7 +2,7 @@ version: 2 updates: - package-ecosystem: "github-actions" directory: "/" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -20,7 +20,7 @@ updates: - "/sda-download" - "/sda-sftp-inbox" - "/sda-doa" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -33,7 +33,7 @@ updates: - "/sda-validator/orchestrator" - "/sda" - "/sda-download" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -43,7 +43,7 @@ updates: - package-ecosystem: maven directory: "/sda-sftp-inbox" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -53,7 +53,7 @@ updates: - package-ecosystem: maven directory: "/sda-doa" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -65,7 +65,7 @@ updates: - package-ecosystem: "github-actions" target-branch: release_v1 directory: "/" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -84,7 +84,7 @@ updates: - "/sda-download" - "/sda-sftp-inbox" - "/sda-doa" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -97,7 +97,7 @@ updates: - package-ecosystem: maven target-branch: release_v1 directory: "/sda-doa" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -110,7 +110,7 @@ updates: - package-ecosystem: maven target-branch: release_v1 directory: "/sda-sftp-inbox" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -125,7 +125,7 @@ updates: directories: - "/sda" - "/sda-download" - open-pull-requests-limit: 10 + open-pull-requests-limit: 1 schedule: interval: weekly groups: @@ -133,4 +133,4 @@ updates: patterns: - "*" commit-message: - prefix: "[release_v1]" \ No newline at end of file + prefix: "[release_v1]" From c963ebfa6826231c26d390b839a460c6be253a42 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 02:39:04 +0000 Subject: [PATCH 320/530] build(deps): bump google.golang.org/grpc in /sda-download Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.79.2 to 1.79.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.2...v1.79.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.79.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 2a95c5e8d4..af8725e863 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -23,7 +23,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.49.0 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa - google.golang.org/grpc v1.79.2 + google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.11 ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 0958bd3c5d..aa38bfe805 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -217,8 +217,8 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= -google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 7be4cc47909591d1ea40eae4661c4df504779acd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 02:39:09 +0000 Subject: [PATCH 321/530] build(deps): bump google.golang.org/grpc in /sda-validator/orchestrator Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.79.2 to 1.79.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.2...v1.79.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.79.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- sda-validator/orchestrator/go.mod | 2 +- sda-validator/orchestrator/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index fa1f6e7549..130097279b 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -16,7 +16,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - google.golang.org/grpc v1.79.2 + google.golang.org/grpc v1.79.3 ) require ( diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index 1b6cc6587c..4dd8f9b0b3 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -194,8 +194,8 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= -google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 40f67636d87bb4413c1bb0ed21b39446923c8f75 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 02:39:11 +0000 Subject: [PATCH 322/530] build(deps): bump google.golang.org/grpc from 1.79.2 to 1.79.3 in /sda Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.79.2 to 1.79.3. - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.2...v1.79.3) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.79.3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 42e874c1b9..10edab954c 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -34,7 +34,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.49.0 golang.org/x/oauth2 v0.36.0 - google.golang.org/grpc v1.79.2 + google.golang.org/grpc v1.79.3 google.golang.org/protobuf v1.36.11 ) diff --git a/sda/go.sum b/sda/go.sum index 62db0b3b34..11a399aade 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -496,8 +496,8 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.2 h1:fRMD94s2tITpyJGtBBn7MkMseNpOZU8ZxgC3MMBaXRU= -google.golang.org/grpc v1.79.2/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= +google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 45a6f243ce3fb69dc71deead6d47912cb5d25059 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Thu, 5 Mar 2026 08:30:25 +0000 Subject: [PATCH 323/530] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 6ddec19db0..45ef30e0ea 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.27 -appVersion: v3.1.25 +version: 2.0.28 +appVersion: v3.1.37 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index f07dd63213..7d38cf01d4 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.1.2 -appVersion: v3.1.25 +version: 2.1.3 +appVersion: v3.1.37 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 2889d5159d..1f4cdd765c 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.2.0 -appVersion: v3.1.25 +version: 3.2.1 +appVersion: v3.1.37 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 3ce2d2863f32c3de1bb393135ac713b4bf034e54 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:03:31 +0000 Subject: [PATCH 324/530] build(deps): bump marocchino/sticky-pull-request-comment from 2 to 3 Bumps [marocchino/sticky-pull-request-comment](https://github.com/marocchino/sticky-pull-request-comment) from 2 to 3. - [Release notes](https://github.com/marocchino/sticky-pull-request-comment/releases) - [Commits](https://github.com/marocchino/sticky-pull-request-comment/compare/v2...v3) --- updated-dependencies: - dependency-name: marocchino/sticky-pull-request-comment dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pr_build_and_test_images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index c17b53f070..4094f503b6 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -133,7 +133,7 @@ jobs: - name: Comment PR if: steps.changes.outputs.changed == 'true' - uses: marocchino/sticky-pull-request-comment@v2 + uses: marocchino/sticky-pull-request-comment@v3 with: header: ${{ matrix.name }} Trivy results path: trivy-comment.md From ace2c263a59b96d0df1c4329542db57c5c9abdca Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:03:34 +0000 Subject: [PATCH 325/530] build(deps): bump codecov/codecov-action from 5.5.2 to 5.5.3 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.5.2 to 5.5.3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.5.2...v5.5.3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 5.5.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/pr_code_tests.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr_code_tests.yml b/.github/workflows/pr_code_tests.yml index a5d2236aec..58683d482c 100644 --- a/.github/workflows/pr_code_tests.yml +++ b/.github/workflows/pr_code_tests.yml @@ -61,7 +61,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.2 + uses: codecov/codecov-action@v5.5.3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -92,7 +92,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.2 + uses: codecov/codecov-action@v5.5.3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -123,7 +123,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.2 + uses: codecov/codecov-action@v5.5.3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt @@ -154,7 +154,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.2 + uses: codecov/codecov-action@v5.5.3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-validator/orchestrator/coverage.txt From 44fa53becdcf29273902a11abe80407deac0e215 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:03:38 +0000 Subject: [PATCH 326/530] build(deps): bump dorny/paths-filter from 3 to 4 Bumps [dorny/paths-filter](https://github.com/dorny/paths-filter) from 3 to 4. - [Release notes](https://github.com/dorny/paths-filter/releases) - [Changelog](https://github.com/dorny/paths-filter/blob/master/CHANGELOG.md) - [Commits](https://github.com/dorny/paths-filter/compare/v3...v4) --- updated-dependencies: - dependency-name: dorny/paths-filter dependency-version: '4' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/pr_build_and_test_images.yml | 2 +- .github/workflows/pr_code_linter.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index c17b53f070..0d883ef1f9 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -106,7 +106,7 @@ jobs: category: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} # Only run trivy scan with markdown output and add PR comment if PR changes the relevant file - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@v4 id: changes with: filters: | diff --git a/.github/workflows/pr_code_linter.yaml b/.github/workflows/pr_code_linter.yaml index 93dd8bb567..5cc01b4e35 100644 --- a/.github/workflows/pr_code_linter.yaml +++ b/.github/workflows/pr_code_linter.yaml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - uses: dorny/paths-filter@v3 + - uses: dorny/paths-filter@v4 id: changes with: filters: | From 590519abab1ae8acb307b5a55726f5ef174c80e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:08:18 +0000 Subject: [PATCH 327/530] build(deps): bump github.com/lib/pq Bumps the all-modules group in /sda-validator/orchestrator with 1 update: [github.com/lib/pq](https://github.com/lib/pq). Updates `github.com/lib/pq` from 1.11.2 to 1.12.0 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.11.2...v1.12.0) --- updated-dependencies: - dependency-name: github.com/lib/pq dependency-version: 1.12.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-validator/orchestrator/go.mod | 2 +- sda-validator/orchestrator/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index 130097279b..50ebd7577c 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -8,7 +8,7 @@ require ( github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.11.2 + github.com/lib/pq v1.12.0 github.com/neicnordic/crypt4gh v1.15.0 github.com/rabbitmq/amqp091-go v1.10.0 github.com/sirupsen/logrus v1.9.4 diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index 4dd8f9b0b3..36bddbeb56 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -90,8 +90,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= -github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.12.0 h1:mC1zeiNamwKBecjHarAr26c/+d8V5w/u4J0I/yASbJo= +github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 9bdc3f6b0b6ee4367c126d7d70c705f263c91acd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:08:41 +0000 Subject: [PATCH 328/530] build(deps): bump the all-modules group in /sda-download with 5 updates Bumps the all-modules group in /sda-download with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.3` | `1.41.4` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.11` | `1.32.12` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.19.11` | `1.19.12` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.97.0` | `1.97.1` | | [github.com/lib/pq](https://github.com/lib/pq) | `1.11.2` | `1.12.0` | Updates `github.com/aws/aws-sdk-go-v2` from 1.41.3 to 1.41.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.3...v1.41.4) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.11 to 1.32.12 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.11...config/v1.32.12) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.11 to 1.19.12 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.11...service/sqs/v1.19.12) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.0 to 1.97.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.0...service/s3/v1.97.1) Updates `github.com/lib/pq` from 1.11.2 to 1.12.0 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.11.2...v1.12.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.97.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/lib/pq dependency-version: 1.12.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 38 +++++++++++------------ sda-download/go.sum | 76 ++++++++++++++++++++++----------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index af8725e863..9b40978ad1 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -5,10 +5,10 @@ go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/Masterminds/semver/v3 v3.4.0 - github.com/aws/aws-sdk-go-v2 v1.41.3 - github.com/aws/aws-sdk-go-v2/config v1.32.11 - github.com/aws/aws-sdk-go-v2/credentials v1.19.11 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/dgraph-io/ristretto v1.0.0 @@ -16,7 +16,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/uuid v1.6.0 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.11.2 + github.com/lib/pq v1.12.0 github.com/neicnordic/crypt4gh v1.15.0 github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 @@ -29,20 +29,20 @@ require ( require ( filippo.io/edwards25519 v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index aa38bfe805..98e63bef33 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -4,42 +4,42 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA= -github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= -github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo= -github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc= -github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 h1:zyKY4OxzUImu+DigelJI9o49QQv8CjREs5E1CywjtIA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= @@ -125,8 +125,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= -github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.12.0 h1:mC1zeiNamwKBecjHarAr26c/+d8V5w/u4J0I/yASbJo= +github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From 454eed3ef5524cea5af39bd02f370e977bd7f9a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Mar 2026 19:10:26 +0000 Subject: [PATCH 329/530] build(deps): bump the all-modules group in /sda with 6 updates Bumps the all-modules group in /sda with 6 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.3` | `1.41.4` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.11` | `1.32.12` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.19.11` | `1.19.12` | | [github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager](https://github.com/aws/aws-sdk-go-v2) | `0.1.8` | `0.1.10` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.97.0` | `1.97.1` | | [github.com/lib/pq](https://github.com/lib/pq) | `1.11.2` | `1.12.0` | Updates `github.com/aws/aws-sdk-go-v2` from 1.41.3 to 1.41.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.3...v1.41.4) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.11 to 1.32.12 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.11...config/v1.32.12) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.11 to 1.19.12 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.11...service/sqs/v1.19.12) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager` from 0.1.8 to 0.1.10 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/transfermanager/v0.1.8...feature/s3/transfermanager/v0.1.10) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.0 to 1.97.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.0...service/s3/v1.97.1) Updates `github.com/lib/pq` from 1.11.2 to 1.12.0 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.11.2...v1.12.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager dependency-version: 0.1.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.97.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/lib/pq dependency-version: 1.12.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 40 +++++++++++++-------------- sda/go.sum | 80 +++++++++++++++++++++++++++--------------------------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 10edab954c..8fcb5688d8 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.25.7 require ( - github.com/aws/aws-sdk-go-v2 v1.41.3 - github.com/aws/aws-sdk-go-v2/config v1.32.11 - github.com/aws/aws-sdk-go-v2/credentials v1.19.11 - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.8 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 + github.com/aws/aws-sdk-go-v2 v1.41.4 + github.com/aws/aws-sdk-go-v2/config v1.32.12 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.10 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 @@ -19,7 +19,7 @@ require ( github.com/iris-contrib/middleware/cors v0.0.0-20251225090426-92c6f28facda github.com/kataras/iris/v12 v12.2.11 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.11.2 + github.com/lib/pq v1.12.0 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 github.com/neicnordic/crypt4gh v1.15.0 @@ -50,20 +50,20 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 // indirect github.com/andybalholm/brotli v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/sda/go.sum b/sda/go.sum index 11a399aade..04d9525c65 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -25,44 +25,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.3 h1:4kQ/fa22KjDt13QCy1+bYADvdgcxpfH18f0zP542kZA= -github.com/aws/aws-sdk-go-v2 v1.41.3/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6 h1:N4lRUXZpZ1KVEUn6hxtco/1d2lgYhNn1fHkkl8WhlyQ= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.6/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.11 h1:ftxI5sgz8jZkckuUHXfC/wMUc8u3fG1vQS0plr2F2Zs= -github.com/aws/aws-sdk-go-v2/config v1.32.11/go.mod h1:twF11+6ps9aNRKEDimksp923o44w/Thk9+8YIlzWMmo= -github.com/aws/aws-sdk-go-v2/credentials v1.19.11 h1:NdV8cwCcAXrCWyxArt58BrvZJ9pZ9Fhf9w6Uh5W3Uyc= -github.com/aws/aws-sdk-go-v2/credentials v1.19.11/go.mod h1:30yY2zqkMPdrvxBqzI9xQCM+WrlrZKSOpSJEsylVU+8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19 h1:INUvJxmhdEbVulJYHI061k4TVuS3jzzthNvjqvVvTKM= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.19/go.mod h1:FpZN2QISLdEBWkayloda+sZjVJL+e9Gl0k1SyTgcswU= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.8 h1:fuVgGvKIwxwXeznHyzVYtHwUZNG+tAoLMyzmP7eBnvk= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.8/go.mod h1:qEZo3wYx35vB0xe9H7S5jXfl/kjhiw0sHnvDusOQ6qU= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19 h1:/sECfyq2JTifMI2JPyZ4bdRN77zJmr6SrS1eL3augIA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.19/go.mod h1:dMf8A5oAqr9/oxOfLkC/c2LU/uMcALP0Rgn2BD5LWn0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19 h1:AWeJMk33GTBf6J20XJe6qZoRSJo0WfUhsMdUKhoODXE= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.19/go.mod h1:+GWrYoaAsV7/4pNHpwh1kiNLXkKaSoppxQq9lbH8Ejw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5 h1:clHU5fm//kWS1C2HgtgWxfQbFbx4b6rx+5jzhgX9HrI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.5/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20 h1:qi3e/dmpdONhj1RyIZdi6DKKpDXS5Lb8ftr3p7cyHJc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.20/go.mod h1:V1K+TeJVD5JOk3D9e5tsX2KUdL7BlB+FV6cBhdobN8c= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6 h1:XAq62tBTJP/85lFD5oqOOe7YYgWxY9LvWq8plyDvDVg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.6/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11 h1:BYf7XNsJMzl4mObARUBUib+j2tf0U//JAAtTnYqvqCw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.11/go.mod h1:aEUS4WrNk/+FxkBZZa7tVgp4pGH+kFGW40Y8rCPqt5g= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19 h1:X1Tow7suZk9UCJHE1Iw9GMZJJl0dAnKXXP1NaSDHwmw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.19/go.mod h1:/rARO8psX+4sfjUQXp5LLifjUt8DuATZ31WptNJTyQA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19 h1:JnQeStZvPHFHeyky/7LbMlyQjUa+jIBj36OlWm0pzIk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.19/go.mod h1:HGyasyHvYdFQeJhvDHfH7HXkHh57htcJGKDZ+7z+I24= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0 h1:zyKY4OxzUImu+DigelJI9o49QQv8CjREs5E1CywjtIA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.0/go.mod h1:NF3JcMGOiARAss1ld3WGORCw71+4ExDD2cbbdKS5PpA= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.7 h1:Y2cAXlClHsXkkOvWZFXATr34b0hxxloeQu/pAZz2row= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.7/go.mod h1:idzZ7gmDeqeNrSPkdbtMp9qWMgcBwykA7P7Rzh5DXVU= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.12 h1:iSsvB9EtQ09YrsmIc44Heqlx5ByGErqhPK1ZQLppias= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.12/go.mod h1:fEWYKTRGoZNl8tZ77i61/ccwOMJdGxwOhWCkp6TXAr0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16 h1:EnUdUqRP1CNzt2DkV67tJx6XDN4xlfBFm+bzeNOQVb0= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.16/go.mod h1:Jic/xv0Rq/pFNCh3WwpH4BEqdbSAl+IyHro8LbibHD8= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.8 h1:XQTQTF75vnug2TXS8m7CVJfC2nniYPZnO1D4Np761Oo= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.8/go.mod h1:Xgx+PR1NUOjNmQY+tRMnouRp83JRM8pRMw/vCaVhPkI= +github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= +github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= +github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= +github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.10 h1:2KCL4TmeiNvpPedtC4Bey5jvjRLD74WUYqGeHJ//aco= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.10/go.mod h1:KwaiUFVO7pG8Z9F5bMGvvrRibdSDaAu8HtlKGKkjZSA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -244,8 +244,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.11.2 h1:x6gxUeu39V0BHZiugWe8LXZYZ+Utk7hSJGThs8sdzfs= -github.com/lib/pq v1.11.2/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.12.0 h1:mC1zeiNamwKBecjHarAr26c/+d8V5w/u4J0I/yASbJo= +github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= From 106090bd9bbe3f7f6f97f9ba47a227baba6ed9f9 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Sun, 15 Mar 2026 23:30:12 +0100 Subject: [PATCH 330/530] fix(sda-download): return 403 when requesting unauthorized or non existing data --- sda-download/api/s3/s3.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go index 6b7dc61913..0cc165477c 100644 --- a/sda-download/api/s3/s3.go +++ b/sda-download/api/s3/s3.go @@ -281,7 +281,12 @@ func parseParams(c *gin.Context) *gin.Context { path = string(protocolPattern.ReplaceAll([]byte(path), []byte("$1/$2"))) } + if path == "" { + return c // This is a legitimate ListBuckets request (e.g., GET /s3/) + } + cache := middleware.GetCacheFromContext(c) + matched := false for _, dataset := range cache.Datasets { // check that the path starts with the dataset name, but also that the // path is only the dataset, or that the following character is a slash. @@ -300,10 +305,17 @@ func parseParams(c *gin.Context) *gin.Context { } c.Params = append(c.Params, gin.Param{Key: key, Value: remainder}) + matched = true + break } } + if !matched && path != "" { + log.Warningf("No matching dataset found for path: %v", path) + c.Set("unauthorized_access", true) + } + return c } @@ -316,6 +328,15 @@ func Download(c *gin.Context) { // Parses the request path into a dataset and a filename c = parseParams(c) + if v, exists := c.Get("unauthorized_access"); exists { + if unauthorized, ok := v.(bool); ok && unauthorized { + // Return 403 to reduce information leakage about existing datasets. + c.AbortWithStatus(http.StatusForbidden) + + return + } + } + // Try to figure out what kind of request we're getting. // S3 request types are described here: // https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html From 87af2855405d984705f30ff500ab7700e435684c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Sun, 15 Mar 2026 23:32:40 +0100 Subject: [PATCH 331/530] fix(sda-download): add tests for the fix of unauthorized access feat(sda-download): catch error from io.ReadAll --- sda-download/api/s3/s3_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go index b83547ba8b..d85c013fd9 100644 --- a/sda-download/api/s3/s3_test.go +++ b/sda-download/api/s3/s3_test.go @@ -284,3 +284,36 @@ func (ts *S3TestSuite) TestParseParams() { assert.Equal(ts.T(), http.StatusAccepted, response.StatusCode, "Request failed") } } + +func (ts *S3TestSuite) TestUnauthorizedAccess() { + // Define paths that should trigger a 403 because they don't match + // the datasets defined in SetupTest (dataset1, dataset10, etc.) + unauthorizedPaths := []string{ + "/not_dataset_exists/file_no.exists", + "/dataset2/somefile.txt", + "/secret-dataset", + } + + for _, path := range unauthorizedPaths { + w := httptest.NewRecorder() + _, router := gin.CreateTestContext(w) + + router.GET("/*path", middleware.TokenMiddleware(), Download) + + req := httptest.NewRequest("GET", path, nil) + router.ServeHTTP(w, req) + + response := w.Result() + + // ASSERT: It should return 403 Forbidden, NOT the ListBuckets XML + assert.Equal(ts.T(), http.StatusForbidden, response.StatusCode, + "Path %s should have returned 403 Forbidden", path) + + // Verify the body is not the XML bucket list + body, err := io.ReadAll(response.Body) + assert.Nil(ts.T(), err, "failed to parse body from location response") + _ = response.Body.Close() + assert.NotContains(ts.T(), string(body), "", + "Response for %s should not contain bucket list", path) + } +} From c75c8fb2bfb793f182dd4e3d62747b0a2b40977a Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 16 Mar 2026 22:56:35 +0100 Subject: [PATCH 332/530] fix(sda-download): add integration test for the fix of unauthorized access --- .../integration/tests/common/90_check_s3_errors.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh index 830573b3eb..6f775f564b 100644 --- a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh +++ b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh @@ -89,4 +89,18 @@ if [ "$resp" -ne 404 ]; then exit 1 fi +# try to download a file from a non existent dataset + +resp=$(curl -s --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/nonexistentdataset/$file") +if [ -n "$resp" ]; then + echo "Incorrect response, expected no error message, got $resp" + exit 1 +fi + +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/nonexistentdataset/$file" -s -o /dev/null -w "%{http_code}") +if [ "$resp" -ne 403 ]; then + echo "Incorrect response, expected 403 got $resp" + exit 1 +fi + echo "OK" \ No newline at end of file From 3f52fe5934ce644339b795b06bdd216b6932a04b Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 17 Mar 2026 11:29:29 +0100 Subject: [PATCH 333/530] feat(sda-download): move c.AbortWithStatus to parseParams --- sda-download/api/s3/s3.go | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go index 0cc165477c..18a12b1315 100644 --- a/sda-download/api/s3/s3.go +++ b/sda-download/api/s3/s3.go @@ -286,7 +286,6 @@ func parseParams(c *gin.Context) *gin.Context { } cache := middleware.GetCacheFromContext(c) - matched := false for _, dataset := range cache.Datasets { // check that the path starts with the dataset name, but also that the // path is only the dataset, or that the following character is a slash. @@ -305,15 +304,13 @@ func parseParams(c *gin.Context) *gin.Context { } c.Params = append(c.Params, gin.Param{Key: key, Value: remainder}) - matched = true - break } } - if !matched && path != "" { - log.Warningf("No matching dataset found for path: %v", path) - c.Set("unauthorized_access", true) + if c.Param("dataset") == "" { + log.Warningf("No matching dataset found for path: %q", path) + c.AbortWithStatus(http.StatusForbidden) } return c @@ -328,13 +325,8 @@ func Download(c *gin.Context) { // Parses the request path into a dataset and a filename c = parseParams(c) - if v, exists := c.Get("unauthorized_access"); exists { - if unauthorized, ok := v.(bool); ok && unauthorized { - // Return 403 to reduce information leakage about existing datasets. - c.AbortWithStatus(http.StatusForbidden) - - return - } + if c.IsAborted() { + return } // Try to figure out what kind of request we're getting. From e1c44e42afd73b4f80768a9a0f3b316549eec929 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 17 Mar 2026 11:42:29 +0100 Subject: [PATCH 334/530] feat(sda-download): return 403 for empty request to the /s3/ endpoint --- sda-download/api/s3/s3.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go index 18a12b1315..cb1b2f4b28 100644 --- a/sda-download/api/s3/s3.go +++ b/sda-download/api/s3/s3.go @@ -281,10 +281,6 @@ func parseParams(c *gin.Context) *gin.Context { path = string(protocolPattern.ReplaceAll([]byte(path), []byte("$1/$2"))) } - if path == "" { - return c // This is a legitimate ListBuckets request (e.g., GET /s3/) - } - cache := middleware.GetCacheFromContext(c) for _, dataset := range cache.Datasets { // check that the path starts with the dataset name, but also that the From 91667a2ff9925cb55c703c621e8050dab09d8c0c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 17 Mar 2026 11:43:17 +0100 Subject: [PATCH 335/530] feat(sda-download): integration test - return 403 for empty request to the /s3/ endpoint --- .../integration/tests/common/90_check_s3_errors.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh index 6f775f564b..69933ea2f3 100644 --- a/sda-download/.github/integration/tests/common/90_check_s3_errors.sh +++ b/sda-download/.github/integration/tests/common/90_check_s3_errors.sh @@ -103,4 +103,12 @@ if [ "$resp" -ne 403 ]; then exit 1 fi +# empty request to the /s3 endpoint should also return 403 + +resp=$(curl --cacert certs/ca.pem -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientkey" -H "SDA-Client-Version: v0.3.0" "https://localhost:9443/s3/" -s -o /dev/null -w "%{http_code}") +if [ "$resp" -ne 403 ]; then + echo "Incorrect response, expected 403 got $resp" + exit 1 +fi + echo "OK" \ No newline at end of file From 3276b52530a5152efcecb570d7801f3f858d5214 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 17 Mar 2026 13:55:09 +0100 Subject: [PATCH 336/530] feat(sda-download): remove non-needed ListBucket Since direct access to the root of the endpoint '/s3', i.e. '/s3/' is disabled, ListBucket is not needed anymore. However, GetBucketLocation by '/s3/?location' is still allowed --- sda-download/api/s3/s3.go | 59 ++++------------------------------ sda-download/api/s3/s3_test.go | 36 --------------------- 2 files changed, 6 insertions(+), 89 deletions(-) diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go index cb1b2f4b28..d198bb7834 100644 --- a/sda-download/api/s3/s3.go +++ b/sda-download/api/s3/s3.go @@ -20,21 +20,11 @@ type LocationConstraint struct { Location string `xml:",innerxml"` } -type Bucket struct { - CreationDate string `xml:"CreationDate,omitempty"` - Name string `xml:"Name"` -} - type Owner struct { DisplayName string `xml:"DisplayName,omitempty"` ID string `xml:"ID,omitempty"` } -type ListAllMyBucketsResult struct { - Buckets []Bucket `xml:"Buckets>Bucket"` - Owner Owner `xml:"Owner"` -} - type Object struct { ChecksumAlgorithm []string `xml:"ChecksumAlgorithm,omitempty"` ETag string `xml:"ETag,omitempty"` @@ -77,44 +67,6 @@ func GetBucketLocation(c *gin.Context) { }) } -// ListBuckets respondes to an S3 ListBuckets request. This request returns the -// available S3 buckets. We use this to list accessible datasets. -// https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html -func ListBuckets(c *gin.Context) { - log.Debug("S3 ListBuckets request") - - // Gin doesn't write the xml header when using c.XML, so we add it manually - _, err := c.Writer.Write([]byte(xml.Header)) - if err != nil { - log.Errorf("Failed writing XML header: %v", err) - c.AbortWithStatus(http.StatusInternalServerError) - - return - } - - buckets := []Bucket{} - cache := middleware.GetCacheFromContext(c) - for _, dataset := range cache.Datasets { - datasetInfo, err := database.GetDatasetInfo(dataset) - if err != nil { - log.Errorf("Failed to get dataset information: %v", err) - c.AbortWithStatus(http.StatusInternalServerError) - - return - } - // TODO: Add real creation date - buckets = append(buckets, Bucket{ - Name: datasetInfo.DatasetID, - CreationDate: datasetInfo.CreatedAt, - }) - } - - c.XML(http.StatusAccepted, ListAllMyBucketsResult{ - Buckets: buckets, - Owner: Owner{DisplayName: "", ID: ""}, - }) -} - // ListObjects respondes to an S3 ListObjects request. This request // lists the contents of an S3 bucket. We use this to return dataset content. // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html @@ -306,7 +258,11 @@ func parseParams(c *gin.Context) *gin.Context { if c.Param("dataset") == "" { log.Warningf("No matching dataset found for path: %q", path) - c.AbortWithStatus(http.StatusForbidden) + // ALLOW the request if it's just asking for the bucket location (?location) + // otherwise, ABORT with 403. + if _, isLocation := c.GetQuery("location"); !isLocation { + c.AbortWithStatus(http.StatusForbidden) + } } return c @@ -335,9 +291,6 @@ func Download(c *gin.Context) { case c.Param("dataset") != "" && c.Param("filename") == "": ListObjects(c) - case c.Param("dataset") == "": - ListBuckets(c) - case c.Param("filename") != "": if config.Config.C4GH.PublicKeyB64 == "" { GetEcnryptedObject(c) @@ -346,7 +299,7 @@ func Download(c *gin.Context) { } default: - log.Warningf("Got unknown S3 request: %v", c.Request) + log.Warningf("Got unknown or unauthorized S3 request: %v", c.Request) c.AbortWithStatus(http.StatusBadRequest) } } diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go index d85c013fd9..e176759005 100644 --- a/sda-download/api/s3/s3_test.go +++ b/sda-download/api/s3/s3_test.go @@ -77,42 +77,6 @@ func (ts *S3TestSuite) TestGetBucketLocation() { assert.Equal(ts.T(), expected, string(body), "Wrong location from S3") } -func (ts *S3TestSuite) TestListBuckets() { - // Setup a mock database to handle queries - - query := `SELECT stable_id, created_at FROM sda.datasets WHERE stable_id = \$1` - ts.Mock.ExpectQuery(query).WithArgs("dataset1"). - WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "nyss")) - ts.Mock.ExpectQuery(query).WithArgs("dataset10"). - WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "nyligen")) - ts.Mock.ExpectQuery(query).WithArgs("https://url/dataset"). - WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "snart")) - - // Send a request through the middleware to get datasets - w := httptest.NewRecorder() - _, router := gin.CreateTestContext(w) - - router.GET("/*path", middleware.TokenMiddleware(), Download) - router.ServeHTTP(w, httptest.NewRequest("GET", "/", nil)) - - response := w.Result() - body, err := io.ReadAll(response.Body) - assert.Nil(ts.T(), err, "failed to parse body from location response") - defer response.Body.Close() - - expected := xml.Header + - "" + - "nyssdataset1" + - "nyligendataset1" + - "snartdataset1" + - "" - - assert.Equal(ts.T(), expected, string(body), "Wrong bucket list from S3") - - err = ts.Mock.ExpectationsWereMet() - assert.Nilf(ts.T(), err, "there were unfulfilled expectations: %s", err) -} - func (ts *S3TestSuite) TestListByPrefix() { // Setup a mock database to handle queries fileInfo := &database.FileInfo{ From e8cc746418929f2f1f31738b0c86cd45a26b6701 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 10:11:56 +0100 Subject: [PATCH 337/530] fix(finalize): ensure backup_location and backup_path are populated when a file is backed up --- sda/cmd/finalize/finalize.go | 8 ++++++-- sda/internal/database/db_functions.go | 13 +++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 66f75c3ac6..e82536a85e 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -317,13 +317,17 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { } }() - _, err = backupWriter.WriteFile(ctx, archiveData.FilePath, contentReader) + backupLocation, err := backupWriter.WriteFile(ctx, archiveData.FilePath, contentReader) if err != nil { return fmt.Errorf("failed to write file to backup storage, reason: %v", err) } _ = contentReader.Close() - // Mark file as "backed up" + // Mark file as "backed up" and populated backup path and location + if err := db.SetBackedUp(backupLocation, archiveData.FilePath, fileID); err != nil { + return fmt.Errorf("SetBackedUp failed, reason: (%v)", err) + } + if err := db.UpdateFileEventLog(fileID, "backed up", "finalize", "{}", string(delivered.Body)); err != nil { return fmt.Errorf("UpdateFileEventLog failed, reason: (%v)", err) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 6c6477ddbf..4eb8b4d5e4 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1565,3 +1565,16 @@ WHERE (f.submission_location = $1 AND f.file_in_dataset IS NOT TRUE) OR f.archiv return size.V, count.V, nil } + +// SetBackedUp sets the file backup_path and backup_location +func (dbs *SDAdb) SetBackedUp(location, path, fileID string) error { + dbs.checkAndReconnectIfNeeded() + + db := dbs.DB + const setBackedUp = "UPDATE sda.files SET backup_location = $1, backup_path = $2 WHERE id = $3;" + if _, err := db.Exec(setBackedUp, location, path, fileID); err != nil { + return fmt.Errorf("setBackedUp error: %s", err.Error()) + } + + return nil +} From 351c9e9c2102730502da8edc7ada17dce1f50225 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 6 Mar 2026 13:26:50 +0100 Subject: [PATCH 338/530] docs(postgres): add 23_populate_storage_locations.md describing data migration needed after schema migrated to version 23, update README.md to mention migratedb.docs directory --- postgresql/README.md | 12 +- .../23_post_populate_storage_locations.md | 121 ++++++++++++++++++ 2 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 postgresql/data_migration.docs/23_post_populate_storage_locations.md diff --git a/postgresql/README.md b/postgresql/README.md index e6a915c0eb..bf9039abec 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -2,7 +2,7 @@ We use [Postgres 15](https://github.com/docker-library/postgres/tree/master/15/alpine) -and Alpine 3.17. +and Alpine 3.23. Security is hardened: @@ -26,3 +26,13 @@ The following environment variables can be used to configure the database: | POSTGRES_VERIFY_PEER | Enforce client verification | verify-ca | Client verification is enforced if `POSTGRES_VERIFY_PEER` is set to `verify-ca` or `verify-full`. + +# Data migration instructions docs + +In [migratedb.docs](data_migration.docs) directory there are instructions on how to execute the data migration +if upgrading a system with existing data related to specific versions of the schema. + +The file naming convention is as follows: `${SCHEMA_VERSION}_${PRE/POST}_${SHORT_DESCRIPTION}.md`. +* `${SCHEMA_VERSION}` - describes the schema version the data migration instructions relates to. +* `${PRE/POST}` describes if these instructions should be executed before or after the schema migration has taken place. +* `${SHORT_DESCRIPTION}` - short description describing the data migration \ No newline at end of file diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md new file mode 100644 index 0000000000..b94d0385f4 --- /dev/null +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -0,0 +1,121 @@ +# Data Migration Plan POST schema migration version 23 + + +## 1. Ensure schema migration has taken place +Ensure [23_expand_files_table_with_storage_locations.sql](../migratedb.d/23_expand_files_table_with_storage_locations.sql) +has been executed. + +Can be checked by +```sql +SELECT * from sda.dbschema_version WHERE version = 23; +``` + +## 2. Prep +Note: Prep is only needed if you have multiple s3 buckets / posix volumes for a storage + +Repeat steps for each s3 bucket / posix volume + +### 2.1. Get file ids file for a storage + +#### If S3 storage +Get all files in form each s3 bucket +```bash +aws s3api list-objects-v2 --endpoint ${ENDPOINT} --bucket ${BUCKET} > ${BUCKET}_raw +``` + +Transform raw response to just list of ids +```bash +cat ${BUCKET}_raw | jq -r '.Contents.[] | .Key' > ${BUCKET}_ids +``` + +#### If Posix storage +``` bash +find . -type f -exec basename {} \; > ${POSIX_VOLUME}_ids +``` + +### 2.2. Create new temporary tables to support DB migration + +```sql +CREATE TABLE temp_file_in_${BUCKET || POSIX_VOLUME} ( +file_id UUID PRIMARY KEY +); +``` + +### 2.3. Populate tables +```bash +psql -U $user -d sda -At -h $host -p $port -c "\copy sda.temp_file_in_${BUCKET || POSIX_VOLUME} from '/path/to/${BUCKET || POSIX_VOLUME}_ids' with delimiter as ','" +``` + +## 3. Run data migration queries + +### 3.1. Inbox Location + +If posix inbox replace `${INBOX_ENDPOINT}/${INBOX_BUCKET}` with `${INBOX_POSIX_VOLUME}` + +If you only have one inbox storage +```sql +UPDATE sda.files +SET submission_location = '${INBOX_ENDPOINT}/${INBOX_BUCKET}'; +``` + +If you only have multiple inbox storages, repeat following UPDATE statement per bucket/volume you have +```sql +UPDATE sda.files AS f +SET submission_location = '${INBOX_ENDPOINT}/${INBOX_BUCKET}' +FROM temp_file_in_${INBOX_BUCKET} AS in_buk +WHERE f.id = in_buk.file_id; +``` + + +### 3.2. Archive Location + +If posix archive replace `${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}` with `/${ARCHIVE_POSIX_VOLUME}` + +If you only have one archive storage +```sql +UPDATE sda.files +SET archive_location ='${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}' +WHERE archive_file_path != ''; +``` + +If you only have multiple archive storages, repeat following UPDATE statement per bucket/volume you have +```sql +UPDATE sda.files AS f +SET archive_location = '${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}' +FROM temp_file_in_${ARCHIVE_BUCKET} AS in_buk +WHERE f.id = in_buk.file_id; +``` + +### 3.3 Backup location +Skip this if you do not have a backup storage + +If posix archive replace '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' with '/${BACKUP_POSIX_VOLUME}' + +If you only have one backup storage +```sql +UPDATE sda.files +SET backup_location ='${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' +WHERE stable_id IS NOT NULL; +``` +If you only have multiple backup storages, repeat following UPDATE statement per bucket/volume you have +```sql +UPDATE sda.files AS f +SET backup_location = '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' +FROM temp_file_in_${BACKUP_BUCKET} AS in_buk +WHERE f.id = in_buk.file_id; +``` + +## 4. Clean up +Only needed if you did the [1. Prep step](#1-prep) and created temporary tables + +Repeat DROP table statement per temporary table created +```sql +DROP TABLE sda.temp_file_in_${BUCKET || POSIX_VOLUME}; +``` + +## 5. Ensure all files have been updated +```sql +SELECT count(id) FROM sda.files WHERE submission_location IS NULL OR (archive_location IS NULL AND archive_file_path != '') +``` +If there exists rows, then there are issues and the required locations of the files are not known. +To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. \ No newline at end of file From 559d78248d45f0cbbea071c99e6f7a5222a8ded1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 09:12:27 +0100 Subject: [PATCH 339/530] docs(postgres): fix data_migration.docs reference in README.md --- postgresql/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/README.md b/postgresql/README.md index bf9039abec..052f62d32d 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -29,7 +29,7 @@ Client verification is enforced if `POSTGRES_VERIFY_PEER` is set to `verify-ca` # Data migration instructions docs -In [migratedb.docs](data_migration.docs) directory there are instructions on how to execute the data migration +In [data_migration.docs](data_migration.docs) directory there are instructions on how to execute the data migration if upgrading a system with existing data related to specific versions of the schema. The file naming convention is as follows: `${SCHEMA_VERSION}_${PRE/POST}_${SHORT_DESCRIPTION}.md`. From 0f74aac17ed99382e5f3ee110420a49852d76389 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 09:14:22 +0100 Subject: [PATCH 340/530] docs(postgres): fix create temp table statment to create in sda database --- .../data_migration.docs/23_post_populate_storage_locations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index b94d0385f4..85d17526ff 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -36,7 +36,7 @@ find . -type f -exec basename {} \; > ${POSIX_VOLUME}_ids ### 2.2. Create new temporary tables to support DB migration ```sql -CREATE TABLE temp_file_in_${BUCKET || POSIX_VOLUME} ( +CREATE TABLE sda.temp_file_in_${BUCKET || POSIX_VOLUME} ( file_id UUID PRIMARY KEY ); ``` From 7f3d086fbdbeec197e4496349cd271ac3e2f11cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 09:15:10 +0100 Subject: [PATCH 341/530] docs(postgres): fix s3 jq prep step to not use cat and jq native syntax --- .../data_migration.docs/23_post_populate_storage_locations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 85d17526ff..4e79bab323 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -25,7 +25,7 @@ aws s3api list-objects-v2 --endpoint ${ENDPOINT} --bucket ${BUCKET} > ${BUCKET}_ Transform raw response to just list of ids ```bash -cat ${BUCKET}_raw | jq -r '.Contents.[] | .Key' > ${BUCKET}_ids +jq -r '.Contents[].Key' ${BUCKET}_raw > ${BUCKET}_ids ``` #### If Posix storage From c0b6e84059da372e80eddb1a32b178f0bb45a454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 09:20:51 +0100 Subject: [PATCH 342/530] docs(postgres): fix 23_post_populate_storage_locations prep header reference --- .../data_migration.docs/23_post_populate_storage_locations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 4e79bab323..520c0ca931 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -106,7 +106,7 @@ WHERE f.id = in_buk.file_id; ``` ## 4. Clean up -Only needed if you did the [1. Prep step](#1-prep) and created temporary tables +Only needed if you did the [2. Prep step](#2-prep) and created temporary tables Repeat DROP table statement per temporary table created ```sql From daeb4478e47be117b8e540f260749920abf15ded Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 09:33:23 +0100 Subject: [PATCH 343/530] docs(postgres): add instruction to run the 23_post_populate_storage_locations data migration in a transaction --- .../23_post_populate_storage_locations.md | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 520c0ca931..b4d8ac55ea 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -47,8 +47,21 @@ psql -U $user -d sda -At -h $host -p $port -c "\copy sda.temp_file_in_${BUCKET | ``` ## 3. Run data migration queries +Run the following data migrations in a transaction such that transaction can be aborted incase something goes wrong and rollback is desired. -### 3.1. Inbox Location +So if something goes wrong after [3.1 Start transaction](#31-start-transaction) and before +[3.5 Commit transaction](#35-commit) you can run +```sql +ROLLBACK; +``` +to rollback the transaction. + +### 3.1. Start transaction +```sql +BEGIN; +``` + +### 3.2. Inbox Location If posix inbox replace `${INBOX_ENDPOINT}/${INBOX_BUCKET}` with `${INBOX_POSIX_VOLUME}` @@ -66,8 +79,7 @@ FROM temp_file_in_${INBOX_BUCKET} AS in_buk WHERE f.id = in_buk.file_id; ``` - -### 3.2. Archive Location +### 3.3. Archive Location If posix archive replace `${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}` with `/${ARCHIVE_POSIX_VOLUME}` @@ -86,7 +98,7 @@ FROM temp_file_in_${ARCHIVE_BUCKET} AS in_buk WHERE f.id = in_buk.file_id; ``` -### 3.3 Backup location +### 3.4 Backup location Skip this if you do not have a backup storage If posix archive replace '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' with '/${BACKUP_POSIX_VOLUME}' @@ -105,6 +117,12 @@ FROM temp_file_in_${BACKUP_BUCKET} AS in_buk WHERE f.id = in_buk.file_id; ``` +### 3.5 Commit +Commit the transaction +```sql +COMMIT; +``` + ## 4. Clean up Only needed if you did the [2. Prep step](#2-prep) and created temporary tables From 20fff77d874de557f80973054446c6a205ec34ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 09:33:55 +0100 Subject: [PATCH 344/530] docs(postgres): update 23_post_populate_storage_locations add optional verification step 5.1 backup location verification --- .../23_post_populate_storage_locations.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index b4d8ac55ea..bca04d09b9 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -136,4 +136,13 @@ DROP TABLE sda.temp_file_in_${BUCKET || POSIX_VOLUME}; SELECT count(id) FROM sda.files WHERE submission_location IS NULL OR (archive_location IS NULL AND archive_file_path != '') ``` If there exists rows, then there are issues and the required locations of the files are not known. -To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. \ No newline at end of file +To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. + +### 5.1 Backup location verification +Skip this step if you do not have a backup storage + +```sql +SELECT count(id) FROM sda.files WHERE archive_location IS NULL AND stable_id IS NOT NULL) +``` +If there exists rows, then there are issues and the backup locations of the files are not known. +To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. From f92b546bd457094f420e9a169a7a06b6ec25567e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 10:06:37 +0100 Subject: [PATCH 345/530] docs(postgres): add schema rollback strucutre and instructions for rollback of version 23 --- postgresql/README.md | 12 +++- ...s_table_with_storage_locations.rollback.md | 63 +++++++++++++++++++ 2 files changed, 73 insertions(+), 2 deletions(-) create mode 100644 postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md diff --git a/postgresql/README.md b/postgresql/README.md index 052f62d32d..711cdae708 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -32,7 +32,15 @@ Client verification is enforced if `POSTGRES_VERIFY_PEER` is set to `verify-ca` In [data_migration.docs](data_migration.docs) directory there are instructions on how to execute the data migration if upgrading a system with existing data related to specific versions of the schema. -The file naming convention is as follows: `${SCHEMA_VERSION}_${PRE/POST}_${SHORT_DESCRIPTION}.md`. +The file naming convention is as follows:`${SCHEMA_VERSION}_${PRE/POST}_${SHORT_DESCRIPTION}.md`. * `${SCHEMA_VERSION}` - describes the schema version the data migration instructions relates to. * `${PRE/POST}` describes if these instructions should be executed before or after the schema migration has taken place. -* `${SHORT_DESCRIPTION}` - short description describing the data migration \ No newline at end of file +* `${SHORT_DESCRIPTION}` - short description describing the data migration + +# Schema migration rollback + +In [rollback.docs](rollback.docs) directory there are instructions on how to rollback schema migrations. + +The file naming convention is as follows: `${SCHEMA_VERSION}_${SHORT_DESCRIPTION}.rollback.md`. +* `${SCHEMA_VERSION}` - describes the schema version the rollback instructions relates to. +* `${SHORT_DESCRIPTION}` - short description describing the schema migration - should be the same as the schema migration in [migratedb.d](migratedb.d) \ No newline at end of file diff --git a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md new file mode 100644 index 0000000000..abf47c343e --- /dev/null +++ b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md @@ -0,0 +1,63 @@ +# Schema migration rollback version 23 +The following instructions describes the procedure to rollback schema version 23. + +## Ensure current schema version +Ensure current schema version is at: 23 + +```sql +SELECT version from sda.dbschema_version ORDER BY version LIMIT 1; +``` +if result of query is not 23, do not proceed to with instructions. + +## Rollback instructions +The schema rollback can be ran is recommended to be ran in a transaction, as if somethings goes wrong during schema rollback +it can be aborted by rolling back transaction with the following statement +```sql +ROLLBACK; +``` + +### Start transaction +```sql +BEGIN; +``` +### Do schema rollback + +```sql +DROP FUNCTION IF EXISTS sda.register_file; +CREATE FUNCTION sda.register_file(file_id TEXT, submission_file_path TEXT, submission_user TEXT) + RETURNS TEXT AS $register_file$ +DECLARE +file_uuid UUID; +BEGIN + -- Upsert file information. we're not interested in restarted uploads so old + -- overwritten files that haven't been ingested are updated instead of + -- inserting a new row. +INSERT INTO sda.files( id, submission_file_path, submission_user, encryption_method ) +VALUES( COALESCE(CAST(NULLIF(file_id, '') AS UUID), gen_random_uuid()), submission_file_path, submission_user, 'CRYPT4GH' ) + ON CONFLICT ON CONSTRAINT unique_ingested + DO UPDATE SET submission_file_path = EXCLUDED.submission_file_path, + submission_user = EXCLUDED.submission_user, + encryption_method = EXCLUDED.encryption_method + RETURNING id INTO file_uuid; + +-- We add a new event for every registration though, as this might help for +-- debugging. +INSERT INTO sda.file_event_log( file_id, event, user_id ) +VALUES (file_uuid, 'registered', submission_user); + +RETURN file_uuid; +END; +$register_file$ LANGUAGE plpgsql; + +ALTER TABLE sda.files + DROP COLUMN archive_location, + DROP COLUMN backup_location, + DROP COLUMN submission_location; + +DELETE FROM sda.dbschema_version WHERE version = 23; +``` + +### Commit transaction +```sql +COMMIT; +``` \ No newline at end of file From bb52b878631469ffcdb6f573a764c1ce0067d532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 10:11:26 +0100 Subject: [PATCH 346/530] docs(postgres): update 23_post_populate_storage_locations queries with additioanl checks when migration based on temporary tables --- .../23_post_populate_storage_locations.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index bca04d09b9..c6f71cb8d3 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -95,7 +95,8 @@ If you only have multiple archive storages, repeat following UPDATE statement pe UPDATE sda.files AS f SET archive_location = '${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}' FROM temp_file_in_${ARCHIVE_BUCKET} AS in_buk -WHERE f.id = in_buk.file_id; +WHERE f.id = in_buk.file_id +AND archive_file_path != ''; ``` ### 3.4 Backup location @@ -114,7 +115,8 @@ If you only have multiple backup storages, repeat following UPDATE statement per UPDATE sda.files AS f SET backup_location = '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' FROM temp_file_in_${BACKUP_BUCKET} AS in_buk -WHERE f.id = in_buk.file_id; +WHERE f.id = in_buk.file_id +AND stable_id IS NOT NULL; ``` ### 3.5 Commit From 010a6da51fee3c02a2cda84ca03446d3a1be525f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 10:12:45 +0100 Subject: [PATCH 347/530] docs(postgres): add DESC when ordering from dbschema_version --- .../23_expand_files_table_with_storage_locations.rollback.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md index abf47c343e..ecc9bf6ea4 100644 --- a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md +++ b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md @@ -5,7 +5,7 @@ The following instructions describes the procedure to rollback schema version 23 Ensure current schema version is at: 23 ```sql -SELECT version from sda.dbschema_version ORDER BY version LIMIT 1; +SELECT version from sda.dbschema_version ORDER BY version LIMIT 1 DESC; ``` if result of query is not 23, do not proceed to with instructions. From fde32c2615ca7398be85385d43d144d867eb6aea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 10:16:21 +0100 Subject: [PATCH 348/530] docs(postgres): fix typo --- .../23_expand_files_table_with_storage_locations.rollback.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md index ecc9bf6ea4..3d1902e159 100644 --- a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md +++ b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md @@ -7,7 +7,7 @@ Ensure current schema version is at: 23 ```sql SELECT version from sda.dbschema_version ORDER BY version LIMIT 1 DESC; ``` -if result of query is not 23, do not proceed to with instructions. +if result of query is not 23, do not proceed with instructions. ## Rollback instructions The schema rollback can be ran is recommended to be ran in a transaction, as if somethings goes wrong during schema rollback From d213056c4b9eb24c3b650a8608c0496089d1ef69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 16:08:55 +0100 Subject: [PATCH 349/530] docs(postgres): fix 23_post_populate_storage_locations.md backup verification query --- .../data_migration.docs/23_post_populate_storage_locations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index c6f71cb8d3..b6b146f5db 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -144,7 +144,7 @@ To resolve you could either manually delete those sda.files entries or ensure th Skip this step if you do not have a backup storage ```sql -SELECT count(id) FROM sda.files WHERE archive_location IS NULL AND stable_id IS NOT NULL) +SELECT count(id) FROM sda.files WHERE backup_location IS NULL AND stable_id IS NOT NULL ``` If there exists rows, then there are issues and the backup locations of the files are not known. To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. From 5c0a35542fee1e5b56fbdf821235a7aaf184951b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 9 Mar 2026 16:15:43 +0100 Subject: [PATCH 350/530] docs(postgres): fix typos / sentancing in 23_expand_files_table_with_storage_locations.rollback.md --- .../23_expand_files_table_with_storage_locations.rollback.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md index 3d1902e159..fb2236621d 100644 --- a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md +++ b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md @@ -10,7 +10,7 @@ SELECT version from sda.dbschema_version ORDER BY version LIMIT 1 DESC; if result of query is not 23, do not proceed with instructions. ## Rollback instructions -The schema rollback can be ran is recommended to be ran in a transaction, as if somethings goes wrong during schema rollback +The schema rollback is recommended to be executed in a transaction, as if something goes wrong during the rollback it can be aborted by rolling back transaction with the following statement ```sql ROLLBACK; From e48e9eb72b7b253a472de536984c6fea42aaf26c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Mar 2026 15:15:04 +0100 Subject: [PATCH 351/530] docs(postgres): address PR comments for multiple storage data migration - Fix typos - Add missing database name in queries - Update step 1 in 23_post_populate_storage_locations.md to check that current version is 23 - Replace ${BUCKET || POSIX_VOLUME} with ${STORAGE_NAME} - Update md headers in README.md --- postgresql/README.md | 8 +++--- .../23_post_populate_storage_locations.md | 28 ++++++++++--------- ...s_table_with_storage_locations.rollback.md | 4 +-- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/postgresql/README.md b/postgresql/README.md index 711cdae708..417fdbf77f 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -27,17 +27,17 @@ The following environment variables can be used to configure the database: Client verification is enforced if `POSTGRES_VERIFY_PEER` is set to `verify-ca` or `verify-full`. -# Data migration instructions docs +## Data migration instructions docs In [data_migration.docs](data_migration.docs) directory there are instructions on how to execute the data migration if upgrading a system with existing data related to specific versions of the schema. -The file naming convention is as follows:`${SCHEMA_VERSION}_${PRE/POST}_${SHORT_DESCRIPTION}.md`. +The file naming convention is as follows:`${SCHEMA_VERSION}_${pre/post}_${SHORT_DESCRIPTION}.md`. * `${SCHEMA_VERSION}` - describes the schema version the data migration instructions relates to. -* `${PRE/POST}` describes if these instructions should be executed before or after the schema migration has taken place. +* `${pre/post}` describes if these instructions should be executed before or after the schema migration has taken place. * `${SHORT_DESCRIPTION}` - short description describing the data migration -# Schema migration rollback +## Schema migration rollback In [rollback.docs](rollback.docs) directory there are instructions on how to rollback schema migrations. diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index b6b146f5db..2d170a73ba 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -2,13 +2,14 @@ ## 1. Ensure schema migration has taken place -Ensure [23_expand_files_table_with_storage_locations.sql](../migratedb.d/23_expand_files_table_with_storage_locations.sql) +Ensure schema version is at 23 and [23_expand_files_table_with_storage_locations.sql](../migratedb.d/23_expand_files_table_with_storage_locations.sql) has been executed. -Can be checked by +Checked by ```sql -SELECT * from sda.dbschema_version WHERE version = 23; +SELECT max(version) AS current_version FROM sda.dbschema_version; ``` +Proceed only if current_version = 23 ## 2. Prep Note: Prep is only needed if you have multiple s3 buckets / posix volumes for a storage @@ -20,30 +21,31 @@ Repeat steps for each s3 bucket / posix volume #### If S3 storage Get all files in form each s3 bucket ```bash -aws s3api list-objects-v2 --endpoint ${ENDPOINT} --bucket ${BUCKET} > ${BUCKET}_raw +aws s3api list-objects-v2 --endpoint ${ENDPOINT} --bucket ${BUCKET} > ${STORAGE_NAME}_raw ``` Transform raw response to just list of ids ```bash -jq -r '.Contents[].Key' ${BUCKET}_raw > ${BUCKET}_ids +jq -r '.Contents[].Key' ${STORAGE_NAME}_raw > ${STORAGE_NAME}_ids ``` #### If Posix storage -``` bash -find . -type f -exec basename {} \; > ${POSIX_VOLUME}_ids +```bash +find . -type f -exec basename {} \; > ${STORAGE_NAME}_ids ``` ### 2.2. Create new temporary tables to support DB migration + ```sql -CREATE TABLE sda.temp_file_in_${BUCKET || POSIX_VOLUME} ( +CREATE TABLE sda.temp_file_in_${STORAGE_NAME} ( file_id UUID PRIMARY KEY ); ``` ### 2.3. Populate tables ```bash -psql -U $user -d sda -At -h $host -p $port -c "\copy sda.temp_file_in_${BUCKET || POSIX_VOLUME} from '/path/to/${BUCKET || POSIX_VOLUME}_ids' with delimiter as ','" +psql -U $user -d sda -At -h $host -p $port -c "\copy sda.temp_file_in_${STORAGE_NAME} from '/path/to/${STORAGE_NAME}_ids' with delimiter as ','" ``` ## 3. Run data migration queries @@ -75,7 +77,7 @@ If you only have multiple inbox storages, repeat following UPDATE statement per ```sql UPDATE sda.files AS f SET submission_location = '${INBOX_ENDPOINT}/${INBOX_BUCKET}' -FROM temp_file_in_${INBOX_BUCKET} AS in_buk +FROM sda.temp_file_in_${INBOX_BUCKET} AS in_buk WHERE f.id = in_buk.file_id; ``` @@ -94,7 +96,7 @@ If you only have multiple archive storages, repeat following UPDATE statement pe ```sql UPDATE sda.files AS f SET archive_location = '${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}' -FROM temp_file_in_${ARCHIVE_BUCKET} AS in_buk +FROM sda.temp_file_in_${ARCHIVE_BUCKET} AS in_buk WHERE f.id = in_buk.file_id AND archive_file_path != ''; ``` @@ -114,7 +116,7 @@ If you only have multiple backup storages, repeat following UPDATE statement per ```sql UPDATE sda.files AS f SET backup_location = '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' -FROM temp_file_in_${BACKUP_BUCKET} AS in_buk +FROM sda.temp_file_in_${BACKUP_BUCKET} AS in_buk WHERE f.id = in_buk.file_id AND stable_id IS NOT NULL; ``` @@ -130,7 +132,7 @@ Only needed if you did the [2. Prep step](#2-prep) and created temporary tables Repeat DROP table statement per temporary table created ```sql -DROP TABLE sda.temp_file_in_${BUCKET || POSIX_VOLUME}; +DROP TABLE sda.temp_file_in_${STORAGE_NAME}; ``` ## 5. Ensure all files have been updated diff --git a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md index fb2236621d..931014475b 100644 --- a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md +++ b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md @@ -1,11 +1,11 @@ # Schema migration rollback version 23 -The following instructions describes the procedure to rollback schema version 23. +The following instructions describe the procedure to rollback schema version 23. ## Ensure current schema version Ensure current schema version is at: 23 ```sql -SELECT version from sda.dbschema_version ORDER BY version LIMIT 1 DESC; +SELECT max(version) AS current_version FROM sda.dbschema_version; ``` if result of query is not 23, do not proceed with instructions. From 5db514d72829a557515d002f268bd5b17149ad68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 12 Mar 2026 15:55:16 +0100 Subject: [PATCH 352/530] docs(postgres): fix some minor typos, sentacing, and spacing --- postgresql/README.md | 2 +- .../23_post_populate_storage_locations.md | 6 +++--- ...23_expand_files_table_with_storage_locations.rollback.md | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/postgresql/README.md b/postgresql/README.md index 417fdbf77f..93c52e9002 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -32,7 +32,7 @@ Client verification is enforced if `POSTGRES_VERIFY_PEER` is set to `verify-ca` In [data_migration.docs](data_migration.docs) directory there are instructions on how to execute the data migration if upgrading a system with existing data related to specific versions of the schema. -The file naming convention is as follows:`${SCHEMA_VERSION}_${pre/post}_${SHORT_DESCRIPTION}.md`. +The file naming convention is as follows: `${SCHEMA_VERSION}_${pre/post}_${SHORT_DESCRIPTION}.md`. * `${SCHEMA_VERSION}` - describes the schema version the data migration instructions relates to. * `${pre/post}` describes if these instructions should be executed before or after the schema migration has taken place. * `${SHORT_DESCRIPTION}` - short description describing the data migration diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 2d170a73ba..561f93a1e7 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -9,7 +9,7 @@ Checked by ```sql SELECT max(version) AS current_version FROM sda.dbschema_version; ``` -Proceed only if current_version = 23 +If result of query is not 23, do not proceed with instructions. ## 2. Prep Note: Prep is only needed if you have multiple s3 buckets / posix volumes for a storage @@ -19,7 +19,7 @@ Repeat steps for each s3 bucket / posix volume ### 2.1. Get file ids file for a storage #### If S3 storage -Get all files in form each s3 bucket +Get all files from each s3 bucket ```bash aws s3api list-objects-v2 --endpoint ${ENDPOINT} --bucket ${BUCKET} > ${STORAGE_NAME}_raw ``` @@ -49,7 +49,7 @@ psql -U $user -d sda -At -h $host -p $port -c "\copy sda.temp_file_in_${STORAGE_ ``` ## 3. Run data migration queries -Run the following data migrations in a transaction such that transaction can be aborted incase something goes wrong and rollback is desired. +Run the following data migrations in a transaction such that transaction can be aborted in case something goes wrong and rollback is desired. So if something goes wrong after [3.1 Start transaction](#31-start-transaction) and before [3.5 Commit transaction](#35-commit) you can run diff --git a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md index 931014475b..c119d95999 100644 --- a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md +++ b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md @@ -7,7 +7,7 @@ Ensure current schema version is at: 23 ```sql SELECT max(version) AS current_version FROM sda.dbschema_version; ``` -if result of query is not 23, do not proceed with instructions. +If result of query is not 23, do not proceed with instructions. ## Rollback instructions The schema rollback is recommended to be executed in a transaction, as if something goes wrong during the rollback From 6aab2dbaf06cddd5fa561a0551f6ea73c0009e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 09:17:59 +0100 Subject: [PATCH 353/530] docs(postgres): update README.md to include recommneded sequence when deploying schema migrations --- postgresql/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/postgresql/README.md b/postgresql/README.md index 93c52e9002..e547e6b21a 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -37,6 +37,19 @@ The file naming convention is as follows: `${SCHEMA_VERSION}_${pre/post}_${SHORT * `${pre/post}` describes if these instructions should be executed before or after the schema migration has taken place. * `${SHORT_DESCRIPTION}` - short description describing the data migration +Before upgrading the schema ensure if there are any data migrations needed for any version to be applied. +If the data migration is a **pre** it needs to be applied **before** that specific schema version migration, +and if it is a **post** it needs to be applied **after** that. + +Recommended sequence when deploying: + +1. Check the currently applied schema version and the target schema version after deployment. +2. Check if there are any data migration instructions for any of the schema versions to be applied. +3. For each schema version to be applied: + 1. Run pre data migrations ***if present***. + 2. Apply schema migration. + 3. Run post data migrations ***if present***. + ## Schema migration rollback In [rollback.docs](rollback.docs) directory there are instructions on how to rollback schema migrations. From f4e7bcd97fe532824167e5d10bdb1f7500580fcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 09:29:19 +0100 Subject: [PATCH 354/530] docs(postgres): fix some typos in 23_post_populate_storage_locations.md instructions --- .../23_post_populate_storage_locations.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 561f93a1e7..44ac47fd63 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -65,7 +65,7 @@ BEGIN; ### 3.2. Inbox Location -If posix inbox replace `${INBOX_ENDPOINT}/${INBOX_BUCKET}` with `${INBOX_POSIX_VOLUME}` +If posix inbox replace `${INBOX_ENDPOINT}/${INBOX_BUCKET}` with `/${INBOX_POSIX_VOLUME}` If you only have one inbox storage ```sql @@ -77,7 +77,7 @@ If you only have multiple inbox storages, repeat following UPDATE statement per ```sql UPDATE sda.files AS f SET submission_location = '${INBOX_ENDPOINT}/${INBOX_BUCKET}' -FROM sda.temp_file_in_${INBOX_BUCKET} AS in_buk +FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk WHERE f.id = in_buk.file_id; ``` @@ -96,7 +96,7 @@ If you only have multiple archive storages, repeat following UPDATE statement pe ```sql UPDATE sda.files AS f SET archive_location = '${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}' -FROM sda.temp_file_in_${ARCHIVE_BUCKET} AS in_buk +FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk WHERE f.id = in_buk.file_id AND archive_file_path != ''; ``` @@ -116,7 +116,7 @@ If you only have multiple backup storages, repeat following UPDATE statement per ```sql UPDATE sda.files AS f SET backup_location = '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' -FROM sda.temp_file_in_${BACKUP_BUCKET} AS in_buk +FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk WHERE f.id = in_buk.file_id AND stable_id IS NOT NULL; ``` From 9a5c41008e80fd9d90f6aeafa118f4126828f9ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 09:31:21 +0100 Subject: [PATCH 355/530] docs(postgres): update semantics in 23_post_populate_storage_locations.md to call the tables created for the data migration staging tables instead of temporary to reduce risk of conufsion with postgres temporary tables --- .../23_post_populate_storage_locations.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 44ac47fd63..aa4172b755 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -34,7 +34,7 @@ jq -r '.Contents[].Key' ${STORAGE_NAME}_raw > ${STORAGE_NAME}_ids find . -type f -exec basename {} \; > ${STORAGE_NAME}_ids ``` -### 2.2. Create new temporary tables to support DB migration +### 2.2. Create new staging tables to support DB migration ```sql @@ -128,9 +128,9 @@ COMMIT; ``` ## 4. Clean up -Only needed if you did the [2. Prep step](#2-prep) and created temporary tables +Only needed if you did the [2. Prep step](#2-prep) and created staging tables -Repeat DROP table statement per temporary table created +Repeat DROP table statement per staging table created ```sql DROP TABLE sda.temp_file_in_${STORAGE_NAME}; ``` From f732584177067267561c69924f0004aeb2a2e477 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 09:44:40 +0100 Subject: [PATCH 356/530] docs(postgres): update 23_post_populate_storage_locations.md to populate backup_path and for backup_location to populated to be based on backup_path --- .../23_post_populate_storage_locations.md | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index aa4172b755..90483ccdf6 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -102,15 +102,26 @@ AND archive_file_path != ''; ``` ### 3.4 Backup location -Skip this if you do not have a backup storage +Skip these steps if you do not have a backup storage +#### 3.4.1 Populate backup_path +Currently the `backup_path` was not being populated when a file was backed up, this is now needed. Currently the `backup_path` +is always the same as `archive_file_path`. + +```sql +UPDATE sda.files +SET backup_path = archive_file_path +WHERE archive_file_path != ''; +``` + +#### 3.4.2 Populate backup_location If posix archive replace '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' with '/${BACKUP_POSIX_VOLUME}' If you only have one backup storage ```sql UPDATE sda.files SET backup_location ='${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' -WHERE stable_id IS NOT NULL; +WHERE backup_path != ''; ``` If you only have multiple backup storages, repeat following UPDATE statement per bucket/volume you have ```sql @@ -118,7 +129,7 @@ UPDATE sda.files AS f SET backup_location = '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk WHERE f.id = in_buk.file_id -AND stable_id IS NOT NULL; +AND backup_path != ''; ``` ### 3.5 Commit From 55e4305cc0869229c2cb518b6fc756b0bf269686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 09:49:11 +0100 Subject: [PATCH 357/530] docs(postgres): update 23_post_populate_storage_locations.md add missing semi coloings in queries --- .../23_post_populate_storage_locations.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 90483ccdf6..2965009536 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -105,7 +105,7 @@ AND archive_file_path != ''; Skip these steps if you do not have a backup storage #### 3.4.1 Populate backup_path -Currently the `backup_path` was not being populated when a file was backed up, this is now needed. Currently the `backup_path` +Currently, the `backup_path` was not being populated when a file was backed up, this is now needed. Currently the `backup_path` is always the same as `archive_file_path`. ```sql @@ -148,7 +148,7 @@ DROP TABLE sda.temp_file_in_${STORAGE_NAME}; ## 5. Ensure all files have been updated ```sql -SELECT count(id) FROM sda.files WHERE submission_location IS NULL OR (archive_location IS NULL AND archive_file_path != '') +SELECT count(id) FROM sda.files WHERE submission_location IS NULL OR (archive_location IS NULL AND archive_file_path != ''); ``` If there exists rows, then there are issues and the required locations of the files are not known. To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. @@ -157,7 +157,7 @@ To resolve you could either manually delete those sda.files entries or ensure th Skip this step if you do not have a backup storage ```sql -SELECT count(id) FROM sda.files WHERE backup_location IS NULL AND stable_id IS NOT NULL +SELECT count(id) FROM sda.files WHERE backup_location IS NULL AND stable_id IS NOT NULL; ``` If there exists rows, then there are issues and the backup locations of the files are not known. To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. From d28a0703133b999db3cea55d0a4dc823ea34102f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 10:02:05 +0100 Subject: [PATCH 358/530] docs(postgres): update 23_post_populate_storage_locations.md inbox data migration when multiple storages to be based on submission user and submission file path concatinated as thats what we have in the storage --- .../data_migration.docs/23_post_populate_storage_locations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 2965009536..8f064ec09a 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -78,7 +78,7 @@ If you only have multiple inbox storages, repeat following UPDATE statement per UPDATE sda.files AS f SET submission_location = '${INBOX_ENDPOINT}/${INBOX_BUCKET}' FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk -WHERE f.id = in_buk.file_id; +WHERE CONCAT(REPLACE(submission_user, '@', '_'), '/', submission_file_path) = in_buk.file_id; ``` ### 3.3. Archive Location From fc76741aeec594b22e9f3f32d4d06ac5e6cf1da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 10:16:55 +0100 Subject: [PATCH 359/530] docs(postgres): update 23_post_populate_storage_locations.md staging tables to use TEXT column instead of UUID for inbox storages to function --- .../data_migration.docs/23_post_populate_storage_locations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 8f064ec09a..2eb9ccae94 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -39,7 +39,7 @@ find . -type f -exec basename {} \; > ${STORAGE_NAME}_ids ```sql CREATE TABLE sda.temp_file_in_${STORAGE_NAME} ( -file_id UUID PRIMARY KEY +file_id TEXT PRIMARY KEY ); ``` From 630b658b9dc2d4cabed2cd1abcafd904decc6057 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 10:17:50 +0100 Subject: [PATCH 360/530] docs(postgres): update 23_post_populate_storage_locations.md fix posix prep step to include sub directories --- .../data_migration.docs/23_post_populate_storage_locations.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 2eb9ccae94..72842eed92 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -31,7 +31,7 @@ jq -r '.Contents[].Key' ${STORAGE_NAME}_raw > ${STORAGE_NAME}_ids #### If Posix storage ```bash -find . -type f -exec basename {} \; > ${STORAGE_NAME}_ids +find . -type f -print | sed 's|^\./||' > ${STORAGE_NAME}_ids ``` ### 2.2. Create new staging tables to support DB migration From a5492a0455f7479fb99c2893470b78b7c78238c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 10:30:15 +0100 Subject: [PATCH 361/530] docs(postgres): update 23_post_populate_storage_locations.md update staging tables semantics to use key instead of id given inbox storage --- .../23_post_populate_storage_locations.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 72842eed92..668fe4ab85 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -24,14 +24,14 @@ Get all files from each s3 bucket aws s3api list-objects-v2 --endpoint ${ENDPOINT} --bucket ${BUCKET} > ${STORAGE_NAME}_raw ``` -Transform raw response to just list of ids +Transform raw response to just list of keys ```bash -jq -r '.Contents[].Key' ${STORAGE_NAME}_raw > ${STORAGE_NAME}_ids +jq -r '.Contents[].Key' ${STORAGE_NAME}_raw > ${STORAGE_NAME}_keys ``` #### If Posix storage ```bash -find . -type f -print | sed 's|^\./||' > ${STORAGE_NAME}_ids +find . -type f -print | sed 's|^\./||' > ${STORAGE_NAME}_keys ``` ### 2.2. Create new staging tables to support DB migration @@ -39,13 +39,13 @@ find . -type f -print | sed 's|^\./||' > ${STORAGE_NAME}_ids ```sql CREATE TABLE sda.temp_file_in_${STORAGE_NAME} ( -file_id TEXT PRIMARY KEY +key TEXT PRIMARY KEY ); ``` ### 2.3. Populate tables ```bash -psql -U $user -d sda -At -h $host -p $port -c "\copy sda.temp_file_in_${STORAGE_NAME} from '/path/to/${STORAGE_NAME}_ids' with delimiter as ','" +psql -U $user -d sda -At -h $host -p $port -c "\copy sda.temp_file_in_${STORAGE_NAME} from '/path/to/${STORAGE_NAME}_keys' with delimiter as ','" ``` ## 3. Run data migration queries @@ -78,7 +78,7 @@ If you only have multiple inbox storages, repeat following UPDATE statement per UPDATE sda.files AS f SET submission_location = '${INBOX_ENDPOINT}/${INBOX_BUCKET}' FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk -WHERE CONCAT(REPLACE(submission_user, '@', '_'), '/', submission_file_path) = in_buk.file_id; +WHERE CONCAT(REPLACE(submission_user, '@', '_'), '/', submission_file_path) = in_buk.key; ``` ### 3.3. Archive Location @@ -97,7 +97,7 @@ If you only have multiple archive storages, repeat following UPDATE statement pe UPDATE sda.files AS f SET archive_location = '${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}' FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk -WHERE f.id = in_buk.file_id +WHERE f.id = in_buk.key AND archive_file_path != ''; ``` @@ -128,7 +128,7 @@ If you only have multiple backup storages, repeat following UPDATE statement per UPDATE sda.files AS f SET backup_location = '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk -WHERE f.id = in_buk.file_id +WHERE f.id = in_buk.key AND backup_path != ''; ``` From dc6fa14d97bd5ec9a4075a9ba4f76ecd6f9504a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 10:36:37 +0100 Subject: [PATCH 362/530] docs(postgres): update 23_post_populate_storage_locations.md fix UUID to TEXT comparision, fix typo --- .../23_post_populate_storage_locations.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 668fe4ab85..89aa923add 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -97,7 +97,7 @@ If you only have multiple archive storages, repeat following UPDATE statement pe UPDATE sda.files AS f SET archive_location = '${ARCHIVE_ENDPOINT}/${ARCHIVE_BUCKET}' FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk -WHERE f.id = in_buk.key +WHERE f.id::text = in_buk.key AND archive_file_path != ''; ``` @@ -115,7 +115,7 @@ WHERE archive_file_path != ''; ``` #### 3.4.2 Populate backup_location -If posix archive replace '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' with '/${BACKUP_POSIX_VOLUME}' +If posix backup replace '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' with '/${BACKUP_POSIX_VOLUME}' If you only have one backup storage ```sql @@ -128,7 +128,7 @@ If you only have multiple backup storages, repeat following UPDATE statement per UPDATE sda.files AS f SET backup_location = '${BACKUP_ENDPOINT}/${BACKUP_BUCKET}' FROM sda.temp_file_in_${STORAGE_NAME} AS in_buk -WHERE f.id = in_buk.key +WHERE f.id::text = in_buk.key AND backup_path != ''; ``` From 93fffa1e4ac738c68ebf283951ab97441850c0aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 11:15:12 +0100 Subject: [PATCH 363/530] docs(postgres): fix some grammer, trailing spaces, and backup verification query --- postgresql/README.md | 5 ++--- .../23_post_populate_storage_locations.md | 2 +- .../23_expand_files_table_with_storage_locations.rollback.md | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/postgresql/README.md b/postgresql/README.md index e547e6b21a..39ddaff442 100644 --- a/postgresql/README.md +++ b/postgresql/README.md @@ -37,9 +37,8 @@ The file naming convention is as follows: `${SCHEMA_VERSION}_${pre/post}_${SHORT * `${pre/post}` describes if these instructions should be executed before or after the schema migration has taken place. * `${SHORT_DESCRIPTION}` - short description describing the data migration -Before upgrading the schema ensure if there are any data migrations needed for any version to be applied. -If the data migration is a **pre** it needs to be applied **before** that specific schema version migration, -and if it is a **post** it needs to be applied **after** that. +Before upgrading the schema, check whether any data migrations are required for the schema versions you plan to apply. +Apply any **pre** data migrations **before** running the corresponding schema migration, and any **post** data migrations **after** it. Recommended sequence when deploying: diff --git a/postgresql/data_migration.docs/23_post_populate_storage_locations.md b/postgresql/data_migration.docs/23_post_populate_storage_locations.md index 89aa923add..cf7322ea9f 100644 --- a/postgresql/data_migration.docs/23_post_populate_storage_locations.md +++ b/postgresql/data_migration.docs/23_post_populate_storage_locations.md @@ -157,7 +157,7 @@ To resolve you could either manually delete those sda.files entries or ensure th Skip this step if you do not have a backup storage ```sql -SELECT count(id) FROM sda.files WHERE backup_location IS NULL AND stable_id IS NOT NULL; +SELECT count(id) FROM sda.files WHERE backup_location IS NULL AND backup_path != ''; ``` If there exists rows, then there are issues and the backup locations of the files are not known. To resolve you could either manually delete those sda.files entries or ensure the files are uploaded to the expected locations. diff --git a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md index c119d95999..62d9cd506b 100644 --- a/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md +++ b/postgresql/rollback.docs/23_expand_files_table_with_storage_locations.rollback.md @@ -54,7 +54,7 @@ ALTER TABLE sda.files DROP COLUMN backup_location, DROP COLUMN submission_location; -DELETE FROM sda.dbschema_version WHERE version = 23; +DELETE FROM sda.dbschema_version WHERE version = 23; ``` ### Commit transaction From 7b31f4a3d3e4c140794e1607e075947fc08681fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 11:02:33 +0100 Subject: [PATCH 364/530] fix(database): update SetBackedUp to return sql.ErrNoRows if no rows affected, add unit tests for SetBackedUp, fix typo in finalize --- sda/cmd/finalize/finalize.go | 2 +- sda/internal/database/db_functions.go | 12 ++++++- sda/internal/database/db_functions_test.go | 38 ++++++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index e82536a85e..7378376d3d 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -323,7 +323,7 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { } _ = contentReader.Close() - // Mark file as "backed up" and populated backup path and location + // Mark file as "backed up" and populate backup path and location if err := db.SetBackedUp(backupLocation, archiveData.FilePath, fileID); err != nil { return fmt.Errorf("SetBackedUp failed, reason: (%v)", err) } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 4eb8b4d5e4..b3edf8b85c 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -1572,9 +1572,19 @@ func (dbs *SDAdb) SetBackedUp(location, path, fileID string) error { db := dbs.DB const setBackedUp = "UPDATE sda.files SET backup_location = $1, backup_path = $2 WHERE id = $3;" - if _, err := db.Exec(setBackedUp, location, path, fileID); err != nil { + r, err := db.Exec(setBackedUp, location, path, fileID) + if err != nil { return fmt.Errorf("setBackedUp error: %s", err.Error()) } + rowsAffected, err := r.RowsAffected() + if err != nil { + return fmt.Errorf("setBackedUp error: %s", err.Error()) + } + + if rowsAffected == 0 { + return sql.ErrNoRows + } + return nil } diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index da1ab2d8f0..c787f68e12 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -3,6 +3,7 @@ package database import ( "context" "crypto/sha256" + "database/sql" "encoding/hex" "fmt" "testing" @@ -1868,3 +1869,40 @@ func (suite *DatabaseTests) TestIsFileInDataset_Yes() { assert.NoError(suite.T(), err) assert.True(suite.T(), inDataset) } + +func (suite *DatabaseTests) TestSetBackedUp() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + // register a file in the database + fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestSetArchived.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + assert.NoError(suite.T(), db.SetArchived("/archive", FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, fileID, fmt.Sprintf("%x", sha256.New()), -1, fmt.Sprintf("%x", sha256.New())}, fileID)) + + assert.NoError(suite.T(), db.SetBackedUp("/backup", fileID, fileID)) + + // Ensure backup_location and backup_path are set + archiveData, err := db.getArchived(fileID) + + assert.NoError(suite.T(), err) + if archiveData == nil { + suite.FailNow("archive data not found") + + return + } + + suite.Equal("/backup", archiveData.BackupLocation) + suite.Equal(fileID, archiveData.BackupFilePath) + + db.Close() +} +func (suite *DatabaseTests) TestSetBackedUp_FileID_Not_Exists() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + + notExistingFileID := uuid.NewString() + assert.EqualError(suite.T(), db.SetBackedUp("/backup", notExistingFileID, notExistingFileID), sql.ErrNoRows.Error()) + + db.Close() +} From cba6a16fed8fc78d85c7d721b966cac72a197fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 11:20:08 +0100 Subject: [PATCH 365/530] fix(finalize): close content reader on backup writer error --- sda/cmd/finalize/finalize.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 7378376d3d..7c7101596b 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -319,6 +319,8 @@ func backupFile(ctx context.Context, delivered amqp.Delivery) error { backupLocation, err := backupWriter.WriteFile(ctx, archiveData.FilePath, contentReader) if err != nil { + _ = contentReader.Close() + return fmt.Errorf("failed to write file to backup storage, reason: %v", err) } _ = contentReader.Close() From e77b89116364fad9b709f5ab8a37029edac28ba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 23 Mar 2026 11:32:38 +0100 Subject: [PATCH 366/530] fix(database): update TestSetBackedUp to defer db close --- sda/internal/database/db_functions_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index c787f68e12..98d35513d7 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1873,6 +1873,7 @@ func (suite *DatabaseTests) TestIsFileInDataset_Yes() { func (suite *DatabaseTests) TestSetBackedUp() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) + defer db.Close() // register a file in the database fileID, err := db.RegisterFile(nil, "/inbox", "/testuser/TestSetArchived.c4gh", "testuser") @@ -1894,15 +1895,12 @@ func (suite *DatabaseTests) TestSetBackedUp() { suite.Equal("/backup", archiveData.BackupLocation) suite.Equal(fileID, archiveData.BackupFilePath) - - db.Close() } func (suite *DatabaseTests) TestSetBackedUp_FileID_Not_Exists() { db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) + defer db.Close() notExistingFileID := uuid.NewString() assert.EqualError(suite.T(), db.SetBackedUp("/backup", notExistingFileID, notExistingFileID), sql.ErrNoRows.Error()) - - db.Close() } From 082dbd83e2b586e8c242273f6ce93665d74f3abc Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 26 Mar 2026 10:08:45 +0100 Subject: [PATCH 367/530] fix(ci): add codecov configuration to prevent false check failures Add .codecov.yml with: - Project threshold of 1% to tolerate small coverage drops on large PRs - Patch target of 20% appropriate for mixed unit+integration test strategy - Carryforward on unittests flag as defensive config - Ignore benchmark CLI tool (integration-tested, not unit-tested) Closes #2264 --- .codecov.yml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .codecov.yml diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000000..3c3624934a --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,35 @@ +# Codecov configuration +# Documentation: https://docs.codecov.io/docs/codecovyml-reference + +# Files to exclude from coverage reporting +ignore: + # Main entry points - tested via integration tests + - "sda/cmd/*/main.go" + + # Benchmark CLI tool - tested via integration tests + - "sda/cmd/download/benchmark/benchmark.go" + + # Generated files + - "**/*_gen.go" + - "**/mocks/**" + + # Test files (already excluded by default, but explicit) + - "**/*_test.go" + +coverage: + status: + project: + default: + threshold: 1% + patch: + default: + target: 20% + +flags: + unittests: + carryforward: true + +comment: + layout: "reach,diff,flags,files" + behavior: default + require_changes: false From eb2f04c7643d71f5459f4e69695cca1b65625387 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 12:44:54 +0000 Subject: [PATCH 368/530] build(deps): bump requests Bumps the pip group with 1 update in the /.github/integration/tests/sda/auth directory: [requests](https://github.com/psf/requests). Updates `requests` from 2.32.4 to 2.33.0 - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.32.4...v2.33.0) --- updated-dependencies: - dependency-name: requests dependency-version: 2.33.0 dependency-type: direct:production dependency-group: pip ... Signed-off-by: dependabot[bot] --- .github/integration/tests/sda/auth/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/auth/requirements.txt b/.github/integration/tests/sda/auth/requirements.txt index 8073666c6d..d49a6a67ef 100644 --- a/.github/integration/tests/sda/auth/requirements.txt +++ b/.github/integration/tests/sda/auth/requirements.txt @@ -1,3 +1,3 @@ -requests==2.32.4 +requests==2.33.0 tox==4.11.3 pytest==7.4.3 From 6d6050d5fcb7b7ad0628adda32cdff6532bf97ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 19:05:18 +0000 Subject: [PATCH 369/530] build(deps): bump the github-actions group with 3 updates Bumps the github-actions group with 3 updates: [DavidAnson/markdownlint-cli2-action](https://github.com/davidanson/markdownlint-cli2-action), [azure/setup-helm](https://github.com/azure/setup-helm) and [codecov/codecov-action](https://github.com/codecov/codecov-action). Updates `DavidAnson/markdownlint-cli2-action` from 22 to 23 - [Release notes](https://github.com/davidanson/markdownlint-cli2-action/releases) - [Commits](https://github.com/davidanson/markdownlint-cli2-action/compare/v22...v23) Updates `azure/setup-helm` from 4 to 5 - [Release notes](https://github.com/azure/setup-helm/releases) - [Changelog](https://github.com/Azure/setup-helm/blob/main/CHANGELOG.md) - [Commits](https://github.com/azure/setup-helm/compare/v4...v5) Updates `codecov/codecov-action` from 5.5.3 to 6.0.0 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.5.3...v6.0.0) --- updated-dependencies: - dependency-name: DavidAnson/markdownlint-cli2-action dependency-version: '23' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: azure/setup-helm dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: codecov/codecov-action dependency-version: 6.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/adr_lint_markdown.yml | 2 +- .github/workflows/pr_build_and_test_images.yml | 2 +- .github/workflows/pr_code_tests.yml | 8 ++++---- .github/workflows/release_charts.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/adr_lint_markdown.yml b/.github/workflows/adr_lint_markdown.yml index 8e04b1eb9c..321fc28e34 100644 --- a/.github/workflows/adr_lint_markdown.yml +++ b/.github/workflows/adr_lint_markdown.yml @@ -17,7 +17,7 @@ jobs: uses: actions/checkout@v6 - name: Lint decision records - uses: DavidAnson/markdownlint-cli2-action@v22 + uses: DavidAnson/markdownlint-cli2-action@v23 with: globs: | docs/decisions/**/*.md diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index 872b1f4951..177ea64457 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -307,7 +307,7 @@ jobs: uses: actions/checkout@v6 - name: Install Helm - uses: azure/setup-helm@v4 + uses: azure/setup-helm@v5 - name: Initialise k3d id: initK3D diff --git a/.github/workflows/pr_code_tests.yml b/.github/workflows/pr_code_tests.yml index 58683d482c..c39c2ca418 100644 --- a/.github/workflows/pr_code_tests.yml +++ b/.github/workflows/pr_code_tests.yml @@ -61,7 +61,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.3 + uses: codecov/codecov-action@v6.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -92,7 +92,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.3 + uses: codecov/codecov-action@v6.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -123,7 +123,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.3 + uses: codecov/codecov-action@v6.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt @@ -154,7 +154,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.3 + uses: codecov/codecov-action@v6.0.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-validator/orchestrator/coverage.txt diff --git a/.github/workflows/release_charts.yml b/.github/workflows/release_charts.yml index 7c8fdf96df..709d20d7c6 100644 --- a/.github/workflows/release_charts.yml +++ b/.github/workflows/release_charts.yml @@ -26,7 +26,7 @@ jobs: git config user.email "$GITHUB_ACTOR@users.noreply.github.com" - name: Install Helm - uses: azure/setup-helm@v4 + uses: azure/setup-helm@v5 - name: Run chart-releaser uses: helm/chart-releaser-action@v1.7.0 From 4b8d2b11c3842fa5d60d1d55b92bb746b324fe72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 26 Mar 2026 19:17:25 +0000 Subject: [PATCH 370/530] build(deps): bump the go group across 2 directories with 5 updates Bumps the go group with 3 updates in the /sda directory: [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager](https://github.com/aws/aws-sdk-go-v2). Bumps the go group with 3 updates in the /sda-download directory: [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2). Updates `github.com/aws/aws-sdk-go-v2` from 1.41.4 to 1.41.5 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.4...v1.41.5) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.12 to 1.32.13 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.12...config/v1.32.13) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.12 to 1.19.13 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/sqs/v1.19.12...credentials/v1.19.13) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager` from 0.1.10 to 0.1.12 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/transfermanager/v0.1.10...feature/s3/transfermanager/v0.1.12) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.1 to 1.97.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.1...service/s3/v1.97.3) Updates `github.com/aws/aws-sdk-go-v2` from 1.41.4 to 1.41.5 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.4...v1.41.5) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.12 to 1.32.13 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.12...config/v1.32.13) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.12 to 1.19.13 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/sqs/v1.19.12...credentials/v1.19.13) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.1 to 1.97.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.1...service/s3/v1.97.3) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.13 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.13 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager dependency-version: 0.1.12 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.97.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.13 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.13 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.97.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 32 +++++++-------- sda-download/go.sum | 64 ++++++++++++++--------------- sda-validator/orchestrator/go.mod | 6 +-- sda-validator/orchestrator/go.sum | 16 ++++---- sda/go.mod | 34 ++++++++-------- sda/go.sum | 68 +++++++++++++++---------------- 6 files changed, 110 insertions(+), 110 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 9b40978ad1..79721cc529 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -5,10 +5,10 @@ go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/Masterminds/semver/v3 v3.4.0 - github.com/aws/aws-sdk-go-v2 v1.41.4 - github.com/aws/aws-sdk-go-v2/config v1.32.12 - github.com/aws/aws-sdk-go-v2/credentials v1.19.12 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 + github.com/aws/aws-sdk-go-v2 v1.41.5 + github.com/aws/aws-sdk-go-v2/config v1.32.13 + github.com/aws/aws-sdk-go-v2/credentials v1.19.13 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/dgraph-io/ristretto v1.0.0 @@ -29,20 +29,20 @@ require ( require ( filippo.io/edwards25519 v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 98e63bef33..8414bcb834 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -4,42 +4,42 @@ github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7Oputl github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= -github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= -github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= -github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= -github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg= +github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index 50ebd7577c..f889788428 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -66,10 +66,10 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect - golang.org/x/crypto v0.48.0 // indirect + golang.org/x/crypto v0.49.0 // indirect golang.org/x/net v0.51.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect + golang.org/x/sys v0.42.0 // indirect + golang.org/x/text v0.35.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index 36bddbeb56..04d4013057 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -174,21 +174,21 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= +golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= +golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/term v0.40.0 h1:36e4zGLqU4yhjlmxEaagx2KuYbJq3EwY8K943ZsHcvg= -golang.org/x/term v0.40.0/go.mod h1:w2P8uVp06p2iyKKuvXIm7N/y0UCRt3UfJTfZ7oOpglM= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= +golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= diff --git a/sda/go.mod b/sda/go.mod index 8fcb5688d8..83dbc34b7a 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.25.7 require ( - github.com/aws/aws-sdk-go-v2 v1.41.4 - github.com/aws/aws-sdk-go-v2/config v1.32.12 - github.com/aws/aws-sdk-go-v2/credentials v1.19.12 - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.10 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 + github.com/aws/aws-sdk-go-v2 v1.41.5 + github.com/aws/aws-sdk-go-v2/config v1.32.13 + github.com/aws/aws-sdk-go-v2/credentials v1.19.13 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.12 + github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 github.com/aws/smithy-go v1.24.2 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 @@ -50,20 +50,20 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 // indirect github.com/andybalholm/brotli v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect diff --git a/sda/go.sum b/sda/go.sum index 04d9525c65..c64c726aef 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -25,44 +25,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= -github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7 h1:3kGOqnh1pPeddVa/E37XNTaWJ8W6vrbYV9lJEkCnhuY= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.7/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= -github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= -github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= -github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.10 h1:2KCL4TmeiNvpPedtC4Bey5jvjRLD74WUYqGeHJ//aco= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.10/go.mod h1:KwaiUFVO7pG8Z9F5bMGvvrRibdSDaAu8HtlKGKkjZSA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= +github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg= +github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.12 h1:JNrcxEplOFBfHDrbqIxf2pMzmuszlvXp8QTCz8Vl5Po= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.12/go.mod h1:sp1ofAB9uYALR+Q7s3BddGbb6Pne0Kucm9lNh45e/Uo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21 h1:SwGMTMLIlvDNyhMteQ6r8IJSBPlRdXX5d4idhIGbkXA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.21/go.mod h1:UUxgWxofmOdAMuqEsSppbDtGKLfR04HGsD0HXzvhI1k= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12 h1:qtJZ70afD3ISKWnoX3xB0J2otEqu3LqicRcDBqsj0hQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.12/go.mod h1:v2pNpJbRNl4vEUWEh5ytQok0zACAKfdmKS51Hotc3pQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20 h1:siU1A6xjUZ2N8zjTHSXFhB9L/2OY8Dqs0xXiLjF30jA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.20/go.mod h1:4TLZCmVJDM3FOu5P5TJP0zOlu9zWgDWU7aUxWbr+rcw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1 h1:csi9NLpFZXb9fxY7rS1xVzgPRGMt7MSNWeQ6eo247kE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.1/go.mod h1:qXVal5H0ChqXP63t6jze5LmFalc7+ZE7wOdLtZ0LCP0= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= From 0623dbc160d317cd20b63fe46f003a7f3bdcdd97 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:06:05 +0100 Subject: [PATCH 371/530] =?UTF-8?q?feat(download):=20add=20service=20found?= =?UTF-8?q?ation=20=E2=80=94=20config,=20health,=20streaming,=20reencrypt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New download service at sda/cmd/download/ replacing sda-download/. Core components: - main.go: entry point with production safety guards, TLS, graceful shutdown - config/: 50+ config flags via internal/config/v2 (pflag/viper framework) - health/: gRPC health server for K8s probes + HTTP ready/live endpoints - streaming/: Range header parsing (RFC 9110), If-Range ETag support, combined header+body and body-only streaming with seek-based positioning - reencrypt/: gRPC client for crypt4gh header re-encryption with lazy connection and TLS support - internal/config/v2/: shared config registration framework - swagger_v2.yml: OpenAPI spec for the v2 REST API --- .codecov.yml | 4 + .gitignore | 5 +- sda/cmd/download/TESTING.md | 270 ++++++ sda/cmd/download/config/config.go | 920 +++++++++++++++++++ sda/cmd/download/dev_config.yaml | 59 ++ sda/cmd/download/download.md | 353 +++++++ sda/cmd/download/health/health.go | 51 + sda/cmd/download/health/health_test.go | 31 + sda/cmd/download/main.go | 340 +++++++ sda/cmd/download/reencrypt/reencrypt.go | 214 +++++ sda/cmd/download/reencrypt/reencrypt_test.go | 44 + sda/cmd/download/streaming/ifrange.go | 37 + sda/cmd/download/streaming/ifrange_test.go | 40 + sda/cmd/download/streaming/streaming.go | 361 ++++++++ sda/cmd/download/streaming/streaming_test.go | 567 ++++++++++++ sda/cmd/download/swagger_v2.yml | 76 +- sda/cmd/reencrypt/reencrypt.go | 16 +- sda/go.mod | 7 +- sda/go.sum | 6 + sda/internal/config/v2/config.go | 157 ++++ sda/internal/config/v2/config_test.go | 41 + 21 files changed, 3549 insertions(+), 50 deletions(-) create mode 100644 sda/cmd/download/TESTING.md create mode 100644 sda/cmd/download/config/config.go create mode 100644 sda/cmd/download/dev_config.yaml create mode 100644 sda/cmd/download/download.md create mode 100644 sda/cmd/download/health/health.go create mode 100644 sda/cmd/download/health/health_test.go create mode 100644 sda/cmd/download/main.go create mode 100644 sda/cmd/download/reencrypt/reencrypt.go create mode 100644 sda/cmd/download/reencrypt/reencrypt_test.go create mode 100644 sda/cmd/download/streaming/ifrange.go create mode 100644 sda/cmd/download/streaming/ifrange_test.go create mode 100644 sda/cmd/download/streaming/streaming.go create mode 100644 sda/cmd/download/streaming/streaming_test.go create mode 100644 sda/internal/config/v2/config.go create mode 100644 sda/internal/config/v2/config_test.go diff --git a/.codecov.yml b/.codecov.yml index 3c3624934a..bc082589af 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -6,6 +6,10 @@ ignore: # Main entry points - tested via integration tests - "sda/cmd/*/main.go" + # Configuration files - tested via integration tests + - "sda/cmd/download/config/config.go" + - "sda/internal/config/v2/config.go" + # Benchmark CLI tool - tested via integration tests - "sda/cmd/download/benchmark/benchmark.go" diff --git a/.gitignore b/.gitignore index d62acf38b3..3643b0c99d 100644 --- a/.gitignore +++ b/.gitignore @@ -198,4 +198,7 @@ db/ *.sec.pem # sda-admin binary -sda-admin/sda-admin \ No newline at end of file +sda-admin/sda-admin + +# test coverage files +sda/coverage.txt diff --git a/sda/cmd/download/TESTING.md b/sda/cmd/download/TESTING.md new file mode 100644 index 0000000000..9e56ccc7f3 --- /dev/null +++ b/sda/cmd/download/TESTING.md @@ -0,0 +1,270 @@ +# Local Testing Guide for sda/cmd/download + +This guide explains how to test the new download service locally against the integration test infrastructure. + +## Integration Tests + +There are two ways to run integration tests for the download service: + +### Option 1: Standalone Go-based Tests (Recommended) + +The download service has its own Docker Compose-based Go integration tests: + +```bash +# From repository root +docker compose -f .github/integration/sda-download-integration.yml run integration_test +``` + +This runs a minimal environment with: +- PostgreSQL (with seeded test data) +- MinIO (S3 storage) +- Mock OIDC provider +- Reencrypt gRPC service +- Download service +- Go test suite + +**Tests included:** +- Health endpoints (live, ready) +- Authentication (valid/invalid tokens) +- List datasets and files +- File download with re-encryption +- Range requests +- Access control + +### Option 2: Full Pipeline Tests + +The download service is also tested as part of the full SDA integration suite: + +```bash +# Run full S3 integration tests (includes download tests) +make integrationtest-sda-s3-run +``` + +This tests the download service with real data that has been processed through the full ingest pipeline (upload → ingest → verify → finalize → mapper). + +## Local Development Testing + +### Prerequisites + +- Docker and Docker Compose v2+ +- Go 1.20+ +- `make` utility +- `curl` and `jq` for testing + +## Quick Start + +### Step 1: Build and Start Infrastructure + +From the repository root: + +```bash +# Build all containers +make build-all + +# Start the S3 integration environment +make sda-s3-up +``` + +Wait for all services to be healthy (about 30-60 seconds). You can check with: + +```bash +docker ps +``` + +Services available: + +| Service | Port | Description | +| ---------- | ----------------------- | ------------------------ | +| PostgreSQL | 15432 | Database | +| MinIO (S3) | 19000 (API), 19001 (UI) | Object storage | +| RabbitMQ | 5672, 15672 | Message broker | +| ls-aai-mock| 8800 | OIDC/JWKS provider | +| auth-aai | 8801 | Auth service with login | +| reencrypt | 50051 | gRPC reencryption | +| api | 8090 | SDA API service | + +### Step 2: Run Integration Tests to Populate Test Data + +The integration tests upload files, ingest them, and create datasets: + +```bash +# Run the full integration test suite +make integrationtest-sda-s3-run +``` + +After tests complete, you'll have: + +- Files with stable IDs like `EGAF74900000001`, `EGAF74900000002` +- Dataset `EGAD74900000101` mapped to these files +- User `test_dummy.org` who submitted the files + +### Step 3: Start the Download Service Locally + +```bash +cd sda +CONFIGFILE=cmd/download/dev_config.yaml go run cmd/download/main.go +``` + +The service starts on: + +- HTTP API: http://localhost:8080 +- gRPC Health: localhost:8081 + +### Step 4: Get an Access Token + +```bash +# Option 1: Use pre-generated token from integration tests +TOKEN=$(docker exec integration_test cat /shared/token 2>/dev/null) + +# Option 2: Get token from ls-aai-mock +TOKEN=$(curl -s http://localhost:8800/oidc/token | jq -r '.access_token') + +# Option 3: Log in via browser at http://localhost:8801 and copy the token +``` + +**Note:** The download service uses JWKS-based JWT validation. The token must be signed +with a key that matches the configured `jwt.pubkey-path` or `jwt.pubkey-url`. + +### Permission Model + +The service supports two permission modes (configured via `jwt.allow-all-data`): + +| Mode | Description | +| --------------------------- | -------------------------------------------------- | +| `allow-all-data: true` | All authenticated users can access all datasets | +| `allow-all-data: false` | Users can only access datasets they submitted | + +The default integration test config uses `allow-all-data: true` for easier testing. + +### Step 5: Test the API + +#### Health Check + +```bash +curl http://localhost:8080/health/live +# {"status":"ok"} + +curl http://localhost:8080/health/ready +# {"status":"ok","services":{"database":"ok",...}} +``` + +#### List Datasets (requires authentication) + +```bash +curl -H "Authorization: Bearer $TOKEN" \ + http://localhost:8080/info/datasets +``` + +#### Get Dataset Info + +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/info/dataset?dataset=EGAD74900000101" +``` + +#### List Files in Dataset + +```bash +curl -H "Authorization: Bearer $TOKEN" \ + "http://localhost:8080/info/dataset/files?dataset=EGAD74900000101" +``` + +#### Download a File + +```bash +# Generate a crypt4gh key pair if needed +# crypt4gh generate --name mykey + +# Full file download +curl -H "Authorization: Bearer $TOKEN" \ + -H "public_key: $(base64 -w0 mykey.pub.pem)" \ + http://localhost:8080/file/EGAF74900000001 \ + -o downloaded_file.c4gh + +# Partial download (Range header) +curl -H "Authorization: Bearer $TOKEN" \ + -H "public_key: $(base64 -w0 mykey.pub.pem)" \ + -H "Range: bytes=0-1023" \ + http://localhost:8080/file/EGAF74900000001 +``` + +## Database Queries for Debugging + +Connect to the database: + +```bash +docker exec -it postgres psql -U postgres -d sda +``` + +Useful queries: + +```sql +-- List all files with stable IDs +SELECT id, stable_id, submission_file_path, submission_user +FROM sda.files +WHERE stable_id IS NOT NULL; + +-- List all datasets +SELECT id, stable_id, title +FROM sda.datasets; + +-- Files in a dataset +SELECT f.stable_id, f.submission_file_path, f.submission_user +FROM sda.files f +JOIN sda.file_dataset fd ON f.id = fd.file_id +JOIN sda.datasets d ON fd.dataset_id = d.id +WHERE d.stable_id = 'EGAD74900000101'; + +-- Datasets owned by a user (data ownership model) +SELECT DISTINCT d.stable_id +FROM sda.datasets d +JOIN sda.file_dataset fd ON d.id = fd.dataset_id +JOIN sda.files f ON fd.file_id = f.id +WHERE f.submission_user = 'test_dummy.org'; +``` + +## MinIO Console + +Access MinIO at http://localhost:19001 with: + +- Username: `access` +- Password: `secretKey` + +## Cleanup + +```bash +# Stop and remove all containers +make sda-s3-down +``` + +## Troubleshooting + +### "Config File Not Found" + +This is a viper warning, not an error. The service loads config from `CONFIGFILE` environment variable. + +### Database Connection Issues + +Ensure PostgreSQL is running: + +```bash +docker logs postgres +``` + +### Storage Reader Errors + +Check that storage configuration in `dev_config.yaml` matches the running Docker environment. + +### JWT/Token Issues + +1. Ensure ls-aai-mock is running: `docker logs ls-aai-mock` +2. Check JWKS endpoint: `curl http://localhost:8800/oidc/jwk` +3. Verify token is valid: `curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/info/datasets` + +### No Datasets Returned + +If `/info/datasets` returns empty: + +1. Check integration tests completed successfully +2. Verify files have stable IDs: `SELECT stable_id FROM sda.files WHERE stable_id IS NOT NULL;` +3. If using `allow-all-data: false`, ensure you're authenticated as the user who submitted the files diff --git a/sda/cmd/download/config/config.go b/sda/cmd/download/config/config.go new file mode 100644 index 0000000000..bc04838387 --- /dev/null +++ b/sda/cmd/download/config/config.go @@ -0,0 +1,920 @@ +// Package config provides configuration for the download service. +package config + +import ( + config "github.com/neicnordic/sensitive-data-archive/internal/config/v2" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +var ( + // Server configuration + apiHost string + apiPort int + apiServerCert string + apiServerKey string + healthPort int + + // Database configuration + dbHost string + dbPort int + dbUser string + dbPassword string + dbDatabase string + dbSSLMode string + dbCACert string + + // Storage configuration + storageBackend string + + // gRPC reencrypt service configuration + grpcHost string + grpcPort int + grpcTimeout int + grpcCACert string + grpcClientCert string + grpcClientKey string + + // JWT configuration (for token validation) + jwtPubKeyPath string + jwtPubKeyURL string + jwtAllowAllData bool // For testing: allow authenticated users access to all datasets + + // OIDC configuration (optional, for discovery) + oidcIssuer string + oidcAudience string + + // Session configuration + sessionExpiration int + sessionDomain string + sessionSecure bool + sessionHTTPOnly bool + sessionName string + + // Cache configuration + cacheEnabled bool + cacheFileTTL int + cachePermissionTTL int + cacheDatasetTTL int + + // Permission model configuration + permissionModel string // "ownership" | "visa" | "combined" + + // Auth configuration + authAllowOpaque bool // Allow userinfo-based auth for opaque tokens + + // Visa (GA4GH) configuration + visaEnabled bool + visaSource string // "userinfo" | "token" + visaUserinfoURL string + visaTrustedIssuers string // Path to JSON file with trusted issuer+JKU pairs + visaDatasetIDMode string // "raw" | "suffix" + visaIdentityMode string // "broker-bound" | "strict-sub" | "strict-iss-sub" + visaValidateAsserted bool + visaMaxVisas int + visaMaxJWKSPerReq int + visaMaxVisaSize int + visaCacheTokenTTL int + visaCacheJWKTTL int + visaCacheValidTTL int + visaCacheUserinfoTTL int + visaAllowInsecureJKU bool + + // Application environment + appEnvironment string + + // Audit configuration + auditRequired bool + + // Pagination configuration + paginationHMACSecret string +) + +func init() { + config.RegisterFlags( + // Server flags + &config.Flag{ + Name: "api.host", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "0.0.0.0", "Host address to bind the API server to") + }, + Required: false, + AssignFunc: func(flagName string) { + apiHost = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "api.port", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 8080, "Port to host the API server at") + }, + Required: false, + AssignFunc: func(flagName string) { + apiPort = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "api.server-cert", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to the server certificate file for TLS") + }, + Required: false, + AssignFunc: func(flagName string) { + apiServerCert = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "api.server-key", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to the server key file for TLS") + }, + Required: false, + AssignFunc: func(flagName string) { + apiServerKey = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "health.port", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 8081, "Port for gRPC health check server") + }, + Required: false, + AssignFunc: func(flagName string) { + healthPort = viper.GetInt(flagName) + }, + }, + + // Database flags + &config.Flag{ + Name: "db.host", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "localhost", "Database host") + }, + Required: true, + AssignFunc: func(flagName string) { + dbHost = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "db.port", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 5432, "Database port") + }, + Required: false, + AssignFunc: func(flagName string) { + dbPort = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "db.user", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Database user") + }, + Required: true, + AssignFunc: func(flagName string) { + dbUser = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "db.password", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Database password") + }, + Required: true, + AssignFunc: func(flagName string) { + dbPassword = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "db.database", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "sda", "Database name") + }, + Required: false, + AssignFunc: func(flagName string) { + dbDatabase = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "db.sslmode", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "prefer", "Database SSL mode (disable, allow, prefer, require, verify-ca, verify-full)") + }, + Required: false, + AssignFunc: func(flagName string) { + dbSSLMode = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "db.cacert", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to CA certificate for database TLS") + }, + Required: false, + AssignFunc: func(flagName string) { + dbCACert = viper.GetString(flagName) + }, + }, + + // Storage flags + &config.Flag{ + Name: "storage.backend", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "archive", "Storage backend name (used for storage/v2 configuration)") + }, + Required: false, + AssignFunc: func(flagName string) { + storageBackend = viper.GetString(flagName) + }, + }, + + // gRPC reencrypt service flags + &config.Flag{ + Name: "grpc.host", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "gRPC reencrypt service host") + }, + Required: true, + AssignFunc: func(flagName string) { + grpcHost = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "grpc.port", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 50051, "gRPC reencrypt service port") + }, + Required: false, + AssignFunc: func(flagName string) { + grpcPort = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "grpc.timeout", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 10, "gRPC request timeout in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + grpcTimeout = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "grpc.cacert", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to CA certificate for gRPC TLS") + }, + Required: false, + AssignFunc: func(flagName string) { + grpcCACert = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "grpc.client-cert", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to client certificate for gRPC mTLS") + }, + Required: false, + AssignFunc: func(flagName string) { + grpcClientCert = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "grpc.client-key", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to client key for gRPC mTLS") + }, + Required: false, + AssignFunc: func(flagName string) { + grpcClientKey = viper.GetString(flagName) + }, + }, + + // JWT flags (for token validation - at least one of path or url must be set) + &config.Flag{ + Name: "jwt.pubkey-path", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to directory containing JWT public key files (PEM format)") + }, + Required: false, + AssignFunc: func(flagName string) { + jwtPubKeyPath = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "jwt.pubkey-url", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "URL to JWKS endpoint for JWT verification") + }, + Required: false, + AssignFunc: func(flagName string) { + jwtPubKeyURL = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "jwt.allow-all-data", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, false, "Allow authenticated users access to all datasets (for testing only)") + }, + Required: false, + AssignFunc: func(flagName string) { + jwtAllowAllData = viper.GetBool(flagName) + }, + }, + + // OIDC flags (optional, for token validation settings) + &config.Flag{ + Name: "oidc.issuer", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Expected OIDC issuer in tokens (optional)") + }, + Required: false, + AssignFunc: func(flagName string) { + oidcIssuer = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "oidc.audience", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Expected audience in tokens (optional)") + }, + Required: false, + AssignFunc: func(flagName string) { + oidcAudience = viper.GetString(flagName) + }, + }, + + // Session flags + &config.Flag{ + Name: "session.expiration", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 3600, "Session expiration time in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + sessionExpiration = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "session.domain", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Cookie domain for session") + }, + Required: false, + AssignFunc: func(flagName string) { + sessionDomain = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "session.secure", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, true, "Use secure cookies (HTTPS only)") + }, + Required: false, + AssignFunc: func(flagName string) { + sessionSecure = viper.GetBool(flagName) + }, + }, + &config.Flag{ + Name: "session.http-only", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, true, "HTTP only cookies (not accessible via JavaScript)") + }, + Required: false, + AssignFunc: func(flagName string) { + sessionHTTPOnly = viper.GetBool(flagName) + }, + }, + &config.Flag{ + Name: "session.name", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "sda_session", "Session cookie name") + }, + Required: false, + AssignFunc: func(flagName string) { + sessionName = viper.GetString(flagName) + }, + }, + + // Cache flags + &config.Flag{ + Name: "cache.enabled", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, true, "Enable database query caching") + }, + Required: false, + AssignFunc: func(flagName string) { + cacheEnabled = viper.GetBool(flagName) + }, + }, + &config.Flag{ + Name: "cache.file-ttl", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 300, "TTL for file query cache in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + cacheFileTTL = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "cache.permission-ttl", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 120, "TTL for permission check cache in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + cachePermissionTTL = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "cache.dataset-ttl", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 300, "TTL for dataset query cache in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + cacheDatasetTTL = viper.GetInt(flagName) + }, + }, + + // Permission model flag + &config.Flag{ + Name: "permission.model", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "combined", "Permission model: ownership, visa, or combined") + }, + Required: false, + AssignFunc: func(flagName string) { + permissionModel = viper.GetString(flagName) + }, + }, + + // Auth flags + &config.Flag{ + Name: "auth.allow-opaque", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, true, "Allow userinfo-based auth for opaque (non-JWT) tokens") + }, + Required: false, + AssignFunc: func(flagName string) { + authAllowOpaque = viper.GetBool(flagName) + }, + }, + + // Visa (GA4GH) flags + &config.Flag{ + Name: "visa.enabled", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, false, "Enable GA4GH visa support for dataset access") + }, + Required: false, + AssignFunc: func(flagName string) { + visaEnabled = viper.GetBool(flagName) + }, + }, + &config.Flag{ + Name: "visa.source", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "userinfo", "Visa source: userinfo (GA4GH compliant) or token (legacy)") + }, + Required: false, + AssignFunc: func(flagName string) { + visaSource = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "visa.userinfo-url", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Userinfo endpoint URL (optional, discovered from oidc.issuer if not set)") + }, + Required: false, + AssignFunc: func(flagName string) { + visaUserinfoURL = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "visa.trusted-issuers-path", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to JSON file with trusted issuer+JKU pairs") + }, + Required: false, + AssignFunc: func(flagName string) { + visaTrustedIssuers = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "visa.dataset-id-mode", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "raw", "Dataset ID extraction mode: raw (exact visa value) or suffix (last URL/URN segment)") + }, + Required: false, + AssignFunc: func(flagName string) { + visaDatasetIDMode = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "visa.identity.mode", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "broker-bound", "Identity binding mode: broker-bound, strict-sub, or strict-iss-sub") + }, + Required: false, + AssignFunc: func(flagName string) { + visaIdentityMode = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "visa.validate-asserted", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, true, "Reject visas where asserted timestamp is in the future") + }, + Required: false, + AssignFunc: func(flagName string) { + visaValidateAsserted = viper.GetBool(flagName) + }, + }, + &config.Flag{ + Name: "visa.limits.max-visas", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 200, "Maximum number of visas to process per passport") + }, + Required: false, + AssignFunc: func(flagName string) { + visaMaxVisas = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "visa.limits.max-jwks-per-request", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 10, "Maximum distinct JKU fetches per single request") + }, + Required: false, + AssignFunc: func(flagName string) { + visaMaxJWKSPerReq = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "visa.limits.max-visa-size", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 16384, "Maximum visa JWT size in bytes") + }, + Required: false, + AssignFunc: func(flagName string) { + visaMaxVisaSize = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "visa.cache.token-ttl", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 3600, "Token-keyed cache TTL in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + visaCacheTokenTTL = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "visa.cache.jwk-ttl", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 300, "JWK cache TTL in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + visaCacheJWKTTL = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "visa.cache.validation-ttl", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 120, "Visa validation cache TTL in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + visaCacheValidTTL = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "visa.cache.userinfo-ttl", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Int(flagName, 60, "Userinfo cache TTL in seconds") + }, + Required: false, + AssignFunc: func(flagName string) { + visaCacheUserinfoTTL = viper.GetInt(flagName) + }, + }, + &config.Flag{ + Name: "visa.allow-insecure-jku", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, false, "Allow HTTP (non-TLS) JKU URLs in trusted issuers (for testing only)") + }, + Required: false, + AssignFunc: func(flagName string) { + visaAllowInsecureJKU = viper.GetBool(flagName) + }, + }, + + // Application environment flag + &config.Flag{ + Name: "app.environment", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Application environment (set to 'production' to enable production guards)") + }, + Required: false, + AssignFunc: func(flagName string) { + appEnvironment = viper.GetString(flagName) + }, + }, + + // Audit flags + &config.Flag{ + Name: "audit.required", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.Bool(flagName, false, "Require a real audit logger (fail startup if noop)") + }, + Required: false, + AssignFunc: func(flagName string) { + auditRequired = viper.GetBool(flagName) + }, + }, + + // Pagination flags + &config.Flag{ + Name: "pagination.hmac-secret", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "HMAC secret for signing page tokens (must be same across replicas)") + }, + Required: false, + AssignFunc: func(flagName string) { + paginationHMACSecret = viper.GetString(flagName) + }, + }, + ) +} + +// APIHost returns the host address for the API server. +func APIHost() string { + return apiHost +} + +// APIPort returns the port for the API server. +func APIPort() int { + return apiPort +} + +// APIServerCert returns the path to the server certificate. +func APIServerCert() string { + return apiServerCert +} + +// APIServerKey returns the path to the server key. +func APIServerKey() string { + return apiServerKey +} + +// HealthPort returns the port for the gRPC health check server. +func HealthPort() int { + return healthPort +} + +// DBHost returns the database host. +func DBHost() string { + return dbHost +} + +// DBPort returns the database port. +func DBPort() int { + return dbPort +} + +// DBUser returns the database user. +func DBUser() string { + return dbUser +} + +// DBPassword returns the database password. +func DBPassword() string { + return dbPassword +} + +// DBDatabase returns the database name. +func DBDatabase() string { + return dbDatabase +} + +// DBSSLMode returns the database SSL mode. +func DBSSLMode() string { + return dbSSLMode +} + +// DBCACert returns the path to the database CA certificate. +func DBCACert() string { + return dbCACert +} + +// StorageBackend returns the storage backend name. +func StorageBackend() string { + return storageBackend +} + +// GRPCHost returns the gRPC service host. +func GRPCHost() string { + return grpcHost +} + +// GRPCPort returns the gRPC service port. +func GRPCPort() int { + return grpcPort +} + +// GRPCTimeout returns the gRPC request timeout in seconds. +func GRPCTimeout() int { + return grpcTimeout +} + +// GRPCCACert returns the path to the gRPC CA certificate. +func GRPCCACert() string { + return grpcCACert +} + +// GRPCClientCert returns the path to the gRPC client certificate. +func GRPCClientCert() string { + return grpcClientCert +} + +// GRPCClientKey returns the path to the gRPC client key. +func GRPCClientKey() string { + return grpcClientKey +} + +// JWTPubKeyPath returns the path to the JWT public key directory. +func JWTPubKeyPath() string { + return jwtPubKeyPath +} + +// JWTPubKeyURL returns the JWKS URL for JWT verification. +func JWTPubKeyURL() string { + return jwtPubKeyURL +} + +// JWTAllowAllData returns whether authenticated users have access to all datasets. +// This should only be used for testing purposes. +func JWTAllowAllData() bool { + return jwtAllowAllData +} + +// OIDCIssuer returns the expected OIDC issuer. +func OIDCIssuer() string { + return oidcIssuer +} + +// OIDCAudience returns the expected OIDC audience. +func OIDCAudience() string { + return oidcAudience +} + +// SessionExpiration returns the session expiration time in seconds. +func SessionExpiration() int { + return sessionExpiration +} + +// SessionDomain returns the cookie domain for sessions. +func SessionDomain() string { + return sessionDomain +} + +// SessionSecure returns whether to use secure cookies. +func SessionSecure() bool { + return sessionSecure +} + +// SessionHTTPOnly returns whether cookies are HTTP only. +func SessionHTTPOnly() bool { + return sessionHTTPOnly +} + +// SessionName returns the session cookie name. +func SessionName() string { + return sessionName +} + +// CacheEnabled returns whether database query caching is enabled. +func CacheEnabled() bool { + return cacheEnabled +} + +// CacheFileTTL returns the TTL for file query cache in seconds. +func CacheFileTTL() int { + return cacheFileTTL +} + +// CachePermissionTTL returns the TTL for permission check cache in seconds. +func CachePermissionTTL() int { + return cachePermissionTTL +} + +// CacheDatasetTTL returns the TTL for dataset query cache in seconds. +func CacheDatasetTTL() int { + return cacheDatasetTTL +} + +// PermissionModel returns the permission model (ownership, visa, or combined). +func PermissionModel() string { + return permissionModel +} + +// AuthAllowOpaque returns whether opaque (non-JWT) tokens are allowed via userinfo. +func AuthAllowOpaque() bool { + return authAllowOpaque +} + +// VisaEnabled returns whether GA4GH visa support is enabled. +func VisaEnabled() bool { + return visaEnabled +} + +// VisaSource returns the visa source (userinfo or token). +func VisaSource() string { + return visaSource +} + +// VisaUserinfoURL returns the configured userinfo endpoint URL. +func VisaUserinfoURL() string { + return visaUserinfoURL +} + +// VisaTrustedIssuersPath returns the path to the trusted issuers JSON file. +func VisaTrustedIssuersPath() string { + return visaTrustedIssuers +} + +// VisaDatasetIDMode returns the dataset ID extraction mode (raw or suffix). +func VisaDatasetIDMode() string { + return visaDatasetIDMode +} + +// VisaIdentityMode returns the identity binding mode. +func VisaIdentityMode() string { + return visaIdentityMode +} + +// VisaValidateAsserted returns whether to reject visas with future asserted timestamps. +func VisaValidateAsserted() bool { + return visaValidateAsserted +} + +// VisaMaxVisas returns the maximum number of visas to process per passport. +func VisaMaxVisas() int { + return visaMaxVisas +} + +// VisaMaxJWKSPerRequest returns the maximum distinct JKU fetches per request. +func VisaMaxJWKSPerRequest() int { + return visaMaxJWKSPerReq +} + +// VisaMaxVisaSize returns the maximum visa JWT size in bytes. +func VisaMaxVisaSize() int { + return visaMaxVisaSize +} + +// VisaCacheTokenTTL returns the token-keyed cache TTL in seconds. +func VisaCacheTokenTTL() int { + return visaCacheTokenTTL +} + +// VisaCacheJWKTTL returns the JWK cache TTL in seconds. +func VisaCacheJWKTTL() int { + return visaCacheJWKTTL +} + +// VisaCacheValidationTTL returns the visa validation cache TTL in seconds. +func VisaCacheValidationTTL() int { + return visaCacheValidTTL +} + +// VisaCacheUserinfoTTL returns the userinfo cache TTL in seconds. +func VisaCacheUserinfoTTL() int { + return visaCacheUserinfoTTL +} + +// VisaAllowInsecureJKU returns whether HTTP (non-TLS) JKU URLs are allowed. +// This should only be used for testing purposes. +func VisaAllowInsecureJKU() bool { + return visaAllowInsecureJKU +} + +// Environment returns the application environment (e.g. "production", "staging"). +func Environment() string { + return appEnvironment +} + +// AuditRequired returns whether audit logging is required. +func AuditRequired() bool { + return auditRequired +} + +// PaginationHMACSecret returns the HMAC secret for signing pagination tokens. +func PaginationHMACSecret() string { + return paginationHMACSecret +} diff --git a/sda/cmd/download/dev_config.yaml b/sda/cmd/download/dev_config.yaml new file mode 100644 index 0000000000..7288d9e328 --- /dev/null +++ b/sda/cmd/download/dev_config.yaml @@ -0,0 +1,59 @@ +# Development configuration for sda/cmd/download +# Use with: CONFIGFILE=cmd/download/dev_config.yaml go run cmd/download/main.go + +# API Server configuration +api: + host: "0.0.0.0" + port: 8080 + +# gRPC Health check server +health: + port: 8081 + +# Database configuration (connects to integration test PostgreSQL) +db: + host: "localhost" + port: 15432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +# gRPC reencrypt service +grpc: + host: "localhost" + port: 50051 + timeout: 10 + +# JWT configuration +# At least one of pubkey-path or pubkey-url must be set for token validation +jwt: + # For local development against integration test environment, use JWKS URL: + pubkey-url: "http://localhost:8800/oidc/jwk" + # For testing: allow authenticated users access to all datasets + allow-all-data: true + +# OIDC configuration (optional token validation settings) +# oidc: +# issuer: "http://localhost:8800/oidc/" +# audience: "" + +# Session configuration +session: + expiration: 3600 + secure: false + http-only: true + name: "sda_session" + +# Storage configuration for storage/v2 +# The backend name is "archive" by default (see config.StorageBackend()) +storage: + backend: "archive" + archive: + s3: + - endpoint: "localhost:19000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true diff --git a/sda/cmd/download/download.md b/sda/cmd/download/download.md new file mode 100644 index 0000000000..5fbe4f0adc --- /dev/null +++ b/sda/cmd/download/download.md @@ -0,0 +1,353 @@ +# Download + +The Download service provides secure file download functionality for archived data. +Users are authenticated with a JWT token, and files are re-encrypted with the user's +public key before download. + +## Service Description + +This service implements the Data Out API for the NeIC Sensitive Data Archive. It allows +authenticated users to browse datasets and download files that have been archived and +verified. + +### Authentication + +All endpoints except health checks require JWT authentication. The token must be: + +- Provided in the `Authorization: Bearer ` header +- Signed with a key matching the configured JWKS (via `jwt.pubkey-path` or `jwt.pubkey-url`) + +### Permission Model + +The service supports two permission modes (configured via `jwt.allow-all-data`): + +| Mode | Description | +| ----------------------- | --------------------------------------------------------- | +| `allow-all-data: true` | All authenticated users can access all datasets (testing) | +| `allow-all-data: false` | Users can only access datasets they submitted | + +In production, the data ownership model is used: users have access to datasets +containing files where they are the `submission_user`. + +## Endpoints + +### Health Endpoints + +#### `/health/ready` + +- accepts `GET` requests +- Returns readiness status with dependency checks + + Example: + + ```bash + curl https://HOSTNAME/health/ready + {"status":"ok","services":{"database":"ok","storage":"ok","grpc":"ok","oidc":"ok"}} + ``` + +#### `/health/live` + +- accepts `GET` requests +- Returns simple liveness status + + Example: + + ```bash + curl https://HOSTNAME/health/live + {"status":"ok"} + ``` + +### Info Endpoints + +#### `/info/datasets` + +- accepts `GET` requests +- Returns all datasets the authenticated user has access to + +- Error codes + - `200` Query executed successfully + - `401` Invalid or missing token + - `500` Internal error due to DB failures + + Example: + + ```bash + curl -H "Authorization: Bearer $token" https://HOSTNAME/info/datasets + [{"id":"EGAD74900000101","title":"Example Dataset"}] + ``` + +#### `/info/dataset` + +- accepts `GET` requests with query parameter `dataset` +- Returns metadata for a specific dataset + +- Error codes + - `200` Query executed successfully + - `400` Missing dataset parameter + - `401` Invalid or missing token + - `403` Access denied to dataset + - `404` Dataset not found + - `500` Internal error due to DB failures + + Example: + + ```bash + curl -H "Authorization: Bearer $token" "https://HOSTNAME/info/dataset?dataset=EGAD74900000101" + {"id":"EGAD74900000101","title":"Example Dataset","fileCount":2,"totalSize":15242998} + ``` + +#### `/info/dataset/files` + +- accepts `GET` requests with query parameter `dataset` +- Returns list of files in the specified dataset + +- Error codes + - `200` Query executed successfully + - `400` Missing dataset parameter + - `401` Invalid or missing token + - `403` Access denied to dataset + - `500` Internal error due to DB failures + + Example: + + ```bash + curl -H "Authorization: Bearer $token" "https://HOSTNAME/info/dataset/files?dataset=EGAD74900000101" + [{"id":"EGAF74900000001","path":"data/file1.c4gh","size":1024000,"checksum":"abc123...","checksumType":"SHA256"}] + ``` + +### File Download Endpoints + +File downloads require a `public_key` header containing the user's Crypt4GH public key +(base64-encoded). The archived file is re-encrypted with this key before streaming to +the client. + +#### `/file/:fileId` + +- accepts `GET` requests with file stable ID in path +- Downloads a file by its stable ID (e.g., `EGAF74900000001`) +- Supports HTTP Range requests for partial downloads + +- Request Headers + - `Authorization: Bearer ` (required) + - `public_key: ` (required) + - `Range: bytes=START-END` (optional, for partial downloads) + +- Response Headers + - `Content-Type: application/octet-stream` + - `Content-Disposition: attachment; filename=".c4gh"` + - `Accept-Ranges: bytes` + - `X-File-Id: ` + - `X-Decrypted-Size: ` + - `X-Decrypted-Checksum: ` (if available) + - `X-Decrypted-Checksum-Type: ` (if available) + +- Error codes + - `200` Download successful + - `206` Partial content (Range request) + - `400` Missing fileId or public_key header + - `401` Invalid or missing token + - `403` Access denied to file + - `404` File not found + - `500` Internal error (storage, reencrypt, or streaming failures) + + Example: + + ```bash + curl -H "Authorization: Bearer $token" \ + -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ + https://HOSTNAME/file/EGAF74900000001 \ + -o downloaded_file.c4gh + ``` + + Example with Range header: + + ```bash + curl -H "Authorization: Bearer $token" \ + -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ + -H "Range: bytes=0-1023" \ + https://HOSTNAME/file/EGAF74900000001 + ``` + +#### `/file` + +- accepts `GET` requests with query parameters +- Downloads a file by dataset and either fileId or filePath +- Supports HTTP Range requests for partial downloads + +- Query Parameters + - `dataset` (required): Dataset stable ID + - `fileId` (optional): File stable ID within the dataset + - `filePath` (optional): File path within the dataset + - Note: Either `fileId` or `filePath` must be provided, but not both + +- Request Headers: Same as `/file/:fileId` +- Response Headers: Same as `/file/:fileId` + +- Error codes + - `200` Download successful + - `206` Partial content (Range request) + - `400` Missing required parameters or invalid combination + - `401` Invalid or missing token + - `403` Access denied to file + - `404` File not found + - `500` Internal error + + Example by fileId: + + ```bash + curl -H "Authorization: Bearer $token" \ + -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ + "https://HOSTNAME/file?dataset=EGAD74900000101&fileId=EGAF74900000001" \ + -o downloaded_file.c4gh + ``` + + Example by filePath: + + ```bash + curl -H "Authorization: Bearer $token" \ + -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ + "https://HOSTNAME/file?dataset=EGAD74900000101&filePath=data/sample.c4gh" \ + -o downloaded_file.c4gh + ``` + +## Configuration + +The service is configured via YAML config file or environment variables. + +### API Server Settings + +| Variable | Config Key | Description | Default | +| ---------- | ---------- | ----------------- | --------- | +| `API_HOST` | `api.host` | Host to bind to | `0.0.0.0` | +| `API_PORT` | `api.port` | Port to listen on | `8080` | + +### gRPC Health Check Settings + +| Variable | Config Key | Description | Default | +| ------------- | ------------- | ---------------------- | ------- | +| `HEALTH_PORT` | `health.port` | gRPC health check port | `8081` | + +### Database Settings + +| Variable | Config Key | Description | +| ------------- | ------------- | ------------------------------ | +| `DB_HOST` | `db.host` | PostgreSQL hostname | +| `DB_PORT` | `db.port` | PostgreSQL port | +| `DB_USER` | `db.user` | Database username | +| `DB_PASSWORD` | `db.password` | Database password | +| `DB_DATABASE` | `db.database` | Database name | +| `DB_SSLMODE` | `db.sslmode` | SSL mode (disable/require/etc) | + +### JWT/Authentication Settings + +| Variable | Config Key | Description | +| -------------------- | -------------------- | ------------------------------------ | +| `JWT_PUBKEY_PATH` | `jwt.pubkey-path` | Path to directory with public keys | +| `JWT_PUBKEY_URL` | `jwt.pubkey-url` | JWKS URL for key fetching | +| `JWT_ALLOW_ALL_DATA` | `jwt.allow-all-data` | Allow all authenticated users access | + +At least one of `jwt.pubkey-path` or `jwt.pubkey-url` must be configured. + +### gRPC Reencrypt Service Settings + +| Variable | Config Key | Description | +| -------------- | -------------- | -------------------------- | +| `GRPC_HOST` | `grpc.host` | Reencrypt service hostname | +| `GRPC_PORT` | `grpc.port` | Reencrypt service port | +| `GRPC_TIMEOUT` | `grpc.timeout` | Request timeout in seconds | + +### Storage Settings + +Storage is configured using the `storage/v2` package. See [storage/v2 README](../../internal/storage/v2/README.md) for details. + +Example S3 configuration: + +```yaml +storage: + backend: "archive" + archive: + s3: + - endpoint: "s3.example.com" + access_key: "access" + secret_key: "secret" + region: "us-east-1" + bucket_prefix: "archive" +``` + +### Session Settings + +| Variable | Config Key | Description | +| -------------------- | -------------------- | ----------------------------- | +| `SESSION_EXPIRATION` | `session.expiration` | Session expiration in seconds | +| `SESSION_SECURE` | `session.secure` | Use secure cookies | +| `SESSION_HTTP_ONLY` | `session.http-only` | HTTP-only cookies | +| `SESSION_NAME` | `session.name` | Session cookie name | + +### Cache Settings + +Database query results are cached in-memory to reduce database roundtrips, which is +particularly beneficial for streaming use cases where the same file metadata may be +requested multiple times (e.g., HTTP Range requests). + +| Variable | Config Key | Description | Default | +| ---------------------- | ---------------------- | ----------------------------------- | ------- | +| `CACHE_ENABLED` | `cache.enabled` | Enable database query caching | `true` | +| `CACHE_FILE_TTL` | `cache.file-ttl` | TTL for file queries (seconds) | `300` | +| `CACHE_PERMISSION_TTL` | `cache.permission-ttl` | TTL for permission checks (seconds) | `120` | +| `CACHE_DATASET_TTL` | `cache.dataset-ttl` | TTL for dataset queries (seconds) | `300` | + +The cache uses [ristretto](https://github.com/dgraph-io/ristretto), a high-performance +concurrent cache. Cached queries include: + +- File lookups by ID and path +- Permission checks (user-scoped via dataset IDs) +- Dataset listings and metadata +- Dataset file listings + +Cache keys for permission checks are scoped by the user's accessible datasets, ensuring +that users with different permissions get appropriately cached results. + +#### Cache Behavior and Considerations + +**Cache Invalidation:** The cache uses time-based expiration (TTL) but does not support +explicit invalidation. This means: + +- If a file's permissions change, the cached permission check will be stale until the + TTL expires (default: 2 minutes for permissions, 5 minutes for file/dataset metadata) +- For most use cases this is acceptable, as permission changes are infrequent +- If immediate permission revocation is required, the cache can be disabled or the + service restarted + +**Memory Usage:** The ristretto cache is configured with: + +- `MaxCost: 100,000` - Maximum number of items to cache +- `NumCounters: 1,000,000` - Counters for admission policy + +Estimated memory usage: + +- Base overhead: ~8 MB for counters +- Per-item overhead: varies by cached data size +- Typical total: 50-200 MB depending on active dataset/file counts + +For production deployments, monitor memory usage and adjust `MaxCost` if needed. The +cache automatically evicts least-recently-used items when capacity is reached. + +## Testing + +### Integration Tests + +Run the standalone Go-based integration tests: + +```bash +docker compose -f .github/integration/sda-download-integration.yml run integration_test +``` + +Or run as part of the full SDA pipeline: + +```bash +make integrationtest-sda-s3-run +``` + +### Local Development + +See [TESTING.md](TESTING.md) for detailed local testing instructions. diff --git a/sda/cmd/download/health/health.go b/sda/cmd/download/health/health.go new file mode 100644 index 0000000000..d06cb8fd89 --- /dev/null +++ b/sda/cmd/download/health/health.go @@ -0,0 +1,51 @@ +// Package health provides gRPC health check server for Kubernetes probes. +package health + +import ( + "fmt" + "net" + + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +var healthServer *health.Server +var grpcServer *grpc.Server + +func init() { + healthServer = health.NewServer() + healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) +} + +// Start starts the gRPC health check server on the specified port. +func Start(port int) error { + grpcServer = grpc.NewServer() + + healthpb.RegisterHealthServer(grpcServer, healthServer) + + listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) + if err != nil { + return fmt.Errorf("failed to listen on port %d: %w", port, err) + } + + return grpcServer.Serve(listener) +} + +// SetServingStatus sets the serving status for the health server. +func SetServingStatus(status healthpb.HealthCheckResponse_ServingStatus) { + healthServer.SetServingStatus("", status) +} + +// Stop gracefully stops the gRPC health server. +func Stop() { + if grpcServer != nil { + grpcServer.GracefulStop() + } +} + +// ServingStatus constants for convenience. +const ( + Serving = healthpb.HealthCheckResponse_SERVING + NotServing = healthpb.HealthCheckResponse_NOT_SERVING +) diff --git a/sda/cmd/download/health/health_test.go b/sda/cmd/download/health/health_test.go new file mode 100644 index 0000000000..aef064c218 --- /dev/null +++ b/sda/cmd/download/health/health_test.go @@ -0,0 +1,31 @@ +package health + +import ( + "testing" + + "github.com/stretchr/testify/assert" + healthpb "google.golang.org/grpc/health/grpc_health_v1" +) + +func TestServingStatusConstants(t *testing.T) { + assert.Equal(t, healthpb.HealthCheckResponse_SERVING, Serving) + assert.Equal(t, healthpb.HealthCheckResponse_NOT_SERVING, NotServing) +} + +func TestSetServingStatus(t *testing.T) { + // Test that SetServingStatus doesn't panic + assert.NotPanics(t, func() { + SetServingStatus(Serving) + }) + + assert.NotPanics(t, func() { + SetServingStatus(NotServing) + }) +} + +func TestStop_WhenNotStarted(t *testing.T) { + // Test that Stop doesn't panic when server hasn't been started + assert.NotPanics(t, func() { + Stop() + }) +} diff --git a/sda/cmd/download/main.go b/sda/cmd/download/main.go new file mode 100644 index 0000000000..063a2f9c35 --- /dev/null +++ b/sda/cmd/download/main.go @@ -0,0 +1,340 @@ +// Package main is the entry point for the download service. +package main + +import ( + "context" + "crypto/rand" + "crypto/tls" + "errors" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/neicnordic/sensitive-data-archive/cmd/download/config" + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/neicnordic/sensitive-data-archive/cmd/download/handlers" + "github.com/neicnordic/sensitive-data-archive/cmd/download/health" + "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + "github.com/neicnordic/sensitive-data-archive/cmd/download/reencrypt" + "github.com/neicnordic/sensitive-data-archive/cmd/download/visa" + internalconfig "github.com/neicnordic/sensitive-data-archive/internal/config/v2" + storage "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" +) + +func main() { + log.SetFormatter(&log.JSONFormatter{}) + gin.SetMode(gin.ReleaseMode) + + if err := run(); err != nil { + log.Fatalf("application error: %v", err) + } +} + +func run() error { + // Setup context with cancellation for graceful shutdown + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // Setup signal handling + sigc := make(chan os.Signal, 5) + signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) + + // Load configuration + // Note: config package init() registers flags with internalconfig + _ = config.APIPort() // Ensure config package is imported and init() runs + if err := internalconfig.Load(); err != nil { + return fmt.Errorf("failed to load config: %w", err) + } + + // Start gRPC health server + go func() { + if err := health.Start(config.HealthPort()); err != nil { + log.Errorf("health server error: %v", err) + cancel() + } + }() + defer health.Stop() + + // Initialize database + if err := database.Init(); err != nil { + return fmt.Errorf("failed to initialize database: %w", err) + } + defer func() { + log.Info("closing database connection...") + if err := database.Close(); err != nil { + log.Errorf("database close error: %v", err) + } + }() + + // Wrap database with cache if enabled + if config.CacheEnabled() { + cachedDB, err := database.NewCachedDB(database.GetDB(), database.CacheConfig{ + FileTTL: time.Duration(config.CacheFileTTL()) * time.Second, + PermissionTTL: time.Duration(config.CachePermissionTTL()) * time.Second, + DatasetTTL: time.Duration(config.CacheDatasetTTL()) * time.Second, + }) + if err != nil { + return fmt.Errorf("failed to initialize database cache: %w", err) + } + database.RegisterDatabase(cachedDB) + log.Info("Database caching enabled") + } + + // Production safety guards + if config.Environment() == "production" { + if err := validateProductionConfig(productionConfig{ + AllowAllData: config.JWTAllowAllData(), + HMACSecret: config.PaginationHMACSecret(), + GRPCClientCert: config.GRPCClientCert(), + GRPCClientKey: config.GRPCClientKey(), + }); err != nil { + return err + } + } + + // Initialize audit logger + auditLogger := audit.NewStdoutLogger() + if config.AuditRequired() && auditLogger.IsNoop() { + return errors.New("audit.required is true but audit logger is noop") + } + + // Initialize pagination HMAC secret + if secret := config.PaginationHMACSecret(); secret != "" { + handlers.SetPaginationSecret([]byte(secret)) + } else { + b := make([]byte, 32) + if _, err := rand.Read(b); err != nil { + return fmt.Errorf("failed to generate pagination secret: %w", err) + } + + handlers.SetPaginationSecret(b) + log.Warn("pagination.hmac-secret not configured: auto-generated random key. " + + "Page tokens will not survive restarts or work across replicas. " + + "Set pagination.hmac-secret for multi-replica deployments.") + } + + // Initialize authentication middleware + if err := middleware.InitAuth(); err != nil { + return fmt.Errorf("failed to initialize auth: %w", err) + } + + // Initialize GA4GH visa validator if enabled + var visaValidator *visa.Validator + if config.VisaEnabled() { + vv, err := initVisaValidator() + if err != nil { + return fmt.Errorf("failed to initialize visa validator: %w", err) + } + visaValidator = vv + log.Infof("GA4GH visa support enabled (source=%s, identity=%s, permission=%s)", + config.VisaSource(), config.VisaIdentityMode(), config.PermissionModel()) + } + + // Initialize storage/v2 reader + storageReader, err := storage.NewReader(ctx, config.StorageBackend()) + if err != nil { + return fmt.Errorf("failed to initialize storage reader: %w", err) + } + log.Infof("storage reader initialized for backend: %s", config.StorageBackend()) + + // Initialize gRPC reencrypt client + reencryptOpts := []reencrypt.ClientOption{ + reencrypt.WithTimeout(time.Duration(config.GRPCTimeout()) * time.Second), + } + if config.GRPCClientCert() != "" && config.GRPCClientKey() != "" { + reencryptOpts = append(reencryptOpts, reencrypt.WithTLS( + config.GRPCCACert(), + config.GRPCClientCert(), + config.GRPCClientKey(), + )) + } + reencryptClient := reencrypt.NewClient(config.GRPCHost(), config.GRPCPort(), reencryptOpts...) + defer func() { + if err := reencryptClient.Close(); err != nil { + log.Errorf("failed to close reencrypt client: %v", err) + } + }() + log.Infof("reencrypt client configured for %s:%d", config.GRPCHost(), config.GRPCPort()) + + // Setup HTTP server + router := gin.New() + router.UseRawPath = true // Route on raw URL-encoded path (supports %2F in dataset IDs) + router.UnescapePathValues = true // c.Param() still returns decoded value + router.Use(gin.Recovery()) + + // Add request logging + if log.IsLevelEnabled(log.InfoLevel) { + router.Use(gin.LoggerWithConfig(gin.LoggerConfig{ + Formatter: func(params gin.LogFormatterParams) string { + return fmt.Sprintf(`{"level":"info","method":"%s","path":"%s","status":%d,"latency":"%v","client_ip":"%s","time":"%s"}`+"\n", + params.Method, + params.Path, + params.StatusCode, + params.Latency, + params.ClientIP, + params.TimeStamp.Format(time.RFC3339), + ) + }, + Output: os.Stdout, + SkipPaths: []string{"/health/live"}, + })) + } + + // Create handlers with dependencies + handlerOpts := []func(*handlers.Handlers){ + handlers.WithDatabase(database.GetDB()), + handlers.WithStorageReader(storageReader), + handlers.WithReencryptClient(reencryptClient), + handlers.WithAuditLogger(auditLogger), + } + if visaValidator != nil { + handlerOpts = append(handlerOpts, handlers.WithVisaValidator(visaValidator)) + } + + h, err := handlers.New(handlerOpts...) + if err != nil { + return fmt.Errorf("failed to create handlers: %w", err) + } + h.RegisterRoutes(router) + + // Configure TLS if certificates are provided + tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12} + + srv := &http.Server{ + Addr: fmt.Sprintf("%s:%d", config.APIHost(), config.APIPort()), + Handler: router, + TLSConfig: tlsConfig, + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), + ReadHeaderTimeout: 20 * time.Second, + ReadTimeout: 5 * time.Minute, + WriteTimeout: 30 * time.Minute, // Long timeout for large file downloads + } + + // Start server in goroutine + go func() { + var err error + if config.APIServerCert() != "" && config.APIServerKey() != "" { + log.Infof("server listening at: https://%s", srv.Addr) + err = srv.ListenAndServeTLS(config.APIServerCert(), config.APIServerKey()) + } else { + log.Infof("server listening at: http://%s", srv.Addr) + err = srv.ListenAndServe() + } + + if err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Errorf("server error: %v", err) + cancel() + } + }() + + // Mark service as ready + health.SetServingStatus(health.Serving) + log.Infof("health server listening at: :%d", config.HealthPort()) + + // Wait for shutdown signal + select { + case <-sigc: + log.Info("received shutdown signal") + case <-ctx.Done(): + log.Info("context cancelled") + } + + // Mark service as not serving + health.SetServingStatus(health.NotServing) + + // Graceful shutdown + log.Info("shutting down server...") + + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer shutdownCancel() + + if err := srv.Shutdown(shutdownCtx); err != nil { + log.Errorf("server shutdown error: %v", err) + } + + log.Info("shutdown complete") + + return nil +} + +// initVisaValidator creates and configures the GA4GH visa validator. +func initVisaValidator() (*visa.Validator, error) { + // Load trusted issuers + trustedPath := config.VisaTrustedIssuersPath() + if trustedPath == "" { + return nil, errors.New("visa.trusted-issuers-path is required when visa is enabled") + } + + allowInsecure := config.VisaAllowInsecureJKU() + if allowInsecure { + log.Warn("visa.allow-insecure-jku is enabled: HTTP JKU URLs are permitted (NOT for production use)") + } + + trustedIssuers, err := visa.LoadTrustedIssuers(trustedPath, allowInsecure) + if err != nil { + return nil, fmt.Errorf("failed to load trusted issuers: %w", err) + } + log.Infof("loaded %d trusted issuer+JKU pairs from %s", len(trustedIssuers), trustedPath) + + // Discover userinfo URL if not explicitly configured + userinfoURL := config.VisaUserinfoURL() + if userinfoURL == "" && config.OIDCIssuer() != "" { + discovered, err := visa.DiscoverUserinfoURL(config.OIDCIssuer()) + if err != nil { + log.Warnf("OIDC discovery failed: %v (userinfo will need to be configured explicitly)", err) + } else { + userinfoURL = discovered + log.Infof("discovered userinfo endpoint: %s", userinfoURL) + } + } + + cfg := visa.ValidatorConfig{ + Source: config.VisaSource(), + UserinfoURL: userinfoURL, + DatasetIDMode: config.VisaDatasetIDMode(), + IdentityMode: config.VisaIdentityMode(), + ValidateAsserted: config.VisaValidateAsserted(), + ClockSkew: 30 * time.Second, + MaxVisas: config.VisaMaxVisas(), + MaxJWKSPerReq: config.VisaMaxJWKSPerRequest(), + MaxVisaSize: config.VisaMaxVisaSize(), + JWKCacheTTL: time.Duration(config.VisaCacheJWKTTL()) * time.Second, + ValidationCacheTTL: time.Duration(config.VisaCacheValidationTTL()) * time.Second, + UserinfoCacheTTL: time.Duration(config.VisaCacheUserinfoTTL()) * time.Second, + } + + return visa.NewValidator(cfg, trustedIssuers, database.GetDB()) +} + +// productionConfig holds the values checked by production safety guards. +type productionConfig struct { + AllowAllData bool + HMACSecret string + GRPCClientCert string + GRPCClientKey string +} + +// validateProductionConfig checks that dangerous testing flags are disabled +// and required security configuration is present for production deployments. +func validateProductionConfig(cfg productionConfig) error { + if cfg.AllowAllData { + return errors.New("jwt.allow-all-data must not be enabled in production") + } + + if cfg.HMACSecret == "" { + return errors.New("pagination.hmac-secret is required in production (tokens must be portable across replicas)") + } + + if cfg.GRPCClientCert == "" || cfg.GRPCClientKey == "" { + return errors.New("grpc.client-cert and grpc.client-key are required in production") + } + + return nil +} diff --git a/sda/cmd/download/reencrypt/reencrypt.go b/sda/cmd/download/reencrypt/reencrypt.go new file mode 100644 index 0000000000..1791dd3da1 --- /dev/null +++ b/sda/cmd/download/reencrypt/reencrypt.go @@ -0,0 +1,214 @@ +// Package reencrypt provides a client for the gRPC reencrypt service. +package reencrypt + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "os" + "sync" + "time" + + re "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" + log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" +) + +// Client provides methods for re-encrypting crypt4gh headers. +type Client struct { + host string + port int + timeout time.Duration + caCert string + clientCert string + clientKey string + + mu sync.Mutex + conn *grpc.ClientConn +} + +// ClientOption is a functional option for configuring the Client. +type ClientOption func(*Client) + +// WithTimeout sets the timeout for gRPC requests. +func WithTimeout(timeout time.Duration) ClientOption { + return func(c *Client) { + c.timeout = timeout + } +} + +// WithTLS enables TLS with the given CA certificate, client cert, and client key. +func WithTLS(caCert, clientCert, clientKey string) ClientOption { + return func(c *Client) { + c.caCert = caCert + c.clientCert = clientCert + c.clientKey = clientKey + } +} + +// NewClient creates a new reencrypt client. +func NewClient(host string, port int, opts ...ClientOption) *Client { + c := &Client{ + host: host, + port: port, + timeout: 10 * time.Second, // default timeout + } + + for _, opt := range opts { + opt(c) + } + + return c +} + +// connect establishes the gRPC connection if not already connected. +func (c *Client) connect() error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.conn != nil { + return nil + } + + opts, err := c.buildDialOptions() + if err != nil { + return err + } + + address := fmt.Sprintf("%s:%d", c.host, c.port) + log.Debugf("connecting to reencrypt service at: %s", address) + + conn, err := grpc.NewClient(address, opts...) + if err != nil { + return fmt.Errorf("failed to connect to reencrypt service: %w", err) + } + + c.conn = conn + + return nil +} + +// buildDialOptions creates gRPC dial options based on TLS configuration. +func (c *Client) buildDialOptions() ([]grpc.DialOption, error) { + if c.clientKey == "" || c.clientCert == "" { + return []grpc.DialOption{grpc.WithTransportCredentials(insecure.NewCredentials())}, nil + } + + creds, err := c.buildTLSCredentials() + if err != nil { + return nil, err + } + + return []grpc.DialOption{grpc.WithTransportCredentials(creds)}, nil +} + +// buildTLSCredentials creates TLS credentials for the gRPC connection. +func (c *Client) buildTLSCredentials() (credentials.TransportCredentials, error) { + rootCAs, err := x509.SystemCertPool() + if err != nil { + log.Warnf("failed to read system CAs: %v, using empty pool", err) + rootCAs = x509.NewCertPool() + } + + if c.caCert != "" { + if err := c.appendCACert(rootCAs); err != nil { + return nil, err + } + } + + certs, err := tls.LoadX509KeyPair(c.clientCert, c.clientKey) + if err != nil { + return nil, fmt.Errorf("failed to load client key pair: %w", err) + } + + return credentials.NewTLS(&tls.Config{ + Certificates: []tls.Certificate{certs}, + MinVersion: tls.VersionTLS13, + RootCAs: rootCAs, + }), nil +} + +// appendCACert reads and appends the CA certificate to the pool. +func (c *Client) appendCACert(pool *x509.CertPool) error { + caCertBytes, err := os.ReadFile(c.caCert) + if err != nil { + return fmt.Errorf("failed to read CA certificate: %w", err) + } + + if !pool.AppendCertsFromPEM(caCertBytes) { + return errors.New("failed to append CA certificate") + } + + return nil +} + +// ReencryptHeader re-encrypts a crypt4gh header with a new public key. +// The publicKey should be base64 encoded. +func (c *Client) ReencryptHeader(ctx context.Context, oldHeader []byte, publicKey string) ([]byte, error) { + if err := c.connect(); err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + client := re.NewReencryptClient(c.conn) + + res, err := client.ReencryptHeader(ctx, &re.ReencryptRequest{ + Publickey: publicKey, + Oldheader: oldHeader, + }) + if err != nil { + return nil, fmt.Errorf("reencrypt request failed: %w", err) + } + + return res.GetHeader(), nil +} + +// ReencryptHeaderWithEditList re-encrypts a crypt4gh header with a new public key +// and a data edit list for partial file access. +func (c *Client) ReencryptHeaderWithEditList(ctx context.Context, oldHeader []byte, publicKey string, dataEditList []uint64) ([]byte, error) { + if err := c.connect(); err != nil { + return nil, err + } + + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + client := re.NewReencryptClient(c.conn) + + res, err := client.ReencryptHeader(ctx, &re.ReencryptRequest{ + Publickey: publicKey, + Oldheader: oldHeader, + Dataeditlist: dataEditList, + }) + if err != nil { + return nil, fmt.Errorf("reencrypt request failed: %w", err) + } + + return res.GetHeader(), nil +} + +// HealthCheck verifies the gRPC connection can be established. +func (c *Client) HealthCheck() error { + return c.connect() +} + +// Close closes the gRPC connection. +func (c *Client) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + + if c.conn != nil { + err := c.conn.Close() + c.conn = nil + + return err + } + + return nil +} diff --git a/sda/cmd/download/reencrypt/reencrypt_test.go b/sda/cmd/download/reencrypt/reencrypt_test.go new file mode 100644 index 0000000000..e3685b00d3 --- /dev/null +++ b/sda/cmd/download/reencrypt/reencrypt_test.go @@ -0,0 +1,44 @@ +package reencrypt + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestNewClient(t *testing.T) { + client := NewClient("localhost", 50051) + + assert.NotNil(t, client) + assert.Equal(t, "localhost", client.host) + assert.Equal(t, 50051, client.port) + assert.Equal(t, 10*time.Second, client.timeout) // default +} + +func TestNewClient_WithTimeout(t *testing.T) { + client := NewClient("localhost", 50051, WithTimeout(30*time.Second)) + + assert.NotNil(t, client) + assert.Equal(t, 30*time.Second, client.timeout) +} + +func TestNewClient_WithTLS(t *testing.T) { + client := NewClient("localhost", 50051, WithTLS("/path/ca.crt", "/path/client.crt", "/path/client.key")) + + assert.NotNil(t, client) + assert.Equal(t, "/path/ca.crt", client.caCert) + assert.Equal(t, "/path/client.crt", client.clientCert) + assert.Equal(t, "/path/client.key", client.clientKey) +} + +func TestClient_Close_WhenNotConnected(t *testing.T) { + client := NewClient("localhost", 50051) + + // Should not error when closing without connecting + err := client.Close() + assert.NoError(t, err) +} + +// Note: Integration tests for ReencryptHeader would require a running gRPC server +// These are better suited as integration tests in a test environment diff --git a/sda/cmd/download/streaming/ifrange.go b/sda/cmd/download/streaming/ifrange.go new file mode 100644 index 0000000000..5ed609f007 --- /dev/null +++ b/sda/cmd/download/streaming/ifrange.go @@ -0,0 +1,37 @@ +package streaming + +import ( + "net/http" + "strings" + "time" +) + +// CheckIfRange evaluates the If-Range precondition (RFC 9110 §13.1.5). +// Returns true if Range should be honored, false if full 200 should be sent. +func CheckIfRange(ifRangeHeader string, currentETag string, lastModified time.Time) bool { + if ifRangeHeader == "" { + return true + } + + // Weak ETags are not valid for If-Range (requires strong comparison) + if strings.HasPrefix(ifRangeHeader, "W/") { + return false + } + + // Strong ETag comparison (starts with quote) + if strings.HasPrefix(ifRangeHeader, "\"") { + return ifRangeHeader == currentETag + } + + // Otherwise treat as HTTP date + t, err := http.ParseTime(ifRangeHeader) + if err != nil { + return false + } + + if lastModified.IsZero() || !lastModified.After(t) { + return true + } + + return false +} diff --git a/sda/cmd/download/streaming/ifrange_test.go b/sda/cmd/download/streaming/ifrange_test.go new file mode 100644 index 0000000000..88f09b3d5f --- /dev/null +++ b/sda/cmd/download/streaming/ifrange_test.go @@ -0,0 +1,40 @@ +package streaming + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCheckIfRange_EmptyHeader(t *testing.T) { + assert.True(t, CheckIfRange("", `"abc"`, time.Now())) +} + +func TestCheckIfRange_MatchingStrongETag(t *testing.T) { + assert.True(t, CheckIfRange(`"etag123"`, `"etag123"`, time.Time{})) +} + +func TestCheckIfRange_MismatchingStrongETag(t *testing.T) { + assert.False(t, CheckIfRange(`"etag123"`, `"etag456"`, time.Time{})) +} + +func TestCheckIfRange_WeakETag(t *testing.T) { + assert.False(t, CheckIfRange(`W/"etag123"`, `"etag123"`, time.Time{})) +} + +func TestCheckIfRange_ValidDate_NotModified(t *testing.T) { + lastModified := time.Date(2025, 1, 1, 0, 0, 0, 0, time.UTC) + ifRange := "Wed, 01 Jan 2025 00:00:00 GMT" + assert.True(t, CheckIfRange(ifRange, "", lastModified)) +} + +func TestCheckIfRange_ValidDate_ModifiedAfter(t *testing.T) { + lastModified := time.Date(2025, 6, 1, 0, 0, 0, 0, time.UTC) + ifRange := "Wed, 01 Jan 2025 00:00:00 GMT" + assert.False(t, CheckIfRange(ifRange, "", lastModified)) +} + +func TestCheckIfRange_InvalidDate(t *testing.T) { + assert.False(t, CheckIfRange("not-a-date", "", time.Now())) +} diff --git a/sda/cmd/download/streaming/streaming.go b/sda/cmd/download/streaming/streaming.go new file mode 100644 index 0000000000..ecbeb30d75 --- /dev/null +++ b/sda/cmd/download/streaming/streaming.go @@ -0,0 +1,361 @@ +// Package streaming provides utilities for streaming files to HTTP clients. +package streaming + +import ( + "bytes" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + "strings" + + storage "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" + log "github.com/sirupsen/logrus" +) + +// RangeSpec represents a parsed HTTP Range header. +// Both Start and End are inclusive byte positions (0-indexed). +type RangeSpec struct { + Start int64 + End int64 // Inclusive end position, always clamped to fileSize-1 +} + +// ErrRangeNotSatisfiable indicates the requested range cannot be satisfied. +// The handler should return HTTP 416 Range Not Satisfiable. +var ErrRangeNotSatisfiable = errors.New("range not satisfiable") + +// ErrRangeInvalid indicates the Range header is malformed or unsupported (e.g. multi-range). +// The handler should return HTTP 400 or reject the range. +var ErrRangeInvalid = errors.New("range invalid") + +// ParseRangeHeader parses an RFC 7233 Range header. +// Returns: +// - (nil, nil) if no range is specified - serve full file with 200 +// - (*RangeSpec, nil) if range is valid - serve partial content with 206 +// - (nil, ErrRangeNotSatisfiable) if range is unsatisfiable - return 416 +// - (nil, ErrRangeInvalid) if range format is invalid or multi-range +// +// Only supports a single byte range (not multiple ranges). +func ParseRangeHeader(rangeHeader string, fileSize int64) (*RangeSpec, error) { + if rangeHeader == "" { + return nil, nil + } + + // Detect multi-range (contains comma after "bytes=") + if idx := strings.Index(rangeHeader, "bytes="); idx >= 0 { + if strings.Contains(rangeHeader[idx+6:], ",") { + return nil, ErrRangeInvalid + } + } + + // Match: bytes=START-END or bytes=START- or bytes=-SUFFIX + re := regexp.MustCompile(`^bytes=(\d*)-(\d*)$`) + matches := re.FindStringSubmatch(rangeHeader) + if matches == nil { + log.Warnf("invalid range header format: %s", rangeHeader) + + return nil, ErrRangeInvalid + } + + var start, end int64 + + // bytes=-SUFFIX (last N bytes) + if matches[1] == "" && matches[2] != "" { + suffix, err := strconv.ParseInt(matches[2], 10, 64) + if err != nil { + log.Warnf("invalid range suffix: %s", matches[2]) + + return nil, ErrRangeInvalid + } + + // Handle edge cases: + // - suffix=0 (bytes=-0) is unsatisfiable (requesting 0 bytes) + // - fileSize=0 means no content to serve + if suffix <= 0 || fileSize <= 0 { + log.Warnf("range not satisfiable: suffix=%d, fileSize=%d", suffix, fileSize) + + return nil, ErrRangeNotSatisfiable + } + + start = fileSize - suffix + if start < 0 { + start = 0 + } + end = fileSize - 1 + + return &RangeSpec{Start: start, End: end}, nil + } + + // bytes=START- or bytes=START-END + if matches[1] != "" { + var err error + start, err = strconv.ParseInt(matches[1], 10, 64) + if err != nil { + log.Warnf("invalid range start: %s", matches[1]) + + return nil, ErrRangeInvalid + } + } + + if matches[2] != "" { + var err error + end, err = strconv.ParseInt(matches[2], 10, 64) + if err != nil { + log.Warnf("invalid range end: %s", matches[2]) + + return nil, ErrRangeInvalid + } + } else { + end = fileSize - 1 // Until end of file + } + + // Validate range - unsatisfiable ranges should return 416 + if start >= fileSize { + log.Warnf("range not satisfiable: start=%d >= fileSize=%d", start, fileSize) + + return nil, ErrRangeNotSatisfiable + } + + if start > end { + log.Warnf("range not satisfiable: start=%d > end=%d", start, end) + + return nil, ErrRangeNotSatisfiable + } + + // Clamp end to file size + if end >= fileSize { + end = fileSize - 1 + } + + return &RangeSpec{Start: start, End: end}, nil +} + +// StreamConfig holds configuration for streaming a file. +type StreamConfig struct { + // Writer is the HTTP response writer + Writer http.ResponseWriter + // NewHeader is the re-encrypted crypt4gh header to prepend + NewHeader []byte + // FileReader is the reader for the encrypted file body in the archive. + // Note: In the SDA pipeline, archive files are header-stripped during ingest, + // so this reader typically starts at the encrypted body data, not the header. + FileReader io.ReadSeekCloser + // ArchiveFileSize is the size of the file in the archive. + // For header-stripped archives (standard SDA), this is the body size only. + // For archives with headers, this is header + body size. + ArchiveFileSize int64 + // OriginalHeaderSize is the size of the original crypt4gh header to skip. + // Set to 0 if the archive file is header-stripped (standard SDA). + // Set to the header size if the archive contains the original header. + OriginalHeaderSize int64 + // Range is the optional byte range to stream (nil for whole file) + Range *RangeSpec +} + +// StreamFile streams a file to the HTTP response writer. +// It combines the new header with the file body and handles range requests. +// If OriginalHeaderSize > 0, the original crypt4gh header in the archive file is skipped. +func StreamFile(cfg StreamConfig) error { + // Guard against nil FileReader to prevent panic on defer + if cfg.FileReader == nil { + return errors.New("invalid config: FileReader cannot be nil") + } + defer cfg.FileReader.Close() + + // Validate configuration to prevent negative sizes + if cfg.OriginalHeaderSize < 0 { + return fmt.Errorf("invalid config: OriginalHeaderSize cannot be negative (%d)", cfg.OriginalHeaderSize) + } + if cfg.ArchiveFileSize < 0 { + return fmt.Errorf("invalid config: ArchiveFileSize cannot be negative (%d)", cfg.ArchiveFileSize) + } + if cfg.OriginalHeaderSize > cfg.ArchiveFileSize { + return fmt.Errorf("invalid config: OriginalHeaderSize (%d) cannot exceed ArchiveFileSize (%d)", + cfg.OriginalHeaderSize, cfg.ArchiveFileSize) + } + + newHeaderSize := int64(len(cfg.NewHeader)) + // Body size is archive size minus any original header that needs to be skipped + bodySize := cfg.ArchiveFileSize - cfg.OriginalHeaderSize + totalSize := newHeaderSize + bodySize + + // Skip the original header in the archive file (if present) + // For header-stripped archives (standard SDA), OriginalHeaderSize is 0 and no seek is needed + if cfg.OriginalHeaderSize > 0 { + if _, err := cfg.FileReader.Seek(cfg.OriginalHeaderSize, io.SeekStart); err != nil { + return fmt.Errorf("failed to skip original header: %w", err) + } + } + + // Create reader for the new header + headerReader := bytes.NewReader(cfg.NewHeader) + + if cfg.Range == nil { + // Stream whole file + cfg.Writer.Header().Set("Content-Length", fmt.Sprintf("%d", totalSize)) + cfg.Writer.Header().Set("Content-Type", "application/octet-stream") + + // Stream new header then body (with original header already skipped) + if _, err := io.Copy(cfg.Writer, headerReader); err != nil { + return fmt.Errorf("failed to stream header: %w", err) + } + if _, err := io.Copy(cfg.Writer, cfg.FileReader); err != nil { + return fmt.Errorf("failed to stream body: %w", err) + } + + return nil + } + + // Validate range bounds (defensive - ParseRangeHeader should already ensure this) + if cfg.Range.Start < 0 || cfg.Range.End < 0 { + return fmt.Errorf("invalid range: negative values (start=%d, end=%d)", cfg.Range.Start, cfg.Range.End) + } + if cfg.Range.Start > cfg.Range.End { + return fmt.Errorf("invalid range: start (%d) > end (%d)", cfg.Range.Start, cfg.Range.End) + } + if cfg.Range.End >= totalSize { + return fmt.Errorf("invalid range: end (%d) >= totalSize (%d)", cfg.Range.End, totalSize) + } + + // Handle range request + rangeLength := cfg.Range.End - cfg.Range.Start + 1 + + cfg.Writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", cfg.Range.Start, cfg.Range.End, totalSize)) + cfg.Writer.Header().Set("Content-Length", fmt.Sprintf("%d", rangeLength)) + cfg.Writer.Header().Set("Content-Type", "application/octet-stream") + cfg.Writer.WriteHeader(http.StatusPartialContent) + + // Stream the requested range (body reader already positioned after original header) + return streamRange(cfg.Writer, headerReader, cfg.FileReader, newHeaderSize, cfg.Range) +} + +// streamRange streams a specific byte range from combined header + body. +func streamRange(w io.Writer, header *bytes.Reader, body io.ReadCloser, headerSize int64, r *RangeSpec) error { + bytesRemaining := r.End - r.Start + 1 + + // Stream from header if range overlaps + if r.Start < headerSize { + headerBytesStreamed, err := streamHeaderRange(w, header, r.Start, headerSize, bytesRemaining) + if err != nil { + return err + } + bytesRemaining -= headerBytesStreamed + } + + // Stream from body if range extends into it + if bytesRemaining > 0 { + return streamBodyRange(w, body, r.Start, headerSize, bytesRemaining) + } + + return nil +} + +// streamHeaderRange streams bytes from the header and returns the number of bytes streamed. +func streamHeaderRange(w io.Writer, header *bytes.Reader, start, headerSize, bytesRemaining int64) (int64, error) { + if _, err := header.Seek(start, io.SeekStart); err != nil { + return 0, fmt.Errorf("failed to seek header: %w", err) + } + + headerBytesToRead := headerSize - start + if headerBytesToRead > bytesRemaining { + headerBytesToRead = bytesRemaining + } + + if _, err := io.CopyN(w, header, headerBytesToRead); err != nil { + return 0, fmt.Errorf("failed to stream header range: %w", err) + } + + return headerBytesToRead, nil +} + +// streamBodyRange streams bytes from the body starting at the appropriate offset. +func streamBodyRange(w io.Writer, body io.ReadCloser, start, headerSize, bytesRemaining int64) error { + bodyOffset := start - headerSize + if bodyOffset < 0 { + bodyOffset = 0 + } + + if err := seekOrSkipBody(body, bodyOffset); err != nil { + return err + } + + if _, err := io.CopyN(w, body, bytesRemaining); err != nil && err != io.EOF { + return fmt.Errorf("failed to stream body range: %w", err) + } + + return nil +} + +// seekOrSkipBody positions the body reader at the given offset from the current position. +// Note: This uses io.SeekCurrent because the body reader may have already been positioned +// past the original crypt4gh header. +func seekOrSkipBody(body io.Reader, offset int64) error { + if offset == 0 { + return nil + } + + if seeker, ok := body.(io.Seeker); ok { + if _, err := seeker.Seek(offset, io.SeekCurrent); err != nil { + return fmt.Errorf("failed to seek body: %w", err) + } + + return nil + } + + // Skip bytes if not seekable + if _, err := io.CopyN(io.Discard, body, offset); err != nil { + return fmt.Errorf("failed to skip body bytes: %w", err) + } + + return nil +} + +// StreamBodyConfig holds configuration for streaming just the file body. +type StreamBodyConfig struct { + Writer http.ResponseWriter + FileReader io.ReadSeekCloser + ArchiveFileSize int64 + Range *RangeSpec +} + +// StreamBodyOnly streams the archive body with optional Range support, no header logic. +func StreamBodyOnly(cfg StreamBodyConfig) error { + if cfg.FileReader == nil { + return errors.New("invalid config: FileReader cannot be nil") + } + defer cfg.FileReader.Close() + + if cfg.Range == nil { + // Full body + cfg.Writer.Header().Set("Content-Length", fmt.Sprintf("%d", cfg.ArchiveFileSize)) + cfg.Writer.Header().Set("Content-Type", "application/octet-stream") + if _, err := io.Copy(cfg.Writer, cfg.FileReader); err != nil { + return fmt.Errorf("failed to stream body: %w", err) + } + + return nil + } + + // Range request over body only + rangeLength := cfg.Range.End - cfg.Range.Start + 1 + cfg.Writer.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", cfg.Range.Start, cfg.Range.End, cfg.ArchiveFileSize)) + cfg.Writer.Header().Set("Content-Length", fmt.Sprintf("%d", rangeLength)) + cfg.Writer.Header().Set("Content-Type", "application/octet-stream") + cfg.Writer.WriteHeader(http.StatusPartialContent) + + // Seek to start position + if err := seekOrSkipBody(cfg.FileReader, cfg.Range.Start); err != nil { + return err + } + + if _, err := io.CopyN(cfg.Writer, cfg.FileReader, rangeLength); err != nil && err != io.EOF { + return fmt.Errorf("failed to stream body range: %w", err) + } + + return nil +} + +// SeekableMultiReader is a re-export of storage.SeekableMultiReader for convenience. +var SeekableMultiReader = storage.SeekableMultiReader diff --git a/sda/cmd/download/streaming/streaming_test.go b/sda/cmd/download/streaming/streaming_test.go new file mode 100644 index 0000000000..ab6b9c0f97 --- /dev/null +++ b/sda/cmd/download/streaming/streaming_test.go @@ -0,0 +1,567 @@ +package streaming + +import ( + "bytes" + "fmt" + "io" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// readSeekCloser wraps bytes.Reader to implement io.ReadSeekCloser +type readSeekCloser struct { + *bytes.Reader +} + +func (r *readSeekCloser) Close() error { + return nil +} + +// newReadSeekCloser creates a new io.ReadSeekCloser from a byte slice +func newReadSeekCloser(data []byte) *readSeekCloser { + return &readSeekCloser{Reader: bytes.NewReader(data)} +} + +func TestParseRangeHeader_Empty(t *testing.T) { + result, err := ParseRangeHeader("", 1000) + assert.NoError(t, err) + assert.Nil(t, result) +} + +func TestParseRangeHeader_FullRange(t *testing.T) { + result, err := ParseRangeHeader("bytes=0-499", 1000) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, int64(0), result.Start) + assert.Equal(t, int64(499), result.End) +} + +func TestParseRangeHeader_OpenEndedRange(t *testing.T) { + result, err := ParseRangeHeader("bytes=500-", 1000) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, int64(500), result.Start) + assert.Equal(t, int64(999), result.End) // End of file +} + +func TestParseRangeHeader_SuffixRange(t *testing.T) { + result, err := ParseRangeHeader("bytes=-100", 1000) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, int64(900), result.Start) + assert.Equal(t, int64(999), result.End) +} + +func TestParseRangeHeader_SuffixLargerThanFile(t *testing.T) { + result, err := ParseRangeHeader("bytes=-2000", 1000) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, int64(0), result.Start) // Clamped to 0 + assert.Equal(t, int64(999), result.End) +} + +func TestParseRangeHeader_EndBeyondFileSize(t *testing.T) { + result, err := ParseRangeHeader("bytes=500-2000", 1000) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, int64(500), result.Start) + assert.Equal(t, int64(999), result.End) // Clamped to file size - 1 +} + +func TestParseRangeHeader_StartBeyondFileSize(t *testing.T) { + result, err := ParseRangeHeader("bytes=2000-3000", 1000) + assert.ErrorIs(t, err, ErrRangeNotSatisfiable) + assert.Nil(t, result) +} + +func TestParseRangeHeader_StartGreaterThanEnd(t *testing.T) { + result, err := ParseRangeHeader("bytes=500-100", 1000) + assert.ErrorIs(t, err, ErrRangeNotSatisfiable) + assert.Nil(t, result) +} + +func TestParseRangeHeader_InvalidFormat(t *testing.T) { + result, err := ParseRangeHeader("invalid", 1000) + assert.ErrorIs(t, err, ErrRangeInvalid) + assert.Nil(t, result) +} + +func TestParseRangeHeader_WrongUnit(t *testing.T) { + result, err := ParseRangeHeader("chars=0-100", 1000) + assert.ErrorIs(t, err, ErrRangeInvalid) + assert.Nil(t, result) +} + +func TestParseRangeHeader_MultipleRanges(t *testing.T) { + result, err := ParseRangeHeader("bytes=0-100,200-300", 1000) + assert.ErrorIs(t, err, ErrRangeInvalid) + assert.Nil(t, result) +} + +func TestParseRangeHeader_SuffixZero(t *testing.T) { + // bytes=-0 requests 0 bytes, which is unsatisfiable + result, err := ParseRangeHeader("bytes=-0", 1000) + assert.ErrorIs(t, err, ErrRangeNotSatisfiable) + assert.Nil(t, result) +} + +func TestParseRangeHeader_ZeroFileSize(t *testing.T) { + // Empty file - suffix range is unsatisfiable + result, err := ParseRangeHeader("bytes=-100", 0) + assert.ErrorIs(t, err, ErrRangeNotSatisfiable) + assert.Nil(t, result) +} + +func TestParseRangeHeader_SingleByte(t *testing.T) { + result, err := ParseRangeHeader("bytes=0-0", 1000) + assert.NoError(t, err) + assert.NotNil(t, result) + assert.Equal(t, int64(0), result.Start) + assert.Equal(t, int64(0), result.End) +} + +// StreamFile tests + +func TestStreamFile_WholeFile(t *testing.T) { + header := []byte("CRYPT4GH_HEADER_DATA") + body := []byte("FILE_BODY_CONTENT_HERE") + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: nil, + }) + + require.NoError(t, err) + assert.Equal(t, "42", recorder.Header().Get("Content-Length")) + assert.Equal(t, "application/octet-stream", recorder.Header().Get("Content-Type")) + assert.Equal(t, string(header)+string(body), recorder.Body.String()) +} + +func TestStreamFile_RangeRequest_HeaderOnly(t *testing.T) { + header := []byte("CRYPT4GH_HEADER_DATA") // 20 bytes + body := []byte("FILE_BODY_CONTENT") // 17 bytes + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: 0, End: 9}, // First 10 bytes (header only) + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + assert.Equal(t, "bytes 0-9/37", recorder.Header().Get("Content-Range")) + assert.Equal(t, "10", recorder.Header().Get("Content-Length")) + assert.Equal(t, "CRYPT4GH_H", recorder.Body.String()) +} + +func TestStreamFile_RangeRequest_BodyOnly(t *testing.T) { + header := []byte("CRYPT4GH_HEADER_DATA") // 20 bytes + body := []byte("FILE_BODY_CONTENT") // 17 bytes + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: 25, End: 30}, // Bytes 25-30 (body only) + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + assert.Equal(t, "bytes 25-30/37", recorder.Header().Get("Content-Range")) + assert.Equal(t, "6", recorder.Header().Get("Content-Length")) + // bodyOffset = 25 - 20 = 5, so we read body[5:11] = "BODY_C" (wait, body is "FILE_BODY_CONTENT") + // body[5:11] = "BODY_C" - correct! + assert.Equal(t, "BODY_C", recorder.Body.String()) +} + +func TestStreamFile_RangeRequest_SpanningHeaderAndBody(t *testing.T) { + header := []byte("CRYPT4GH_HEADER_DATA") // 20 bytes + body := []byte("FILE_BODY_CONTENT") // 17 bytes + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: 15, End: 24}, // Last 5 bytes of header + first 5 bytes of body + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + assert.Equal(t, "bytes 15-24/37", recorder.Header().Get("Content-Range")) + assert.Equal(t, "10", recorder.Header().Get("Content-Length")) + assert.Equal(t, "_DATAFILE_", recorder.Body.String()) // header[15:20] + body[0:5] +} + +func TestStreamFile_RangeRequest_LastBytes(t *testing.T) { + header := []byte("HEADER") + body := []byte("BODY_DATA") + + recorder := httptest.NewRecorder() + totalSize := int64(len(header) + len(body)) // 15 + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: totalSize - 5, End: totalSize - 1}, // Last 5 bytes + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + assert.Equal(t, "_DATA", recorder.Body.String()) +} + +// Tests for OriginalHeaderSize (skipping original crypt4gh header) + +func TestStreamFile_SkipsOriginalHeader(t *testing.T) { + // Simulates the real scenario where archive file contains [ORIGINAL_HEADER][BODY] + // and we want to stream [NEW_HEADER][BODY] + newHeader := []byte("NEW_CRYPT4GH_HDR") + originalHeader := []byte("OLD_HDR_DATA") // 12 bytes to skip + body := []byte("ENCRYPTED_FILE_BODY") + archiveFile := make([]byte, 0, len(originalHeader)+len(body)) + archiveFile = append(archiveFile, originalHeader...) + archiveFile = append(archiveFile, body...) // [OLD_HDR_DATA][ENCRYPTED_FILE_BODY] + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: newHeader, + FileReader: newReadSeekCloser(archiveFile), + ArchiveFileSize: int64(len(archiveFile)), + OriginalHeaderSize: int64(len(originalHeader)), + Range: nil, + }) + + require.NoError(t, err) + + // Total size should be: newHeader + (archiveFile - originalHeader) + // = 16 + (31 - 12) = 16 + 19 = 35 + expectedSize := len(newHeader) + len(body) + assert.Equal(t, fmt.Sprintf("%d", expectedSize), recorder.Header().Get("Content-Length")) + + // Output should be [NEW_HEADER][BODY], not [NEW_HEADER][OLD_HEADER][BODY] + expectedOutput := string(newHeader) + string(body) + assert.Equal(t, expectedOutput, recorder.Body.String()) + + // Verify the original header is NOT in the output + assert.NotContains(t, recorder.Body.String(), string(originalHeader)) +} + +func TestStreamFile_SkipsOriginalHeader_RangeInBody(t *testing.T) { + newHeader := []byte("NEW_HDR") // 7 bytes + originalHeader := []byte("ORIGINAL_HDR") // 12 bytes to skip + body := []byte("THE_ACTUAL_BODY_DATA") // 20 bytes + archiveFile := make([]byte, 0, len(originalHeader)+len(body)) + archiveFile = append(archiveFile, originalHeader...) + archiveFile = append(archiveFile, body...) + + recorder := httptest.NewRecorder() + + // Request bytes 10-19, which should be in the body portion + // New header is 7 bytes, so bytes 10-19 means body[3:13] + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: newHeader, + FileReader: newReadSeekCloser(archiveFile), + ArchiveFileSize: int64(len(archiveFile)), + OriginalHeaderSize: int64(len(originalHeader)), + Range: &RangeSpec{Start: 10, End: 19}, + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + + // totalSize = 7 + 20 = 27 + assert.Equal(t, "bytes 10-19/27", recorder.Header().Get("Content-Range")) + assert.Equal(t, "10", recorder.Header().Get("Content-Length")) + // body[3:13] = "_ACTUAL_BO" + assert.Equal(t, "_ACTUAL_BO", recorder.Body.String()) +} + +func TestStreamFile_SkipsOriginalHeader_RangeSpanningHeaderAndBody(t *testing.T) { + newHeader := []byte("NEW_HDR") // 7 bytes + originalHeader := []byte("ORIGINAL_HDR") // 12 bytes to skip + body := []byte("BODY_CONTENT") // 12 bytes + archiveFile := make([]byte, 0, len(originalHeader)+len(body)) + archiveFile = append(archiveFile, originalHeader...) + archiveFile = append(archiveFile, body...) + + recorder := httptest.NewRecorder() + + // Request bytes 5-10, spanning new header (5-6) and body (7-10 -> body[0:4]) + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: newHeader, + FileReader: newReadSeekCloser(archiveFile), + ArchiveFileSize: int64(len(archiveFile)), + OriginalHeaderSize: int64(len(originalHeader)), + Range: &RangeSpec{Start: 5, End: 10}, + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + + // totalSize = 7 + 12 = 19 + assert.Equal(t, "bytes 5-10/19", recorder.Header().Get("Content-Range")) + assert.Equal(t, "6", recorder.Header().Get("Content-Length")) + // newHeader[5:7] = "DR" + body[0:4] = "BODY" -> "DRBODY" + assert.Equal(t, "DRBODY", recorder.Body.String()) +} + +// StreamBodyOnly tests + +func TestStreamBodyOnly_WholeFile(t *testing.T) { + body := []byte("FULL_BODY_CONTENT_DATA") + + recorder := httptest.NewRecorder() + + err := StreamBodyOnly(StreamBodyConfig{ + Writer: recorder, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: nil, + }) + + require.NoError(t, err) + assert.Equal(t, fmt.Sprintf("%d", len(body)), recorder.Header().Get("Content-Length")) + assert.Equal(t, "application/octet-stream", recorder.Header().Get("Content-Type")) + assert.Equal(t, string(body), recorder.Body.String()) +} + +func TestStreamBodyOnly_RangeRequest(t *testing.T) { + body := []byte("ABCDEFGHIJKLMNOPQRST") // 20 bytes + + recorder := httptest.NewRecorder() + + err := StreamBodyOnly(StreamBodyConfig{ + Writer: recorder, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: 5, End: 14}, + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + assert.Equal(t, "bytes 5-14/20", recorder.Header().Get("Content-Range")) + assert.Equal(t, "10", recorder.Header().Get("Content-Length")) + assert.Equal(t, "FGHIJKLMNO", recorder.Body.String()) +} + +func TestStreamBodyOnly_RangeLastBytes(t *testing.T) { + body := []byte("ABCDEFGHIJ") // 10 bytes + + recorder := httptest.NewRecorder() + + err := StreamBodyOnly(StreamBodyConfig{ + Writer: recorder, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: 7, End: 9}, + }) + + require.NoError(t, err) + assert.Equal(t, 206, recorder.Code) + assert.Equal(t, "HIJ", recorder.Body.String()) +} + +func TestStreamBodyOnly_NilFileReader(t *testing.T) { + recorder := httptest.NewRecorder() + + err := StreamBodyOnly(StreamBodyConfig{ + Writer: recorder, + FileReader: nil, + ArchiveFileSize: 100, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "FileReader cannot be nil") +} + +// seekOrSkipBody tests + +type nonSeekableReader struct { + *bytes.Reader +} + +func (r *nonSeekableReader) Read(p []byte) (n int, err error) { + return r.Reader.Read(p) +} + +func TestSeekOrSkipBody_ZeroOffset(t *testing.T) { + body := bytes.NewReader([]byte("test data")) + err := seekOrSkipBody(body, 0) + require.NoError(t, err) + + // Should still be at the beginning + data, _ := io.ReadAll(body) + assert.Equal(t, "test data", string(data)) +} + +func TestSeekOrSkipBody_Seekable(t *testing.T) { + body := bytes.NewReader([]byte("test data")) + err := seekOrSkipBody(body, 5) + require.NoError(t, err) + + // Should be at position 5 + data, _ := io.ReadAll(body) + assert.Equal(t, "data", string(data)) +} + +func TestSeekOrSkipBody_NonSeekable(t *testing.T) { + body := &nonSeekableReader{bytes.NewReader([]byte("test data"))} + err := seekOrSkipBody(body, 5) + require.NoError(t, err) + + // Should have skipped 5 bytes + data, _ := io.ReadAll(body) + assert.Equal(t, "data", string(data)) +} + +// StreamFile validation tests + +func TestStreamFile_InvalidConfig_NegativeOriginalHeaderSize(t *testing.T) { + header := []byte("HEADER") + body := []byte("BODY") + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + OriginalHeaderSize: -1, // Invalid + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "OriginalHeaderSize cannot be negative") +} + +func TestStreamFile_InvalidConfig_NilFileReader(t *testing.T) { + header := []byte("HEADER") + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: nil, // nil would panic without guard + ArchiveFileSize: 100, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "FileReader cannot be nil") +} + +func TestStreamFile_InvalidConfig_NegativeArchiveFileSize(t *testing.T) { + header := []byte("HEADER") + body := []byte("BODY") + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: -1, // Invalid + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "ArchiveFileSize cannot be negative") +} + +func TestStreamFile_InvalidConfig_HeaderSizeExceedsArchiveSize(t *testing.T) { + header := []byte("HEADER") + body := []byte("BODY") + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: 10, + OriginalHeaderSize: 20, // Exceeds archive size + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "OriginalHeaderSize") + assert.Contains(t, err.Error(), "cannot exceed ArchiveFileSize") +} + +func TestStreamFile_InvalidRange_EndExceedsTotalSize(t *testing.T) { + header := []byte("HEADER") // 6 bytes + body := []byte("BODY") // 4 bytes, totalSize = 10 + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: 0, End: 100}, // End exceeds totalSize (10) + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "end") + assert.Contains(t, err.Error(), "totalSize") +} + +func TestStreamFile_InvalidRange_NegativeStart(t *testing.T) { + header := []byte("HEADER") + body := []byte("BODY") + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: -1, End: 5}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "negative values") +} + +func TestStreamFile_InvalidRange_StartGreaterThanEnd(t *testing.T) { + header := []byte("HEADER") + body := []byte("BODY") + + recorder := httptest.NewRecorder() + + err := StreamFile(StreamConfig{ + Writer: recorder, + NewHeader: header, + FileReader: newReadSeekCloser(body), + ArchiveFileSize: int64(len(body)), + Range: &RangeSpec{Start: 8, End: 5}, + }) + + require.Error(t, err) + assert.Contains(t, err.Error(), "start") + assert.Contains(t, err.Error(), "> end") +} diff --git a/sda/cmd/download/swagger_v2.yml b/sda/cmd/download/swagger_v2.yml index 9eec484f89..2669453fa9 100644 --- a/sda/cmd/download/swagger_v2.yml +++ b/sda/cmd/download/swagger_v2.yml @@ -85,16 +85,16 @@ paths: schema: $ref: "#/components/schemas/ServiceInfo" example: - id: org.neic.sda.download - name: SDA Data Out API + id: se.nbis.sda.download + name: SDA Download type: - group: org.neic - artifact: sda-download - version: 2.0.0 + group: org.ga4gh + artifact: drs + version: 1.0.0 description: Sensitive Data Archive file download service organization: - name: NeIC - url: https://neic.no + name: NBIS + url: https://nbis.se version: 2.0.0 environment: production "500": @@ -298,8 +298,6 @@ paths: $ref: "#/components/headers/ContentLength" ETag: $ref: "#/components/headers/ETag" - Last-Modified: - $ref: "#/components/headers/LastModified" Content-Disposition: $ref: "#/components/headers/ContentDisposition" "400": @@ -358,8 +356,6 @@ paths: $ref: "#/components/headers/ContentLength" ETag: $ref: "#/components/headers/ETag" - Last-Modified: - $ref: "#/components/headers/LastModified" Content-Disposition: $ref: "#/components/headers/ContentDisposition" content: @@ -378,8 +374,6 @@ paths: $ref: "#/components/headers/ContentRange" ETag: $ref: "#/components/headers/ETag" - Last-Modified: - $ref: "#/components/headers/LastModified" Content-Disposition: $ref: "#/components/headers/ContentDisposition" content: @@ -541,8 +535,6 @@ paths: $ref: "#/components/headers/ContentLength" ETag: $ref: "#/components/headers/ContentETag" - Last-Modified: - $ref: "#/components/headers/LastModified" "401": $ref: "#/components/responses/Unauthorized" "403": @@ -587,8 +579,6 @@ paths: $ref: "#/components/headers/ContentLength" ETag: $ref: "#/components/headers/ContentETag" - Last-Modified: - $ref: "#/components/headers/LastModified" content: application/octet-stream: schema: @@ -605,8 +595,6 @@ paths: $ref: "#/components/headers/ContentRange" ETag: $ref: "#/components/headers/ContentETag" - Last-Modified: - $ref: "#/components/headers/LastModified" content: application/octet-stream: schema: @@ -658,10 +646,11 @@ paths: schema: $ref: "#/components/schemas/ReadinessStatus" example: - c4ghService: responding - database: responding - oidcServer: responding - storageBackend: responding + status: ok + services: + database: ok + grpc: ok + storage: ok "503": description: Unhealthy service @@ -792,7 +781,7 @@ components: id: type: string description: Unique service identifier in reverse domain name format - example: org.neic.sda.download + example: se.nbis.sda.download name: type: string description: Human-readable service name @@ -802,13 +791,13 @@ components: properties: group: type: string - example: org.neic + example: org.ga4gh artifact: type: string - example: sda-download + example: drs version: type: string - example: 2.0.0 + example: 1.0.0 required: [group, artifact, version] description: type: string @@ -818,11 +807,11 @@ components: properties: name: type: string - example: NeIC + example: NBIS url: type: string format: uri - example: https://neic.no + example: https://nbis.se required: [name, url] version: type: string @@ -871,16 +860,21 @@ components: ReadinessStatus: type: object - additionalProperties: true properties: - c4ghService: - type: string - database: - type: string - oidcServer: - type: string - storageBackend: + status: type: string + description: Overall status (ok or degraded) + example: ok + services: + type: object + additionalProperties: + type: string + description: Per-service health status + example: + database: ok + grpc: ok + storage: ok + required: [status] parameters: DatasetIdPath: @@ -967,7 +961,7 @@ components: schema: type: string description: | - Makes the Range request conditional on the entity tag (ETag) or Last-Modified value. + Makes the Range request conditional on the entity tag (ETag). Commonly used to resume downloads safely. example: "\"686897696a7c876b7e\"" @@ -1067,12 +1061,6 @@ components: type: string example: "\"a1b2c3d4e5f6g7h8\"" - LastModified: - description: Last modification time of the resource (HTTP-date format per RFC 9110). - schema: - type: string - example: "Fri, 14 Feb 2025 14:51:26 GMT" - ContentDisposition: description: Suggested filename for saving the download. schema: diff --git a/sda/cmd/reencrypt/reencrypt.go b/sda/cmd/reencrypt/reencrypt.go index e3ade4da87..479d69945c 100644 --- a/sda/cmd/reencrypt/reencrypt.go +++ b/sda/cmd/reencrypt/reencrypt.go @@ -82,10 +82,18 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r extraHeaderPackets = append(extraHeaderPackets, dataEditListPacket) } - reader := bytes.NewReader(publicKey) - newReaderPublicKey, err := keys.ReadPublicKey(reader) - if err != nil { - return nil, status.Error(400, err.Error()) + var newReaderPublicKey [chacha20poly1305.KeySize]byte + if len(publicKey) == chacha20poly1305.KeySize { + // Raw 32-byte X25519 key — use directly + copy(newReaderPublicKey[:], publicKey) + } else { + // Legacy: PEM text — pass to crypt4gh key parser + reader := bytes.NewReader(publicKey) + parsedKey, err := keys.ReadPublicKey(reader) + if err != nil { + return nil, status.Error(400, err.Error()) + } + newReaderPublicKey = parsedKey } newReaderPublicKeyList := [][chacha20poly1305.KeySize]byte{} newReaderPublicKeyList = append(newReaderPublicKeyList, newReaderPublicKey) diff --git a/sda/go.mod b/sda/go.mod index 83dbc34b7a..de52b21fe7 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,6 +3,7 @@ module github.com/neicnordic/sensitive-data-archive go 1.25.7 require ( + github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/config v1.32.13 github.com/aws/aws-sdk-go-v2/credentials v1.19.13 @@ -12,6 +13,7 @@ require ( github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 github.com/coreos/go-oidc/v3 v3.17.0 + github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de github.com/gin-gonic/gin v1.12.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/uuid v1.6.0 @@ -30,6 +32,8 @@ require ( github.com/rabbitmq/amqp091-go v1.10.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.4 + github.com/spf13/cobra v1.10.2 + github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.49.0 @@ -72,6 +76,7 @@ require ( github.com/casbin/govaluate v1.10.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/containerd/continuity v0.4.5 // indirect @@ -105,6 +110,7 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -146,7 +152,6 @@ require ( github.com/segmentio/asm v1.2.1 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect - github.com/spf13/pflag v1.0.10 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tdewolff/minify/v2 v2.24.10 // indirect diff --git a/sda/go.sum b/sda/go.sum index c64c726aef..99f583566b 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -99,6 +99,7 @@ github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151X github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -194,6 +195,8 @@ github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQ github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE= github.com/iris-contrib/middleware/cors v0.0.0-20251225090426-92c6f28facda h1:wtN/fj3B3xmjLLuDmcykBwx/IlUpwypYBZe+HG1tN5o= @@ -337,6 +340,9 @@ github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= diff --git a/sda/internal/config/v2/config.go b/sda/internal/config/v2/config.go new file mode 100644 index 0000000000..c670456062 --- /dev/null +++ b/sda/internal/config/v2/config.go @@ -0,0 +1,157 @@ +// Package config provides a flag-based configuration framework that allows +// applications to register their own configuration options. +// +// Usage: +// +// // In your app's config package (e.g., cmd/download/config/config.go): +// func init() { +// config.RegisterFlags( +// &config.Flag{ +// Name: "api.port", +// RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { +// flagSet.Int(flagName, 8080, "Port to host the API server at") +// }, +// Required: false, +// AssignFunc: func(flagName string) { +// apiPort = viper.GetInt(flagName) +// }, +// }, +// ) +// } +// +// // In your main.go: +// if err := config.Load(); err != nil { +// log.Fatalf("failed to load config: %v", err) +// } +package config + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + "text/tabwriter" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/spf13/viper" +) + +// Flag represents a configuration flag that can be registered by applications. +type Flag struct { + // Name is the flag name, e.g., "api.port" or "db.host" + Name string + // RegisterFunc registers the flag with the given FlagSet + RegisterFunc func(flagSet *pflag.FlagSet, flagName string) + // Required indicates if the flag must be set + Required bool + // AssignFunc assigns the flag value to the application's config variable + AssignFunc func(flagName string) +} + +var registeredFlags []*Flag + +// RegisterFlags registers configuration flags. Call this from your app's config +// package init() function. +func RegisterFlags(flags ...*Flag) { + registeredFlags = append(registeredFlags, flags...) +} + +var command = &cobra.Command{ + Run: func(_ *cobra.Command, _ []string) { + // Empty func such that cobra will evaluate flags, etc + }, +} + +func init() { + viper.SetConfigName("config") + + viper.AutomaticEnv() + viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_")) + viper.SetConfigType("yaml") + + command.Flags().String("config-path", ".", "Set the path viper will look for the config file at") + command.Flags().String("config-file", "", "Set the direct path to the config file") + command.Flags().String("log.level", "INFO", "Set the log level, supported levels: PANIC, FATAL, ERROR, WARN, INFO, DEBUG, TRACE") + + command.SetHelpFunc(func(cmd *cobra.Command, _ []string) { + fmt.Println("Flags:") + writer := tabwriter.NewWriter(os.Stdout, 1, 1, 8, ' ', 0) + _, _ = fmt.Fprintln(writer, " Name:\tEnv variable:\tType:\tUsage:\tDefault Value:\t") + cmd.Flags().VisitAll(func(flag *pflag.Flag) { + if flag.Name == "help" { + return + } + + flagType, usage := pflag.UnquoteUsage(flag) + envVar := strings.ToUpper(strings.ReplaceAll(strings.ReplaceAll(flag.Name, "-", "_"), ".", "_")) + _, _ = fmt.Fprintf(writer, " --%s\t%s\t%s\t%s\t%v\t\n", flag.Name, envVar, flagType, usage, flag.DefValue) + }) + + _ = writer.Flush() + + os.Exit(0) + }) +} + +// Load loads configuration from flags, environment variables, and config files. +// It validates that all required flags are set and assigns values to application +// config variables via their AssignFunc. +func Load() error { + for _, flag := range registeredFlags { + flag.RegisterFunc(command.Flags(), flag.Name) + } + + if err := viper.BindPFlags(command.Flags()); err != nil { + panic(err) + } + + if err := command.Execute(); err != nil { + return err + } + + if viper.IsSet("config-path") { + configPath := viper.GetString("config-path") + splitPath := strings.Split(strings.TrimLeft(configPath, "/"), "/") + viper.AddConfigPath(path.Join(splitPath...)) + } + + if viper.IsSet("config-file") { + viper.SetConfigFile(viper.GetString("config-file")) + } + + _ = viper.ReadInConfig() + + var missingFlags error + for _, flag := range registeredFlags { + if !flag.Required { + continue + } + + if !viper.IsSet(flag.Name) { + missingFlags = errors.Join(missingFlags, fmt.Errorf("missing required flag: %s", flag.Name)) + + continue + } + } + if missingFlags != nil { + return missingFlags + } + + for _, flag := range registeredFlags { + flag.AssignFunc(flag.Name) + } + + stringLevel := viper.GetString("log.level") + logLevel, err := log.ParseLevel(stringLevel) + if err != nil { + log.Debugf("Log level '%s' not supported, setting to 'trace'", stringLevel) + logLevel = log.TraceLevel + } + log.SetLevel(logLevel) + log.Infof("Setting log level to '%s'", stringLevel) + + return nil +} diff --git a/sda/internal/config/v2/config_test.go b/sda/internal/config/v2/config_test.go new file mode 100644 index 0000000000..e3733497b8 --- /dev/null +++ b/sda/internal/config/v2/config_test.go @@ -0,0 +1,41 @@ +package config + +import ( + "testing" + + "github.com/spf13/pflag" + "github.com/stretchr/testify/assert" +) + +func TestRegisterFlags(t *testing.T) { + // Save and restore state + oldFlags := registeredFlags + defer func() { registeredFlags = oldFlags }() + registeredFlags = nil + + RegisterFlags( + &Flag{ + Name: "test.flag", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "default", "Test flag") + }, + Required: false, + AssignFunc: func(_ string) { + // noop for this test + }, + }, + ) + + assert.Len(t, registeredFlags, 1) + assert.Equal(t, "test.flag", registeredFlags[0].Name) +} + +// Note: Tests for Load() are skipped because cobra.Command.Execute() modifies +// global state that cannot be easily reset between tests. The Load() function +// is tested via integration tests instead. +// +// The following scenarios are covered by integration tests: +// - Missing required flags +// - Optional flags with defaults +// - Environment variable binding +// - Config file loading From 4497104b0f8d48bb812d12ba2a29308b4b22c0cc Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:06:46 +0100 Subject: [PATCH 372/530] feat(download): add database layer with caching and keyset pagination Database components: - database.go: PostgreSQL interface with prepared statements, keyset cursor pagination via (submission_file_path, stable_id) composite cursor, LATERAL json_agg for checksum aggregation (prevents row multiplication), LIKE prefix escaping, and CheckDatasetExists for no-existence-leakage pattern - cache.go: Ristretto-based cache wrapper (lock-free) for file lookups, permission checks, and dataset queries. Paginated queries bypass cache due to cursor variability. --- sda/cmd/download/database/cache.go | 335 ++++++++ sda/cmd/download/database/cache_test.go | 573 ++++++++++++++ sda/cmd/download/database/database.go | 764 +++++++++++++++++++ sda/cmd/download/database/database_test.go | 839 +++++++++++++++++++++ 4 files changed, 2511 insertions(+) create mode 100644 sda/cmd/download/database/cache.go create mode 100644 sda/cmd/download/database/cache_test.go create mode 100644 sda/cmd/download/database/database.go create mode 100644 sda/cmd/download/database/database_test.go diff --git a/sda/cmd/download/database/cache.go b/sda/cmd/download/database/cache.go new file mode 100644 index 0000000000..19010ad5ed --- /dev/null +++ b/sda/cmd/download/database/cache.go @@ -0,0 +1,335 @@ +// Package database provides database operations for the download service. +package database + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "time" + + "github.com/dgraph-io/ristretto" + log "github.com/sirupsen/logrus" +) + +// CacheConfig holds configuration for the database cache. +type CacheConfig struct { + // FileTTL is the time-to-live for file query results. + FileTTL time.Duration + // PermissionTTL is the time-to-live for permission check results. + PermissionTTL time.Duration + // DatasetTTL is the time-to-live for dataset query results. + DatasetTTL time.Duration +} + +// DefaultCacheConfig returns sensible defaults for cache configuration. +func DefaultCacheConfig() CacheConfig { + return CacheConfig{ + FileTTL: 5 * time.Minute, + PermissionTTL: 2 * time.Minute, + DatasetTTL: 5 * time.Minute, + } +} + +// CachedDB wraps a Database implementation with ristretto caching. +// It implements the Database interface and caches query results +// to reduce database roundtrips, particularly beneficial for streaming +// use cases where the same file may be requested multiple times. +type CachedDB struct { + db Database + cache *ristretto.Cache + config CacheConfig +} + +// NewCachedDB creates a new CachedDB wrapping the provided database. +// The cache is configured with sensible defaults for a download service: +// - NumCounters: 1e6 (recommended: expected max items * 10) +// - MaxCost: 100000 (max items, each with cost 1) +// - BufferItems: 64 +func NewCachedDB(db Database, cfg CacheConfig) (*CachedDB, error) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + if err != nil { + return nil, fmt.Errorf("failed to create cache: %w", err) + } + + log.Info("Database cache initialized") + log.Debugf("Cache config: FileTTL=%v, PermissionTTL=%v, DatasetTTL=%v", + cfg.FileTTL, cfg.PermissionTTL, cfg.DatasetTTL) + + return &CachedDB{ + db: db, + cache: cache, + config: cfg, + }, nil +} + +// Ping verifies the database connection is alive. +// This delegates directly to the underlying database without caching. +func (c *CachedDB) Ping(ctx context.Context) error { + return c.db.Ping(ctx) +} + +// Close closes the underlying database connection and the cache. +func (c *CachedDB) Close() error { + c.cache.Close() + + return c.db.Close() +} + +// GetAllDatasets returns all datasets (for allow-all-data mode). +// Results are cached with DatasetTTL. +func (c *CachedDB) GetAllDatasets(ctx context.Context) ([]Dataset, error) { + const key = "datasets:all" + + if val, found := c.cache.Get(key); found { + if rval, ok := val.([]Dataset); ok { + log.Debug("cache hit: GetAllDatasets") + + return rval, nil + } + } + + log.Debug("cache miss: GetAllDatasets") + datasets, err := c.db.GetAllDatasets(ctx) + if err != nil { + return nil, err + } + + c.cache.SetWithTTL(key, datasets, 1, c.config.DatasetTTL) + + return datasets, nil +} + +// GetDatasetIDsByUser returns dataset IDs where the user is the submission_user. +// Results are cached per user with DatasetTTL. +func (c *CachedDB) GetDatasetIDsByUser(ctx context.Context, user string) ([]string, error) { + key := "datasets:user:" + user + + if val, found := c.cache.Get(key); found { + if rval, ok := val.([]string); ok { + log.Debugf("cache hit: GetDatasetIDsByUser(%s)", user) + + return rval, nil + } + } + + log.Debugf("cache miss: GetDatasetIDsByUser(%s)", user) + datasetIDs, err := c.db.GetDatasetIDsByUser(ctx, user) + if err != nil { + return nil, err + } + + c.cache.SetWithTTL(key, datasetIDs, 1, c.config.DatasetTTL) + + return datasetIDs, nil +} + +// GetUserDatasets returns datasets the user has access to. +// Results are cached based on the hash of dataset IDs with DatasetTTL. +func (c *CachedDB) GetUserDatasets(ctx context.Context, datasetIDs []string) ([]Dataset, error) { + key := "datasets:ids:" + hashStrings(datasetIDs) + + if val, found := c.cache.Get(key); found { + if rval, ok := val.([]Dataset); ok { + log.Debug("cache hit: GetUserDatasets") + + return rval, nil + } + } + + log.Debug("cache miss: GetUserDatasets") + datasets, err := c.db.GetUserDatasets(ctx, datasetIDs) + if err != nil { + return nil, err + } + + c.cache.SetWithTTL(key, datasets, 1, c.config.DatasetTTL) + + return datasets, nil +} + +// GetDatasetInfo returns metadata for a specific dataset. +// Results are cached with DatasetTTL. +func (c *CachedDB) GetDatasetInfo(ctx context.Context, datasetID string) (*DatasetInfo, error) { + key := "dataset:info:" + datasetID + + if val, found := c.cache.Get(key); found { + if rval, ok := val.(*DatasetInfo); ok { + log.Debugf("cache hit: GetDatasetInfo(%s)", datasetID) + + return rval, nil + } + } + + log.Debugf("cache miss: GetDatasetInfo(%s)", datasetID) + info, err := c.db.GetDatasetInfo(ctx, datasetID) + if err != nil { + return nil, err + } + + if info != nil { + c.cache.SetWithTTL(key, info, 1, c.config.DatasetTTL) + } + + return info, nil +} + +// GetDatasetFiles returns files in a dataset. +// Results are cached with DatasetTTL. +func (c *CachedDB) GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { + key := "dataset:files:" + datasetID + + if val, found := c.cache.Get(key); found { + if rval, ok := val.([]File); ok { + log.Debugf("cache hit: GetDatasetFiles(%s)", datasetID) + + return rval, nil + } + } + + log.Debugf("cache miss: GetDatasetFiles(%s)", datasetID) + files, err := c.db.GetDatasetFiles(ctx, datasetID) + if err != nil { + return nil, err + } + + c.cache.SetWithTTL(key, files, 1, c.config.DatasetTTL) + + return files, nil +} + +// GetFileByID returns file information by stable ID. +// Results are cached with FileTTL. This is particularly important +// for streaming use cases where the same file may be requested +// multiple times (e.g., range requests). +func (c *CachedDB) GetFileByID(ctx context.Context, fileID string) (*File, error) { + key := "file:id:" + fileID + + if val, found := c.cache.Get(key); found { + if rval, ok := val.(*File); ok { + log.Debugf("cache hit: GetFileByID(%s)", fileID) + + return rval, nil + } + } + + log.Debugf("cache miss: GetFileByID(%s)", fileID) + file, err := c.db.GetFileByID(ctx, fileID) + if err != nil { + return nil, err + } + + if file != nil { + c.cache.SetWithTTL(key, file, 1, c.config.FileTTL) + } + + return file, nil +} + +// GetFileByPath returns file information by dataset and submitted path. +// Results are cached with FileTTL. +func (c *CachedDB) GetFileByPath(ctx context.Context, datasetID, filePath string) (*File, error) { + key := "file:path:" + datasetID + ":" + filePath + + if val, found := c.cache.Get(key); found { + if rval, ok := val.(*File); ok { + log.Debugf("cache hit: GetFileByPath(%s, %s)", datasetID, filePath) + + return rval, nil + } + } + + log.Debugf("cache miss: GetFileByPath(%s, %s)", datasetID, filePath) + file, err := c.db.GetFileByPath(ctx, datasetID, filePath) + if err != nil { + return nil, err + } + + if file != nil { + c.cache.SetWithTTL(key, file, 1, c.config.FileTTL) + } + + return file, nil +} + +// CheckFilePermission verifies the user has access to download a file. +// Results are cached with PermissionTTL. The cache key includes a hash +// of the sorted datasetIDs to ensure user-scoped caching. +func (c *CachedDB) CheckFilePermission(ctx context.Context, fileID string, datasetIDs []string) (bool, error) { + key := "perm:" + fileID + ":" + hashStrings(datasetIDs) + + if val, found := c.cache.Get(key); found { + if rval, ok := val.(bool); ok { + log.Debugf("cache hit: CheckFilePermission(%s)", fileID) + + return rval, nil + } + } + + log.Debugf("cache miss: CheckFilePermission(%s)", fileID) + hasPermission, err := c.db.CheckFilePermission(ctx, fileID, datasetIDs) + if err != nil { + return false, err + } + + c.cache.SetWithTTL(key, hasPermission, 1, c.config.PermissionTTL) + + return hasPermission, nil +} + +// CheckDatasetExists checks if a dataset with the given stable_id exists. +// Results are cached with DatasetTTL. +func (c *CachedDB) CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { + key := "dataset:exists:" + datasetID + + if val, found := c.cache.Get(key); found { + if rval, ok := val.(bool); ok { + log.Debugf("cache hit: CheckDatasetExists(%s)", datasetID) + + return rval, nil + } + } + + log.Debugf("cache miss: CheckDatasetExists(%s)", datasetID) + exists, err := c.db.CheckDatasetExists(ctx, datasetID) + if err != nil { + return false, err + } + + c.cache.SetWithTTL(key, exists, 1, c.config.DatasetTTL) + + return exists, nil +} + +// GetDatasetFilesPaginated delegates to the underlying database without caching. +// Paginated queries use ephemeral cursors, making caching impractical. +func (c *CachedDB) GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) { + return c.db.GetDatasetFilesPaginated(ctx, datasetID, opts) +} + +// hashStrings creates a deterministic hash of a string slice. +// The slice is sorted before hashing to ensure consistent keys +// regardless of input order. +func hashStrings(strs []string) string { + if len(strs) == 0 { + return "empty" + } + + // Sort a copy to avoid modifying the input + sorted := make([]string, len(strs)) + copy(sorted, strs) + sort.Strings(sorted) + + h := sha256.New() + for _, s := range sorted { + h.Write([]byte(s)) + h.Write([]byte{0}) // Separator to avoid collision between ["ab", "c"] and ["a", "bc"] + } + + return hex.EncodeToString(h.Sum(nil))[:24] // Use first 24 chars (96 bits) for low collision probability +} diff --git a/sda/cmd/download/database/cache_test.go b/sda/cmd/download/database/cache_test.go new file mode 100644 index 0000000000..32fbd295db --- /dev/null +++ b/sda/cmd/download/database/cache_test.go @@ -0,0 +1,573 @@ +package database + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" +) + +// MockDatabase is a mock implementation of the Database interface +type MockDatabase struct { + mock.Mock +} + +func (m *MockDatabase) Ping(ctx context.Context) error { + args := m.Called(ctx) + + return args.Error(0) +} + +func (m *MockDatabase) Close() error { + args := m.Called() + + return args.Error(0) +} + +func (m *MockDatabase) GetAllDatasets(ctx context.Context) ([]Dataset, error) { + args := m.Called(ctx) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]Dataset), args.Error(1) +} + +func (m *MockDatabase) GetDatasetIDsByUser(ctx context.Context, user string) ([]string, error) { + args := m.Called(ctx, user) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]string), args.Error(1) +} + +func (m *MockDatabase) GetUserDatasets(ctx context.Context, datasetIDs []string) ([]Dataset, error) { + args := m.Called(ctx, datasetIDs) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]Dataset), args.Error(1) +} + +func (m *MockDatabase) GetDatasetInfo(ctx context.Context, datasetID string) (*DatasetInfo, error) { + args := m.Called(ctx, datasetID) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*DatasetInfo), args.Error(1) +} + +func (m *MockDatabase) GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { + args := m.Called(ctx, datasetID) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]File), args.Error(1) +} + +func (m *MockDatabase) GetFileByID(ctx context.Context, fileID string) (*File, error) { + args := m.Called(ctx, fileID) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*File), args.Error(1) +} + +func (m *MockDatabase) GetFileByPath(ctx context.Context, datasetID, filePath string) (*File, error) { + args := m.Called(ctx, datasetID, filePath) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).(*File), args.Error(1) +} + +func (m *MockDatabase) CheckFilePermission(ctx context.Context, fileID string, datasetIDs []string) (bool, error) { + args := m.Called(ctx, fileID, datasetIDs) + + return args.Bool(0), args.Error(1) +} + +func (m *MockDatabase) CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { + args := m.Called(ctx, datasetID) + + return args.Bool(0), args.Error(1) +} + +func (m *MockDatabase) GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) { + args := m.Called(ctx, datasetID, opts) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]File), args.Error(1) +} + +func TestNewCachedDB(t *testing.T) { + mockDB := new(MockDatabase) + cfg := DefaultCacheConfig() + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + require.NotNil(t, cachedDB) + + mockDB.On("Close").Return(nil) + err = cachedDB.Close() + assert.NoError(t, err) + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetFileByID_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + expectedFile := &File{ + ID: "file123", + DatasetID: "dataset1", + SubmittedPath: "/path/to/file.txt", + ArchiveSize: 1024, + DecryptedSize: 1000, + } + + // First call should hit the database + mockDB.On("GetFileByID", ctx, "file123").Return(expectedFile, nil).Once() + + file1, err := cachedDB.GetFileByID(ctx, "file123") + require.NoError(t, err) + assert.Equal(t, expectedFile, file1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit the cache (no additional DB call) + file2, err := cachedDB.GetFileByID(ctx, "file123") + require.NoError(t, err) + assert.Equal(t, expectedFile, file2) + + // Verify the database was only called once + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetFileByID_NotFound(t *testing.T) { + mockDB := new(MockDatabase) + cfg := DefaultCacheConfig() + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + + // Database returns nil for not found + mockDB.On("GetFileByID", ctx, "nonexistent").Return(nil, nil).Once() + + file, err := cachedDB.GetFileByID(ctx, "nonexistent") + require.NoError(t, err) + assert.Nil(t, file) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetFileByID_Error(t *testing.T) { + mockDB := new(MockDatabase) + cfg := DefaultCacheConfig() + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + expectedErr := errors.New("database error") + + mockDB.On("GetFileByID", ctx, "file123").Return(nil, expectedErr).Once() + + file, err := cachedDB.GetFileByID(ctx, "file123") + assert.Error(t, err) + assert.Equal(t, expectedErr, err) + assert.Nil(t, file) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetFileByPath_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + expectedFile := &File{ + ID: "file456", + DatasetID: "dataset1", + SubmittedPath: "/path/to/file.txt", + } + + // First call should hit the database + mockDB.On("GetFileByPath", ctx, "dataset1", "/path/to/file.txt").Return(expectedFile, nil).Once() + + file1, err := cachedDB.GetFileByPath(ctx, "dataset1", "/path/to/file.txt") + require.NoError(t, err) + assert.Equal(t, expectedFile, file1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit the cache + file2, err := cachedDB.GetFileByPath(ctx, "dataset1", "/path/to/file.txt") + require.NoError(t, err) + assert.Equal(t, expectedFile, file2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_CheckFilePermission_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + datasetIDs := []string{"dataset1", "dataset2"} + + // First call should hit the database + mockDB.On("CheckFilePermission", ctx, "file123", datasetIDs).Return(true, nil).Once() + + hasPermission1, err := cachedDB.CheckFilePermission(ctx, "file123", datasetIDs) + require.NoError(t, err) + assert.True(t, hasPermission1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit the cache + hasPermission2, err := cachedDB.CheckFilePermission(ctx, "file123", datasetIDs) + require.NoError(t, err) + assert.True(t, hasPermission2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_CheckFilePermission_DifferentDatasets(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + datasetIDs1 := []string{"dataset1", "dataset2"} + datasetIDs2 := []string{"dataset3"} + + // Each unique set of datasets should result in a separate cache entry + mockDB.On("CheckFilePermission", ctx, "file123", datasetIDs1).Return(true, nil).Once() + mockDB.On("CheckFilePermission", ctx, "file123", datasetIDs2).Return(false, nil).Once() + + hasPermission1, err := cachedDB.CheckFilePermission(ctx, "file123", datasetIDs1) + require.NoError(t, err) + assert.True(t, hasPermission1) + + hasPermission2, err := cachedDB.CheckFilePermission(ctx, "file123", datasetIDs2) + require.NoError(t, err) + assert.False(t, hasPermission2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_CheckFilePermission_OrderIndependent(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + // Different order, same datasets - should produce same cache key + datasetIDs1 := []string{"dataset2", "dataset1"} + datasetIDs2 := []string{"dataset1", "dataset2"} + + // Only one DB call expected because cache key should be the same + mockDB.On("CheckFilePermission", ctx, "file123", datasetIDs1).Return(true, nil).Once() + + hasPermission1, err := cachedDB.CheckFilePermission(ctx, "file123", datasetIDs1) + require.NoError(t, err) + assert.True(t, hasPermission1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call with different order should hit cache + hasPermission2, err := cachedDB.CheckFilePermission(ctx, "file123", datasetIDs2) + require.NoError(t, err) + assert.True(t, hasPermission2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetDatasetInfo_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + expectedInfo := &DatasetInfo{ + ID: "dataset1", + Title: "Test Dataset", + FileCount: 10, + TotalSize: 1024000, + } + + mockDB.On("GetDatasetInfo", ctx, "dataset1").Return(expectedInfo, nil).Once() + + info1, err := cachedDB.GetDatasetInfo(ctx, "dataset1") + require.NoError(t, err) + assert.Equal(t, expectedInfo, info1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit cache + info2, err := cachedDB.GetDatasetInfo(ctx, "dataset1") + require.NoError(t, err) + assert.Equal(t, expectedInfo, info2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetDatasetFiles_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + expectedFiles := []File{ + {ID: "file1", DatasetID: "dataset1", SubmittedPath: "/file1.txt"}, + {ID: "file2", DatasetID: "dataset1", SubmittedPath: "/file2.txt"}, + } + + mockDB.On("GetDatasetFiles", ctx, "dataset1").Return(expectedFiles, nil).Once() + + files1, err := cachedDB.GetDatasetFiles(ctx, "dataset1") + require.NoError(t, err) + assert.Equal(t, expectedFiles, files1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit cache + files2, err := cachedDB.GetDatasetFiles(ctx, "dataset1") + require.NoError(t, err) + assert.Equal(t, expectedFiles, files2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetAllDatasets_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + expectedDatasets := []Dataset{ + {ID: "dataset1", Title: "Dataset 1"}, + {ID: "dataset2", Title: "Dataset 2"}, + } + + mockDB.On("GetAllDatasets", ctx).Return(expectedDatasets, nil).Once() + + datasets1, err := cachedDB.GetAllDatasets(ctx) + require.NoError(t, err) + assert.Equal(t, expectedDatasets, datasets1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit cache + datasets2, err := cachedDB.GetAllDatasets(ctx) + require.NoError(t, err) + assert.Equal(t, expectedDatasets, datasets2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetDatasetIDsByUser_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + expectedIDs := []string{"dataset1", "dataset2"} + + mockDB.On("GetDatasetIDsByUser", ctx, "user@example.com").Return(expectedIDs, nil).Once() + + ids1, err := cachedDB.GetDatasetIDsByUser(ctx, "user@example.com") + require.NoError(t, err) + assert.Equal(t, expectedIDs, ids1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit cache + ids2, err := cachedDB.GetDatasetIDsByUser(ctx, "user@example.com") + require.NoError(t, err) + assert.Equal(t, expectedIDs, ids2) + + mockDB.AssertExpectations(t) +} + +func TestCachedDB_GetUserDatasets_CacheHit(t *testing.T) { + mockDB := new(MockDatabase) + cfg := CacheConfig{ + FileTTL: 1 * time.Minute, + PermissionTTL: 1 * time.Minute, + DatasetTTL: 1 * time.Minute, + } + + cachedDB, err := NewCachedDB(mockDB, cfg) + require.NoError(t, err) + + ctx := context.Background() + datasetIDs := []string{"dataset1", "dataset2"} + expectedDatasets := []Dataset{ + {ID: "dataset1", Title: "Dataset 1"}, + {ID: "dataset2", Title: "Dataset 2"}, + } + + mockDB.On("GetUserDatasets", ctx, datasetIDs).Return(expectedDatasets, nil).Once() + + datasets1, err := cachedDB.GetUserDatasets(ctx, datasetIDs) + require.NoError(t, err) + assert.Equal(t, expectedDatasets, datasets1) + + // Wait for ristretto to process the set + time.Sleep(10 * time.Millisecond) + + // Second call should hit cache + datasets2, err := cachedDB.GetUserDatasets(ctx, datasetIDs) + require.NoError(t, err) + assert.Equal(t, expectedDatasets, datasets2) + + mockDB.AssertExpectations(t) +} + +func TestHashStrings(t *testing.T) { + tests := []struct { + name string + input1 []string + input2 []string + equal bool + }{ + { + name: "same order", + input1: []string{"a", "b", "c"}, + input2: []string{"a", "b", "c"}, + equal: true, + }, + { + name: "different order same elements", + input1: []string{"c", "a", "b"}, + input2: []string{"a", "b", "c"}, + equal: true, + }, + { + name: "different elements", + input1: []string{"a", "b", "c"}, + input2: []string{"a", "b", "d"}, + equal: false, + }, + { + name: "empty slices", + input1: []string{}, + input2: []string{}, + equal: true, + }, + { + name: "single element", + input1: []string{"single"}, + input2: []string{"single"}, + equal: true, + }, + { + name: "collision prevention - different concatenations", + input1: []string{"ab", "c"}, + input2: []string{"a", "bc"}, + equal: false, // Should not collide due to separator + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + hash1 := hashStrings(tt.input1) + hash2 := hashStrings(tt.input2) + + if tt.equal { + assert.Equal(t, hash1, hash2, "hashes should be equal for %v and %v", tt.input1, tt.input2) + } else { + assert.NotEqual(t, hash1, hash2, "hashes should be different for %v and %v", tt.input1, tt.input2) + } + }) + } +} + +func TestDefaultCacheConfig(t *testing.T) { + cfg := DefaultCacheConfig() + assert.Equal(t, 5*time.Minute, cfg.FileTTL) + assert.Equal(t, 2*time.Minute, cfg.PermissionTTL) + assert.Equal(t, 5*time.Minute, cfg.DatasetTTL) +} diff --git a/sda/cmd/download/database/database.go b/sda/cmd/download/database/database.go new file mode 100644 index 0000000000..768ff72312 --- /dev/null +++ b/sda/cmd/download/database/database.go @@ -0,0 +1,764 @@ +// Package database provides database operations for the download service. +package database + +import ( + "context" + "database/sql" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/lib/pq" + "github.com/neicnordic/sensitive-data-archive/cmd/download/config" + log "github.com/sirupsen/logrus" +) + +// Query name constants +const ( + getAllDatasetsQuery = "getAllDatasets" + getDatasetIDsByUserQuery = "getDatasetIDsByUser" + getUserDatasetsQuery = "getUserDatasets" + getDatasetInfoQuery = "getDatasetInfo" + getDatasetFilesQuery = "getDatasetFiles" + getFileByIDQuery = "getFileByID" + getFileByPathQuery = "getFileByPath" + checkFilePermissionQuery = "checkFilePermission" + checkDatasetExistsQuery = "checkDatasetExists" + getDatasetFilesPageQuery = "getDatasetFilesPage" + getDatasetFilesPageByPathQuery = "getDatasetFilesPageByPath" + getDatasetFilesPageByPrefixQuery = "getDatasetFilesPageByPrefix" +) + +// paginatedFileBase is the shared SELECT+JOIN+LATERAL block for keyset-paginated +// dataset file queries. Each variant appends its own WHERE filter and LIMIT clause. +const paginatedFileBase = ` + SELECT f.stable_id, f.submission_file_path, f.archive_file_size, + f.decrypted_file_size, cs.checksums + FROM sda.files f + INNER JOIN sda.file_dataset fd ON f.id = fd.file_id + INNER JOIN sda.datasets d ON fd.dataset_id = d.id + LEFT JOIN LATERAL ( + SELECT json_agg(json_build_object('type', lower(c.type::text), 'checksum', c.checksum)) AS checksums + FROM sda.checksums c WHERE c.file_id = f.id AND c.source = 'UNENCRYPTED' + ) cs ON true + WHERE d.stable_id = $1 + AND f.stable_id IS NOT NULL` + +// queries contains all SQL queries used by the download service. +// These are prepared at startup to verify correctness and improve performance. +var queries = map[string]string{ + // getAllDatasets returns all datasets (for allow-all-data mode) + getAllDatasetsQuery: ` + SELECT DISTINCT d.stable_id, d.title, d.description, d.created_at + FROM sda.datasets d + ORDER BY d.created_at DESC`, + + // getDatasetIDsByUser returns dataset stable_ids where the user is the submission_user + // This follows the data ownership model used by the API service + getDatasetIDsByUserQuery: ` + SELECT DISTINCT d.stable_id + FROM sda.datasets d + INNER JOIN sda.file_dataset fd ON d.id = fd.dataset_id + INNER JOIN sda.files f ON fd.file_id = f.id + WHERE f.submission_user = $1 AND f.stable_id IS NOT NULL`, + + // getUserDatasets returns datasets where the stable_id matches any of the allowed dataset IDs + getUserDatasetsQuery: ` + SELECT DISTINCT d.stable_id, d.title, d.description, d.created_at + FROM sda.datasets d + WHERE d.stable_id = ANY($1) + ORDER BY d.created_at DESC`, + + getDatasetInfoQuery: ` + SELECT + d.stable_id, + d.title, + d.description, + d.created_at, + COUNT(f.id) as file_count, + COALESCE(SUM(f.decrypted_file_size), 0) as total_size + FROM sda.datasets d + LEFT JOIN sda.file_dataset fd ON d.id = fd.dataset_id + LEFT JOIN sda.files f ON fd.file_id = f.id + WHERE d.stable_id = $1 + GROUP BY d.id, d.stable_id, d.title, d.description, d.created_at`, + + // getDatasetFiles returns files in a dataset + // Checksums are stored in sda.checksums table, not in files table + getDatasetFilesQuery: ` + SELECT + f.stable_id, + d.stable_id as dataset_id, + f.submission_file_path, + f.archive_file_path, + f.archive_location, + f.archive_file_size, + f.decrypted_file_size, + c.checksum as decrypted_checksum, + c.type as decrypted_checksum_type + FROM sda.files f + INNER JOIN sda.file_dataset fd ON f.id = fd.file_id + INNER JOIN sda.datasets d ON fd.dataset_id = d.id + LEFT JOIN sda.checksums c ON f.id = c.file_id AND c.source = 'UNENCRYPTED' + WHERE d.stable_id = $1 AND f.stable_id IS NOT NULL + ORDER BY f.submission_file_path`, + + getFileByIDQuery: ` + SELECT + f.stable_id, + d.stable_id as dataset_id, + f.submission_file_path, + f.archive_file_path, + f.archive_location, + f.archive_file_size, + f.decrypted_file_size, + c.checksum as decrypted_checksum, + c.type as decrypted_checksum_type, + f.header + FROM sda.files f + INNER JOIN sda.file_dataset fd ON f.id = fd.file_id + INNER JOIN sda.datasets d ON fd.dataset_id = d.id + LEFT JOIN sda.checksums c ON f.id = c.file_id AND c.source = 'UNENCRYPTED' + WHERE f.stable_id = $1`, + + getFileByPathQuery: ` + SELECT + f.stable_id, + d.stable_id as dataset_id, + f.submission_file_path, + f.archive_file_path, + f.archive_location, + f.archive_file_size, + f.decrypted_file_size, + c.checksum as decrypted_checksum, + c.type as decrypted_checksum_type, + f.header + FROM sda.files f + INNER JOIN sda.file_dataset fd ON f.id = fd.file_id + INNER JOIN sda.datasets d ON fd.dataset_id = d.id + LEFT JOIN sda.checksums c ON f.id = c.file_id AND c.source = 'UNENCRYPTED' + WHERE d.stable_id = $1 AND f.submission_file_path = $2`, + + // checkFilePermission verifies user has access to the file's dataset + // by checking if the dataset stable_id is in the user's visas + checkFilePermissionQuery: ` + SELECT EXISTS( + SELECT 1 + FROM sda.files f + INNER JOIN sda.file_dataset fd ON f.id = fd.file_id + INNER JOIN sda.datasets d ON fd.dataset_id = d.id + WHERE f.stable_id = $1 AND d.stable_id = ANY($2) + )`, + + // checkDatasetExists checks if a dataset with the given stable_id exists + checkDatasetExistsQuery: ` + SELECT EXISTS( + SELECT 1 + FROM sda.datasets d + WHERE d.stable_id = $1 + )`, + + // Keyset-paginated file queries compose from paginatedFileBase (defined below). + + // getDatasetFilesPage returns paginated files in a dataset (no path filter). + // Keyset cursor on (submission_file_path, stable_id). $2='' means first page. + getDatasetFilesPageQuery: paginatedFileBase + ` + AND ($2 = '' OR (f.submission_file_path, f.stable_id) > ($2, $3)) + ORDER BY f.submission_file_path, f.stable_id + LIMIT $4`, + + // getDatasetFilesPageByPath returns a file by exact path (at most 1 result). + getDatasetFilesPageByPathQuery: paginatedFileBase + ` + AND f.submission_file_path = $2 + LIMIT $3`, + + // getDatasetFilesPageByPrefix returns paginated files matching a path prefix. + // Keyset cursor on (submission_file_path, stable_id). $3='' means first page. + getDatasetFilesPageByPrefixQuery: paginatedFileBase + ` + AND f.submission_file_path LIKE $2 ESCAPE '\' + AND ($3 = '' OR (f.submission_file_path, f.stable_id) > ($3, $4)) + ORDER BY f.submission_file_path, f.stable_id + LIMIT $5`, +} + +// Checksum represents a file checksum with its algorithm type. +type Checksum struct { + Type string `json:"type"` + Checksum string `json:"checksum"` +} + +// FileListOptions controls filtering and pagination for GetDatasetFilesPaginated. +type FileListOptions struct { + FilePath string // exact match (mutually exclusive with PathPrefix) + PathPrefix string // prefix match + Limit int // pageSize + 1 (fetch one extra to detect next page) + CursorPath string // last-seen submission_file_path ("" for first page) + CursorID string // tie-breaker: last-seen stable_id ("" for first page) +} + +// Database defines the interface for download service database operations. +type Database interface { + // Ping verifies the database connection is alive. + Ping(ctx context.Context) error + + // Close closes the database connection. + Close() error + + // GetAllDatasets returns all datasets (for allow-all-data mode). + GetAllDatasets(ctx context.Context) ([]Dataset, error) + + // GetDatasetIDsByUser returns dataset IDs where the user is the submission_user (data owner). + // This follows the data ownership model - users have access to datasets containing files they submitted. + GetDatasetIDsByUser(ctx context.Context, user string) ([]string, error) + + // GetUserDatasets returns datasets the user has access to based on their allowed dataset IDs. + GetUserDatasets(ctx context.Context, datasetIDs []string) ([]Dataset, error) + + // GetDatasetInfo returns metadata for a specific dataset. + GetDatasetInfo(ctx context.Context, datasetID string) (*DatasetInfo, error) + + // GetDatasetFiles returns files in a dataset. + GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) + + // GetFileByID returns file information by stable ID. + GetFileByID(ctx context.Context, fileID string) (*File, error) + + // GetFileByPath returns file information by dataset and submitted path. + GetFileByPath(ctx context.Context, datasetID, filePath string) (*File, error) + + // CheckFilePermission verifies the user has access to download a file. + CheckFilePermission(ctx context.Context, fileID string, datasetIDs []string) (bool, error) + + // CheckDatasetExists checks if a dataset with the given stable_id exists. + CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) + + // GetDatasetFilesPaginated returns files in a dataset with keyset cursor pagination. + // Files are returned with aggregated checksums. Use FileListOptions to filter and paginate. + GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) +} + +// Dataset represents a dataset the user has access to. +type Dataset struct { + ID string `json:"id"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + CreatedAt time.Time `json:"createdAt"` +} + +// DatasetInfo contains metadata about a dataset. +type DatasetInfo struct { + ID string `json:"id"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` + FileCount int `json:"fileCount"` + TotalSize int64 `json:"totalSize"` + CreatedAt time.Time `json:"createdAt"` +} + +// File represents a file in the archive. +type File struct { + ID string `json:"fileId"` + DatasetID string `json:"datasetId"` + SubmittedPath string `json:"filePath"` + ArchivePath string `json:"-"` + ArchiveLocation string `json:"-"` // Storage backend location (e.g., "s3:9000/archive" or "/archive") + ArchiveSize int64 `json:"archiveSize"` + DecryptedSize int64 `json:"decryptedSize"` + DecryptedChecksum string `json:"decryptedChecksum"` + DecryptedChecksumType string `json:"decryptedChecksumType"` + Header []byte `json:"-"` + Checksums []Checksum `json:"checksums,omitempty"` // Aggregated checksums (from paginated queries) +} + +// PostgresDB implements the Database interface for PostgreSQL. +type PostgresDB struct { + db *sql.DB + preparedStatements map[string]*sql.Stmt +} + +var db Database + +// RegisterDatabase registers the database implementation to be used. +func RegisterDatabase(d Database) { + db = d +} + +// GetDB returns the registered database instance. +func GetDB() Database { + return db +} + +// Init initializes the database connection using configuration values. +// All queries are prepared at startup to verify correctness and improve runtime performance. +func Init() error { + connStr := fmt.Sprintf( + "host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", + config.DBHost(), + config.DBPort(), + config.DBUser(), + config.DBPassword(), + config.DBDatabase(), + config.DBSSLMode(), + ) + + if config.DBCACert() != "" { + connStr += fmt.Sprintf(" sslrootcert=%s", config.DBCACert()) + } + + sqlDB, err := sql.Open("postgres", connStr) + if err != nil { + return fmt.Errorf("failed to open database connection: %w", err) + } + + // Configure connection pool + sqlDB.SetMaxOpenConns(10) + sqlDB.SetMaxIdleConns(5) + sqlDB.SetConnMaxLifetime(5 * time.Minute) + + // Verify connection + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if err := sqlDB.PingContext(ctx); err != nil { + return fmt.Errorf("failed to ping database: %w", err) + } + + log.Info("Database connection established") + + // Prepare all statements to verify correctness and improve performance + preparedStatements := make(map[string]*sql.Stmt) + for queryName, query := range queries { + preparedStmt, err := sqlDB.PrepareContext(ctx, query) + if err != nil { + log.Errorf("failed to prepare query: %s, due to: %v", queryName, err) + + return errors.Join(fmt.Errorf("failed to prepare query: %s", queryName), err) + } + preparedStatements[queryName] = preparedStmt + log.Debugf("Prepared query: %s", queryName) + } + + log.Infof("Successfully prepared %d database queries", len(preparedStatements)) + + RegisterDatabase(&PostgresDB{ + db: sqlDB, + preparedStatements: preparedStatements, + }) + + return nil +} + +// Close closes the database connection. +func Close() error { + if db != nil { + return db.Close() + } + + return nil +} + +// Ping verifies the database connection is alive. +func (p *PostgresDB) Ping(ctx context.Context) error { + return p.db.PingContext(ctx) +} + +// Close closes the PostgreSQL database connection and all prepared statements. +func (p *PostgresDB) Close() error { + // Close all prepared statements first + for name, stmt := range p.preparedStatements { + if err := stmt.Close(); err != nil { + log.Warnf("failed to close prepared statement %s: %v", name, err) + } + } + + return p.db.Close() +} + +// GetAllDatasets returns all datasets (for allow-all-data mode). +func (p *PostgresDB) GetAllDatasets(ctx context.Context) ([]Dataset, error) { + stmt := p.preparedStatements[getAllDatasetsQuery] + rows, err := stmt.QueryContext(ctx) + if err != nil { + return nil, fmt.Errorf("failed to query all datasets: %w", err) + } + defer rows.Close() + + var datasets []Dataset + for rows.Next() { + var d Dataset + var title, description sql.NullString + if err := rows.Scan(&d.ID, &title, &description, &d.CreatedAt); err != nil { + return nil, fmt.Errorf("failed to scan dataset row: %w", err) + } + d.Title = title.String + d.Description = description.String + datasets = append(datasets, d) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating dataset rows: %w", err) + } + + return datasets, nil +} + +// GetDatasetIDsByUser returns dataset IDs where the user is the submission_user (data owner). +func (p *PostgresDB) GetDatasetIDsByUser(ctx context.Context, user string) ([]string, error) { + stmt := p.preparedStatements[getDatasetIDsByUserQuery] + rows, err := stmt.QueryContext(ctx, user) + if err != nil { + return nil, fmt.Errorf("failed to query datasets by user: %w", err) + } + defer rows.Close() + + var datasetIDs []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("failed to scan dataset ID: %w", err) + } + datasetIDs = append(datasetIDs, id) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating dataset ID rows: %w", err) + } + + return datasetIDs, nil +} + +// GetUserDatasets returns datasets the user has access to based on their allowed dataset IDs. +func (p *PostgresDB) GetUserDatasets(ctx context.Context, datasetIDs []string) ([]Dataset, error) { + stmt := p.preparedStatements[getUserDatasetsQuery] + rows, err := stmt.QueryContext(ctx, pq.Array(datasetIDs)) + if err != nil { + return nil, fmt.Errorf("failed to query datasets: %w", err) + } + defer rows.Close() + + var datasets []Dataset + for rows.Next() { + var d Dataset + var title, description sql.NullString + if err := rows.Scan(&d.ID, &title, &description, &d.CreatedAt); err != nil { + return nil, fmt.Errorf("failed to scan dataset row: %w", err) + } + d.Title = title.String + d.Description = description.String + datasets = append(datasets, d) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating dataset rows: %w", err) + } + + return datasets, nil +} + +// GetDatasetInfo returns metadata for a specific dataset. +func (p *PostgresDB) GetDatasetInfo(ctx context.Context, datasetID string) (*DatasetInfo, error) { + stmt := p.preparedStatements[getDatasetInfoQuery] + + var info DatasetInfo + var title, description sql.NullString + err := stmt.QueryRowContext(ctx, datasetID).Scan( + &info.ID, + &title, + &description, + &info.CreatedAt, + &info.FileCount, + &info.TotalSize, + ) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to query dataset info: %w", err) + } + + info.Title = title.String + info.Description = description.String + + return &info, nil +} + +// GetDatasetFiles returns files in a dataset. +func (p *PostgresDB) GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { + stmt := p.preparedStatements[getDatasetFilesQuery] + rows, err := stmt.QueryContext(ctx, datasetID) + if err != nil { + return nil, fmt.Errorf("failed to query dataset files: %w", err) + } + defer rows.Close() + + var files []File + for rows.Next() { + var f File + var archivePath, archiveLocation, decryptedChecksum, decryptedChecksumType sql.NullString + var archiveSize, decryptedSize sql.NullInt64 + if err := rows.Scan( + &f.ID, + &f.DatasetID, + &f.SubmittedPath, + &archivePath, + &archiveLocation, + &archiveSize, + &decryptedSize, + &decryptedChecksum, + &decryptedChecksumType, + ); err != nil { + return nil, fmt.Errorf("failed to scan file row: %w", err) + } + f.ArchivePath = archivePath.String + f.ArchiveLocation = archiveLocation.String + f.ArchiveSize = archiveSize.Int64 + f.DecryptedSize = decryptedSize.Int64 + f.DecryptedChecksum = decryptedChecksum.String + f.DecryptedChecksumType = decryptedChecksumType.String + files = append(files, f) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating file rows: %w", err) + } + + return files, nil +} + +// GetFileByID returns file information by stable ID. +func (p *PostgresDB) GetFileByID(ctx context.Context, fileID string) (*File, error) { + stmt := p.preparedStatements[getFileByIDQuery] + + var f File + var archivePath, archiveLocation, decryptedChecksum, decryptedChecksumType, headerHex sql.NullString + var archiveSize, decryptedSize sql.NullInt64 + err := stmt.QueryRowContext(ctx, fileID).Scan( + &f.ID, + &f.DatasetID, + &f.SubmittedPath, + &archivePath, + &archiveLocation, + &archiveSize, + &decryptedSize, + &decryptedChecksum, + &decryptedChecksumType, + &headerHex, + ) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to query file by ID: %w", err) + } + + f.ArchivePath = archivePath.String + f.ArchiveLocation = archiveLocation.String + f.ArchiveSize = archiveSize.Int64 + f.DecryptedSize = decryptedSize.Int64 + f.DecryptedChecksum = decryptedChecksum.String + f.DecryptedChecksumType = decryptedChecksumType.String + + // Header is stored as hex string in database, decode it. + // The ingest pipeline (sda/internal/database/db_functions.go:StoreHeader) stores the + // crypt4gh file header as a hex-encoded string using hex.EncodeToString(). + // After decoding, the header bytes represent the complete crypt4gh header which includes: + // - magic number (8 bytes: "crypt4gh") + // - version (4 bytes) + // - header packet count (4 bytes) + // - header packets (variable length, contains encryption info) + // NOTE: If you encounter "encoding/hex: invalid byte" errors, ensure the header was + // stored using hex encoding, not base64. The SDA pipeline consistently uses hex. + if headerHex.Valid && headerHex.String != "" { + f.Header, err = hex.DecodeString(headerHex.String) + if err != nil { + return nil, fmt.Errorf("failed to decode header from hex (expected hex-encoded string, got %d chars): %w", + len(headerHex.String), err) + } + // Validate header has minimum crypt4gh structure (magic + version + count = 16 bytes minimum) + if len(f.Header) < 16 { + return nil, fmt.Errorf("decoded header too small (%d bytes), expected at least 16 bytes for crypt4gh", len(f.Header)) + } + } + + return &f, nil +} + +// GetFileByPath returns file information by dataset and submitted path. +func (p *PostgresDB) GetFileByPath(ctx context.Context, datasetID, filePath string) (*File, error) { + stmt := p.preparedStatements[getFileByPathQuery] + + var f File + var archivePath, archiveLocation, decryptedChecksum, decryptedChecksumType, headerHex sql.NullString + var archiveSize, decryptedSize sql.NullInt64 + err := stmt.QueryRowContext(ctx, datasetID, filePath).Scan( + &f.ID, + &f.DatasetID, + &f.SubmittedPath, + &archivePath, + &archiveLocation, + &archiveSize, + &decryptedSize, + &decryptedChecksum, + &decryptedChecksumType, + &headerHex, + ) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to query file by path: %w", err) + } + + f.ArchivePath = archivePath.String + f.ArchiveLocation = archiveLocation.String + f.ArchiveSize = archiveSize.Int64 + f.DecryptedSize = decryptedSize.Int64 + f.DecryptedChecksum = decryptedChecksum.String + f.DecryptedChecksumType = decryptedChecksumType.String + + // Header is stored as hex string in database, decode it. + // See GetFileByID for details on the header format and encoding. + if headerHex.Valid && headerHex.String != "" { + f.Header, err = hex.DecodeString(headerHex.String) + if err != nil { + return nil, fmt.Errorf("failed to decode header from hex (expected hex-encoded string, got %d chars): %w", + len(headerHex.String), err) + } + // Validate header has minimum crypt4gh structure + if len(f.Header) < 16 { + return nil, fmt.Errorf("decoded header too small (%d bytes), expected at least 16 bytes for crypt4gh", len(f.Header)) + } + } + + return &f, nil +} + +// CheckFilePermission verifies the user has access to download a file. +func (p *PostgresDB) CheckFilePermission(ctx context.Context, fileID string, datasetIDs []string) (bool, error) { + stmt := p.preparedStatements[checkFilePermissionQuery] + + var hasPermission bool + err := stmt.QueryRowContext(ctx, fileID, pq.Array(datasetIDs)).Scan(&hasPermission) + if err != nil { + return false, fmt.Errorf("failed to check file permission: %w", err) + } + + return hasPermission, nil +} + +// CheckDatasetExists checks if a dataset with the given stable_id exists. +func (p *PostgresDB) CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { + stmt := p.preparedStatements[checkDatasetExistsQuery] + + var exists bool + err := stmt.QueryRowContext(ctx, datasetID).Scan(&exists) + if err != nil { + return false, fmt.Errorf("failed to check dataset existence: %w", err) + } + + return exists, nil +} + +// GetDatasetFilesPaginated returns files with keyset cursor pagination and aggregated checksums. +func (p *PostgresDB) GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) { + var rows *sql.Rows + var err error + + switch { + case opts.FilePath != "": + stmt := p.preparedStatements[getDatasetFilesPageByPathQuery] + rows, err = stmt.QueryContext(ctx, datasetID, opts.FilePath, opts.Limit) + case opts.PathPrefix != "": + stmt := p.preparedStatements[getDatasetFilesPageByPrefixQuery] + escaped := escapeLikePrefix(opts.PathPrefix) + rows, err = stmt.QueryContext(ctx, datasetID, escaped, opts.CursorPath, opts.CursorID, opts.Limit) + default: + stmt := p.preparedStatements[getDatasetFilesPageQuery] + rows, err = stmt.QueryContext(ctx, datasetID, opts.CursorPath, opts.CursorID, opts.Limit) + } + + if err != nil { + return nil, fmt.Errorf("failed to query paginated dataset files: %w", err) + } + defer rows.Close() + + var files []File + for rows.Next() { + var f File + var archiveSize, decryptedSize sql.NullInt64 + var checksumsJSON []byte + + if err := rows.Scan(&f.ID, &f.SubmittedPath, &archiveSize, &decryptedSize, &checksumsJSON); err != nil { + return nil, fmt.Errorf("failed to scan paginated file row: %w", err) + } + + f.ArchiveSize = archiveSize.Int64 + f.DecryptedSize = decryptedSize.Int64 + + if checksumsJSON != nil { + if err := json.Unmarshal(checksumsJSON, &f.Checksums); err != nil { + return nil, fmt.Errorf("failed to unmarshal checksums for file %s: %w", f.ID, err) + } + } + + files = append(files, f) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating paginated file rows: %w", err) + } + + return files, nil +} + +// escapeLikePrefix escapes SQL LIKE wildcards in a prefix and appends %. +func escapeLikePrefix(prefix string) string { + r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`) + + return r.Replace(prefix) + "%" +} + +// Package-level functions that delegate to the registered database + +// GetAllDatasets returns all datasets (for allow-all-data mode). +func GetAllDatasets(ctx context.Context) ([]Dataset, error) { + return db.GetAllDatasets(ctx) +} + +// GetUserDatasets returns datasets the user has access to based on their allowed dataset IDs. +func GetUserDatasets(ctx context.Context, datasetIDs []string) ([]Dataset, error) { + return db.GetUserDatasets(ctx, datasetIDs) +} + +// GetDatasetInfo returns metadata for a specific dataset. +func GetDatasetInfo(ctx context.Context, datasetID string) (*DatasetInfo, error) { + return db.GetDatasetInfo(ctx, datasetID) +} + +// GetDatasetFiles returns files in a dataset. +func GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { + return db.GetDatasetFiles(ctx, datasetID) +} + +// GetFileByID returns file information by stable ID. +func GetFileByID(ctx context.Context, fileID string) (*File, error) { + return db.GetFileByID(ctx, fileID) +} + +// GetFileByPath returns file information by dataset and submitted path. +func GetFileByPath(ctx context.Context, datasetID, filePath string) (*File, error) { + return db.GetFileByPath(ctx, datasetID, filePath) +} + +// CheckFilePermission verifies the user has access to download a file. +func CheckFilePermission(ctx context.Context, fileID string, visas []string) (bool, error) { + return db.CheckFilePermission(ctx, fileID, visas) +} + +// CheckDatasetExists checks if a dataset with the given stable_id exists. +func CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { + return db.CheckDatasetExists(ctx, datasetID) +} diff --git a/sda/cmd/download/database/database_test.go b/sda/cmd/download/database/database_test.go new file mode 100644 index 0000000000..324305dfb7 --- /dev/null +++ b/sda/cmd/download/database/database_test.go @@ -0,0 +1,839 @@ +package database + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/lib/pq" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setupMockDB creates a mock database with prepared statements for testing. +// It prepares all queries defined in the queries map and returns the mock for expectations. +func setupMockDB(t *testing.T) (*PostgresDB, sqlmock.Sqlmock, func()) { + t.Helper() + + // Use QueryMatcherEqual to avoid regex issues with SQL queries + mockDB, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) + require.NoError(t, err) + + // Prepare all statements + preparedStatements := make(map[string]*sql.Stmt) + for queryName, query := range queries { + mock.ExpectPrepare(query) + stmt, err := mockDB.Prepare(query) + require.NoError(t, err, "failed to prepare query: %s", queryName) + preparedStatements[queryName] = stmt + } + + db := &PostgresDB{ + db: mockDB, + preparedStatements: preparedStatements, + } + + cleanup := func() { + mockDB.Close() + } + + return db, mock, cleanup +} + +func TestGetAllDatasets(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"stable_id", "title", "description", "created_at"}). + AddRow("dataset-1", "Test Dataset", "Description", time.Now()). + AddRow("dataset-2", "Another Dataset", nil, time.Now()) + + mock.ExpectQuery(queries[getAllDatasetsQuery]). + WillReturnRows(rows) + + datasets, err := db.GetAllDatasets(context.Background()) + + assert.NoError(t, err) + assert.Len(t, datasets, 2) + assert.Equal(t, "dataset-1", datasets[0].ID) + assert.Equal(t, "dataset-2", datasets[1].ID) + + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetAllDatasets_Empty(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"stable_id", "title", "description", "created_at"}) + + mock.ExpectQuery(queries[getAllDatasetsQuery]). + WillReturnRows(rows) + + datasets, err := db.GetAllDatasets(context.Background()) + + assert.NoError(t, err) + assert.Empty(t, datasets) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetIDsByUser(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"stable_id"}). + AddRow("dataset-1"). + AddRow("dataset-2"). + AddRow("dataset-3") + + mock.ExpectQuery(queries[getDatasetIDsByUserQuery]). + WithArgs("test-user"). + WillReturnRows(rows) + + datasetIDs, err := db.GetDatasetIDsByUser(context.Background(), "test-user") + + assert.NoError(t, err) + assert.Len(t, datasetIDs, 3) + assert.Equal(t, "dataset-1", datasetIDs[0]) + assert.Equal(t, "dataset-2", datasetIDs[1]) + assert.Equal(t, "dataset-3", datasetIDs[2]) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetIDsByUser_Empty(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"stable_id"}) + + mock.ExpectQuery(queries[getDatasetIDsByUserQuery]). + WithArgs("test-user"). + WillReturnRows(rows) + + datasetIDs, err := db.GetDatasetIDsByUser(context.Background(), "test-user") + + assert.NoError(t, err) + assert.Empty(t, datasetIDs) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetUserDatasets(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + // Setup expected query - use the full query from the queries map + rows := sqlmock.NewRows([]string{"stable_id", "title", "description", "created_at"}). + AddRow("dataset-1", "Test Dataset", "Description", time.Now()). + AddRow("dataset-2", nil, nil, time.Now()) + + mock.ExpectQuery(queries[getUserDatasetsQuery]). + WithArgs(pq.Array([]string{"visa1", "visa2"})). + WillReturnRows(rows) + + // Execute + datasets, err := db.GetUserDatasets(context.Background(), []string{"visa1", "visa2"}) + + // Assert + assert.NoError(t, err) + assert.Len(t, datasets, 2) + assert.Equal(t, "dataset-1", datasets[0].ID) + assert.Equal(t, "Test Dataset", datasets[0].Title) + assert.Equal(t, "dataset-2", datasets[1].ID) + assert.Empty(t, datasets[1].Title) + + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetUserDatasets_Empty(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"stable_id", "title", "description", "created_at"}) + + mock.ExpectQuery(queries[getUserDatasetsQuery]). + WithArgs(pq.Array([]string{})). + WillReturnRows(rows) + + datasets, err := db.GetUserDatasets(context.Background(), []string{}) + + assert.NoError(t, err) + assert.Empty(t, datasets) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetInfo(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + createdAt := time.Now() + rows := sqlmock.NewRows([]string{"stable_id", "title", "description", "created_at", "file_count", "total_size"}). + AddRow("dataset-1", "Test Dataset", "Description", createdAt, 5, int64(1024000)) + + mock.ExpectQuery(queries[getDatasetInfoQuery]). + WithArgs("dataset-1"). + WillReturnRows(rows) + + info, err := db.GetDatasetInfo(context.Background(), "dataset-1") + + assert.NoError(t, err) + require.NotNil(t, info) + assert.Equal(t, "dataset-1", info.ID) + assert.Equal(t, "Test Dataset", info.Title) + assert.Equal(t, 5, info.FileCount) + assert.Equal(t, int64(1024000), info.TotalSize) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetInfo_NotFound(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"stable_id", "title", "description", "created_at", "file_count", "total_size"}) + + mock.ExpectQuery(queries[getDatasetInfoQuery]). + WithArgs("nonexistent"). + WillReturnRows(rows) + + info, err := db.GetDatasetInfo(context.Background(), "nonexistent") + + assert.NoError(t, err) + assert.Nil(t, info) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetFiles(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", "decrypted_file_checksum_type", + }). + AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", int64(1024), int64(900), "abc123", "SHA256"). + AddRow("file-2", "dataset-1", "/path/to/file2.txt", nil, nil, nil, nil, nil, nil) + + mock.ExpectQuery(queries[getDatasetFilesQuery]). + WithArgs("dataset-1"). + WillReturnRows(rows) + + files, err := db.GetDatasetFiles(context.Background(), "dataset-1") + + assert.NoError(t, err) + assert.Len(t, files, 2) + assert.Equal(t, "file-1", files[0].ID) + assert.Equal(t, "/path/to/file.txt", files[0].SubmittedPath) + assert.Equal(t, "s3:9000/archive", files[0].ArchiveLocation) + assert.Equal(t, int64(1024), files[0].ArchiveSize) + assert.Equal(t, "file-2", files[1].ID) + assert.Empty(t, files[1].ArchivePath) + assert.Empty(t, files[1].ArchiveLocation) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByID(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + // Header is stored as hex string in database + // Minimum valid crypt4gh header: magic (8 bytes) + version (4 bytes) + packet count (4 bytes) = 16 bytes + headerBytes := []byte{ + 0x63, 0x72, 0x79, 0x70, 0x74, 0x34, 0x67, 0x68, // "crypt4gh" magic + 0x01, 0x00, 0x00, 0x00, // version 1 + 0x01, 0x00, 0x00, 0x00, // 1 header packet + } + headerHex := "63727970743467680100000001000000" // 32 hex chars = 16 bytes + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }). + AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", + int64(1024), int64(900), "abc123", "SHA256", headerHex) + + mock.ExpectQuery(queries[getFileByIDQuery]). + WithArgs("file-1"). + WillReturnRows(rows) + + file, err := db.GetFileByID(context.Background(), "file-1") + + assert.NoError(t, err) + require.NotNil(t, file) + assert.Equal(t, "file-1", file.ID) + assert.Equal(t, "dataset-1", file.DatasetID) + assert.Equal(t, "s3:9000/archive", file.ArchiveLocation) + assert.Equal(t, headerBytes, file.Header) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByID_NotFound(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }) + + mock.ExpectQuery(queries[getFileByIDQuery]). + WithArgs("nonexistent"). + WillReturnRows(rows) + + file, err := db.GetFileByID(context.Background(), "nonexistent") + + assert.NoError(t, err) + assert.Nil(t, file) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByID_HeaderTooSmall(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + // Header with only 8 bytes (too small, minimum is 16 bytes for crypt4gh) + headerHex := "6372797074346768" // only "crypt4gh" magic, missing version and packet count + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }). + AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", + int64(1024), int64(900), "abc123", "SHA256", headerHex) + + mock.ExpectQuery(queries[getFileByIDQuery]). + WithArgs("file-1"). + WillReturnRows(rows) + + file, err := db.GetFileByID(context.Background(), "file-1") + + assert.Error(t, err) + assert.Nil(t, file) + assert.Contains(t, err.Error(), "decoded header too small") + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByID_InvalidHexHeader(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + // Invalid hex string (contains non-hex characters) + headerHex := "ZZZ_NOT_VALID_HEX" + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }). + AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", + int64(1024), int64(900), "abc123", "SHA256", headerHex) + + mock.ExpectQuery(queries[getFileByIDQuery]). + WithArgs("file-1"). + WillReturnRows(rows) + + file, err := db.GetFileByID(context.Background(), "file-1") + + assert.Error(t, err) + assert.Nil(t, file) + assert.Contains(t, err.Error(), "failed to decode header from hex") + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByPath(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + // Header is stored as hex string in database + // Minimum valid crypt4gh header: magic (8 bytes) + version (4 bytes) + packet count (4 bytes) = 16 bytes + headerHex := "63727970743467680100000001000000" + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }). + AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", + int64(1024), int64(900), "abc123", "SHA256", headerHex) + + mock.ExpectQuery(queries[getFileByPathQuery]). + WithArgs("dataset-1", "/path/to/file.txt"). + WillReturnRows(rows) + + file, err := db.GetFileByPath(context.Background(), "dataset-1", "/path/to/file.txt") + + assert.NoError(t, err) + require.NotNil(t, file) + assert.Equal(t, "file-1", file.ID) + assert.Equal(t, "/path/to/file.txt", file.SubmittedPath) + assert.Equal(t, "s3:9000/archive", file.ArchiveLocation) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByPath_HeaderTooSmall(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + // Header with only 8 bytes (too small, minimum is 16 bytes for crypt4gh) + headerHex := "6372797074346768" // only "crypt4gh" magic, missing version and packet count + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }). + AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", + int64(1024), int64(900), "abc123", "SHA256", headerHex) + + mock.ExpectQuery(queries[getFileByPathQuery]). + WithArgs("dataset-1", "/path/to/file.txt"). + WillReturnRows(rows) + + file, err := db.GetFileByPath(context.Background(), "dataset-1", "/path/to/file.txt") + + assert.Error(t, err) + assert.Nil(t, file) + assert.Contains(t, err.Error(), "decoded header too small") + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByPath_InvalidHexHeader(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + // Invalid hex string (contains non-hex characters) - guards against corrupt DB data + headerHex := "GHIJ_CORRUPT_DATA" + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }). + AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", + int64(1024), int64(900), "abc123", "SHA256", headerHex) + + mock.ExpectQuery(queries[getFileByPathQuery]). + WithArgs("dataset-1", "/path/to/file.txt"). + WillReturnRows(rows) + + file, err := db.GetFileByPath(context.Background(), "dataset-1", "/path/to/file.txt") + + assert.Error(t, err) + assert.Nil(t, file) + assert.Contains(t, err.Error(), "failed to decode header from hex") + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestCheckFilePermission(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"exists"}).AddRow(true) + + mock.ExpectQuery(queries[checkFilePermissionQuery]). + WithArgs("file-1", pq.Array([]string{"visa1"})). + WillReturnRows(rows) + + hasPermission, err := db.CheckFilePermission(context.Background(), "file-1", []string{"visa1"}) + + assert.NoError(t, err) + assert.True(t, hasPermission) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestCheckFilePermission_Denied(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{"exists"}).AddRow(false) + + mock.ExpectQuery(queries[checkFilePermissionQuery]). + WithArgs("file-1", pq.Array([]string{"wrong-visa"})). + WillReturnRows(rows) + + hasPermission, err := db.CheckFilePermission(context.Background(), "file-1", []string{"wrong-visa"}) + + assert.NoError(t, err) + assert.False(t, hasPermission) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +// Error handling tests + +func TestGetUserDatasets_QueryError(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + mock.ExpectQuery(queries[getUserDatasetsQuery]). + WithArgs(pq.Array([]string{"visa1"})). + WillReturnError(sql.ErrConnDone) + + datasets, err := db.GetUserDatasets(context.Background(), []string{"visa1"}) + + assert.Error(t, err) + assert.Nil(t, datasets) + assert.Contains(t, err.Error(), "failed to query datasets") +} + +func TestGetDatasetInfo_QueryError(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + mock.ExpectQuery(queries[getDatasetInfoQuery]). + WithArgs("dataset-1"). + WillReturnError(sql.ErrConnDone) + + info, err := db.GetDatasetInfo(context.Background(), "dataset-1") + + assert.Error(t, err) + assert.Nil(t, info) + assert.Contains(t, err.Error(), "failed to query dataset info") +} + +func TestGetDatasetFiles_QueryError(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + mock.ExpectQuery(queries[getDatasetFilesQuery]). + WithArgs("dataset-1"). + WillReturnError(sql.ErrConnDone) + + files, err := db.GetDatasetFiles(context.Background(), "dataset-1") + + assert.Error(t, err) + assert.Nil(t, files) + assert.Contains(t, err.Error(), "failed to query dataset files") +} + +func TestGetDatasetFiles_Empty(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", "decrypted_file_checksum_type", + }) + + mock.ExpectQuery(queries[getDatasetFilesQuery]). + WithArgs("empty-dataset"). + WillReturnRows(rows) + + files, err := db.GetDatasetFiles(context.Background(), "empty-dataset") + + assert.NoError(t, err) + assert.Empty(t, files) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetFileByID_QueryError(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + mock.ExpectQuery(queries[getFileByIDQuery]). + WithArgs("file-1"). + WillReturnError(sql.ErrConnDone) + + file, err := db.GetFileByID(context.Background(), "file-1") + + assert.Error(t, err) + assert.Nil(t, file) + assert.Contains(t, err.Error(), "failed to query file by ID") +} + +func TestGetFileByPath_QueryError(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + mock.ExpectQuery(queries[getFileByPathQuery]). + WithArgs("dataset-1", "/path/file.txt"). + WillReturnError(sql.ErrConnDone) + + file, err := db.GetFileByPath(context.Background(), "dataset-1", "/path/file.txt") + + assert.Error(t, err) + assert.Nil(t, file) + assert.Contains(t, err.Error(), "failed to query file by path") +} + +func TestGetFileByPath_NotFound(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{ + "stable_id", "dataset_id", "submission_file_path", "archive_file_path", + "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", + }) + + mock.ExpectQuery(queries[getFileByPathQuery]). + WithArgs("dataset-1", "/nonexistent"). + WillReturnRows(rows) + + file, err := db.GetFileByPath(context.Background(), "dataset-1", "/nonexistent") + + assert.NoError(t, err) + assert.Nil(t, file) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestCheckFilePermission_QueryError(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + mock.ExpectQuery(queries[checkFilePermissionQuery]). + WithArgs("file-1", pq.Array([]string{"visa1"})). + WillReturnError(sql.ErrConnDone) + + hasPermission, err := db.CheckFilePermission(context.Background(), "file-1", []string{"visa1"}) + + assert.Error(t, err) + assert.False(t, hasPermission) + assert.Contains(t, err.Error(), "failed to check file permission") +} + +// Package-level function tests + +func TestRegisterAndGetDB(t *testing.T) { + mockDB := &mockTestDatabase{} + + RegisterDatabase(mockDB) + defer RegisterDatabase(nil) + + retrieved := GetDB() + assert.Equal(t, mockDB, retrieved) +} + +func TestClose_WithNilDB(t *testing.T) { + // Save current db + oldDB := db + defer func() { db = oldDB }() + + db = nil + err := Close() + assert.NoError(t, err) +} + +func TestClose_WithDB(t *testing.T) { + // Save current db + oldDB := db + defer func() { db = oldDB }() + + mockDB := &mockTestDatabase{} + db = mockDB + err := Close() + assert.NoError(t, err) + assert.True(t, mockDB.closed) +} + +// mockTestDatabase for package-level function tests +type mockTestDatabase struct { + closed bool +} + +func (m *mockTestDatabase) Ping(_ context.Context) error { + return nil +} + +func (m *mockTestDatabase) Close() error { + m.closed = true + + return nil +} + +func (m *mockTestDatabase) GetAllDatasets(_ context.Context) ([]Dataset, error) { + return nil, nil +} + +func (m *mockTestDatabase) GetDatasetIDsByUser(_ context.Context, _ string) ([]string, error) { + return nil, nil +} + +func (m *mockTestDatabase) GetUserDatasets(_ context.Context, _ []string) ([]Dataset, error) { + return nil, nil +} + +func (m *mockTestDatabase) GetDatasetInfo(_ context.Context, _ string) (*DatasetInfo, error) { + return nil, nil +} + +func (m *mockTestDatabase) GetDatasetFiles(_ context.Context, _ string) ([]File, error) { + return nil, nil +} + +func (m *mockTestDatabase) GetFileByID(_ context.Context, _ string) (*File, error) { + return nil, nil +} + +func (m *mockTestDatabase) GetFileByPath(_ context.Context, _, _ string) (*File, error) { + return nil, nil +} + +func (m *mockTestDatabase) CheckFilePermission(_ context.Context, _ string, _ []string) (bool, error) { + return false, nil +} + +func (m *mockTestDatabase) CheckDatasetExists(_ context.Context, _ string) (bool, error) { + return false, nil +} + +func (m *mockTestDatabase) GetDatasetFilesPaginated(_ context.Context, _ string, _ FileListOptions) ([]File, error) { + return nil, nil +} + +func TestGetDatasetFilesPaginated_NoFilter(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{ + "stable_id", "submission_file_path", "archive_file_size", "decrypted_file_size", "checksums", + }). + AddRow("file-1", "/path/a.txt", int64(1024), int64(900), nil). + AddRow("file-2", "/path/b.txt", int64(2048), int64(1800), nil) + + mock.ExpectQuery(queries[getDatasetFilesPageQuery]). + WithArgs("dataset-1", "", "", 3). + WillReturnRows(rows) + + files, err := db.GetDatasetFilesPaginated(context.Background(), "dataset-1", FileListOptions{Limit: 3}) + + assert.NoError(t, err) + assert.Len(t, files, 2) + assert.Equal(t, "file-1", files[0].ID) + assert.Equal(t, "/path/a.txt", files[0].SubmittedPath) + assert.Equal(t, int64(1024), files[0].ArchiveSize) + assert.Empty(t, files[0].Checksums) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetFilesPaginated_WithCursor(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{ + "stable_id", "submission_file_path", "archive_file_size", "decrypted_file_size", "checksums", + }). + AddRow("file-3", "/path/c.txt", int64(512), int64(400), nil) + + mock.ExpectQuery(queries[getDatasetFilesPageQuery]). + WithArgs("dataset-1", "/path/b.txt", "file-2", 2). + WillReturnRows(rows) + + files, err := db.GetDatasetFilesPaginated(context.Background(), "dataset-1", FileListOptions{ + CursorPath: "/path/b.txt", + CursorID: "file-2", + Limit: 2, + }) + + assert.NoError(t, err) + assert.Len(t, files, 1) + assert.Equal(t, "file-3", files[0].ID) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetFilesPaginated_ByPath(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + checksumJSON := []byte(`[{"type":"sha256","checksum":"abc123"}]`) + rows := sqlmock.NewRows([]string{ + "stable_id", "submission_file_path", "archive_file_size", "decrypted_file_size", "checksums", + }). + AddRow("file-1", "/exact/path.txt", int64(1024), int64(900), checksumJSON) + + mock.ExpectQuery(queries[getDatasetFilesPageByPathQuery]). + WithArgs("dataset-1", "/exact/path.txt", 2). + WillReturnRows(rows) + + files, err := db.GetDatasetFilesPaginated(context.Background(), "dataset-1", FileListOptions{ + FilePath: "/exact/path.txt", + Limit: 2, + }) + + assert.NoError(t, err) + assert.Len(t, files, 1) + assert.Equal(t, "file-1", files[0].ID) + require.Len(t, files[0].Checksums, 1) + assert.Equal(t, "sha256", files[0].Checksums[0].Type) + assert.Equal(t, "abc123", files[0].Checksums[0].Checksum) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetFilesPaginated_ByPrefix(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + rows := sqlmock.NewRows([]string{ + "stable_id", "submission_file_path", "archive_file_size", "decrypted_file_size", "checksums", + }). + AddRow("file-1", "/data/sample1.txt", int64(100), int64(90), nil). + AddRow("file-2", "/data/sample2.txt", int64(200), int64(180), nil) + + mock.ExpectQuery(queries[getDatasetFilesPageByPrefixQuery]). + WithArgs("dataset-1", `/data/%`, "", "", 3). + WillReturnRows(rows) + + files, err := db.GetDatasetFilesPaginated(context.Background(), "dataset-1", FileListOptions{ + PathPrefix: "/data/", + Limit: 3, + }) + + assert.NoError(t, err) + assert.Len(t, files, 2) + assert.Equal(t, "/data/sample1.txt", files[0].SubmittedPath) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetFilesPaginated_MultipleChecksums(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + checksumJSON := []byte(`[{"type":"sha256","checksum":"abc"},{"type":"md5","checksum":"def"}]`) + rows := sqlmock.NewRows([]string{ + "stable_id", "submission_file_path", "archive_file_size", "decrypted_file_size", "checksums", + }). + AddRow("file-1", "/path/file.txt", int64(1024), int64(900), checksumJSON) + + mock.ExpectQuery(queries[getDatasetFilesPageQuery]). + WithArgs("dataset-1", "", "", 2). + WillReturnRows(rows) + + files, err := db.GetDatasetFilesPaginated(context.Background(), "dataset-1", FileListOptions{Limit: 2}) + + assert.NoError(t, err) + require.Len(t, files, 1) + require.Len(t, files[0].Checksums, 2) + assert.Equal(t, "sha256", files[0].Checksums[0].Type) + assert.Equal(t, "md5", files[0].Checksums[1].Type) + assert.NoError(t, mock.ExpectationsWereMet()) +} + +func TestGetDatasetFilesPaginated_QueryError(t *testing.T) { + db, mock, cleanup := setupMockDB(t) + defer cleanup() + + mock.ExpectQuery(queries[getDatasetFilesPageQuery]). + WithArgs("dataset-1", "", "", 2). + WillReturnError(sql.ErrConnDone) + + files, err := db.GetDatasetFilesPaginated(context.Background(), "dataset-1", FileListOptions{Limit: 2}) + + assert.Error(t, err) + assert.Nil(t, files) + assert.Contains(t, err.Error(), "failed to query paginated dataset files") +} + +func TestEscapeLikePrefix(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"/data/", `/data/%`}, + {"/path_with_underscores/", `/path\_with\_underscores/%`}, + {"100%/dir/", `100\%/dir/%`}, + {`back\slash/`, `back\\slash/%`}, + {"", "%"}, + } + + for _, tt := range tests { + t.Run(tt.input, func(t *testing.T) { + assert.Equal(t, tt.expected, escapeLikePrefix(tt.input)) + }) + } +} From c68d80f23d0ba38e94374cb0275415eed474cc2b Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:07:05 +0100 Subject: [PATCH 373/530] =?UTF-8?q?feat(download):=20add=20auth=20middlewa?= =?UTF-8?q?re=20=E2=80=94=20JWT,=20opaque=20tokens,=20session=20cache?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Authentication middleware: - Structure-based JWT detection via looksLikeJWT() (3 dot-segments with base64url JSON header+payload) - JWT path: validate locally via loaded keyset, optional issuer match - Opaque path: call UserinfoClient.FetchUserinfo for subject resolution - Session cookie cache (sda_session + legacy sda_session_key) - Token-keyed cache (sha256(token)) with TTL bounded by min(token.exp, min(visa.exp), configTTL) - Permission model: combined/visa/ownership dataset population - SameSite=Lax on session cookies - Audit denial events (download.denied) on all 401 paths --- sda/cmd/download/middleware/auth.go | 763 ++++++++++++++++++ sda/cmd/download/middleware/auth_test.go | 265 ++++++ sda/cmd/download/middleware/auth_visa_test.go | 151 ++++ sda/cmd/download/middleware/cache_ttl_test.go | 61 ++ .../middleware/config_linkname_test.go | 45 ++ .../middleware/config_testhelper_test.go | 35 + .../middleware/visa_middleware_test.go | 421 ++++++++++ 7 files changed, 1741 insertions(+) create mode 100644 sda/cmd/download/middleware/auth.go create mode 100644 sda/cmd/download/middleware/auth_test.go create mode 100644 sda/cmd/download/middleware/auth_visa_test.go create mode 100644 sda/cmd/download/middleware/cache_ttl_test.go create mode 100644 sda/cmd/download/middleware/config_linkname_test.go create mode 100644 sda/cmd/download/middleware/config_testhelper_test.go create mode 100644 sda/cmd/download/middleware/visa_middleware_test.go diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go new file mode 100644 index 0000000000..2150d79ddb --- /dev/null +++ b/sda/cmd/download/middleware/auth.go @@ -0,0 +1,763 @@ +// Package middleware provides HTTP middleware for the download service. +package middleware + +import ( + "context" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/dgraph-io/ristretto" + "github.com/gin-gonic/gin" + "github.com/google/uuid" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jws" + "github.com/lestrrat-go/jwx/v2/jwt" + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/neicnordic/sensitive-data-archive/cmd/download/config" + "github.com/neicnordic/sensitive-data-archive/cmd/download/visa" + log "github.com/sirupsen/logrus" +) + +// ContextKey is the key used to store auth data in the request context. +const ContextKey = "authContext" + +// DatasetLookup is an interface for looking up datasets by user. +// This allows the middleware to populate user datasets without depending on the full database package. +type DatasetLookup interface { + // GetDatasetIDsByUser returns dataset IDs where the user is the data owner (submission_user). + GetDatasetIDsByUser(ctx context.Context, user string) ([]string, error) +} + +// AuthContext holds the authenticated user's information. +type AuthContext struct { + // Issuer is the token issuer (from JWT iss claim or configured OIDC issuer for opaque tokens) + Issuer string + // Subject is the user identifier from the JWT token or userinfo + Subject string + // OwnedDatasets are datasets where the user is the submission_user + OwnedDatasets []string + // VisaDatasets are datasets granted by GA4GH visas + VisaDatasets []string + // Datasets is the combined (deduplicated) union of OwnedDatasets and VisaDatasets + Datasets []string + // Token is the parsed JWT token (nil for opaque token auth) + Token jwt.Token + // AuthSource indicates how the user was authenticated ("jwt" or "userinfo") + AuthSource string +} + +// Authenticator validates JWT tokens. +type Authenticator struct { + Keyset jwk.Set + mu sync.RWMutex +} + +// SessionCache holds cached session data using ristretto. +// No mutex needed — ristretto is designed for lock-free concurrent access. +type SessionCache struct { + cache *ristretto.Cache +} + +var ( + // auth is the global authenticator instance. + auth *Authenticator + + // sessionCache is the in-memory session cache. + sessionCache *SessionCache + + // tokenCache is the token-keyed cache for non-cookie clients. + tokenCache *SessionCache + + // legacyCookieName is the old cookie name for dual-read compatibility. + legacyCookieName = "sda_session_key" +) + +// InitAuthForTesting initializes the auth middleware with an empty keyset for testing. +// This allows middleware to run without valid keys, causing authentication to fail. +func InitAuthForTesting() error { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + if err != nil { + return err + } + sessionCache = &SessionCache{cache: cache} + + tokenCacheInst, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + if err != nil { + return err + } + tokenCache = &SessionCache{cache: tokenCacheInst} + + auth = &Authenticator{ + Keyset: jwk.NewSet(), + } + + return nil +} + +// InitAuth initializes the authentication middleware. +// This should be called during application startup. +// It loads JWT public keys from either a local path or remote JWKS URL. +func InitAuth() error { + // Initialize session cache + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + if err != nil { + return fmt.Errorf("failed to initialize session cache: %w", err) + } + sessionCache = &SessionCache{cache: cache} + + // Initialize token cache + tokenCacheInst, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + if err != nil { + return fmt.Errorf("failed to initialize token cache: %w", err) + } + tokenCache = &SessionCache{cache: tokenCacheInst} + + // Initialize authenticator with empty keyset + auth = &Authenticator{ + Keyset: jwk.NewSet(), + } + + // Load keys from path or URL + pubKeyPath := config.JWTPubKeyPath() + pubKeyURL := config.JWTPubKeyURL() + + if pubKeyPath == "" && pubKeyURL == "" { + // When allow-opaque is enabled and no JWT keys are configured, + // all tokens will go through userinfo. This is valid. + if config.AuthAllowOpaque() { + log.Warn("no JWT keys configured; all tokens will be treated as opaque") + + return nil + } + + return errors.New("either jwt.pubkey-path or jwt.pubkey-url must be configured") + } + + // Load from local path if configured + if pubKeyPath != "" { + if err := auth.loadKeysFromPath(pubKeyPath); err != nil { + return fmt.Errorf("failed to load JWT keys from path: %w", err) + } + log.Infof("JWT keys loaded from path: %s", pubKeyPath) + } + + // Load from URL if configured (can be used in addition to or instead of path) + if pubKeyURL != "" { + if err := auth.loadKeysFromURL(pubKeyURL); err != nil { + return fmt.Errorf("failed to load JWT keys from URL: %w", err) + } + log.Infof("JWT keys loaded from URL: %s", pubKeyURL) + } + + return nil +} + +// loadKeysFromPath loads public keys from a directory containing PEM files. +func (a *Authenticator) loadKeysFromPath(keyPath string) error { + a.mu.Lock() + defer a.mu.Unlock() + + return filepath.Walk(keyPath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if info.Mode().IsRegular() { + log.Debugf("Loading key from file: %s", path) + + keyData, err := os.ReadFile(path) + if err != nil { + return fmt.Errorf("failed to read key file %s: %w", path, err) + } + + key, err := jwk.ParseKey(keyData, jwk.WithPEM(true)) + if err != nil { + log.Warnf("failed to parse key from %s: %v (skipping)", path, err) + + return nil + } + + if err := jwk.AssignKeyID(key); err != nil { + return fmt.Errorf("failed to assign key ID: %w", err) + } + + if err := a.Keyset.AddKey(key); err != nil { + return fmt.Errorf("failed to add key to set: %w", err) + } + + log.Debugf("Loaded key with ID: %s", key.KeyID()) + } + + return nil + }) +} + +// loadKeysFromURL fetches keys from a JWKS endpoint. +func (a *Authenticator) loadKeysFromURL(jwksURL string) error { + a.mu.Lock() + defer a.mu.Unlock() + + parsedURL, err := url.ParseRequestURI(jwksURL) + if err != nil || parsedURL.Scheme == "" || parsedURL.Host == "" { + return fmt.Errorf("invalid JWKS URL: %s", jwksURL) + } + + keySet, err := jwk.Fetch(context.Background(), jwksURL) + if err != nil { + return fmt.Errorf("failed to fetch JWKS: %w", err) + } + + // Add all keys from the fetched keyset + for iter := keySet.Keys(context.Background()); iter.Next(context.Background()); { + key, ok := iter.Pair().Value.(jwk.Key) + if !ok { + log.Warn("unexpected key type in JWKS response, skipping") + + continue + } + if err := jwk.AssignKeyID(key); err != nil { + log.Warnf("failed to assign key ID: %v", err) + + continue + } + if err := a.Keyset.AddKey(key); err != nil { + log.Warnf("failed to add key: %v", err) + + continue + } + } + + return nil +} + +// looksLikeJWT returns true if token has JWT structure: +// 3 dot-separated segments where segments 1+2 base64url-decode into valid JSON objects. +// This avoids misclassifying "dotty" opaque tokens (e.g., Azure AD opaque tokens +// that contain dots but aren't JWTs) as JWT-shaped. +func looksLikeJWT(token string) bool { + parts := strings.Split(token, ".") + if len(parts) != 3 { + return false + } + // Both header and payload must be valid base64url-encoded JSON objects + for _, part := range parts[:2] { + decoded, err := base64.RawURLEncoding.DecodeString(part) + if err != nil { + return false + } + var obj map[string]any + if err := json.Unmarshal(decoded, &obj); err != nil { + return false + } + } + + return true +} + +// verifyJWT validates a JWT token string against the loaded keyset. +func (a *Authenticator) verifyJWT(tokenStr string) (jwt.Token, error) { + a.mu.RLock() + defer a.mu.RUnlock() + + if a.Keyset.Len() == 0 { + return nil, errors.New("no keys configured for token validation") + } + + token, err := jwt.Parse( + []byte(tokenStr), + jwt.WithKeySet(a.Keyset, + jws.WithInferAlgorithmFromKey(true), + jws.WithRequireKid(false), + ), + jwt.WithValidate(true), + ) + if err != nil { + return nil, fmt.Errorf("token validation failed: %w", err) + } + + // Validate issuer if configured + if issuer := config.OIDCIssuer(); issuer != "" { + tokenIssuer := token.Issuer() + if tokenIssuer != issuer && tokenIssuer != strings.TrimSuffix(issuer, "/") { + return nil, fmt.Errorf("invalid issuer: got %s, expected %s", tokenIssuer, issuer) + } + } + + // Validate audience if configured + if audience := config.OIDCAudience(); audience != "" { + aud := token.Audience() + found := false + for _, a := range aud { + if a == audience { + found = true + + break + } + } + if !found { + return nil, errors.New("invalid audience") + } + } + + // Validate issuer is a valid URI + iss, err := url.ParseRequestURI(token.Issuer()) + if err != nil || iss.Hostname() == "" { + return nil, fmt.Errorf("invalid issuer in token: %v", token.Issuer()) + } + + return token, nil +} + +// Authenticate verifies a token and returns the parsed JWT. +// This is the legacy method that always tries JWT verification. +func (a *Authenticator) Authenticate(r *http.Request) (jwt.Token, error) { + tokenStr, _, err := GetToken(r.Header) + if err != nil { + return nil, err + } + + return a.verifyJWT(tokenStr) +} + +// authResult holds the result of structure-based authentication. +type authResult struct { + Issuer string + Subject string + AuthSource string + Token jwt.Token +} + +// authenticateStructureBased performs structure-based authentication. +// JWT-shaped tokens are validated locally; opaque tokens go through userinfo. +func authenticateStructureBased(rawToken string, userinfoClient *visa.UserinfoClient) (*authResult, error) { + if looksLikeJWT(rawToken) { + // JWT-shaped: MUST validate locally, no userinfo fallback + token, err := auth.verifyJWT(rawToken) + if err != nil { + return nil, fmt.Errorf("jwt validation failed: %w", err) + } + + return &authResult{ + Issuer: token.Issuer(), + Subject: token.Subject(), + AuthSource: "jwt", + Token: token, + }, nil + } + + // Opaque token + if !config.AuthAllowOpaque() { + return nil, errors.New("opaque tokens not allowed") + } + + // Userinfo fallback for opaque tokens only + if userinfoClient == nil { + return nil, errors.New("userinfo client not configured for opaque token auth") + } + + userinfo, err := userinfoClient.FetchUserinfo(rawToken) + if err != nil || userinfo.Sub == "" { + if err != nil { + return nil, fmt.Errorf("userinfo failed: %w", err) + } + + return nil, errors.New("userinfo returned empty subject") + } + + // For opaque tokens, issuer comes from configured OIDC issuer + issuer := config.OIDCIssuer() + + return &authResult{ + Issuer: issuer, + Subject: userinfo.Sub, + AuthSource: "userinfo", + Token: nil, + }, nil +} + +// TokenMiddleware performs access token verification and validation. +// The authenticated user's subject is stored in the request context. +// If db is provided and allow-all-data is disabled, the user's datasets are populated from the database. +// If visaValidator is provided, GA4GH visa-based access is also computed based on permission.model. +func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogger audit.Logger) gin.HandlerFunc { + // Determine userinfo client for opaque token support. + // The visa validator's userinfo client can be reused for opaque token auth. + var userinfoClient *visa.UserinfoClient + if visaValidator != nil && config.AuthAllowOpaque() { + userinfoClient = visaValidator.UserinfoClient() + } + + auditDenied := func(c *gin.Context, status int) { + auditLogger.Log(c.Request.Context(), audit.Event{ + Event: "download.denied", + CorrelationID: c.GetString("correlationId"), + Endpoint: c.Request.URL.Path, + HTTPStatus: status, + }) + } + + return func(c *gin.Context) { + // Check for cached session (try configured name, then legacy name) + sessionCookie, err := c.Cookie(config.SessionName()) + if err != nil || sessionCookie == "" { + // Try legacy cookie name for compatibility with sda-cli + sessionCookie, _ = c.Cookie(legacyCookieName) + } + if sessionCookie != "" { + if authCtx, exists := sessionCache.Get(sessionCookie); exists { + log.Debug("session found in cache") + c.Set(ContextKey, authCtx) + c.Next() + + return + } + } + + // Extract raw token + rawToken, _, err := GetToken(c.Request.Header) + if err != nil { + c.Header("Content-Type", "application/problem+json") + c.JSON(http.StatusUnauthorized, gin.H{ + "title": "Unauthorized", + "status": http.StatusUnauthorized, + "detail": "Missing, invalid, or expired bearer token.", + }) + auditDenied(c, http.StatusUnauthorized) + c.Abort() + + return + } + + // Check token cache + tokenKey := sha256Hex(rawToken) + if cached, exists := tokenCache.Get(tokenKey); exists { + log.Debug("token cache hit") + c.Set(ContextKey, cached) + c.Next() + + return + } + + permModel := config.PermissionModel() + needsVisa := visaValidator != nil && (permModel == "visa" || permModel == "combined") + + // Authenticate: structure-based if visa support is active, legacy otherwise + var authCtx AuthContext + if needsVisa || config.AuthAllowOpaque() { + result, authErr := authenticateStructureBased(rawToken, userinfoClient) + if authErr != nil { + log.Debugf("authentication failed: %v", authErr) + c.Header("Content-Type", "application/problem+json") + c.JSON(http.StatusUnauthorized, gin.H{ + "title": "Unauthorized", + "status": http.StatusUnauthorized, + "detail": "Missing, invalid, or expired bearer token.", + }) + auditDenied(c, http.StatusUnauthorized) + c.Abort() + + return + } + authCtx = AuthContext{ + Issuer: result.Issuer, + Subject: result.Subject, + Token: result.Token, + AuthSource: result.AuthSource, + } + } else { + // Legacy JWT-only auth + token, authErr := auth.Authenticate(c.Request) + if authErr != nil { + log.Debugf("authentication failed: %v", authErr) + c.Header("Content-Type", "application/problem+json") + c.JSON(http.StatusUnauthorized, gin.H{ + "title": "Unauthorized", + "status": http.StatusUnauthorized, + "detail": "Missing, invalid, or expired bearer token.", + }) + auditDenied(c, http.StatusUnauthorized) + c.Abort() + + return + } + authCtx = AuthContext{ + Issuer: token.Issuer(), + Subject: token.Subject(), + Token: token, + AuthSource: "jwt", + } + } + + skipCache := false + + // Populate owned datasets (ownership model) + if db != nil && !config.JWTAllowAllData() && (permModel == "ownership" || permModel == "combined") { + datasets, err := db.GetDatasetIDsByUser(c.Request.Context(), authCtx.Subject) + if err != nil { + log.Warnf("failed to get datasets for user %s: %v", authCtx.Subject, err) + } else { + authCtx.OwnedDatasets = datasets + } + } + + // Populate visa datasets + var visaMinExpiry *time.Time + if needsVisa { + datasets, minExp, visaErr := processVisas(c.Request.Context(), visaValidator, authCtx, rawToken) + if visaErr != nil { + log.Warnf("visa processing failed: %v", visaErr) + if permModel == "visa" { + c.Header("Content-Type", "application/problem+json") + c.JSON(http.StatusUnauthorized, gin.H{ + "title": "Unauthorized", + "status": http.StatusUnauthorized, + "detail": "Visa validation failed.", + }) + auditDenied(c, http.StatusUnauthorized) + c.Abort() + + return + } + // Combined mode: continue with ownership only, but don't cache + skipCache = true + } else { + authCtx.VisaDatasets = datasets + visaMinExpiry = minExp + } + } + + // Merge datasets (deduplicated union) + authCtx.Datasets = mergeDatasets(authCtx.OwnedDatasets, authCtx.VisaDatasets) + + // Create session and token cache entries + if !skipCache { + // Extract JWT token expiry for cache TTL bounding + var tokenExp *time.Time + if authCtx.Token != nil { + exp := authCtx.Token.Expiration() + if !exp.IsZero() { + tokenExp = &exp + } + } + + cacheTTL := computeCacheTTL(tokenExp, visaMinExpiry) + + // Guard against zero or negative TTL (expired token/visa) + // Ristretto's behavior with TTL=0 is undefined, and caching + // an already-expired auth makes no sense. + if cacheTTL <= 0 { + log.Debug("skipping cache: computed TTL is zero or negative (token/visa already expired)") + } else { + sessionKey := uuid.New().String() + sessionCache.Set(sessionKey, authCtx, cacheTTL) + + // Set session cookie + c.SetSameSite(http.SameSiteLaxMode) + c.SetCookie( + config.SessionName(), + sessionKey, + int(cacheTTL.Seconds()), + "/", + config.SessionDomain(), + config.SessionSecure(), + config.SessionHTTPOnly(), + ) + + // Token-keyed cache + tokenCache.Set(tokenKey, authCtx, cacheTTL) + } + } + + // Store in request context + c.Set(ContextKey, authCtx) + log.Debugf("authentication successful for subject: %s (datasets: %d)", authCtx.Subject, len(authCtx.Datasets)) + + c.Next() + } +} + +// GetToken extracts the access token from request headers. +// Supports both Authorization: Bearer and X-Amz-Security-Token headers. +var GetToken = func(headers http.Header) (string, int, error) { + // Check X-Amz-Security-Token header first (for S3 compatibility) + if token := headers.Get("X-Amz-Security-Token"); token != "" { + return token, 0, nil + } + + // Check Authorization header + authHeader := headers.Get("Authorization") + if authHeader == "" { + return "", http.StatusUnauthorized, errors.New("access token must be provided") + } + + // Check Bearer scheme + parts := strings.Split(authHeader, " ") + if len(parts) != 2 { + return "", http.StatusBadRequest, errors.New("invalid authorization header format") + } + + if !strings.EqualFold(parts[0], "Bearer") { + return "", http.StatusBadRequest, errors.New("authorization scheme must be bearer") + } + + return parts[1], 0, nil +} + +// GetAuthContext retrieves the auth context from a gin context. +func GetAuthContext(c *gin.Context) (AuthContext, bool) { + val, exists := c.Get(ContextKey) + if !exists { + return AuthContext{}, false + } + + authCtx, ok := val.(AuthContext) + + return authCtx, ok +} + +// Session cache methods + +// Get retrieves a value from the session cache. +func (sc *SessionCache) Get(key string) (AuthContext, bool) { + val, found := sc.cache.Get(key) + if !found { + return AuthContext{}, false + } + + authCtx, ok := val.(AuthContext) + if !ok { + return AuthContext{}, false + } + + return authCtx, true +} + +// Set stores a value in the session cache with the specified TTL. +func (sc *SessionCache) Set(key string, value AuthContext, ttl time.Duration) { + sc.cache.SetWithTTL(key, value, 1, ttl) +} + +// mergeDatasets returns a deduplicated union of two string slices. +func mergeDatasets(a, b []string) []string { + seen := make(map[string]bool, len(a)+len(b)) + var result []string + + for _, s := range a { + if !seen[s] { + seen[s] = true + result = append(result, s) + } + } + for _, s := range b { + if !seen[s] { + seen[s] = true + result = append(result, s) + } + } + + return result +} + +// processVisas calls the visa validator and extracts datasets and the earliest visa expiry. +// Returns (datasets, minExpiry, error). minExpiry is nil if no accepted visas had an expiry. +func processVisas(ctx context.Context, vv *visa.Validator, authCtx AuthContext, rawToken string) ([]string, *time.Time, error) { + identity := visa.Identity{Issuer: authCtx.Issuer, Subject: authCtx.Subject} + + visaResult, err := vv.GetVisaDatasets(ctx, identity, rawToken, authCtx.AuthSource) + if err != nil { + return nil, nil, err + } + + if visaResult == nil { + return nil, nil, nil + } + + for _, w := range visaResult.Warnings { + log.Warn(w) + } + + var minExp *time.Time + if !visaResult.MinExpiry.IsZero() { + minExp = &visaResult.MinExpiry + } + + return visaResult.Datasets, minExp, nil +} + +// sha256Hex returns the hex-encoded SHA-256 hash of a string. +func sha256Hex(s string) string { + h := sha256.Sum256([]byte(s)) + + return hex.EncodeToString(h[:]) +} + +// computeCacheTTL calculates the appropriate cache TTL bounded by token and visa expiry. +// +// Per GA4GH spec, cache TTL must never outlive visa validity: +// +// JWT + accepted visas: min(configTTL, token.exp - now, min(visa.exp) - now) +// JWT + no accepted visas: min(configTTL, token.exp - now) +// Opaque + accepted visas: min(configTTL, min(visa.exp) - now) +// Opaque + no accepted visas: configTTL +func computeCacheTTL(tokenExp *time.Time, visaMinExpiry *time.Time) time.Duration { + configTTL := time.Duration(config.VisaCacheTokenTTL()) * time.Second + if configTTL == 0 { + configTTL = time.Duration(config.SessionExpiration()) * time.Second + } + + minTTL := configTTL + now := time.Now() + + // Bound by JWT token expiry if available + if tokenExp != nil { + remaining := tokenExp.Sub(now) + if remaining <= 0 { + return 0 + } + + if remaining < minTTL { + minTTL = remaining + } + } + + // Bound by earliest visa expiry (GA4GH requirement: never cache beyond visa validity) + if visaMinExpiry != nil && !visaMinExpiry.IsZero() { + remaining := visaMinExpiry.Sub(now) + if remaining <= 0 { + return 0 + } + + if remaining < minTTL { + minTTL = remaining + } + } + + return minTTL +} diff --git a/sda/cmd/download/middleware/auth_test.go b/sda/cmd/download/middleware/auth_test.go new file mode 100644 index 0000000000..2a4e667827 --- /dev/null +++ b/sda/cmd/download/middleware/auth_test.go @@ -0,0 +1,265 @@ +package middleware + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "context" + "sync" + + "github.com/dgraph-io/ristretto" + "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + gin.SetMode(gin.TestMode) +} + +func TestGetToken_BearerToken(t *testing.T) { + headers := http.Header{} + headers.Set("Authorization", "Bearer test-token-123") + + token, code, err := GetToken(headers) + + assert.NoError(t, err) + assert.Equal(t, 0, code) + assert.Equal(t, "test-token-123", token) +} + +func TestGetToken_AmzSecurityToken(t *testing.T) { + headers := http.Header{} + headers.Set("X-Amz-Security-Token", "amz-token-456") + + token, code, err := GetToken(headers) + + assert.NoError(t, err) + assert.Equal(t, 0, code) + assert.Equal(t, "amz-token-456", token) +} + +func TestGetToken_AmzSecurityTokenPriority(t *testing.T) { + // X-Amz-Security-Token should take priority over Authorization + headers := http.Header{} + headers.Set("X-Amz-Security-Token", "amz-token") + headers.Set("Authorization", "Bearer bearer-token") + + token, code, err := GetToken(headers) + + assert.NoError(t, err) + assert.Equal(t, 0, code) + assert.Equal(t, "amz-token", token) +} + +func TestGetToken_MissingToken(t *testing.T) { + headers := http.Header{} + + token, code, err := GetToken(headers) + + assert.Error(t, err) + assert.Equal(t, http.StatusUnauthorized, code) + assert.Empty(t, token) + assert.Contains(t, err.Error(), "access token must be provided") +} + +func TestGetToken_InvalidScheme(t *testing.T) { + headers := http.Header{} + headers.Set("Authorization", "Basic dXNlcjpwYXNz") + + token, code, err := GetToken(headers) + + assert.Error(t, err) + assert.Equal(t, http.StatusBadRequest, code) + assert.Empty(t, token) + assert.Contains(t, err.Error(), "bearer") +} + +func TestGetToken_InvalidFormat(t *testing.T) { + headers := http.Header{} + headers.Set("Authorization", "BearerNoSpace") + + token, code, err := GetToken(headers) + + assert.Error(t, err) + assert.Equal(t, http.StatusBadRequest, code) + assert.Empty(t, token) +} + +func TestGetToken_CaseInsensitiveBearer(t *testing.T) { + testCases := []string{"Bearer", "bearer", "BEARER", "BeArEr"} + + for _, scheme := range testCases { + headers := http.Header{} + headers.Set("Authorization", scheme+" test-token") + + token, code, err := GetToken(headers) + + assert.NoError(t, err, "scheme: %s", scheme) + assert.Equal(t, 0, code, "scheme: %s", scheme) + assert.Equal(t, "test-token", token, "scheme: %s", scheme) + } +} + +func TestGetAuthContext_Exists(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + expected := AuthContext{ + Subject: "user@example.com", + Datasets: []string{"dataset1", "dataset2"}, + } + c.Set(ContextKey, expected) + + authCtx, exists := GetAuthContext(c) + + assert.True(t, exists) + assert.Equal(t, expected.Subject, authCtx.Subject) + assert.Equal(t, expected.Datasets, authCtx.Datasets) +} + +func TestGetAuthContext_NotExists(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + authCtx, exists := GetAuthContext(c) + + assert.False(t, exists) + assert.Empty(t, authCtx.Subject) + assert.Empty(t, authCtx.Datasets) +} + +func TestSessionCache_SetAndGet(t *testing.T) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + assert.NoError(t, err) + + sc := &SessionCache{cache: cache} + + authCtx := AuthContext{ + Subject: "user@example.com", + Datasets: []string{"test-dataset"}, + } + + sc.Set("test-key", authCtx, time.Hour) + + // Wait for cache to process + time.Sleep(10 * time.Millisecond) + + retrieved, exists := sc.Get("test-key") + + assert.True(t, exists) + assert.Equal(t, authCtx.Subject, retrieved.Subject) + assert.Equal(t, authCtx.Datasets, retrieved.Datasets) +} + +func TestSessionCache_GetNotExists(t *testing.T) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + assert.NoError(t, err) + + sc := &SessionCache{cache: cache} + + _, exists := sc.Get("nonexistent-key") + + assert.False(t, exists) +} + +func TestTokenMiddleware_NoToken(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest("GET", "/test", nil) + + // Initialize auth middleware for testing + err := InitAuthForTesting() + assert.NoError(t, err) + + handler := TokenMiddleware(nil, nil, audit.NoopLogger{}) // nil database, nil visa - not testing dataset lookup + handler(c) + + assert.Equal(t, http.StatusUnauthorized, w.Code) +} + +// capturingLogger records audit events for test assertions. +type capturingLogger struct { + events []audit.Event + mu sync.Mutex +} + +func (l *capturingLogger) Log(_ context.Context, event audit.Event) { + l.mu.Lock() + defer l.mu.Unlock() + l.events = append(l.events, event) +} + +func (l *capturingLogger) IsNoop() bool { return false } + +func TestTokenMiddleware_NoToken_EmitsAuditDenied(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest("GET", "/datasets", nil) + c.Set("correlationId", "test-corr-123") + + err := InitAuthForTesting() + assert.NoError(t, err) + + logger := &capturingLogger{} + handler := TokenMiddleware(nil, nil, logger) + handler(c) + + assert.Equal(t, http.StatusUnauthorized, w.Code) + require.Len(t, logger.events, 1) + evt := logger.events[0] + assert.Equal(t, "download.denied", evt.Event) + assert.Equal(t, http.StatusUnauthorized, evt.HTTPStatus) + assert.Equal(t, "/datasets", evt.Endpoint) + assert.Equal(t, "test-corr-123", evt.CorrelationID) + assert.Empty(t, evt.UserID, "no user identity on 401") +} + +func TestTokenMiddleware_WithSessionCookie(t *testing.T) { + // This test requires config to be initialized with session name + // In a real scenario, config.SessionName() would return the cookie name + // For unit testing, we skip this as it requires full config initialization + t.Skip("Requires full config initialization - covered by integration tests") +} + +// Integration test helper - creates a test server with middleware +func setupTestRouter() *gin.Engine { + gin.SetMode(gin.TestMode) + r := gin.New() + + // Initialize session cache + cache, _ := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e6, + MaxCost: 100000, + BufferItems: 64, + }) + sessionCache = &SessionCache{cache: cache} + + return r +} + +func TestAuthenticator_NoKeys(t *testing.T) { + auth := &Authenticator{ + Keyset: jwk.NewSet(), // Empty keyset + } + + req := httptest.NewRequest("GET", "/test", nil) + req.Header.Set("Authorization", "Bearer test-token") + + _, err := auth.Authenticate(req) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "no keys configured") +} diff --git a/sda/cmd/download/middleware/auth_visa_test.go b/sda/cmd/download/middleware/auth_visa_test.go new file mode 100644 index 0000000000..9b95f4f4ba --- /dev/null +++ b/sda/cmd/download/middleware/auth_visa_test.go @@ -0,0 +1,151 @@ +//go:build visas +// +build visas + +package middleware + +import ( + "crypto/rand" + "crypto/rsa" + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jws" + "github.com/lestrrat-go/jwx/v2/jwt" + "github.com/neicnordic/sensitive-data-archive/cmd/download/visa" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLooksLikeJWT(t *testing.T) { + header := map[string]interface{}{"alg": "RS256", "typ": "JWT"} + payload := map[string]interface{}{"sub": "user", "exp": time.Now().Add(1 * time.Hour).Unix()} + + token := b64JSON(t, header) + "." + b64JSON(t, payload) + ".signature" + assert.True(t, looksLikeJWT(token)) + + opaque := "abc." + base64.RawURLEncoding.EncodeToString([]byte("not-json")) + ".xyz" + assert.False(t, looksLikeJWT(opaque)) +} + +func TestAuthenticate_JWTShapedInvalid_NoUserinfoFallback(t *testing.T) { + ensureTestConfig(t) + + userinfoCalls := int32(0) + userinfoServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&userinfoCalls, 1) + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"sub":"user-123"}`)) + })) + t.Cleanup(userinfoServer.Close) + + _, pubA := newKeyPair(t) + privB, _ := newKeyPair(t) + + keyset := jwk.NewSet() + require.NoError(t, keyset.AddKey(pubA)) + + badToken := signJWT(t, privB, "https://issuer.example", "user-123", time.Now().Add(1*time.Hour)) + + auth = &Authenticator{Keyset: keyset} + + userinfoClient, err := visa.NewUserinfoClient(userinfoServer.URL, time.Second) + require.NoError(t, err) + + _, err = authenticateStructureBased(badToken, userinfoClient) + assert.Error(t, err) + assert.Equal(t, int32(0), atomic.LoadInt32(&userinfoCalls)) +} + +func TestAuthenticate_OpaqueToken_UserinfoFlow(t *testing.T) { + ensureTestConfig(t) + + userinfoServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"sub":"user-123"}`)) + })) + t.Cleanup(userinfoServer.Close) + + userinfoClient, err := visa.NewUserinfoClient(userinfoServer.URL, time.Second) + require.NoError(t, err) + + authCtx, err := authenticateStructureBased("opaque-token", userinfoClient) + require.NoError(t, err) + assert.Equal(t, "user-123", authCtx.Subject) +} + +func TestAuthenticate_DottyOpaque_UserinfoFlow(t *testing.T) { + ensureTestConfig(t) + + userinfoCalls := int32(0) + userinfoServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&userinfoCalls, 1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"sub":"user-123"}`)) + })) + t.Cleanup(userinfoServer.Close) + + userinfoClient, err := visa.NewUserinfoClient(userinfoServer.URL, time.Second) + require.NoError(t, err) + + token := "abc." + base64.RawURLEncoding.EncodeToString([]byte("not-json")) + ".xyz" + authCtx, err := authenticateStructureBased(token, userinfoClient) + require.NoError(t, err) + assert.Equal(t, "user-123", authCtx.Subject) + assert.Equal(t, int32(1), atomic.LoadInt32(&userinfoCalls)) +} + +func TestAuthenticate_OpaqueToken_Disallowed(t *testing.T) { + ensureTestConfig(t) + setAuthAllowOpaque(t, false) + + _, err := authenticateStructureBased("opaque-token", nil) + assert.Error(t, err) +} + +func b64JSON(t *testing.T, value interface{}) string { + t.Helper() + + data, err := json.Marshal(value) + require.NoError(t, err) + + return base64.RawURLEncoding.EncodeToString(data) +} + +func newKeyPair(t *testing.T) (*rsa.PrivateKey, jwk.Key) { + t.Helper() + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + pub, err := jwk.FromRaw(&priv.PublicKey) + require.NoError(t, err) + require.NoError(t, jwk.AssignKeyID(pub)) + + return priv, pub +} + +func signJWT(t *testing.T, priv *rsa.PrivateKey, iss, sub string, exp time.Time) string { + t.Helper() + + token := jwt.New() + require.NoError(t, token.Set(jwt.IssuerKey, iss)) + require.NoError(t, token.Set(jwt.SubjectKey, sub)) + require.NoError(t, token.Set(jwt.ExpirationKey, exp)) + require.NoError(t, token.Set(jwt.IssuedAtKey, time.Now().Add(-1*time.Minute))) + require.NoError(t, token.Set(jwt.NotBeforeKey, time.Now().Add(-1*time.Minute))) + + headers := jws.NewHeaders() + require.NoError(t, headers.Set(jwk.KeyIDKey, "test-key")) + + signed, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, priv, jws.WithProtectedHeaders(headers))) + require.NoError(t, err) + + return string(signed) +} diff --git a/sda/cmd/download/middleware/cache_ttl_test.go b/sda/cmd/download/middleware/cache_ttl_test.go new file mode 100644 index 0000000000..25af944993 --- /dev/null +++ b/sda/cmd/download/middleware/cache_ttl_test.go @@ -0,0 +1,61 @@ +//go:build visas +// +build visas + +package middleware + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestComputeCacheTTL_BoundedByTokenExp(t *testing.T) { + ensureTestConfig(t) + + tokenExp := time.Now().Add(30 * time.Minute) + + ttl := computeCacheTTL(&tokenExp, nil) + + assert.InDelta(t, float64(30*time.Minute), float64(ttl), float64(time.Second)) +} + +func TestComputeCacheTTL_BoundedByVisaExpiry(t *testing.T) { + ensureTestConfig(t) + + tokenExp := time.Now().Add(2 * time.Hour) + visaExp := time.Now().Add(20 * time.Minute) + + ttl := computeCacheTTL(&tokenExp, &visaExp) + + assert.InDelta(t, float64(20*time.Minute), float64(ttl), float64(time.Second)) +} + +func TestComputeCacheTTL_UsesConfigTTLWhenNoToken(t *testing.T) { + ensureTestConfig(t) + + ttl := computeCacheTTL(nil, nil) + + assert.Equal(t, 2*time.Hour, ttl) +} + +func TestComputeCacheTTL_ExpiredToken(t *testing.T) { + ensureTestConfig(t) + + tokenExp := time.Now().Add(-1 * time.Minute) + + ttl := computeCacheTTL(&tokenExp, nil) + + assert.Equal(t, time.Duration(0), ttl, "expired token should return TTL=0") +} + +func TestComputeCacheTTL_ExpiredVisa(t *testing.T) { + ensureTestConfig(t) + + tokenExp := time.Now().Add(1 * time.Hour) + visaExp := time.Now().Add(-5 * time.Minute) + + ttl := computeCacheTTL(&tokenExp, &visaExp) + + assert.Equal(t, time.Duration(0), ttl, "expired visa should return TTL=0 even if token is valid") +} diff --git a/sda/cmd/download/middleware/config_linkname_test.go b/sda/cmd/download/middleware/config_linkname_test.go new file mode 100644 index 0000000000..d615f38953 --- /dev/null +++ b/sda/cmd/download/middleware/config_linkname_test.go @@ -0,0 +1,45 @@ +//go:build visas +// +build visas + +package middleware + +import ( + "testing" + "unsafe" +) + +//go:linkname permissionModel github.com/neicnordic/sensitive-data-archive/cmd/download/config.permissionModel +var permissionModel string + +//go:linkname authAllowOpaque github.com/neicnordic/sensitive-data-archive/cmd/download/config.authAllowOpaque +var authAllowOpaque bool + +//go:linkname oidcIssuer github.com/neicnordic/sensitive-data-archive/cmd/download/config.oidcIssuer +var oidcIssuer string + +// Use unsafe to ensure go:linkname is permitted. +var _ = unsafe.Pointer(nil) + +func setPermissionModel(t *testing.T, value string) { + prev := permissionModel + permissionModel = value + if t != nil { + t.Cleanup(func() { permissionModel = prev }) + } +} + +func setAuthAllowOpaque(t *testing.T, value bool) { + prev := authAllowOpaque + authAllowOpaque = value + if t != nil { + t.Cleanup(func() { authAllowOpaque = prev }) + } +} + +func setOIDCIssuer(t *testing.T, value string) { + prev := oidcIssuer + oidcIssuer = value + if t != nil { + t.Cleanup(func() { oidcIssuer = prev }) + } +} diff --git a/sda/cmd/download/middleware/config_testhelper_test.go b/sda/cmd/download/middleware/config_testhelper_test.go new file mode 100644 index 0000000000..2cc5abf2dd --- /dev/null +++ b/sda/cmd/download/middleware/config_testhelper_test.go @@ -0,0 +1,35 @@ +//go:build visas +// +build visas + +package middleware + +import ( + "os" + "sync" + "testing" + + configv2 "github.com/neicnordic/sensitive-data-archive/internal/config/v2" + "github.com/stretchr/testify/require" +) + +var loadConfigOnce sync.Once + +func ensureTestConfig(t *testing.T) { + t.Helper() + + loadConfigOnce.Do(func() { + t.Setenv("DB_HOST", "localhost") + t.Setenv("DB_USER", "test") + t.Setenv("DB_PASSWORD", "test") + t.Setenv("GRPC_HOST", "localhost") + t.Setenv("AUTH_ALLOW_OPAQUE", "true") + t.Setenv("OIDC_ISSUER", "https://issuer.example") + t.Setenv("VISA_CACHE_TOKEN_TTL", "7200") + + oldArgs := os.Args + os.Args = []string{oldArgs[0]} + defer func() { os.Args = oldArgs }() + + require.NoError(t, configv2.Load()) + }) +} diff --git a/sda/cmd/download/middleware/visa_middleware_test.go b/sda/cmd/download/middleware/visa_middleware_test.go new file mode 100644 index 0000000000..2ba1feb257 --- /dev/null +++ b/sda/cmd/download/middleware/visa_middleware_test.go @@ -0,0 +1,421 @@ +//go:build visas +// +build visas + +package middleware + +import ( + "bytes" + "context" + "crypto/rsa" + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/dgraph-io/ristretto" + "github.com/gin-gonic/gin" + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jws" + "github.com/lestrrat-go/jwx/v2/jwt" + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/neicnordic/sensitive-data-archive/cmd/download/visa" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type fakeDatasetLookup struct { + datasets []string + calls int32 +} + +func (f *fakeDatasetLookup) GetDatasetIDsByUser(ctx context.Context, user string) ([]string, error) { + atomic.AddInt32(&f.calls, 1) + return f.datasets, nil +} + +type fakeDatasetChecker struct { + existing map[string]bool +} + +func (f *fakeDatasetChecker) CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { + return f.existing[datasetID], nil +} + +type authResponse struct { + Subject string `json:"subject"` + Datasets []string `json:"datasets"` + Owned []string `json:"owned"` + Visa []string `json:"visa"` +} + +func TestTokenMiddleware_PermissionModels(t *testing.T) { + ensureTestConfig(t) + + token, _ := setupAuthAndToken(t, "user-123") + + visaValidator := newVisaValidator(t, "visa-1", "https://visa-issuer.example") + + t.Run("ownership", func(t *testing.T) { + setPermissionModel(t, "ownership") + db := &fakeDatasetLookup{datasets: []string{"owned-1"}} + + r := newTestRouter(t, db, visaValidator) + resp := doRequest(t, r, token, "") + + assert.Equal(t, http.StatusOK, resp.Code) + got := decodeAuthResponse(t, resp.Body) + assert.ElementsMatch(t, []string{"owned-1"}, got.Datasets) + assert.Empty(t, got.Visa) + assert.ElementsMatch(t, []string{"owned-1"}, got.Owned) + }) + + t.Run("visa", func(t *testing.T) { + setPermissionModel(t, "visa") + db := &fakeDatasetLookup{datasets: []string{"owned-1"}} + + r := newTestRouter(t, db, visaValidator) + resp := doRequest(t, r, token, "") + + assert.Equal(t, http.StatusOK, resp.Code) + got := decodeAuthResponse(t, resp.Body) + assert.ElementsMatch(t, []string{"visa-1"}, got.Datasets) + assert.ElementsMatch(t, []string{"visa-1"}, got.Visa) + assert.Empty(t, got.Owned) + }) + + t.Run("combined", func(t *testing.T) { + setPermissionModel(t, "combined") + db := &fakeDatasetLookup{datasets: []string{"owned-1"}} + + r := newTestRouter(t, db, visaValidator) + resp := doRequest(t, r, token, "") + + assert.Equal(t, http.StatusOK, resp.Code) + got := decodeAuthResponse(t, resp.Body) + assert.ElementsMatch(t, []string{"owned-1", "visa-1"}, got.Datasets) + assert.ElementsMatch(t, []string{"visa-1"}, got.Visa) + assert.ElementsMatch(t, []string{"owned-1"}, got.Owned) + }) +} + +func TestTokenMiddleware_TokenCacheHit(t *testing.T) { + ensureTestConfig(t) + setPermissionModel(t, "combined") + + token, _ := setupAuthAndToken(t, "user-123") + db := &fakeDatasetLookup{datasets: []string{"owned-1"}} + + validator, userinfoCalls := newVisaValidatorWithUserinfoCounter(t, "visa-1", "https://visa-issuer.example") + + r := newTestRouter(t, db, validator) + + resp1 := doRequest(t, r, token, "") + assert.Equal(t, http.StatusOK, resp1.Code) + time.Sleep(10 * time.Millisecond) + resp2 := doRequest(t, r, token, "") + assert.Equal(t, http.StatusOK, resp2.Code) + + assert.Equal(t, int32(1), atomic.LoadInt32(&db.calls)) + assert.Equal(t, int32(1), atomic.LoadInt32(userinfoCalls)) +} + +func TestTokenMiddleware_VisaFailureCachingPolicyCombined(t *testing.T) { + ensureTestConfig(t) + setPermissionModel(t, "combined") + + token, _ := setupAuthAndToken(t, "user-123") + db := &fakeDatasetLookup{datasets: []string{"owned-1"}} + + validator, userinfoCalls := newVisaValidatorWithUserinfoFailure(t) + + r := newTestRouter(t, db, validator) + + resp1 := doRequest(t, r, token, "") + assert.Equal(t, http.StatusOK, resp1.Code) + resp2 := doRequest(t, r, token, "") + assert.Equal(t, http.StatusOK, resp2.Code) + + assert.Equal(t, int32(2), atomic.LoadInt32(&db.calls)) + assert.Equal(t, int32(2), atomic.LoadInt32(userinfoCalls)) +} + +func TestTokenMiddleware_VisaFailureReturns401InVisaMode(t *testing.T) { + ensureTestConfig(t) + setPermissionModel(t, "visa") + + token, _ := setupAuthAndToken(t, "user-123") + validator, _ := newVisaValidatorWithUserinfoFailure(t) + + r := newTestRouter(t, nil, validator) + + resp := doRequest(t, r, token, "") + assert.Equal(t, http.StatusUnauthorized, resp.Code) +} + +func TestTokenMiddleware_LegacyCookieFallback(t *testing.T) { + ensureTestConfig(t) + setPermissionModel(t, "combined") + + initCaches(t) + cached := AuthContext{Subject: "cached-user", Datasets: []string{"ds-1"}} + sessionCache.Set("legacy", cached, time.Hour) + time.Sleep(10 * time.Millisecond) + + r := gin.New() + r.Use(TokenMiddleware(nil, nil, audit.NoopLogger{})) + r.GET("/test", func(c *gin.Context) { + ctx, _ := GetAuthContext(c) + c.JSON(http.StatusOK, gin.H{"subject": ctx.Subject, "datasets": ctx.Datasets}) + }) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + req.Header.Set("Cookie", "sda_session_key=legacy") + resp := httptest.NewRecorder() + r.ServeHTTP(resp, req) + + assert.Equal(t, http.StatusOK, resp.Code) + var body struct { + Subject string `json:"subject"` + Datasets []string `json:"datasets"` + } + require.NoError(t, json.Unmarshal(resp.Body.Bytes(), &body)) + assert.Equal(t, "cached-user", body.Subject) + assert.ElementsMatch(t, []string{"ds-1"}, body.Datasets) +} + +func TestTokenMiddleware_UserinfoFailureReturns401(t *testing.T) { + ensureTestConfig(t) + setPermissionModel(t, "ownership") + + validator, _ := newVisaValidatorWithUserinfoFailure(t) + + r := newTestRouter(t, nil, validator) + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + req.Header.Set("Authorization", "Bearer opaque-token") + resp := httptest.NewRecorder() + r.ServeHTTP(resp, req) + + assert.Equal(t, http.StatusUnauthorized, resp.Code) +} + +func newTestRouter(t *testing.T, db DatasetLookup, vv *visa.Validator) *gin.Engine { + t.Helper() + gin.SetMode(gin.TestMode) + initCaches(t) + + r := gin.New() + r.Use(TokenMiddleware(db, vv, audit.NoopLogger{})) + r.GET("/test", func(c *gin.Context) { + authCtx, _ := GetAuthContext(c) + c.JSON(http.StatusOK, authResponse{ + Subject: authCtx.Subject, + Datasets: authCtx.Datasets, + Owned: authCtx.OwnedDatasets, + Visa: authCtx.VisaDatasets, + }) + }) + + return r +} + +func initCaches(t *testing.T) { + t.Helper() + + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e5, + MaxCost: 10000, + BufferItems: 64, + }) + require.NoError(t, err) + sessionCache = &SessionCache{cache: cache} + + tokenCacheInst, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e5, + MaxCost: 10000, + BufferItems: 64, + }) + require.NoError(t, err) + tokenCache = &SessionCache{cache: tokenCacheInst} +} + +func setupAuthAndToken(t *testing.T, subject string) (string, *rsa.PrivateKey) { + t.Helper() + + priv, pub := newKeyPair(t) + keyset := jwk.NewSet() + require.NoError(t, keyset.AddKey(pub)) + + auth = &Authenticator{Keyset: keyset} + + token := signJWT(t, priv, "https://issuer.example", subject, time.Now().Add(time.Hour)) + + return token, priv +} + +func doRequest(t *testing.T, r *gin.Engine, token string, cookie string) *httptest.ResponseRecorder { + t.Helper() + + req := httptest.NewRequest(http.MethodGet, "/test", nil) + if token != "" { + req.Header.Set("Authorization", "Bearer "+token) + } + if cookie != "" { + req.Header.Set("Cookie", cookie) + } + resp := httptest.NewRecorder() + r.ServeHTTP(resp, req) + + return resp +} + +func decodeAuthResponse(t *testing.T, body *bytes.Buffer) authResponse { + t.Helper() + + var resp authResponse + require.NoError(t, json.Unmarshal(body.Bytes(), &resp)) + + return resp +} + +func newVisaValidator(t *testing.T, datasetID, issuer string) *visa.Validator { + t.Helper() + + validator, _ := newVisaValidatorWithUserinfoCounter(t, datasetID, issuer) + return validator +} + +func newVisaValidatorWithUserinfoCounter(t *testing.T, datasetID, issuer string) (*visa.Validator, *int32) { + t.Helper() + + priv, pub := newKeyPair(t) + jwksServer := newJWKSServer(t, pub) + t.Cleanup(jwksServer.Close) + + visaJWT := signVisaJWT(t, priv, jwksServer.URL, "visa-kid", issuer, "user-123", visaClaim(datasetID), time.Now().Add(time.Hour)) + + var calls int32 + userinfoServer := newUserinfoServer(t, []string{visaJWT}, http.StatusOK, &calls) + t.Cleanup(userinfoServer.Close) + + checker := &fakeDatasetChecker{existing: map[string]bool{datasetID: true}} + + cfg := visa.ValidatorConfig{ + Source: "userinfo", + UserinfoURL: userinfoServer.URL, + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: 0, + } + + validator, err := visa.NewValidator(cfg, []visa.TrustedIssuer{{ISS: issuer, JKU: jwksServer.URL}}, checker) + require.NoError(t, err) + + return validator, &calls +} + +func newVisaValidatorWithUserinfoFailure(t *testing.T) (*visa.Validator, *int32) { + t.Helper() + + var calls int32 + userinfoServer := newUserinfoServer(t, nil, http.StatusInternalServerError, &calls) + t.Cleanup(userinfoServer.Close) + + checker := &fakeDatasetChecker{existing: map[string]bool{}} + cfg := visa.ValidatorConfig{ + Source: "userinfo", + UserinfoURL: userinfoServer.URL, + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + } + + validator, err := visa.NewValidator(cfg, nil, checker) + require.NoError(t, err) + + return validator, &calls +} + +func newUserinfoServer(t *testing.T, passports []string, status int, calls *int32) *httptest.Server { + t.Helper() + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if calls != nil { + atomic.AddInt32(calls, 1) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if status != http.StatusOK { + return + } + payload, _ := json.Marshal(map[string]interface{}{ + "sub": "user-123", + "ga4gh_passport_v1": passports, + }) + _, _ = w.Write(payload) + })) +} + +func newJWKSServer(t *testing.T, key jwk.Key) *httptest.Server { + t.Helper() + + set := jwk.NewSet() + require.NoError(t, set.AddKey(key)) + + payload, err := json.Marshal(set) + require.NoError(t, err) + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(payload) + })) +} + +func visaClaim(dataset string) map[string]interface{} { + return map[string]interface{}{ + "type": "ControlledAccessGrants", + "by": "https://example.org/issuer", + "value": dataset, + "source": "https://example.org/source", + "asserted": time.Now().Add(-1 * time.Hour).Unix(), + "conditions": []string{}, + } +} + +func signVisaJWT(t *testing.T, priv *rsa.PrivateKey, jku, kid, iss, sub string, visaClaim map[string]interface{}, exp time.Time) string { + t.Helper() + + token := jwt.New() + require.NoError(t, token.Set(jwt.IssuerKey, iss)) + require.NoError(t, token.Set(jwt.SubjectKey, sub)) + require.NoError(t, token.Set(jwt.ExpirationKey, exp)) + require.NoError(t, token.Set(jwt.IssuedAtKey, time.Now().Add(-1*time.Minute))) + require.NoError(t, token.Set(jwt.NotBeforeKey, time.Now().Add(-1*time.Minute))) + require.NoError(t, token.Set("ga4gh_visa_v1", visaClaim)) + + headers := jws.NewHeaders() + require.NoError(t, headers.Set(jwk.KeyIDKey, kid)) + require.NoError(t, headers.Set("jku", jku)) + + signed, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, priv, jws.WithProtectedHeaders(headers))) + require.NoError(t, err) + + return string(signed) +} From da94a9095ff4da96d88aa607709c87c919ae4403 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:07:26 +0100 Subject: [PATCH 374/530] feat(download): add GA4GH visa validation and identity binding GA4GH visa support: - validator.go: GetVisaDatasets() extracts datasets from visa JWTs, enforces (iss, jku) allowlist, verifies signatures via cached JWKS, validates ControlledAccessGrants (by, value, source, conditions, asserted), supports broker-bound/strict-sub/strict-iss-sub identity binding modes, detects multi-identity scenarios - trust.go: LoadTrustedIssuers from JSON with conditional HTTPS enforcement for JKU URLs - userinfo.go: UserinfoClient with HTTP cache, io.LimitReader safety, GA4GH passport v1 claim extraction - jwks_cache.go: JWK cache with per-request fetch limits and (iss, jku) allowlist enforcement - types.go: Identity, TrustedIssuer, VisaClaim, UserinfoResponse - Pre-validation limits: max-visas (200), max-visa-size (16KB), max-jwks-per-request (10) --- sda/cmd/download/visa/extract_test.go | 54 ++ sda/cmd/download/visa/jwks_cache.go | 138 ++++++ sda/cmd/download/visa/testhelpers_test.go | 103 ++++ sda/cmd/download/visa/trust.go | 76 +++ sda/cmd/download/visa/trust_test.go | 43 ++ sda/cmd/download/visa/types.go | 70 +++ sda/cmd/download/visa/userinfo.go | 184 +++++++ sda/cmd/download/visa/userinfo_test.go | 47 ++ sda/cmd/download/visa/validator.go | 464 ++++++++++++++++++ sda/cmd/download/visa/validator_extra_test.go | 368 ++++++++++++++ sda/cmd/download/visa/validator_test.go | 156 ++++++ 11 files changed, 1703 insertions(+) create mode 100644 sda/cmd/download/visa/extract_test.go create mode 100644 sda/cmd/download/visa/jwks_cache.go create mode 100644 sda/cmd/download/visa/testhelpers_test.go create mode 100644 sda/cmd/download/visa/trust.go create mode 100644 sda/cmd/download/visa/trust_test.go create mode 100644 sda/cmd/download/visa/types.go create mode 100644 sda/cmd/download/visa/userinfo.go create mode 100644 sda/cmd/download/visa/userinfo_test.go create mode 100644 sda/cmd/download/visa/validator.go create mode 100644 sda/cmd/download/visa/validator_extra_test.go create mode 100644 sda/cmd/download/visa/validator_test.go diff --git a/sda/cmd/download/visa/extract_test.go b/sda/cmd/download/visa/extract_test.go new file mode 100644 index 0000000000..b69d37b452 --- /dev/null +++ b/sda/cmd/download/visa/extract_test.go @@ -0,0 +1,54 @@ +//go:build visas +// +build visas + +package visa + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExtractDatasetID_Raw(t *testing.T) { + value := "urn:sampledataset:gdi:testcase" + + id := ExtractDatasetID(value, "raw") + + assert.Equal(t, value, id) +} + +func TestExtractDatasetID_Suffix(t *testing.T) { + cases := []struct { + name string + value string + want string + }{ + { + name: "urn", + value: "urn:sampledataset:gdi:testcase", + want: "testcase", + }, + { + name: "url", + value: "https://example.org/datasets/EGAD001", + want: "EGAD001", + }, + { + name: "url-with-query", + value: "https://example.org/datasets/EGAD001?foo=bar", + want: "EGAD001", + }, + { + name: "plain", + value: "EGAD001", + want: "EGAD001", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + id := ExtractDatasetID(tc.value, "suffix") + assert.Equal(t, tc.want, id) + }) + } +} diff --git a/sda/cmd/download/visa/jwks_cache.go b/sda/cmd/download/visa/jwks_cache.go new file mode 100644 index 0000000000..213b001efe --- /dev/null +++ b/sda/cmd/download/visa/jwks_cache.go @@ -0,0 +1,138 @@ +package visa + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "sync" + "time" + + "github.com/dgraph-io/ristretto" + "github.com/lestrrat-go/jwx/v2/jwk" + log "github.com/sirupsen/logrus" +) + +const maxJWKSResponseSize = 512 * 1024 // 512KB + +// JWKSCache caches JWKS fetched from trusted JKU URLs. +// It enforces the trusted issuer allowlist and limits on JKU fetches per request. +type JWKSCache struct { + trusted []TrustedIssuer + cache *ristretto.Cache + cacheTTL time.Duration + httpClient *http.Client + mu sync.RWMutex +} + +// NewJWKSCache creates a new JWKS cache with the given trusted issuers. +func NewJWKSCache(trusted []TrustedIssuer, cacheTTL time.Duration) (*JWKSCache, error) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e5, + MaxCost: 10000, + BufferItems: 64, + }) + if err != nil { + return nil, fmt.Errorf("failed to create JWKS cache: %w", err) + } + + return &JWKSCache{ + trusted: trusted, + cache: cache, + cacheTTL: cacheTTL, + httpClient: &http.Client{ + Timeout: httpClientTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && req.URL.Host != via[0].URL.Host { + return fmt.Errorf("JWKS redirect to different host not allowed: %s", req.URL.Host) + } + if len(via) >= 3 { + return errors.New("too many redirects") + } + + return nil + }, + }, + }, nil +} + +// GetKeySet returns the JWKS for the given issuer+JKU pair. +// It validates the pair is trusted, checks the cache, and fetches if needed. +// The jkuTracker is used to enforce per-request limits on distinct JKU fetches. +func (jc *JWKSCache) GetKeySet(issuer, jkuURL string, jkuTracker map[string]bool, maxJKU int) (jwk.Set, error) { + // Validate the issuer+JKU pair is trusted + if !IsTrusted(jc.trusted, issuer, jkuURL) { + return nil, fmt.Errorf("untrusted issuer+JKU pair: iss=%s, jku=%s", issuer, jkuURL) + } + + normalizedJKU := NormalizeURL(jkuURL) + + // Check cache + if val, found := jc.cache.Get(normalizedJKU); found { + if ks, ok := val.(jwk.Set); ok { + log.Debugf("JWKS cache hit for %s", normalizedJKU) + + return ks, nil + } + } + + // Check per-request JKU limit and track attempt BEFORE fetching. + // This prevents unlimited retry attempts when fetches fail, as failed + // attempts still count against the limit. + if !jkuTracker[normalizedJKU] { + if len(jkuTracker) >= maxJKU { + return nil, fmt.Errorf("exceeded max JWKS fetches per request (%d)", maxJKU) + } + // Mark as attempted before fetch to count against limit even on failure + jkuTracker[normalizedJKU] = true + } + + // Fetch JWKS + keySet, err := jc.fetchJWKS(normalizedJKU) + if err != nil { + return nil, err + } + + // Cache the result + jc.cache.SetWithTTL(normalizedJKU, keySet, 1, jc.cacheTTL) + + return keySet, nil +} + +func (jc *JWKSCache) fetchJWKS(jkuURL string) (jwk.Set, error) { + jc.mu.RLock() + defer jc.mu.RUnlock() + + log.Debugf("fetching JWKS from %s", jkuURL) + + req, err := http.NewRequestWithContext(context.Background(), "GET", jkuURL, nil) + if err != nil { + return nil, fmt.Errorf("failed to create JWKS request: %w", err) + } + req.Header.Set("Accept", "application/json") + + resp, err := jc.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("JWKS fetch failed for %s: %w", jkuURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("JWKS fetch returned status %d for %s", resp.StatusCode, jkuURL) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxJWKSResponseSize)) + if err != nil { + return nil, fmt.Errorf("failed to read JWKS response from %s: %w", jkuURL, err) + } + + keySet, err := jwk.Parse(body) + if err != nil { + return nil, fmt.Errorf("failed to parse JWKS from %s: %w", jkuURL, err) + } + + log.Debugf("fetched JWKS from %s with %d keys", jkuURL, keySet.Len()) + + return keySet, nil +} diff --git a/sda/cmd/download/visa/testhelpers_test.go b/sda/cmd/download/visa/testhelpers_test.go new file mode 100644 index 0000000000..063d5f2c96 --- /dev/null +++ b/sda/cmd/download/visa/testhelpers_test.go @@ -0,0 +1,103 @@ +//go:build visas +// +build visas + +package visa + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/lestrrat-go/jwx/v2/jwa" + "github.com/lestrrat-go/jwx/v2/jwk" + "github.com/lestrrat-go/jwx/v2/jws" + "github.com/lestrrat-go/jwx/v2/jwt" + "github.com/stretchr/testify/require" +) + +type fakeDatasetChecker struct { + existing map[string]bool +} + +func (f *fakeDatasetChecker) CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { + return f.existing[datasetID], nil +} + +func newRSAKeyPair(t *testing.T) (*rsa.PrivateKey, jwk.Key, string) { + t.Helper() + + priv, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + pubKey, err := jwk.FromRaw(&priv.PublicKey) + require.NoError(t, err) + + kid := "test-key" + require.NoError(t, pubKey.Set(jwk.KeyIDKey, kid)) + + return priv, pubKey, kid +} + +func newJWKSServer(t *testing.T, key jwk.Key) *httptest.Server { + t.Helper() + + set := jwk.NewSet() + require.NoError(t, set.AddKey(key)) + + payload, err := json.Marshal(set) + require.NoError(t, err) + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(payload) + })) +} + +func newFailingJWKSServer(t *testing.T) *httptest.Server { + t.Helper() + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusInternalServerError) + })) +} + +func newUserinfoServer(t *testing.T, passports []string) *httptest.Server { + t.Helper() + + payload, err := json.Marshal(map[string]interface{}{ + "sub": "user-123", + "ga4gh_passport_v1": passports, + }) + require.NoError(t, err) + + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write(payload) + })) +} + +func signVisaJWT(t *testing.T, priv *rsa.PrivateKey, jku, kid, iss, sub string, visa map[string]interface{}, exp time.Time) string { + t.Helper() + + token := jwt.New() + require.NoError(t, token.Set(jwt.IssuerKey, iss)) + require.NoError(t, token.Set(jwt.SubjectKey, sub)) + require.NoError(t, token.Set(jwt.ExpirationKey, exp)) + require.NoError(t, token.Set(jwt.IssuedAtKey, time.Now().Add(-1*time.Minute))) + require.NoError(t, token.Set(jwt.NotBeforeKey, time.Now().Add(-1*time.Minute))) + require.NoError(t, token.Set("ga4gh_visa_v1", visa)) + + headers := jws.NewHeaders() + require.NoError(t, headers.Set(jwk.KeyIDKey, kid)) + require.NoError(t, headers.Set("jku", jku)) + + signed, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, priv, jws.WithProtectedHeaders(headers))) + require.NoError(t, err) + + return string(signed) +} diff --git a/sda/cmd/download/visa/trust.go b/sda/cmd/download/visa/trust.go new file mode 100644 index 0000000000..e4ef87899f --- /dev/null +++ b/sda/cmd/download/visa/trust.go @@ -0,0 +1,76 @@ +package visa + +import ( + "encoding/json" + "fmt" + "net/url" + "os" + "path" + "strings" +) + +// LoadTrustedIssuers loads trusted issuer+JKU pairs from a JSON file. +// The file must contain a JSON array of objects with "iss" and "jku" fields. +// Both URLs are normalized before storage. +// If allowInsecureJKU is true, HTTP JKU URLs are permitted (for testing only). +func LoadTrustedIssuers(filePath string, allowInsecureJKU bool) ([]TrustedIssuer, error) { + data, err := os.ReadFile(filePath) + if err != nil { + return nil, fmt.Errorf("failed to read trusted issuers file: %w", err) + } + + var issuers []TrustedIssuer + if err := json.Unmarshal(data, &issuers); err != nil { + return nil, fmt.Errorf("failed to parse trusted issuers JSON: %w", err) + } + + // Validate and normalize + for i := range issuers { + if issuers[i].ISS == "" { + return nil, fmt.Errorf("trusted issuer at index %d has empty 'iss'", i) + } + if issuers[i].JKU == "" { + return nil, fmt.Errorf("trusted issuer at index %d has empty 'jku'", i) + } + + // Require HTTPS for JKU unless explicitly allowed for testing + if !allowInsecureJKU && !strings.HasPrefix(issuers[i].JKU, "https://") { + return nil, fmt.Errorf("trusted issuer at index %d: JKU must use https:// (got %s); set visa.allow-insecure-jku for testing", i, issuers[i].JKU) + } + + issuers[i].ISS = NormalizeURL(issuers[i].ISS) + issuers[i].JKU = NormalizeURL(issuers[i].JKU) + } + + return issuers, nil +} + +// IsTrusted checks if the given issuer+JKU combination is in the trusted list. +func IsTrusted(trusted []TrustedIssuer, issuer, jku string) bool { + normIss := NormalizeURL(issuer) + normJKU := NormalizeURL(jku) + + for _, t := range trusted { + if t.ISS == normIss && t.JKU == normJKU { + return true + } + } + + return false +} + +// NormalizeURL normalizes a URL by cleaning its path component. +// This ensures consistent comparison of URLs with trailing slashes, +// double slashes, or dot segments. +func NormalizeURL(rawURL string) string { + u, err := url.Parse(rawURL) + if err != nil { + return rawURL + } + u.Path = path.Clean(u.Path) + if u.Path == "." { + u.Path = "" + } + + return u.String() +} diff --git a/sda/cmd/download/visa/trust_test.go b/sda/cmd/download/visa/trust_test.go new file mode 100644 index 0000000000..ca554e4a1e --- /dev/null +++ b/sda/cmd/download/visa/trust_test.go @@ -0,0 +1,43 @@ +//go:build visas +// +build visas + +package visa + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadTrustedIssuers_RejectsHTTPJKUByDefault(t *testing.T) { + path := writeTrustedIssuersFile(t, []TrustedIssuer{{ISS: "https://issuer.example", JKU: "http://example.org/jwks"}}) + + _, err := LoadTrustedIssuers(path, false) + assert.Error(t, err) +} + +func TestLoadTrustedIssuers_AllowsHTTPJKUWhenEnabled(t *testing.T) { + path := writeTrustedIssuersFile(t, []TrustedIssuer{{ISS: "https://issuer.example", JKU: "http://example.org/jwks"}}) + + issuers, err := LoadTrustedIssuers(path, true) + require.NoError(t, err) + require.Len(t, issuers, 1) + assert.Equal(t, "http://example.org/jwks", issuers[0].JKU) +} + +func writeTrustedIssuersFile(t *testing.T, issuers []TrustedIssuer) string { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, "trusted-issuers.json") + + payload, err := json.Marshal(issuers) + require.NoError(t, err) + require.NoError(t, os.WriteFile(path, payload, 0o600)) + + return path +} diff --git a/sda/cmd/download/visa/types.go b/sda/cmd/download/visa/types.go new file mode 100644 index 0000000000..d36ce76d20 --- /dev/null +++ b/sda/cmd/download/visa/types.go @@ -0,0 +1,70 @@ +// Package visa provides GA4GH visa validation for the download service. +// It implements ControlledAccessGrants visa processing per the GA4GH Passport spec. +package visa + +import "time" + +// Identity represents an authenticated user's identity as an {issuer, subject} pair. +type Identity struct { + Issuer string + Subject string +} + +// TrustedIssuer represents an allowed (issuer, JKU) pair for visa verification. +type TrustedIssuer struct { + ISS string `json:"iss"` + JKU string `json:"jku"` +} + +// VisaClaim represents the ga4gh_visa_v1 claim within a visa JWT. +type VisaClaim struct { + Type string `json:"type"` + Asserted int64 `json:"asserted"` + Value string `json:"value"` + Source string `json:"source"` + By string `json:"by"` + Conditions any `json:"conditions,omitempty"` +} + +// UserinfoResponse represents the relevant fields from an OIDC userinfo response. +type UserinfoResponse struct { + Sub string `json:"sub"` + Passport []string `json:"ga4gh_passport_v1"` +} + +// ValidatorConfig holds configuration for the visa validator. +type ValidatorConfig struct { + Source string // "userinfo" or "token" + UserinfoURL string // Userinfo endpoint URL + DatasetIDMode string // "raw" or "suffix" + IdentityMode string // "broker-bound", "strict-sub", or "strict-iss-sub" + ValidateAsserted bool + ClockSkew time.Duration + + // Limits + MaxVisas int + MaxJWKSPerReq int + MaxVisaSize int + + // Cache TTLs + JWKCacheTTL time.Duration + ValidationCacheTTL time.Duration + UserinfoCacheTTL time.Duration +} + +// DefaultConfig returns a ValidatorConfig with sensible defaults. +func DefaultConfig() ValidatorConfig { + return ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 30 * time.Second, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16384, + JWKCacheTTL: 5 * time.Minute, + ValidationCacheTTL: 2 * time.Minute, + UserinfoCacheTTL: 1 * time.Minute, + } +} diff --git a/sda/cmd/download/visa/userinfo.go b/sda/cmd/download/visa/userinfo.go new file mode 100644 index 0000000000..9e159e902f --- /dev/null +++ b/sda/cmd/download/visa/userinfo.go @@ -0,0 +1,184 @@ +package visa + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "sync" + "time" + + "github.com/dgraph-io/ristretto" + log "github.com/sirupsen/logrus" +) + +const ( + maxUserinfoResponseSize = 1 << 20 // 1MB + maxDiscoveryResponseSize = 1 << 16 // 64KB + httpClientTimeout = 10 * time.Second +) + +// UserinfoClient handles fetching userinfo from OIDC providers. +type UserinfoClient struct { + httpClient *http.Client + userinfoURL string + cache *ristretto.Cache + cacheTTL time.Duration + mu sync.RWMutex +} + +// NewUserinfoClient creates a new userinfo client. +// If userinfoURL is empty, it must be discovered from the OIDC issuer. +func NewUserinfoClient(userinfoURL string, cacheTTL time.Duration) (*UserinfoClient, error) { + cache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e5, + MaxCost: 10000, + BufferItems: 64, + }) + if err != nil { + return nil, fmt.Errorf("failed to create userinfo cache: %w", err) + } + + // TODO: The CheckRedirect logic below is duplicated in 3 places (here, DiscoverUserinfoURL, + // and jwks_cache.go). If a 4th HTTP client is added, consider extracting a shared + // restrictedRedirectPolicy() helper to reduce duplication and ensure consistent security policy. + return &UserinfoClient{ + httpClient: &http.Client{ + Timeout: httpClientTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && req.URL.Host != via[0].URL.Host { + return fmt.Errorf("userinfo redirect to different host not allowed: %s", req.URL.Host) + } + if len(via) >= 3 { + return errors.New("too many redirects") + } + + return nil + }, + }, + userinfoURL: userinfoURL, + cache: cache, + cacheTTL: cacheTTL, + }, nil +} + +// FetchUserinfo calls the userinfo endpoint with the given access token. +// Results are cached by token hash. +func (uc *UserinfoClient) FetchUserinfo(accessToken string) (*UserinfoResponse, error) { + // Check cache + tokenHash := hashToken(accessToken) + if val, found := uc.cache.Get(tokenHash); found { + if resp, ok := val.(*UserinfoResponse); ok { + log.Debug("userinfo cache hit") + + return resp, nil + } + } + + uc.mu.RLock() + endpoint := uc.userinfoURL + uc.mu.RUnlock() + + if endpoint == "" { + return nil, errors.New("userinfo endpoint URL not configured") + } + + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, fmt.Errorf("failed to create userinfo request: %w", err) + } + req.Header.Set("Authorization", "Bearer "+accessToken) + req.Header.Set("Accept", "application/json") + + resp, err := uc.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("userinfo request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("userinfo returned status %d", resp.StatusCode) + } + + // Check content type - only support JSON + ct := resp.Header.Get("Content-Type") + if strings.Contains(ct, "application/jwt") { + return nil, errors.New("unsupported userinfo content-type: application/jwt; only application/json is supported") + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxUserinfoResponseSize)) + if err != nil { + return nil, fmt.Errorf("failed to read userinfo response: %w", err) + } + + var userinfo UserinfoResponse + if err := json.Unmarshal(body, &userinfo); err != nil { + return nil, fmt.Errorf("failed to parse userinfo response: %w", err) + } + + if userinfo.Sub == "" { + return nil, errors.New("userinfo response missing 'sub' claim") + } + + // Cache the result + uc.cache.SetWithTTL(tokenHash, &userinfo, 1, uc.cacheTTL) + + return &userinfo, nil +} + +// DiscoverUserinfoURL discovers the userinfo endpoint from an OIDC issuer. +func DiscoverUserinfoURL(issuerURL string) (string, error) { + discoveryURL := strings.TrimSuffix(issuerURL, "/") + "/.well-known/openid-configuration" + + client := &http.Client{ + Timeout: httpClientTimeout, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 && req.URL.Host != via[0].URL.Host { + return fmt.Errorf("OIDC discovery redirect to different host not allowed: %s", req.URL.Host) + } + if len(via) >= 3 { + return errors.New("too many redirects") + } + + return nil + }, + } + + resp, err := client.Get(discoveryURL) + if err != nil { + return "", fmt.Errorf("OIDC discovery request failed: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("OIDC discovery returned status %d", resp.StatusCode) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxDiscoveryResponseSize)) + if err != nil { + return "", fmt.Errorf("failed to read discovery response: %w", err) + } + + var discovery struct { + UserinfoEndpoint string `json:"userinfo_endpoint"` + } + if err := json.Unmarshal(body, &discovery); err != nil { + return "", fmt.Errorf("failed to parse discovery response: %w", err) + } + + if discovery.UserinfoEndpoint == "" { + return "", errors.New("discovery response missing userinfo_endpoint") + } + + return discovery.UserinfoEndpoint, nil +} + +func hashToken(token string) string { + h := sha256.Sum256([]byte(token)) + + return hex.EncodeToString(h[:]) +} diff --git a/sda/cmd/download/visa/userinfo_test.go b/sda/cmd/download/visa/userinfo_test.go new file mode 100644 index 0000000000..1d02994bde --- /dev/null +++ b/sda/cmd/download/visa/userinfo_test.go @@ -0,0 +1,47 @@ +//go:build visas +// +build visas + +package visa + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiscoverUserinfoURL_RejectsCrossHostRedirect(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, "https://evil.example/.well-known/openid-configuration", http.StatusFound) + })) + t.Cleanup(server.Close) + + _, err := DiscoverUserinfoURL(server.URL) + require.Error(t, err) + assert.Contains(t, err.Error(), "redirect to different host not allowed") +} + +func TestDiscoverUserinfoURL_AllowsSameHostRedirect(t *testing.T) { + mux := http.NewServeMux() + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + + mux.HandleFunc("/.well-known/openid-configuration", func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, server.URL+"/real-discovery", http.StatusFound) + }) + mux.HandleFunc("/real-discovery", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + resp, _ := json.Marshal(map[string]string{ + "userinfo_endpoint": "https://example.org/userinfo", + }) + _, _ = w.Write(resp) + }) + + url, err := DiscoverUserinfoURL(server.URL) + require.NoError(t, err) + assert.Equal(t, "https://example.org/userinfo", url) +} diff --git a/sda/cmd/download/visa/validator.go b/sda/cmd/download/visa/validator.go new file mode 100644 index 0000000000..3320befeb9 --- /dev/null +++ b/sda/cmd/download/visa/validator.go @@ -0,0 +1,464 @@ +package visa + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "path" + "strings" + "time" + + "github.com/dgraph-io/ristretto" + "github.com/lestrrat-go/jwx/v2/jws" + "github.com/lestrrat-go/jwx/v2/jwt" + log "github.com/sirupsen/logrus" +) + +const maxURLClaimLength = 255 + +// DatasetChecker checks whether a dataset exists in the local database. +type DatasetChecker interface { + CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) +} + +// Validator performs GA4GH visa validation. +type Validator struct { + config ValidatorConfig + trustedIssuers []TrustedIssuer + jwksCache *JWKSCache + userinfoClient *UserinfoClient + validCache *ristretto.Cache + datasetChecker DatasetChecker +} + +// NewValidator creates a new visa validator with the given configuration. +func NewValidator(cfg ValidatorConfig, trustedIssuers []TrustedIssuer, dc DatasetChecker) (*Validator, error) { + jwksCache, err := NewJWKSCache(trustedIssuers, cfg.JWKCacheTTL) + if err != nil { + return nil, fmt.Errorf("failed to create JWKS cache: %w", err) + } + + userinfoClient, err := NewUserinfoClient(cfg.UserinfoURL, cfg.UserinfoCacheTTL) + if err != nil { + return nil, fmt.Errorf("failed to create userinfo client: %w", err) + } + + validCache, err := ristretto.NewCache(&ristretto.Config{ + NumCounters: 1e5, + MaxCost: 10000, + BufferItems: 64, + }) + if err != nil { + return nil, fmt.Errorf("failed to create validation cache: %w", err) + } + + return &Validator{ + config: cfg, + trustedIssuers: trustedIssuers, + jwksCache: jwksCache, + userinfoClient: userinfoClient, + validCache: validCache, + datasetChecker: dc, + }, nil +} + +// UserinfoClient returns the validator's userinfo client for reuse in opaque token authentication. +func (v *Validator) UserinfoClient() *UserinfoClient { + return v.userinfoClient +} + +// VisaResult holds the result of visa processing for a single request. +type VisaResult struct { + Datasets []string // Datasets granted by valid visas + MinExpiry time.Time // Earliest visa expiry (for cache TTL bounding) + Warnings []string // Non-fatal issues (logged, not returned to client) +} + +// GetVisaDatasets retrieves and validates visas, returning granted dataset IDs. +// authSource indicates how the user authenticated ("jwt" or "userinfo"). +// For "jwt" auth with source="userinfo", we call userinfo to get visas. +// For opaque tokens (authSource="userinfo"), we already have the userinfo response. +func (v *Validator) GetVisaDatasets(ctx context.Context, identity Identity, rawToken string, authSource string) (*VisaResult, error) { + // Get passport (visa JWTs) + passport, err := v.getPassport(rawToken, authSource) + if err != nil { + return nil, fmt.Errorf("failed to retrieve passport: %w", err) + } + + if len(passport) == 0 { + return &VisaResult{}, nil + } + + return v.processVisas(ctx, identity, passport) +} + +// getPassport retrieves visa JWTs from the configured source. +func (v *Validator) getPassport(rawToken string, authSource string) ([]string, error) { + switch v.config.Source { + case "userinfo": + // Always call userinfo for visa retrieval (GA4GH compliant) + userinfo, err := v.userinfoClient.FetchUserinfo(rawToken) + if err != nil { + return nil, err + } + + return userinfo.Passport, nil + + case "token": + // For opaque tokens, must use userinfo regardless + if authSource == "userinfo" { + userinfo, err := v.userinfoClient.FetchUserinfo(rawToken) + if err != nil { + return nil, err + } + + return userinfo.Passport, nil + } + + // For JWT tokens in "token" mode, parse visas from token claims. + // Userinfo overrides if also called (per GA4GH spec precedence). + token, err := jwt.Parse([]byte(rawToken), jwt.WithVerify(false)) + if err != nil { + return nil, fmt.Errorf("failed to parse token for visa claims: %w", err) + } + + claims := token.PrivateClaims() + passportClaim, ok := claims["ga4gh_passport_v1"] + if !ok { + return nil, nil + } + + passportSlice, ok := passportClaim.([]any) + if !ok { + return nil, errors.New("ga4gh_passport_v1 claim is not an array") + } + + passport := make([]string, 0, len(passportSlice)) + for _, v := range passportSlice { + s, ok := v.(string) + if !ok { + continue + } + passport = append(passport, s) + } + + return passport, nil + + default: + return nil, fmt.Errorf("unknown visa source: %s", v.config.Source) + } +} + +// processVisas validates individual visa JWTs and extracts dataset grants. +func (v *Validator) processVisas(ctx context.Context, identity Identity, passport []string) (*VisaResult, error) { + result := &VisaResult{} + seen := make(map[string]bool) + jkuTracker := make(map[string]bool) + identities := make(map[string]bool) // Track {iss, sub} pairs for multi-identity detection + + for i, visaJWT := range passport { + // Enforce max visas limit + if i >= v.config.MaxVisas { + result.Warnings = append(result.Warnings, fmt.Sprintf("stopped processing after %d visas (max-visas limit)", v.config.MaxVisas)) + log.Warnf("stopped processing after %d visas (max-visas limit)", v.config.MaxVisas) + + break + } + + // Enforce max visa size + if len(visaJWT) > v.config.MaxVisaSize { + result.Warnings = append(result.Warnings, fmt.Sprintf("skipped visa %d: size %d exceeds max %d", i, len(visaJWT), v.config.MaxVisaSize)) + log.Warnf("skipped visa %d: size %d exceeds max %d", i, len(visaJWT), v.config.MaxVisaSize) + + continue + } + + // Check validation cache + visaHash := hashToken(visaJWT) + if cached, found := v.validCache.Get(visaHash); found { + if datasets, ok := cached.([]string); ok { + for _, ds := range datasets { + if !seen[ds] { + seen[ds] = true + result.Datasets = append(result.Datasets, ds) + } + } + + continue + } + } + + datasets, expiry, err := v.validateSingleVisa(ctx, identity, visaJWT, jkuTracker, identities) + if err != nil { + log.Debugf("visa %d rejected: %v", i, err) + + continue + } + + // Track earliest expiry for cache TTL bounding + if !expiry.IsZero() && (result.MinExpiry.IsZero() || expiry.Before(result.MinExpiry)) { + result.MinExpiry = expiry + } + + // Cache validated visa result + cacheTTL := v.config.ValidationCacheTTL + if !expiry.IsZero() { + remaining := time.Until(expiry) + if remaining < cacheTTL { + cacheTTL = remaining + } + } + if cacheTTL > 0 { + v.validCache.SetWithTTL(visaHash, datasets, 1, cacheTTL) + } + + for _, ds := range datasets { + if !seen[ds] { + seen[ds] = true + result.Datasets = append(result.Datasets, ds) + } + } + } + + // Multi-identity detection + if len(identities) > 1 { + msg := fmt.Sprintf("passport contains visas from %d distinct {iss, sub} pairs", len(identities)) + result.Warnings = append(result.Warnings, msg) + log.Warn(msg) + } + + return result, nil +} + +// validateSingleVisa validates one visa JWT and returns granted datasets and expiry. +func (v *Validator) validateSingleVisa(ctx context.Context, identity Identity, visaJWT string, jkuTracker map[string]bool, identities map[string]bool) ([]string, time.Time, error) { + var zeroTime time.Time + + // 1. Parse JWS header to get JKU + msg, err := jws.Parse([]byte(visaJWT)) + if err != nil { + return nil, zeroTime, fmt.Errorf("failed to parse visa JWS: %w", err) + } + + sigs := msg.Signatures() + if len(sigs) == 0 { + return nil, zeroTime, errors.New("visa has no signatures") + } + + jkuURL := sigs[0].ProtectedHeaders().JWKSetURL() + if jkuURL == "" { + return nil, zeroTime, errors.New("visa missing jku header") + } + + // 2. Parse unverified token to get issuer (needed for trust check) + unverified, err := jwt.Parse([]byte(visaJWT), jwt.WithVerify(false)) + if err != nil { + return nil, zeroTime, fmt.Errorf("failed to parse visa claims: %w", err) + } + + visaIssuer := unverified.Issuer() + visaSubject := unverified.Subject() + + // Track identity for multi-identity detection + identityKey := visaIssuer + "\x00" + visaSubject + identities[identityKey] = true + + // 3. Get JWKS from trusted JKU (enforces allowlist) + keySet, err := v.jwksCache.GetKeySet(visaIssuer, jkuURL, jkuTracker, v.config.MaxJWKSPerReq) + if err != nil { + return nil, zeroTime, fmt.Errorf("failed to get JWKS: %w", err) + } + + // 4. Verify visa signature + verified, err := jwt.Parse([]byte(visaJWT), + jwt.WithKeySet(keySet, jws.WithInferAlgorithmFromKey(true), jws.WithRequireKid(false)), + jwt.WithAcceptableSkew(v.config.ClockSkew), + jwt.WithValidate(true), + ) + if err != nil { + return nil, zeroTime, fmt.Errorf("visa signature/claims validation failed: %w", err) + } + + // 5. Identity binding check + if err := v.checkIdentityBinding(identity, visaIssuer, visaSubject); err != nil { + return nil, zeroTime, err + } + + // 6. Extract and validate ga4gh_visa_v1 claim + visaClaim, err := extractVisaClaim(verified) + if err != nil { + return nil, zeroTime, err + } + + // 7. Only process ControlledAccessGrants + if visaClaim.Type != "ControlledAccessGrants" { + // Unknown visa types are silently ignored (GA4GH compliant) + return nil, zeroTime, nil + } + + // 8. Validate ControlledAccessGrants requirements + if err := v.validateControlledAccessGrant(visaClaim); err != nil { + return nil, zeroTime, err + } + + // 9. Extract dataset ID + datasetID := ExtractDatasetID(visaClaim.Value, v.config.DatasetIDMode) + if datasetID == "" { + return nil, zeroTime, errors.New("empty dataset ID extracted from visa value") + } + + // 10. Verify dataset exists locally + exists, err := v.datasetChecker.CheckDatasetExists(ctx, datasetID) + if err != nil { + log.Debugf("dataset existence check failed for %s: %v", datasetID, err) + + return nil, zeroTime, nil + } + if !exists { + log.Debugf("visa dataset %s not found locally", datasetID) + + return nil, zeroTime, nil + } + + return []string{datasetID}, verified.Expiration(), nil +} + +// checkIdentityBinding enforces the configured identity binding mode. +func (v *Validator) checkIdentityBinding(identity Identity, visaIssuer, visaSubject string) error { + switch v.config.IdentityMode { + case "broker-bound": + // Accept all visas from trusted issuers (no identity check) + return nil + case "strict-sub": + if visaSubject != identity.Subject { + return fmt.Errorf("visa subject %q does not match authenticated subject %q", visaSubject, identity.Subject) + } + + return nil + case "strict-iss-sub": + if visaIssuer != identity.Issuer || visaSubject != identity.Subject { + return fmt.Errorf("visa {iss, sub} {%q, %q} does not match authenticated {%q, %q}", + visaIssuer, visaSubject, identity.Issuer, identity.Subject) + } + + return nil + default: + return fmt.Errorf("unknown identity mode: %s", v.config.IdentityMode) + } +} + +// extractVisaClaim extracts and parses the ga4gh_visa_v1 claim from a validated token. +func extractVisaClaim(token jwt.Token) (*VisaClaim, error) { + claims := token.PrivateClaims() + raw, ok := claims["ga4gh_visa_v1"] + if !ok { + return nil, errors.New("visa missing ga4gh_visa_v1 claim") + } + + // Marshal and unmarshal to get a typed struct + data, err := json.Marshal(raw) + if err != nil { + return nil, fmt.Errorf("failed to marshal visa claim: %w", err) + } + + var vc VisaClaim + if err := json.Unmarshal(data, &vc); err != nil { + return nil, fmt.Errorf("failed to parse visa claim: %w", err) + } + + return &vc, nil +} + +// validateControlledAccessGrant validates GA4GH ControlledAccessGrants requirements. +func (v *Validator) validateControlledAccessGrant(vc *VisaClaim) error { + // by claim must exist and be non-empty + if vc.By == "" { + return errors.New("ControlledAccessGrants visa missing 'by' claim") + } + + // value must be valid URL-claim (max 255 chars) + if len(vc.Value) > maxURLClaimLength { + return fmt.Errorf("visa value exceeds max length (%d > %d)", len(vc.Value), maxURLClaimLength) + } + if vc.Value == "" { + return errors.New("ControlledAccessGrants visa missing 'value' claim") + } + + // source must be valid URL-claim + if vc.Source == "" { + return errors.New("ControlledAccessGrants visa missing 'source' claim") + } + if len(vc.Source) > maxURLClaimLength { + return fmt.Errorf("visa source exceeds max length (%d > %d)", len(vc.Source), maxURLClaimLength) + } + + // asserted is REQUIRED for ControlledAccessGrants (GA4GH requirement) + if vc.Asserted <= 0 { + return errors.New("ControlledAccessGrants visa missing or invalid 'asserted' claim") + } + + // asserted must be <= now (if validation enabled) + if v.config.ValidateAsserted { + assertedTime := time.Unix(vc.Asserted, 0) + if assertedTime.After(time.Now().Add(v.config.ClockSkew)) { + return fmt.Errorf("visa asserted timestamp is in the future: %v", assertedTime) + } + } + + // conditions: if present and non-empty, reject + if vc.Conditions != nil { + if err := rejectNonEmptyConditions(vc.Conditions); err != nil { + return err + } + } + + return nil +} + +// rejectNonEmptyConditions rejects visas with non-empty conditions. +func rejectNonEmptyConditions(conditions any) error { + switch c := conditions.(type) { + case []any: + if len(c) > 0 { + return errors.New("visa has non-empty conditions (not supported)") + } + case map[string]any: + if len(c) > 0 { + return errors.New("visa has non-empty conditions (not supported)") + } + default: + // Non-nil, non-empty conditions of unknown type + return errors.New("visa has conditions of unsupported type") + } + + return nil +} + +// ExtractDatasetID extracts a dataset ID from a visa value based on the configured mode. +func ExtractDatasetID(value string, mode string) string { + if mode == "raw" { + return value + } + + // mode == "suffix": extract last segment from URL/URN + value = strings.TrimSuffix(value, "/") + + // For URLs, use proper parsing + if strings.HasPrefix(value, "http://") || strings.HasPrefix(value, "https://") { + if u, err := url.Parse(value); err == nil { + return path.Base(u.Path) + } + } + + // For URNs or paths, extract last segment + if idx := strings.LastIndex(value, "/"); idx >= 0 { + return value[idx+1:] + } + if idx := strings.LastIndex(value, ":"); idx >= 0 { + return value[idx+1:] + } + + return value +} diff --git a/sda/cmd/download/visa/validator_extra_test.go b/sda/cmd/download/visa/validator_extra_test.go new file mode 100644 index 0000000000..068183cd47 --- /dev/null +++ b/sda/cmd/download/visa/validator_extra_test.go @@ -0,0 +1,368 @@ +//go:build visas +// +build visas + +package visa + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestVisa_UntrustedIssuerRejected(t *testing.T) { + visaClaim := baseVisaClaim("EGAD100") + validator, identity, rawToken := setupValidatorWithUntrustedIssuer(t, visaClaim) + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestVisa_StrictSub_RejectsMismatchedSubject(t *testing.T) { + visaClaim := baseVisaClaim("EGAD101") + validator, identity, rawToken := setupValidatorWithIdentityMode(t, visaClaim, "strict-sub", "other-subject", "https://broker.example") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestVisa_StrictIssSub_RejectsMismatchedIssuer(t *testing.T) { + visaClaim := baseVisaClaim("EGAD102") + validator, identity, rawToken := setupValidatorWithIdentityMode(t, visaClaim, "strict-iss-sub", "user-123", "https://different-issuer.example") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestVisa_UnknownTypeIgnored(t *testing.T) { + visaClaim := baseVisaClaim("EGAD103") + visaClaim["type"] = "AffiliationAndRole" + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestVisa_ValueExceedsMaxLengthRejected(t *testing.T) { + longValue := strings.Repeat("a", 256) + visaClaim := baseVisaClaim(longValue) + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestVisa_SourceExceedsMaxLengthRejected(t *testing.T) { + longSource := strings.Repeat("b", 256) + visaClaim := baseVisaClaim("EGAD104") + visaClaim["source"] = longSource + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestVisa_MultiIdentityWarning(t *testing.T) { + priv, pub, kid := newRSAKeyPair(t) + jwksServer := newJWKSServer(t, pub) + t.Cleanup(jwksServer.Close) + + issuerA := "https://issuer-a.example" + issuerB := "https://issuer-b.example" + jku := jwksServer.URL + + visaA := signVisaJWT(t, priv, jku, kid, issuerA, "user-123", baseVisaClaim("DS1"), time.Now().Add(time.Hour)) + visaB := signVisaJWT(t, priv, jku, kid, issuerB, "user-456", baseVisaClaim("DS2"), time.Now().Add(time.Hour)) + + validator := setupValidatorWithPassports(t, []string{visaA, visaB}, []TrustedIssuer{ + {ISS: issuerA, JKU: jku}, + {ISS: issuerB, JKU: jku}, + }, map[string]bool{"DS1": true, "DS2": true}, ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + }) + + identity := Identity{Issuer: "https://broker.example", Subject: "user-123"} + result, err := validator.GetVisaDatasets(context.Background(), identity, "opaque-token", "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.ElementsMatch(t, []string{"DS1", "DS2"}, result.Datasets) + assert.NotEmpty(t, result.Warnings) + assert.True(t, containsWarning(result.Warnings, "distinct {iss, sub} pairs")) +} + +func TestVisa_MaxJWKSPerRequestEnforced(t *testing.T) { + privA, pubA, kidA := newRSAKeyPair(t) + privB, pubB, kidB := newRSAKeyPair(t) + + jwksServerA := newJWKSServer(t, pubA) + t.Cleanup(jwksServerA.Close) + jwksServerB := newJWKSServer(t, pubB) + t.Cleanup(jwksServerB.Close) + + issuerA := "https://issuer-a.example" + issuerB := "https://issuer-b.example" + + visaA := signVisaJWT(t, privA, jwksServerA.URL, kidA, issuerA, "user-123", baseVisaClaim("DSA"), time.Now().Add(time.Hour)) + visaB := signVisaJWT(t, privB, jwksServerB.URL, kidB, issuerB, "user-123", baseVisaClaim("DSB"), time.Now().Add(time.Hour)) + + validator := setupValidatorWithPassports(t, []string{visaA, visaB}, []TrustedIssuer{ + {ISS: issuerA, JKU: jwksServerA.URL}, + {ISS: issuerB, JKU: jwksServerB.URL}, + }, map[string]bool{"DSA": true, "DSB": true}, ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 1, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + }) + + identity := Identity{Issuer: "https://broker.example", Subject: "user-123"} + result, err := validator.GetVisaDatasets(context.Background(), identity, "opaque-token", "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, 1, len(result.Datasets)) + assert.Equal(t, "DSA", result.Datasets[0]) +} + +func TestVisa_MaxVisasLimit(t *testing.T) { + priv, pub, kid := newRSAKeyPair(t) + jwksServer := newJWKSServer(t, pub) + t.Cleanup(jwksServer.Close) + + issuer := "https://issuer-maxvisas.example" + jku := jwksServer.URL + + passports := make([]string, 0, 201) + existing := make(map[string]bool, 201) + for i := 0; i < 201; i++ { + dataset := fmt.Sprintf("DS%03d", i) + visaJWT := signVisaJWT(t, priv, jku, kid, issuer, "user-123", baseVisaClaim(dataset), time.Now().Add(time.Hour)) + passports = append(passports, visaJWT) + existing[dataset] = true + } + + validator := setupValidatorWithPassports(t, passports, []TrustedIssuer{{ISS: issuer, JKU: jku}}, existing, ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + }) + + identity := Identity{Issuer: "https://broker.example", Subject: "user-123"} + result, err := validator.GetVisaDatasets(context.Background(), identity, "opaque-token", "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Len(t, result.Datasets, 200) +} + +func TestVisa_FailedJKUFetchCountsAgainstLimit(t *testing.T) { + // First JKU server returns 500 (simulates transient failure) + failingServer := newFailingJWKSServer(t) + t.Cleanup(failingServer.Close) + + // Second JKU server works correctly + privB, pubB, kidB := newRSAKeyPair(t) + workingServer := newJWKSServer(t, pubB) + t.Cleanup(workingServer.Close) + + issuerA := "https://issuer-failing.example" + issuerB := "https://issuer-working.example" + + // Sign visa A with a throwaway key (JKU will fail before verification) + privA, _, _ := newRSAKeyPair(t) + visaA := signVisaJWT(t, privA, failingServer.URL, "kid-a", issuerA, "user-123", baseVisaClaim("DSA"), time.Now().Add(time.Hour)) + visaB := signVisaJWT(t, privB, workingServer.URL, kidB, issuerB, "user-123", baseVisaClaim("DSB"), time.Now().Add(time.Hour)) + + validator := setupValidatorWithPassports(t, []string{visaA, visaB}, []TrustedIssuer{ + {ISS: issuerA, JKU: failingServer.URL}, + {ISS: issuerB, JKU: workingServer.URL}, + }, map[string]bool{"DSA": true, "DSB": true}, ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 1, // Only 1 distinct JKU allowed + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + }) + + identity := Identity{Issuer: "https://broker.example", Subject: "user-123"} + result, err := validator.GetVisaDatasets(context.Background(), identity, "opaque-token", "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + + // With the fix: failed JKU fetch counts against limit, blocking second visa's different JKU. + // Without the fix: failed fetch wouldn't count, allowing the second visa's JKU fetch to succeed. + assert.Empty(t, result.Datasets, "failed JKU fetch should count against per-request limit, preventing second JKU fetch") +} + +func TestVisa_MaxVisaSizeLimit(t *testing.T) { + priv, pub, kid := newRSAKeyPair(t) + jwksServer := newJWKSServer(t, pub) + t.Cleanup(jwksServer.Close) + + issuer := "https://issuer-maxsize.example" + jku := jwksServer.URL + + largeValue := strings.Repeat("x", 2000) + visaClaim := baseVisaClaim(largeValue) + visaJWT := signVisaJWT(t, priv, jku, kid, issuer, "user-123", visaClaim, time.Now().Add(time.Hour)) + + validator := setupValidatorWithPassports(t, []string{visaJWT}, []TrustedIssuer{{ISS: issuer, JKU: jku}}, map[string]bool{}, ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 512, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + }) + + identity := Identity{Issuer: "https://broker.example", Subject: "user-123"} + result, err := validator.GetVisaDatasets(context.Background(), identity, "opaque-token", "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func setupValidatorWithPassports(t *testing.T, passports []string, trusted []TrustedIssuer, existing map[string]bool, cfg ValidatorConfig) *Validator { + t.Helper() + + userinfoServer := newUserinfoServer(t, passports) + t.Cleanup(userinfoServer.Close) + + cfg.UserinfoURL = userinfoServer.URL + + checker := &fakeDatasetChecker{existing: existing} + + validator, err := NewValidator(cfg, trusted, checker) + require.NoError(t, err) + + return validator +} + +func setupValidatorWithUntrustedIssuer(t *testing.T, visaClaim map[string]interface{}) (*Validator, Identity, string) { + t.Helper() + + priv, pub, kid := newRSAKeyPair(t) + jwksServer := newJWKSServer(t, pub) + t.Cleanup(jwksServer.Close) + + issuer := "https://trusted-issuer.example" + visaJWT := signVisaJWT(t, priv, jwksServer.URL, kid, issuer, "user-123", visaClaim, time.Now().Add(time.Hour)) + + cfg := ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: "broker-bound", + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + } + + validator := setupValidatorWithPassports(t, []string{visaJWT}, []TrustedIssuer{ + {ISS: "https://different-issuer.example", JKU: jwksServer.URL}, + }, map[string]bool{ExtractDatasetID(visaClaim["value"].(string), "raw"): true}, cfg) + + identity := Identity{Issuer: "https://broker.example", Subject: "user-123"} + + return validator, identity, "opaque-token" +} + +func setupValidatorWithIdentityMode(t *testing.T, visaClaim map[string]interface{}, mode, subject, issuer string) (*Validator, Identity, string) { + t.Helper() + + priv, pub, kid := newRSAKeyPair(t) + jwksServer := newJWKSServer(t, pub) + t.Cleanup(jwksServer.Close) + + visaIssuer := "https://visa-issuer.example" + visaJWT := signVisaJWT(t, priv, jwksServer.URL, kid, visaIssuer, "user-123", visaClaim, time.Now().Add(time.Hour)) + + cfg := ValidatorConfig{ + Source: "userinfo", + DatasetIDMode: "raw", + IdentityMode: mode, + ValidateAsserted: true, + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: time.Minute, + ValidationCacheTTL: time.Minute, + UserinfoCacheTTL: time.Minute, + } + + validator := setupValidatorWithPassports(t, []string{visaJWT}, []TrustedIssuer{ + {ISS: visaIssuer, JKU: jwksServer.URL}, + }, map[string]bool{ExtractDatasetID(visaClaim["value"].(string), "raw"): true}, cfg) + + identity := Identity{Issuer: issuer, Subject: subject} + + return validator, identity, "opaque-token" +} + +func containsWarning(warnings []string, substr string) bool { + for _, w := range warnings { + if strings.Contains(w, substr) { + return true + } + } + + return false +} diff --git a/sda/cmd/download/visa/validator_test.go b/sda/cmd/download/visa/validator_test.go new file mode 100644 index 0000000000..449953b1d9 --- /dev/null +++ b/sda/cmd/download/visa/validator_test.go @@ -0,0 +1,156 @@ +//go:build visas +// +build visas + +package visa + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetVisaDatasets_UserinfoFlow(t *testing.T) { + visaValue := "EGAD001" + visaClaim := baseVisaClaim(visaValue) + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, []string{visaValue}, result.Datasets) +} + +func TestGetVisaDatasets_RejectsMissingBy(t *testing.T) { + visaClaim := baseVisaClaim("EGAD002") + delete(visaClaim, "by") + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestGetVisaDatasets_RejectsConditions(t *testing.T) { + visaClaim := baseVisaClaim("EGAD003") + visaClaim["conditions"] = []string{"consent"} + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestGetVisaDatasets_RejectsFutureAsserted(t *testing.T) { + visaClaim := baseVisaClaim("EGAD004") + visaClaim["asserted"] = time.Now().Add(1 * time.Hour).Unix() + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets) +} + +func TestGetVisaDatasets_RejectsMissingAsserted(t *testing.T) { + visaClaim := baseVisaClaim("EGAD004") + delete(visaClaim, "asserted") // Remove asserted field entirely + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets, "visa without asserted claim should be rejected") +} + +func TestGetVisaDatasets_RejectsZeroAsserted(t *testing.T) { + visaClaim := baseVisaClaim("EGAD004") + visaClaim["asserted"] = 0 // Explicit zero + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "raw") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Empty(t, result.Datasets, "visa with asserted=0 should be rejected") +} + +func TestGetVisaDatasets_SuffixExtraction(t *testing.T) { + visaValue := "https://example.org/datasets/EGAD005?foo=bar" + visaClaim := baseVisaClaim(visaValue) + + validator, identity, rawToken := setupValidatorWithVisa(t, visaClaim, "suffix") + + result, err := validator.GetVisaDatasets(context.Background(), identity, rawToken, "userinfo") + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, []string{"EGAD005"}, result.Datasets) +} + +func baseVisaClaim(value string) map[string]interface{} { + return map[string]interface{}{ + "type": "ControlledAccessGrants", + "by": "https://example.org/issuer", + "value": value, + "source": "https://example.org/source", + "asserted": time.Now().Add(-1 * time.Hour).Unix(), + "conditions": []string{}, + } +} + +func setupValidatorWithVisa(t *testing.T, visaClaim map[string]interface{}, datasetIDMode string) (*Validator, Identity, string) { + t.Helper() + + priv, pub, kid := newRSAKeyPair(t) + jwksServer := newJWKSServer(t, pub) + t.Cleanup(jwksServer.Close) + + issuer := "https://visa-issuer.example" + jku := jwksServer.URL + visaJWT := signVisaJWT(t, priv, jku, kid, issuer, "user-123", visaClaim, time.Now().Add(1*time.Hour)) + + userinfoServer := newUserinfoServer(t, []string{visaJWT}) + t.Cleanup(userinfoServer.Close) + + cfg := ValidatorConfig{ + Source: "userinfo", + UserinfoURL: userinfoServer.URL, + DatasetIDMode: datasetIDMode, + ValidateAsserted: true, + IdentityMode: "broker-bound", + ClockSkew: 0, + MaxVisas: 200, + MaxJWKSPerReq: 10, + MaxVisaSize: 16 * 1024, + JWKCacheTTL: 5 * time.Minute, + ValidationCacheTTL: 2 * time.Minute, + UserinfoCacheTTL: 1 * time.Minute, + } + + checker := &fakeDatasetChecker{existing: map[string]bool{ + ExtractDatasetID(visaClaim["value"].(string), datasetIDMode): true, + }} + + trustedIssuers := []TrustedIssuer{ + { + ISS: issuer, + JKU: jku, + }, + } + + validator, err := NewValidator(cfg, trustedIssuers, checker) + require.NoError(t, err) + + identity := Identity{Issuer: "https://broker.example", Subject: "user-123"} + + return validator, identity, "opaque-token" +} From a9295e6ece05e04e72fba280bea873c3b61c6e3b Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:07:54 +0100 Subject: [PATCH 375/530] =?UTF-8?q?feat(download):=20implement=20v2=20API?= =?UTF-8?q?=20endpoints=20=E2=80=94=20datasets,=20files,=20service-info?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit REST API endpoints: - GET /datasets: paginated dataset list with HMAC-signed page tokens - GET /datasets/:datasetId: dataset metadata (date, files, size) - GET /datasets/:datasetId/files: keyset-paginated file list with filePath/pathPrefix filters (mutually exclusive, 4096 char limit) - HEAD/GET /files/:fileId: combined download with re-encrypted header - HEAD/GET /files/:fileId/header: re-encrypted header only - HEAD/GET /files/:fileId/content: raw archive body (no pubkey needed) - GET /service-info: GA4GH service-info metadata - GET /health/ready, /health/live: health probes Cross-cutting: - RFC 9457 Problem Details for all errors (application/problem+json) - No existence leakage: 403 for both "not found" and "no access" - Content-Disposition with .c4gh extension - Cache-Control headers on all data endpoints - UseRawPath=true for URL-encoded slash dataset IDs - Audit denied/failed events on 403 and server errors - Correlation ID middleware (X-Correlation-ID) --- sda/cmd/download/handlers/datasets.go | 309 +++++++++++ sda/cmd/download/handlers/datasets_test.go | 499 ++++++++++++++++++ sda/cmd/download/handlers/files.go | 484 +++++++++++++++++ sda/cmd/download/handlers/files_test.go | 452 ++++++++++++++++ sda/cmd/download/handlers/handlers.go | 112 ++++ sda/cmd/download/handlers/handlers_test.go | 210 ++++++++ sda/cmd/download/handlers/health.go | 80 +++ .../download/handlers/mock_database_test.go | 126 +++++ sda/cmd/download/handlers/options.go | 58 ++ sda/cmd/download/handlers/options_test.go | 42 ++ sda/cmd/download/handlers/pagination.go | 123 +++++ sda/cmd/download/handlers/pagination_test.go | 197 +++++++ sda/cmd/download/handlers/problem.go | 54 ++ sda/cmd/download/handlers/problem_test.go | 108 ++++ sda/cmd/download/handlers/publickey.go | 22 + sda/cmd/download/handlers/publickey_test.go | 58 ++ sda/cmd/download/handlers/serviceinfo.go | 47 ++ sda/cmd/download/handlers/serviceinfo_test.go | 38 ++ 18 files changed, 3019 insertions(+) create mode 100644 sda/cmd/download/handlers/datasets.go create mode 100644 sda/cmd/download/handlers/datasets_test.go create mode 100644 sda/cmd/download/handlers/files.go create mode 100644 sda/cmd/download/handlers/files_test.go create mode 100644 sda/cmd/download/handlers/handlers.go create mode 100644 sda/cmd/download/handlers/handlers_test.go create mode 100644 sda/cmd/download/handlers/health.go create mode 100644 sda/cmd/download/handlers/mock_database_test.go create mode 100644 sda/cmd/download/handlers/options.go create mode 100644 sda/cmd/download/handlers/options_test.go create mode 100644 sda/cmd/download/handlers/pagination.go create mode 100644 sda/cmd/download/handlers/pagination_test.go create mode 100644 sda/cmd/download/handlers/problem.go create mode 100644 sda/cmd/download/handlers/problem_test.go create mode 100644 sda/cmd/download/handlers/publickey.go create mode 100644 sda/cmd/download/handlers/publickey_test.go create mode 100644 sda/cmd/download/handlers/serviceinfo.go create mode 100644 sda/cmd/download/handlers/serviceinfo_test.go diff --git a/sda/cmd/download/handlers/datasets.go b/sda/cmd/download/handlers/datasets.go new file mode 100644 index 0000000000..6c18e0e5a3 --- /dev/null +++ b/sda/cmd/download/handlers/datasets.go @@ -0,0 +1,309 @@ +package handlers + +import ( + "net/http" + "sort" + + "github.com/gin-gonic/gin" + "github.com/neicnordic/sensitive-data-archive/cmd/download/config" + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + log "github.com/sirupsen/logrus" +) + +// hasDatasetAccess checks if the user has access to a specific dataset. +// In allow-all-data mode, all authenticated users have access to all datasets. +func hasDatasetAccess(userDatasets []string, datasetID string) bool { + if config.JWTAllowAllData() { + return true + } + + for _, d := range userDatasets { + if d == datasetID { + return true + } + } + + return false +} + +// fileInfo is the v2 API response representation of a file within a dataset. +type fileInfo struct { + FileID string `json:"fileId"` + FilePath string `json:"filePath"` + Size int64 `json:"size"` + DecryptedSize int64 `json:"decryptedSize"` + Checksums []database.Checksum `json:"checksums"` + DownloadURL string `json:"downloadUrl"` +} + +// ListDatasets returns a paginated list of dataset IDs the user has access to. +// GET /datasets +func (h *Handlers) ListDatasets(c *gin.Context) { + authCtx, ok := middleware.GetAuthContext(c) + if !ok { + problemJSON(c, http.StatusUnauthorized, "authentication required") + + return + } + + var datasets []database.Dataset + var err error + + if config.JWTAllowAllData() { + datasets, err = h.db.GetAllDatasets(c.Request.Context()) + } else { + datasets, err = h.db.GetUserDatasets(c.Request.Context(), authCtx.Datasets) + } + + if err != nil { + log.Errorf("failed to retrieve datasets: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to retrieve datasets") + + return + } + + // Extract IDs and sort ascending + ids := make([]string, len(datasets)) + for i, d := range datasets { + ids[i] = d.ID + } + sort.Strings(ids) + + // Parse pagination parameters + pageSize, err := parsePageSize(c) + if err != nil { + problemJSON(c, http.StatusBadRequest, err.Error()) + + return + } + + start := 0 + if tokenStr := c.Query("pageToken"); tokenStr != "" { + tok, err := decodePageToken(tokenStr) + if err != nil { + problemJSON(c, http.StatusBadRequest, err.Error()) + + return + } + + if tok.PageSize != pageSize { + problemJSON(c, http.StatusBadRequest, "pageSize must not change between pages") + + return + } + + qh := queryFingerprint("datasets") + if tok.QueryHash != qh { + problemJSON(c, http.StatusBadRequest, "page token does not match this query") + + return + } + + // Find cursor position: start after the cursor value + for i, id := range ids { + if id == tok.Cursor { + start = i + 1 + + break + } + } + } + + // Slice the page + end := start + pageSize + if end > len(ids) { + end = len(ids) + } + + page := ids[start:end] + + // Build next page token if more items remain + var nextPageToken *string + if end < len(ids) { + qh := queryFingerprint("datasets") + tok := encodePageToken(page[len(page)-1], "", pageSize, qh) + nextPageToken = &tok + } + + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + c.JSON(http.StatusOK, gin.H{ + "datasets": page, + "nextPageToken": nextPageToken, + }) +} + +// GetDataset returns metadata for a specific dataset. +// GET /datasets/:datasetId +func (h *Handlers) GetDataset(c *gin.Context) { + datasetID := c.Param("datasetId") + + authCtx, ok := middleware.GetAuthContext(c) + if !ok { + problemJSON(c, http.StatusUnauthorized, "authentication required") + + return + } + + if !hasDatasetAccess(authCtx.Datasets, datasetID) { + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return + } + + info, err := h.db.GetDatasetInfo(c.Request.Context(), datasetID) + if err != nil { + log.Errorf("failed to retrieve dataset info: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to retrieve dataset info") + + return + } + + if info == nil { + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return + } + + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + c.JSON(http.StatusOK, gin.H{ + "datasetId": info.ID, + "date": info.CreatedAt.Format("2006-01-02T15:04:05Z07:00"), + "files": info.FileCount, + "size": info.TotalSize, + }) +} + +// ListDatasetFiles returns a paginated list of files in a dataset. +// GET /datasets/:datasetId/files +func (h *Handlers) ListDatasetFiles(c *gin.Context) { + datasetID := c.Param("datasetId") + + authCtx, ok := middleware.GetAuthContext(c) + if !ok { + problemJSON(c, http.StatusUnauthorized, "authentication required") + + return + } + + if !hasDatasetAccess(authCtx.Datasets, datasetID) { + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return + } + + // Check dataset exists (consistent with GetDataset — no existence leakage) + exists, err := h.db.CheckDatasetExists(c.Request.Context(), datasetID) + if err != nil { + log.Errorf("failed to check dataset existence: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to check dataset") + + return + } + + if !exists { + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return + } + + // Validate filters + filePath := c.Query("filePath") + pathPrefix := c.Query("pathPrefix") + + const maxFilterLen = 4096 + if len(filePath) > maxFilterLen || len(pathPrefix) > maxFilterLen { + problemJSON(c, http.StatusBadRequest, "filter value too long") + + return + } + + if filePath != "" && pathPrefix != "" { + problemJSONWithCode(c, http.StatusBadRequest, + "filePath and pathPrefix are mutually exclusive", "FILTER_CONFLICT") + + return + } + + pageSize, err := parsePageSize(c) + if err != nil { + problemJSON(c, http.StatusBadRequest, err.Error()) + + return + } + + opts := database.FileListOptions{ + FilePath: filePath, + PathPrefix: pathPrefix, + Limit: pageSize + 1, + } + + // Build query fingerprint from stable parameters + qh := queryFingerprint(datasetID, filePath, pathPrefix) + + if tokenStr := c.Query("pageToken"); tokenStr != "" { + tok, err := decodePageToken(tokenStr) + if err != nil { + problemJSON(c, http.StatusBadRequest, err.Error()) + + return + } + + if tok.QueryHash != qh { + problemJSON(c, http.StatusBadRequest, "page token does not match this query") + + return + } + + opts.CursorPath = tok.Cursor + opts.CursorID = tok.CursorID + } + + files, err := h.db.GetDatasetFilesPaginated(c.Request.Context(), datasetID, opts) + if err != nil { + log.Errorf("failed to retrieve dataset files: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to retrieve dataset files") + + return + } + + // Detect next page + hasNext := len(files) > pageSize + if hasNext { + files = files[:pageSize] + } + + // Map to v2 response format + result := make([]fileInfo, len(files)) + for i, f := range files { + checksums := f.Checksums + if checksums == nil { + checksums = []database.Checksum{} + } + result[i] = fileInfo{ + FileID: f.ID, + FilePath: f.SubmittedPath, + Size: f.ArchiveSize, + DecryptedSize: f.DecryptedSize, + Checksums: checksums, + DownloadURL: "/files/" + f.ID, + } + } + + var nextPageToken *string + if hasNext { + last := files[len(files)-1] + tok := encodePageToken(last.SubmittedPath, last.ID, pageSize, qh) + nextPageToken = &tok + } + + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + c.JSON(http.StatusOK, gin.H{ + "files": result, + "nextPageToken": nextPageToken, + }) +} diff --git a/sda/cmd/download/handlers/datasets_test.go b/sda/cmd/download/handlers/datasets_test.go new file mode 100644 index 0000000000..e83695894b --- /dev/null +++ b/sda/cmd/download/handlers/datasets_test.go @@ -0,0 +1,499 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// listDatasetsResponse matches the JSON shape of ListDatasets. +type listDatasetsResponse struct { + Datasets []string `json:"datasets"` + NextPageToken *string `json:"nextPageToken"` +} + +// getDatasetResponse matches the JSON shape of GetDataset. +type getDatasetResponse struct { + DatasetID string `json:"datasetId"` + Date string `json:"date"` + Files int `json:"files"` + Size int64 `json:"size"` +} + +// listDatasetFilesResponse matches the JSON shape of ListDatasetFiles. +type listDatasetFilesResponse struct { + Files []fileInfo `json:"files"` + NextPageToken *string `json:"nextPageToken"` +} + +// --- ListDatasets tests --- + +func TestListDatasets_ReturnsSortedIDs(t *testing.T) { + router := setupTestRouterWithAuth([]string{"c-dataset", "a-dataset", "b-dataset"}) + mockDB := &mockDatabase{ + datasets: []database.Dataset{ + {ID: "c-dataset"}, + {ID: "a-dataset"}, + {ID: "b-dataset"}, + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets", h.ListDatasets) + + req, _ := http.NewRequest(http.MethodGet, "/datasets", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "private, max-age=60, must-revalidate", w.Header().Get("Cache-Control")) + + var resp listDatasetsResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Equal(t, []string{"a-dataset", "b-dataset", "c-dataset"}, resp.Datasets) + assert.Nil(t, resp.NextPageToken) +} + +func TestListDatasets_Pagination(t *testing.T) { + // 5 datasets, page size 2 => pages of [a,b], [c,d], [e] + ids := []string{"e", "d", "c", "b", "a"} + datasets := make([]database.Dataset, len(ids)) + for i, id := range ids { + datasets[i] = database.Dataset{ID: id} + } + + mockDB := &mockDatabase{datasets: datasets} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + // Page 1 + router := setupTestRouterWithAuth(ids) + router.GET("/datasets", h.ListDatasets) + + req, _ := http.NewRequest(http.MethodGet, "/datasets?pageSize=2", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + var resp1 listDatasetsResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp1)) + assert.Equal(t, []string{"a", "b"}, resp1.Datasets) + require.NotNil(t, resp1.NextPageToken) + + // Page 2 + req, _ = http.NewRequest(http.MethodGet, "/datasets?pageSize=2&pageToken="+*resp1.NextPageToken, nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + var resp2 listDatasetsResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp2)) + assert.Equal(t, []string{"c", "d"}, resp2.Datasets) + require.NotNil(t, resp2.NextPageToken) + + // Page 3 (last) + req, _ = http.NewRequest(http.MethodGet, "/datasets?pageSize=2&pageToken="+*resp2.NextPageToken, nil) + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + var resp3 listDatasetsResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp3)) + assert.Equal(t, []string{"e"}, resp3.Datasets) + assert.Nil(t, resp3.NextPageToken) +} + +func TestListDatasets_Unauthenticated(t *testing.T) { + router := gin.New() + h := newTestHandlers(t) + router.GET("/datasets", h.ListDatasets) + + req, _ := http.NewRequest(http.MethodGet, "/datasets", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) +} + +func TestListDatasets_DatabaseError(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{err: assert.AnError} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets", h.ListDatasets) + + req, _ := http.NewRequest(http.MethodGet, "/datasets", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) +} + +func TestListDatasets_EmptyResult(t *testing.T) { + router := setupTestRouterWithAuth([]string{}) + mockDB := &mockDatabase{datasets: []database.Dataset{}} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets", h.ListDatasets) + + req, _ := http.NewRequest(http.MethodGet, "/datasets", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var resp listDatasetsResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Empty(t, resp.Datasets) + assert.Nil(t, resp.NextPageToken) +} + +func TestListDatasets_InvalidPageSize(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{datasets: []database.Dataset{{ID: "ds1"}}} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets", h.ListDatasets) + + req, _ := http.NewRequest(http.MethodGet, "/datasets?pageSize=0", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +// --- GetDataset tests --- + +func TestGetDataset_Success(t *testing.T) { + ts := time.Date(2024, 6, 15, 12, 0, 0, 0, time.UTC) + router := setupTestRouterWithAuth([]string{"EGAD00000000001"}) + mockDB := &mockDatabase{ + datasetInfo: &database.DatasetInfo{ + ID: "EGAD00000000001", + FileCount: 42, + TotalSize: 999999, + CreatedAt: ts, + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId", h.GetDataset) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/EGAD00000000001", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var resp getDatasetResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Equal(t, "EGAD00000000001", resp.DatasetID) + assert.Equal(t, 42, resp.Files) + assert.Equal(t, int64(999999), resp.Size) + assert.Equal(t, "2024-06-15T12:00:00Z", resp.Date) +} + +func TestGetDataset_NoAccess(t *testing.T) { + router := setupTestRouterWithAuth([]string{"other-dataset"}) + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId", h.GetDataset) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/secret-dataset", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestGetDataset_NotFoundReturns403(t *testing.T) { + // User has dataset in their access list, but DB returns nil (doesn't actually exist). + // Should still return 403 to avoid leaking existence. + router := setupTestRouterWithAuth([]string{"ghost-dataset"}) + mockDB := &mockDatabase{datasetInfo: nil} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId", h.GetDataset) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ghost-dataset", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestGetDataset_NoAccess_EmitsAuditDenied(t *testing.T) { + logger := &capturingLogger{} + router := gin.New() + // Wire correlation ID middleware + mock auth with subject + router.Use(correlationIDMiddleware()) + router.Use(func(c *gin.Context) { + c.Set(middleware.ContextKey, middleware.AuthContext{ + Subject: "test-user", + Datasets: []string{"other-dataset"}, + }) + c.Next() + }) + + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB), WithAuditLogger(logger)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId", h.GetDataset) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/secret-dataset", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) + require.Len(t, logger.events, 1) + evt := logger.events[0] + assert.Equal(t, "download.denied", evt.Event) + assert.Equal(t, http.StatusForbidden, evt.HTTPStatus) + assert.Equal(t, "/datasets/secret-dataset", evt.Endpoint) + assert.Equal(t, "test-user", evt.UserID) + assert.NotEmpty(t, evt.CorrelationID) +} + +func TestGetDataset_Unauthenticated(t *testing.T) { + router := gin.New() + h := newTestHandlers(t) + router.GET("/datasets/:datasetId", h.GetDataset) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/any", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) +} + +func TestGetDataset_DatabaseError(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{err: assert.AnError} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId", h.GetDataset) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) +} + +// --- ListDatasetFiles tests --- + +func TestListDatasetFiles_Success(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{ + datasetFilesPaged: []database.File{ + { + ID: "file-1", + SubmittedPath: "data/sample.c4gh", + ArchiveSize: 5000, + DecryptedSize: 4096, + Checksums: []database.Checksum{ + {Type: "sha256", Checksum: "abc123"}, + }, + }, + { + ID: "file-2", + SubmittedPath: "data/other.c4gh", + ArchiveSize: 9000, + DecryptedSize: 8192, + }, + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1/files", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var resp listDatasetFilesResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + require.Len(t, resp.Files, 2) + + // First file has checksums + assert.Equal(t, "file-1", resp.Files[0].FileID) + assert.Equal(t, "data/sample.c4gh", resp.Files[0].FilePath) + assert.Equal(t, int64(5000), resp.Files[0].Size) + assert.Equal(t, int64(4096), resp.Files[0].DecryptedSize) + assert.Equal(t, "/files/file-1", resp.Files[0].DownloadURL) + require.Len(t, resp.Files[0].Checksums, 1) + assert.Equal(t, "sha256", resp.Files[0].Checksums[0].Type) + + // Second file has empty checksums array (not null) + assert.Equal(t, "file-2", resp.Files[1].FileID) + assert.NotNil(t, resp.Files[1].Checksums) + assert.Len(t, resp.Files[1].Checksums, 0) + + assert.Nil(t, resp.NextPageToken) +} + +func TestListDatasetFiles_Pagination(t *testing.T) { + // Mock returns pageSize+1 items to signal next page + files := make([]database.File, 3) + for i := range files { + files[i] = database.File{ + ID: "f" + string(rune('0'+i)), + SubmittedPath: "path/" + string(rune('a'+i)), + } + } + + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{datasetFilesPaged: files} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1/files?pageSize=2", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var resp listDatasetFilesResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + // Should return only pageSize items + assert.Len(t, resp.Files, 2) + // Should have next page token + require.NotNil(t, resp.NextPageToken) +} + +func TestListDatasetFiles_FilterConflict(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1/files?filePath=a&pathPrefix=b", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + var resp ProblemDetails + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Equal(t, "FILTER_CONFLICT", resp.ErrorCode) +} + +func TestListDatasetFiles_NoAccess(t *testing.T) { + router := setupTestRouterWithAuth([]string{"other"}) + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/secret/files", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestListDatasetFiles_NotFoundReturns403(t *testing.T) { + // User has dataset in their access list, but it doesn't exist in DB. + // Should return 403 (consistent with GetDataset — no existence leakage). + router := setupTestRouterWithAuth([]string{"ghost-dataset"}) + mockDB := &mockDatabase{datasetNotFound: true} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ghost-dataset/files", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestListDatasetFiles_Unauthenticated(t *testing.T) { + router := gin.New() + h := newTestHandlers(t) + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1/files", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) +} + +func TestListDatasetFiles_DatabaseError(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{err: assert.AnError} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1/files", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) +} + +func TestListDatasetFiles_InvalidPageToken(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1/files?pageToken=bogus", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func TestListDatasetFiles_EmptyResult(t *testing.T) { + router := setupTestRouterWithAuth([]string{"ds1"}) + mockDB := &mockDatabase{datasetFilesPaged: []database.File{}} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/datasets/:datasetId/files", h.ListDatasetFiles) + + req, _ := http.NewRequest(http.MethodGet, "/datasets/ds1/files", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var resp listDatasetFilesResponse + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &resp)) + assert.Empty(t, resp.Files) + assert.Nil(t, resp.NextPageToken) +} diff --git a/sda/cmd/download/handlers/files.go b/sda/cmd/download/handlers/files.go new file mode 100644 index 0000000000..1bf9667904 --- /dev/null +++ b/sda/cmd/download/handlers/files.go @@ -0,0 +1,484 @@ +package handlers + +import ( + "crypto/sha256" + "errors" + "fmt" + "mime" + "net/http" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/neicnordic/sensitive-data-archive/cmd/download/config" + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + "github.com/neicnordic/sensitive-data-archive/cmd/download/streaming" + log "github.com/sirupsen/logrus" +) + +// resolvedBase holds the shared file resolution state. +type resolvedBase struct { + file *database.File + location string + authCtx middleware.AuthContext +} + +// resolvedFile holds the result of resolving a file for download. +type resolvedFile struct { + resolvedBase + publicKey string + newHeader []byte + etag string +} + +// resolveFileBase performs auth, permission check, file lookup, +// and storage resolution common to both full-download and content-only endpoints. +// Returns (nil, false) if an error response was already sent. +func (h *Handlers) resolveFileBase(c *gin.Context) (*resolvedBase, bool) { + fileID := c.Param("fileId") + + // Get auth context + authCtx, ok := middleware.GetAuthContext(c) + if !ok { + problemJSON(c, http.StatusUnauthorized, "authentication required") + + return nil, false + } + + // Permission check: return 403 for both "no access" AND "not found" (no existence leakage) + if !config.JWTAllowAllData() { + hasPermission, err := h.db.CheckFilePermission(c.Request.Context(), fileID, authCtx.Datasets) + if err != nil { + log.Errorf("failed to check file permission: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to check file permission") + + return nil, false + } + + if !hasPermission { + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return nil, false + } + } + + // Get file from DB + file, err := h.db.GetFileByID(c.Request.Context(), fileID) + if err != nil { + log.Errorf("failed to retrieve file info: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to retrieve file info") + + return nil, false + } + + if file == nil { + // Return 403 (not 404) to avoid leaking file existence + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return nil, false + } + + if file.ArchivePath == "" { + log.Errorf("file %s has no archive path", file.ID) + problemJSON(c, http.StatusInternalServerError, "file not in archive") + + return nil, false + } + + if file.ArchivePath == "" { + log.Errorf("file %s has no archive path", file.ID) + problemJSON(c, http.StatusInternalServerError, "file not in archive") + + return nil, false + } + + // Determine storage location + var location string + if file.ArchiveLocation != "" { + location = file.ArchiveLocation + } else { + if h.storageReader == nil { + log.Error("storage reader not configured") + problemJSON(c, http.StatusInternalServerError, "storage not configured") + + return nil, false + } + + log.Warnf("file %s has no archive_location stored, falling back to FindFile search", file.ID) + + location, err = h.storageReader.FindFile(c.Request.Context(), file.ArchivePath) + if err != nil { + log.Errorf("failed to find file in storage: %v", err) + problemJSON(c, http.StatusInternalServerError, "file not found in storage") + + return nil, false + } + } + + return &resolvedBase{ + file: file, + location: location, + authCtx: authCtx, + }, true +} + +// resolveFileForDownload wraps resolveFileBase, extracts a public key, +// and invokes gRPC re-encryption. +func (h *Handlers) resolveFileForDownload(c *gin.Context) (*resolvedFile, bool) { + // Extract public key from headers + publicKey, errorCode, detail := extractPublicKey(c) + if errorCode != "" { + problemJSONWithCode(c, http.StatusBadRequest, detail, errorCode) + + return nil, false + } + + base, ok := h.resolveFileBase(c) + if !ok { + return nil, false + } + + // Validate file has header + if len(base.file.Header) == 0 { + log.Errorf("file %s has no header", base.file.ID) + problemJSON(c, http.StatusInternalServerError, "file header not available") + + return nil, false + } + + // Re-encrypt header + if h.reencryptClient == nil { + log.Error("reencrypt client not configured") + problemJSON(c, http.StatusInternalServerError, "reencrypt service not configured") + + return nil, false + } + + newHeader, err := h.reencryptClient.ReencryptHeader(c.Request.Context(), base.file.Header, publicKey) + if err != nil { + log.Errorf("failed to reencrypt header: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to prepare file for download") + + return nil, false + } + + // ETag from re-encrypted header (full SHA-256) + hash := sha256.Sum256(newHeader) + etag := fmt.Sprintf(`"%x"`, hash[:]) + + return &resolvedFile{ + resolvedBase: *base, + publicKey: publicKey, + newHeader: newHeader, + etag: etag, + }, true +} + +// contentDisposition returns the Content-Disposition header value for a file download. +// Appends .c4gh extension if not already present. +func contentDisposition(submittedPath string) string { + filename := filepath.Base(submittedPath) + if !strings.HasSuffix(filename, ".c4gh") { + filename += ".c4gh" + } + + return mime.FormatMediaType("attachment", map[string]string{"filename": filename}) +} + +// DownloadFile handles file download by stable ID. +// GET /files/:fileId +func (h *Handlers) DownloadFile(c *gin.Context) { + resolved, ok := h.resolveFileForDownload(c) + if !ok { + return + } + + file := resolved.file + + // Open file reader from storage + if h.storageReader == nil { + log.Error("storage reader not configured") + problemJSON(c, http.StatusInternalServerError, "storage not configured") + h.auditFailed(c, resolved.authCtx, file, "storage not configured") + + return + } + + fileReader, err := h.storageReader.NewFileReadSeeker(c.Request.Context(), resolved.location, file.ArchivePath) + if err != nil { + log.Errorf("failed to open file: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to open file") + h.auditFailed(c, resolved.authCtx, file, "failed to open file") + + return + } + + // Calculate total size + newHeaderSize := int64(len(resolved.newHeader)) + totalSize := newHeaderSize + file.ArchiveSize + + // If-Range check + honorRange := streaming.CheckIfRange(c.GetHeader("If-Range"), resolved.etag, time.Time{}) + + // Parse Range header + var rangeSpec *streaming.RangeSpec + rangeHeader := c.GetHeader("Range") + if rangeHeader != "" && honorRange { + var rangeErr error + rangeSpec, rangeErr = streaming.ParseRangeHeader(rangeHeader, totalSize) + if errors.Is(rangeErr, streaming.ErrRangeInvalid) { + fileReader.Close() + problemJSONWithCode(c, http.StatusBadRequest, "invalid range header", "RANGE_INVALID") + + return + } + if errors.Is(rangeErr, streaming.ErrRangeNotSatisfiable) { + fileReader.Close() + c.Header("Content-Range", fmt.Sprintf("bytes */%d", totalSize)) + problemJSON(c, http.StatusRequestedRangeNotSatisfiable, "range not satisfiable") + + return + } + } + + // Set response headers + c.Header("Accept-Ranges", "bytes") + c.Header("ETag", resolved.etag) + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", contentDisposition(file.SubmittedPath)) + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + + // Stream the file + err = streaming.StreamFile(streaming.StreamConfig{ + Writer: c.Writer, + NewHeader: resolved.newHeader, + FileReader: fileReader, + ArchiveFileSize: file.ArchiveSize, + OriginalHeaderSize: 0, + Range: rangeSpec, + }) + if err != nil { + log.Errorf("error streaming file: %v", err) + h.auditFailed(c, resolved.authCtx, file, "streaming error") + + return + } + + // Audit event on completion + h.auditLogger.Log(c.Request.Context(), audit.Event{ + Event: "download.complete", + UserID: resolved.authCtx.Subject, + FileID: file.ID, + DatasetID: file.DatasetID, + CorrelationID: c.GetString("correlationId"), + Endpoint: c.Request.URL.Path, + HTTPStatus: c.Writer.Status(), + }) +} + +// HeadFile handles HEAD requests for file metadata. +// HEAD /files/:fileId +func (h *Handlers) HeadFile(c *gin.Context) { + resolved, ok := h.resolveFileForDownload(c) + if !ok { + return + } + + file := resolved.file + + // Calculate total size: re-encrypted header + archive body + totalSize := int64(len(resolved.newHeader)) + file.ArchiveSize + + // Set response headers + c.Header("Content-Length", fmt.Sprintf("%d", totalSize)) + c.Header("ETag", resolved.etag) + c.Header("Accept-Ranges", "bytes") + c.Header("Content-Disposition", contentDisposition(file.SubmittedPath)) + c.Header("Content-Type", "application/octet-stream") + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + + c.Status(http.StatusOK) +} + +// resolvedContentFile holds the result of resolving a file for body-only content access. +type resolvedContentFile struct { + resolvedBase + etag string +} + +// contentETag computes a stable ETag from file ID and archive size. +// This is recipient-independent (no re-encrypted header involved). +func contentETag(fileID string, archiveSize int64) string { + hash := sha256.Sum256([]byte(fileID + ":" + strconv.FormatInt(archiveSize, 10))) + + return fmt.Sprintf(`"%x"`, hash[:]) +} + +// resolveFileForContent wraps resolveFileBase and computes the basic content ETag. +func (h *Handlers) resolveFileForContent(c *gin.Context) (*resolvedContentFile, bool) { + base, ok := h.resolveFileBase(c) + if !ok { + return nil, false + } + + etag := contentETag(base.file.ID, base.file.ArchiveSize) + + return &resolvedContentFile{ + resolvedBase: *base, + etag: etag, + }, true +} + +// GetFileHeader handles requests for the re-encrypted file header only. +// GET /files/:fileId/header +func (h *Handlers) GetFileHeader(c *gin.Context) { + resolved, ok := h.resolveFileForDownload(c) + if !ok { + return + } + + file := resolved.file + + // Stable content ETag (same value /content would return) + cETag := contentETag(file.ID, file.ArchiveSize) + + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Length", fmt.Sprintf("%d", len(resolved.newHeader))) + c.Header("SDA-Content-ETag", cETag) + c.Data(http.StatusOK, "application/octet-stream", resolved.newHeader) + + h.auditLogger.Log(c.Request.Context(), audit.Event{ + Event: "download.header", + UserID: resolved.authCtx.Subject, + FileID: file.ID, + DatasetID: file.DatasetID, + CorrelationID: c.GetString("correlationId"), + Endpoint: c.Request.URL.Path, + HTTPStatus: c.Writer.Status(), + }) +} + +// HeadFileHeader handles HEAD requests for the file header metadata. +// HEAD /files/:fileId/header +func (h *Handlers) HeadFileHeader(c *gin.Context) { + resolved, ok := h.resolveFileForDownload(c) + if !ok { + return + } + + file := resolved.file + cETag := contentETag(file.ID, file.ArchiveSize) + + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Length", fmt.Sprintf("%d", len(resolved.newHeader))) + c.Header("SDA-Content-ETag", cETag) + c.Status(http.StatusOK) +} + +// GetFileContent handles requests for the file body (archive data without header). +// GET /files/:fileId/content +func (h *Handlers) GetFileContent(c *gin.Context) { + resolved, ok := h.resolveFileForContent(c) + if !ok { + return + } + + file := resolved.file + + // Open file reader from storage + if h.storageReader == nil { + log.Error("storage reader not configured") + problemJSON(c, http.StatusInternalServerError, "storage not configured") + h.auditFailed(c, resolved.authCtx, file, "storage not configured") + + return + } + + fileReader, err := h.storageReader.NewFileReadSeeker(c.Request.Context(), resolved.location, file.ArchivePath) + if err != nil { + log.Errorf("failed to open file: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to open file") + h.auditFailed(c, resolved.authCtx, file, "failed to open file") + + return + } + + // If-Range check + honorRange := streaming.CheckIfRange(c.GetHeader("If-Range"), resolved.etag, time.Time{}) + + // Parse Range header + var rangeSpec *streaming.RangeSpec + rangeHeader := c.GetHeader("Range") + if rangeHeader != "" && honorRange { + var rangeErr error + rangeSpec, rangeErr = streaming.ParseRangeHeader(rangeHeader, file.ArchiveSize) + if errors.Is(rangeErr, streaming.ErrRangeInvalid) { + fileReader.Close() + problemJSONWithCode(c, http.StatusBadRequest, "invalid range header", "RANGE_INVALID") + + return + } + if errors.Is(rangeErr, streaming.ErrRangeNotSatisfiable) { + fileReader.Close() + c.Header("Content-Range", fmt.Sprintf("bytes */%d", file.ArchiveSize)) + problemJSON(c, http.StatusRequestedRangeNotSatisfiable, "range not satisfiable") + + return + } + } + + // Set response headers + c.Header("Accept-Ranges", "bytes") + c.Header("ETag", resolved.etag) + c.Header("Content-Type", "application/octet-stream") + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + + // Stream the body only + err = streaming.StreamBodyOnly(streaming.StreamBodyConfig{ + Writer: c.Writer, + FileReader: fileReader, + ArchiveFileSize: file.ArchiveSize, + Range: rangeSpec, + }) + if err != nil { + log.Errorf("error streaming file content: %v", err) + h.auditFailed(c, resolved.authCtx, file, "streaming error") + + return + } + + // Audit event on completion + h.auditLogger.Log(c.Request.Context(), audit.Event{ + Event: "download.content", + UserID: resolved.authCtx.Subject, + FileID: file.ID, + DatasetID: file.DatasetID, + CorrelationID: c.GetString("correlationId"), + Endpoint: c.Request.URL.Path, + HTTPStatus: c.Writer.Status(), + }) +} + +// HeadFileContent handles HEAD requests for the file content metadata. +// HEAD /files/:fileId/content +func (h *Handlers) HeadFileContent(c *gin.Context) { + resolved, ok := h.resolveFileForContent(c) + if !ok { + return + } + + file := resolved.file + + c.Header("Content-Length", fmt.Sprintf("%d", file.ArchiveSize)) + c.Header("ETag", resolved.etag) + c.Header("Accept-Ranges", "bytes") + c.Header("Content-Type", "application/octet-stream") + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + c.Status(http.StatusOK) +} diff --git a/sda/cmd/download/handlers/files_test.go b/sda/cmd/download/handlers/files_test.go new file mode 100644 index 0000000000..0ecce2dc21 --- /dev/null +++ b/sda/cmd/download/handlers/files_test.go @@ -0,0 +1,452 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDownloadFile_MissingPublicKey(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + h := newTestHandlers(t) + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file-id", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + var response ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "KEY_MISSING", response.ErrorCode) +} + +func TestDownloadFile_Unauthenticated(t *testing.T) { + // Router without auth middleware => no AuthContext in context + router := setupTestRouter() + h := newTestHandlers(t) + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file-id", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) + + var response ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "Unauthorized", response.Title) +} + +func TestDownloadFile_NonExistentFile_Returns403(t *testing.T) { + // Verify no 404 leakage: permission check fails for non-existent file + router := setupTestRouterWithAuth([]string{"some-dataset"}) + mockDB := &mockDatabase{ + hasPermission: false, // Permission check returns false for non-existent file + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/nonexistent-file", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + // Must be 403, not 404 + assert.Equal(t, http.StatusForbidden, w.Code) + + var response ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "access denied", response.Detail) +} + +func TestDownloadFile_FileNilInDB_Returns403(t *testing.T) { + // Permission check passes (allow-all-data or user has access) but GetFileByID returns nil + router := setupTestRouterWithAuth([]string{"test-dataset"}) + mockDB := &mockDatabase{ + hasPermission: true, + fileByID: nil, // File not found in DB + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/nonexistent-file", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + // Must be 403, not 404 + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestDownloadFile_ReencryptNotConfigured(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + mockDB := &mockDatabase{ + hasPermission: true, + fileByID: &database.File{ + ID: "test-file", + Header: make([]byte, 20), + ArchivePath: "/archive/test.c4gh", + ArchiveLocation: "s3:9000/archive", + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + // reencryptClient is nil + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + + var response ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Contains(t, response.Detail, "reencrypt service not configured") +} + +func TestDownloadFile_FileNoHeader_Returns500(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + mockDB := &mockDatabase{ + hasPermission: true, + fileByID: &database.File{ + ID: "test-file", + Header: nil, // No header + ArchivePath: "/archive/test.c4gh", + ArchiveLocation: "s3:9000/archive", + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + + var response ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Contains(t, response.Detail, "file header not available") +} + +func TestDownloadFile_FileNoArchivePath_Returns500(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + mockDB := &mockDatabase{ + hasPermission: true, + fileByID: &database.File{ + ID: "test-file", + Header: make([]byte, 20), + ArchivePath: "", // No archive path + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) + + var response ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Contains(t, response.Detail, "file not in archive") +} + +func TestDownloadFile_PublicKeyConflict(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + h := newTestHandlers(t) + + router.GET("/files/:fileId", h.DownloadFile) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file-id", nil) + req.Header.Set("X-C4GH-Public-Key", "key1") + req.Header.Set("Htsget-Context-Public-Key", "key2") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + var response ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "KEY_CONFLICT", response.ErrorCode) +} + +// HeadFile tests + +func TestHeadFile_MissingPublicKey(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + h := newTestHandlers(t) + + router.HEAD("/files/:fileId", h.HeadFile) + + req, _ := http.NewRequest(http.MethodHead, "/files/test-file-id", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +func TestHeadFile_NonExistentFile_Returns403(t *testing.T) { + router := setupTestRouterWithAuth([]string{"some-dataset"}) + mockDB := &mockDatabase{ + hasPermission: false, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.HEAD("/files/:fileId", h.HeadFile) + + req, _ := http.NewRequest(http.MethodHead, "/files/nonexistent-file", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestHeadFile_Unauthenticated(t *testing.T) { + router := setupTestRouter() + h := newTestHandlers(t) + + router.HEAD("/files/:fileId", h.HeadFile) + + req, _ := http.NewRequest(http.MethodHead, "/files/test-file-id", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) +} + +func TestHeadFile_ReencryptNotConfigured(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + mockDB := &mockDatabase{ + hasPermission: true, + fileByID: &database.File{ + ID: "test-file", + Header: make([]byte, 20), + ArchivePath: "/archive/test.c4gh", + ArchiveLocation: "s3:9000/archive", + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.HEAD("/files/:fileId", h.HeadFile) + + req, _ := http.NewRequest(http.MethodHead, "/files/test-file", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) +} + +// Content-Disposition tests + +// GetFileHeader tests + +func TestGetFileHeader_MissingPublicKey(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + h := newTestHandlers(t) + + router.GET("/files/:fileId/header", h.GetFileHeader) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file-id/header", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + var response ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "KEY_MISSING", response.ErrorCode) +} + +func TestGetFileHeader_Unauthenticated(t *testing.T) { + router := setupTestRouter() + h := newTestHandlers(t) + + router.GET("/files/:fileId/header", h.GetFileHeader) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file-id/header", nil) + req.Header.Set("X-C4GH-Public-Key", "dGVzdC1wdWJsaWMta2V5") + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) + + var response ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "Unauthorized", response.Title) +} + +// HeadFileHeader tests + +func TestHeadFileHeader_MissingPublicKey(t *testing.T) { + router := setupTestRouterWithAuth([]string{"test-dataset"}) + h := newTestHandlers(t) + + router.HEAD("/files/:fileId/header", h.HeadFileHeader) + + req, _ := http.NewRequest(http.MethodHead, "/files/test-file-id/header", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) +} + +// GetFileContent tests + +func TestGetFileContent_Unauthenticated(t *testing.T) { + router := setupTestRouter() + h := newTestHandlers(t) + + router.GET("/files/:fileId/content", h.GetFileContent) + + req, _ := http.NewRequest(http.MethodGet, "/files/test-file-id/content", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) + + var response ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "Unauthorized", response.Title) +} + +func TestGetFileContent_NonExistentFile_Returns403(t *testing.T) { + router := setupTestRouterWithAuth([]string{"some-dataset"}) + mockDB := &mockDatabase{ + hasPermission: false, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/files/:fileId/content", h.GetFileContent) + + req, _ := http.NewRequest(http.MethodGet, "/files/nonexistent-file/content", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) + + var response ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "access denied", response.Detail) +} + +// HeadFileContent tests + +func TestHeadFileContent_Unauthenticated(t *testing.T) { + router := setupTestRouter() + h := newTestHandlers(t) + + router.HEAD("/files/:fileId/content", h.HeadFileContent) + + req, _ := http.NewRequest(http.MethodHead, "/files/test-file-id/content", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) +} + +func TestHeadFileContent_NoPublicKeyRequired(t *testing.T) { + // Content endpoints do not require a public key — verify HEAD /content works + // with a mock that has permission and file data, but no reencrypt client. + router := setupTestRouterWithAuth([]string{"test-dataset"}) + mockDB := &mockDatabase{ + hasPermission: true, + fileByID: &database.File{ + ID: "test-file", + ArchivePath: "/archive/test.c4gh", + ArchiveLocation: "s3:9000/archive", + ArchiveSize: 1024, + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + // reencryptClient is nil — content endpoints don't need it + + router.HEAD("/files/:fileId/content", h.HeadFileContent) + + // No public key header set + req, _ := http.NewRequest(http.MethodHead, "/files/test-file/content", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "1024", w.Header().Get("Content-Length")) + assert.Equal(t, "application/octet-stream", w.Header().Get("Content-Type")) + assert.Equal(t, "bytes", w.Header().Get("Accept-Ranges")) + assert.NotEmpty(t, w.Header().Get("ETag")) + assert.Equal(t, "private, max-age=60, must-revalidate", w.Header().Get("Cache-Control")) +} + +// Content-Disposition tests + +func TestContentDisposition_AddsC4ghExtension(t *testing.T) { + result := contentDisposition("/path/to/myfile.txt") + assert.Contains(t, result, "myfile.txt.c4gh") +} + +func TestContentDisposition_KeepsExistingC4ghExtension(t *testing.T) { + result := contentDisposition("/path/to/myfile.c4gh") + assert.Contains(t, result, "myfile.c4gh") + // Ensure it's not doubled + assert.NotContains(t, result, "myfile.c4gh.c4gh") +} diff --git a/sda/cmd/download/handlers/handlers.go b/sda/cmd/download/handlers/handlers.go new file mode 100644 index 0000000000..6e076db906 --- /dev/null +++ b/sda/cmd/download/handlers/handlers.go @@ -0,0 +1,112 @@ +// Package handlers provides HTTP handlers for the download service. +package handlers + +import ( + "errors" + + "github.com/gin-gonic/gin" + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + "github.com/neicnordic/sensitive-data-archive/cmd/download/reencrypt" + "github.com/neicnordic/sensitive-data-archive/cmd/download/visa" + storage "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" +) + +// Handlers holds the dependencies for HTTP handlers. +type Handlers struct { + db database.Database + storageReader storage.Reader + reencryptClient *reencrypt.Client + visaValidator *visa.Validator + auditLogger audit.Logger + grpcHost string + grpcPort int +} + +// New creates a new Handlers instance with the given options. +func New(options ...func(*Handlers)) (*Handlers, error) { + h := &Handlers{ + auditLogger: audit.NoopLogger{}, + } + + for _, option := range options { + option(h) + } + + // Validate required dependencies + if h.db == nil { + return nil, errors.New("database is required") + } + + return h, nil +} + +// auditDenied logs a download.denied audit event for 403 responses. +func (h *Handlers) auditDenied(c *gin.Context) { + authCtx, _ := middleware.GetAuthContext(c) + h.auditLogger.Log(c.Request.Context(), audit.Event{ + Event: "download.denied", + UserID: authCtx.Subject, + CorrelationID: c.GetString("correlationId"), + Endpoint: c.Request.URL.Path, + HTTPStatus: c.Writer.Status(), + }) +} + +// auditFailed logs a download.failed audit event for server errors during download. +func (h *Handlers) auditFailed(c *gin.Context, authCtx middleware.AuthContext, file *database.File, reason string) { + var fileID, datasetID string + if file != nil { + fileID = file.ID + datasetID = file.DatasetID + } + + h.auditLogger.Log(c.Request.Context(), audit.Event{ + Event: "download.failed", + UserID: authCtx.Subject, + FileID: fileID, + DatasetID: datasetID, + CorrelationID: c.GetString("correlationId"), + Endpoint: c.Request.URL.Path, + HTTPStatus: c.Writer.Status(), + ErrorReason: reason, + }) +} + +// RegisterRoutes registers all HTTP routes with the given gin engine. +func (h *Handlers) RegisterRoutes(r *gin.Engine) { + // Correlation ID middleware on all routes + r.Use(correlationIDMiddleware()) + + // Health endpoints (no auth required) + health := r.Group("/health") + { + health.GET("/ready", h.HealthReady) + health.GET("/live", h.HealthLive) + } + + // Service info (no auth required) + r.GET("/service-info", h.ServiceInfo) + + // Datasets (auth required) + datasets := r.Group("/datasets") + datasets.Use(middleware.TokenMiddleware(h.db, h.visaValidator, h.auditLogger)) + { + datasets.GET("", h.ListDatasets) + datasets.GET("/:datasetId", h.GetDataset) + datasets.GET("/:datasetId/files", h.ListDatasetFiles) + } + + // Files (auth required) + files := r.Group("/files") + files.Use(middleware.TokenMiddleware(h.db, h.visaValidator, h.auditLogger)) + { + files.HEAD("/:fileId", h.HeadFile) + files.GET("/:fileId", h.DownloadFile) + files.HEAD("/:fileId/header", h.HeadFileHeader) + files.GET("/:fileId/header", h.GetFileHeader) + files.HEAD("/:fileId/content", h.HeadFileContent) + files.GET("/:fileId/content", h.GetFileContent) + } +} diff --git a/sda/cmd/download/handlers/handlers_test.go b/sda/cmd/download/handlers/handlers_test.go new file mode 100644 index 0000000000..4381713cce --- /dev/null +++ b/sda/cmd/download/handlers/handlers_test.go @@ -0,0 +1,210 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + gin.SetMode(gin.TestMode) +} + +// newTestHandlers creates handlers with a mock database for testing. +func newTestHandlers(t *testing.T) *Handlers { + t.Helper() + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + return h +} + +// setupTestRouter creates a router without auth middleware for testing handlers directly +func setupTestRouter() *gin.Engine { + router := gin.New() + + return router +} + +// setupTestRouterWithAuth creates a router with mocked auth that injects auth context +func setupTestRouterWithAuth(datasets []string) *gin.Engine { + router := gin.New() + + // Mock middleware that injects auth context + router.Use(func(c *gin.Context) { + authCtx := middleware.AuthContext{ + Datasets: datasets, + } + c.Set(middleware.ContextKey, authCtx) + c.Next() + }) + + return router +} + +func TestHealthLive(t *testing.T) { + router := gin.New() + h := newTestHandlers(t) + h.RegisterRoutes(router) + + req, _ := http.NewRequest(http.MethodGet, "/health/live", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response map[string]string + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "ok", response["status"]) +} + +func TestHealthReady_Degraded(t *testing.T) { + // Handler with only database (no storage or grpc) should be degraded + router := gin.New() + h := newTestHandlers(t) + h.RegisterRoutes(router) + + req, _ := http.NewRequest(http.MethodGet, "/health/ready", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + + var response HealthStatus + err := json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "degraded", response.Status) + assert.NotEmpty(t, response.Services) + assert.Equal(t, "ok", response.Services["database"]) + assert.Contains(t, response.Services["storage"], "not configured") + assert.Contains(t, response.Services["grpc"], "not configured") +} + +// Test that hasDatasetAccess helper works correctly +func TestHasDatasetAccess(t *testing.T) { + testCases := []struct { + name string + datasets []string + datasetID string + hasAccess bool + }{ + { + name: "has access", + datasets: []string{"dataset1", "dataset2", "dataset3"}, + datasetID: "dataset2", + hasAccess: true, + }, + { + name: "no access", + datasets: []string{"dataset1", "dataset2"}, + datasetID: "dataset3", + hasAccess: false, + }, + { + name: "empty datasets", + datasets: []string{}, + datasetID: "dataset1", + hasAccess: false, + }, + { + name: "single dataset match", + datasets: []string{"dataset1"}, + datasetID: "dataset1", + hasAccess: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result := hasDatasetAccess(tc.datasets, tc.datasetID) + assert.Equal(t, tc.hasAccess, result) + }) + } +} + +// v2 route wiring tests — verify RegisterRoutes applies auth middleware correctly + +func TestRoutes_AuthRequired(t *testing.T) { + err := middleware.InitAuthForTesting() + require.NoError(t, err) + + router := gin.New() + h := newTestHandlers(t) + h.RegisterRoutes(router) + + // All dataset and file routes should return 401 without a token + paths := []struct { + method string + path string + }{ + {http.MethodGet, "/datasets"}, + {http.MethodGet, "/datasets/test-dataset"}, + {http.MethodGet, "/datasets/test-dataset/files"}, + {http.MethodGet, "/files/test-file-id"}, + {http.MethodHead, "/files/test-file-id"}, + {http.MethodGet, "/files/test-file-id/header"}, + {http.MethodHead, "/files/test-file-id/header"}, + {http.MethodGet, "/files/test-file-id/content"}, + {http.MethodHead, "/files/test-file-id/content"}, + } + + for _, p := range paths { + t.Run(p.method+" "+p.path, func(t *testing.T) { + req, _ := http.NewRequest(p.method, p.path, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + assert.Equal(t, http.StatusUnauthorized, w.Code) + }) + } +} + +func TestRoutes_ServiceInfoNoAuth(t *testing.T) { + router := gin.New() + h := newTestHandlers(t) + h.RegisterRoutes(router) + + req, _ := http.NewRequest(http.MethodGet, "/service-info", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + // service-info does NOT require auth + assert.Equal(t, http.StatusOK, w.Code) +} + +func TestRoutes_OldV1PathsReturn404(t *testing.T) { + err := middleware.InitAuthForTesting() + require.NoError(t, err) + + router := gin.New() + h := newTestHandlers(t) + h.RegisterRoutes(router) + + // Old v1 routes should no longer exist + paths := []string{ + "/info/datasets", + "/info/dataset", + "/info/dataset/files", + "/file/test-file", + "/file", + } + + for _, path := range paths { + t.Run(path, func(t *testing.T) { + req, _ := http.NewRequest(http.MethodGet, path, nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, req) + assert.Equal(t, http.StatusNotFound, w.Code) + }) + } +} diff --git a/sda/cmd/download/handlers/health.go b/sda/cmd/download/handlers/health.go new file mode 100644 index 0000000000..0d89767ca3 --- /dev/null +++ b/sda/cmd/download/handlers/health.go @@ -0,0 +1,80 @@ +package handlers + +import ( + "context" + "net/http" + "time" + + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" +) + +// HealthStatus represents the health check response. +type HealthStatus struct { + Status string `json:"status"` + Services map[string]string `json:"services,omitempty"` +} + +// HealthReady performs a detailed health check of all dependencies. +// GET /health/ready +func (h *Handlers) HealthReady(c *gin.Context) { + ctx, cancel := context.WithTimeout(c.Request.Context(), 5*time.Second) + defer cancel() + + services := make(map[string]string) + allHealthy := true + + // Check database + if h.db != nil { + if err := h.db.Ping(ctx); err != nil { + log.Warnf("health check: database ping failed: %v", err) + services["database"] = "error: " + err.Error() + allHealthy = false + } else { + services["database"] = "ok" + } + } else { + services["database"] = "error: not configured" + allHealthy = false + } + + // Check storage reader + if h.storageReader != nil { + services["storage"] = "ok" + } else { + services["storage"] = "error: not configured" + allHealthy = false + } + + // Check gRPC reencrypt client + if h.reencryptClient != nil { + if err := h.reencryptClient.HealthCheck(); err != nil { + log.Warnf("health check: grpc connection failed: %v", err) + services["grpc"] = "error: " + err.Error() + allHealthy = false + } else { + services["grpc"] = "ok" + } + } else { + services["grpc"] = "error: not configured" + allHealthy = false + } + + status := HealthStatus{ + Services: services, + } + + if allHealthy { + status.Status = "ok" + c.JSON(http.StatusOK, status) + } else { + status.Status = "degraded" + c.JSON(http.StatusServiceUnavailable, status) + } +} + +// HealthLive returns a simple liveness probe response. +// GET /health/live +func (h *Handlers) HealthLive(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"status": "ok"}) +} diff --git a/sda/cmd/download/handlers/mock_database_test.go b/sda/cmd/download/handlers/mock_database_test.go new file mode 100644 index 0000000000..ac64feb891 --- /dev/null +++ b/sda/cmd/download/handlers/mock_database_test.go @@ -0,0 +1,126 @@ +package handlers + +import ( + "context" + "sync" + + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" +) + +// capturingLogger records audit events for test assertions. +type capturingLogger struct { + events []audit.Event + mu sync.Mutex +} + +func (l *capturingLogger) Log(_ context.Context, event audit.Event) { + l.mu.Lock() + defer l.mu.Unlock() + l.events = append(l.events, event) +} + +func (l *capturingLogger) IsNoop() bool { return false } + +// mockDatabase is a mock implementation of the database.Database interface for testing. +type mockDatabase struct { + datasets []database.Dataset + datasetIDs []string + datasetInfo *database.DatasetInfo + datasetFiles []database.File + datasetFilesPaged []database.File + fileByID *database.File + fileByPath *database.File + hasPermission bool + datasetNotFound bool + err error + pingErr error +} + +func (m *mockDatabase) Ping(_ context.Context) error { + return m.pingErr +} + +func (m *mockDatabase) Close() error { + return nil +} + +func (m *mockDatabase) GetAllDatasets(_ context.Context) ([]database.Dataset, error) { + if m.err != nil { + return nil, m.err + } + + return m.datasets, nil +} + +func (m *mockDatabase) GetDatasetIDsByUser(_ context.Context, _ string) ([]string, error) { + if m.err != nil { + return nil, m.err + } + + return m.datasetIDs, nil +} + +func (m *mockDatabase) GetUserDatasets(_ context.Context, _ []string) ([]database.Dataset, error) { + if m.err != nil { + return nil, m.err + } + + return m.datasets, nil +} + +func (m *mockDatabase) GetDatasetInfo(_ context.Context, _ string) (*database.DatasetInfo, error) { + if m.err != nil { + return nil, m.err + } + + return m.datasetInfo, nil +} + +func (m *mockDatabase) GetDatasetFiles(_ context.Context, _ string) ([]database.File, error) { + if m.err != nil { + return nil, m.err + } + + return m.datasetFiles, nil +} + +func (m *mockDatabase) GetFileByID(_ context.Context, _ string) (*database.File, error) { + if m.err != nil { + return nil, m.err + } + + return m.fileByID, nil +} + +func (m *mockDatabase) GetFileByPath(_ context.Context, _, _ string) (*database.File, error) { + if m.err != nil { + return nil, m.err + } + + return m.fileByPath, nil +} + +func (m *mockDatabase) CheckFilePermission(_ context.Context, _ string, _ []string) (bool, error) { + if m.err != nil { + return false, m.err + } + + return m.hasPermission, nil +} + +func (m *mockDatabase) CheckDatasetExists(_ context.Context, _ string) (bool, error) { + if m.err != nil { + return false, m.err + } + + return !m.datasetNotFound, nil +} + +func (m *mockDatabase) GetDatasetFilesPaginated(_ context.Context, _ string, _ database.FileListOptions) ([]database.File, error) { + if m.err != nil { + return nil, m.err + } + + return m.datasetFilesPaged, nil +} diff --git a/sda/cmd/download/handlers/options.go b/sda/cmd/download/handlers/options.go new file mode 100644 index 0000000000..fec83e4064 --- /dev/null +++ b/sda/cmd/download/handlers/options.go @@ -0,0 +1,58 @@ +package handlers + +import ( + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/neicnordic/sensitive-data-archive/cmd/download/reencrypt" + "github.com/neicnordic/sensitive-data-archive/cmd/download/visa" + storage "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" +) + +// WithDatabase sets the database for the handlers. +func WithDatabase(db database.Database) func(*Handlers) { + return func(h *Handlers) { + h.db = db + } +} + +// WithStorageReader sets the storage reader for the handlers. +func WithStorageReader(reader storage.Reader) func(*Handlers) { + return func(h *Handlers) { + h.storageReader = reader + } +} + +// WithReencryptClient sets the reencrypt client for the handlers. +func WithReencryptClient(client *reencrypt.Client) func(*Handlers) { + return func(h *Handlers) { + h.reencryptClient = client + } +} + +// WithVisaValidator sets the GA4GH visa validator for the handlers. +func WithVisaValidator(v *visa.Validator) func(*Handlers) { + return func(h *Handlers) { + h.visaValidator = v + } +} + +// WithAuditLogger sets the audit logger for the handlers. +func WithAuditLogger(l audit.Logger) func(*Handlers) { + return func(h *Handlers) { + h.auditLogger = l + } +} + +// WithGRPCReencryptHost sets the gRPC reencrypt service host (deprecated, use WithReencryptClient). +func WithGRPCReencryptHost(host string) func(*Handlers) { + return func(h *Handlers) { + h.grpcHost = host + } +} + +// WithGRPCReencryptPort sets the gRPC reencrypt service port (deprecated, use WithReencryptClient). +func WithGRPCReencryptPort(port int) func(*Handlers) { + return func(h *Handlers) { + h.grpcPort = port + } +} diff --git a/sda/cmd/download/handlers/options_test.go b/sda/cmd/download/handlers/options_test.go new file mode 100644 index 0000000000..be56333ec6 --- /dev/null +++ b/sda/cmd/download/handlers/options_test.go @@ -0,0 +1,42 @@ +package handlers + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNew_WithoutDatabase_ReturnsError(t *testing.T) { + h, err := New() + + assert.Nil(t, h) + assert.Error(t, err) + assert.Contains(t, err.Error(), "database is required") +} + +func TestNew_WithDatabase_Success(t *testing.T) { + mockDB := &mockDatabase{} + + h, err := New(WithDatabase(mockDB)) + + require.NoError(t, err) + assert.NotNil(t, h) + assert.Equal(t, mockDB, h.db) +} + +func TestNew_WithAllOptions(t *testing.T) { + mockDB := &mockDatabase{} + + h, err := New( + WithDatabase(mockDB), + WithGRPCReencryptHost("localhost"), + WithGRPCReencryptPort(50051), + ) + + require.NoError(t, err) + assert.NotNil(t, h) + assert.Equal(t, mockDB, h.db) + assert.Equal(t, "localhost", h.grpcHost) + assert.Equal(t, 50051, h.grpcPort) +} diff --git a/sda/cmd/download/handlers/pagination.go b/sda/cmd/download/handlers/pagination.go new file mode 100644 index 0000000000..26b00ca5e2 --- /dev/null +++ b/sda/cmd/download/handlers/pagination.go @@ -0,0 +1,123 @@ +package handlers + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/gin-gonic/gin" +) + +type pageToken struct { + Cursor string `json:"c"` // last-seen primary sort key + CursorID string `json:"i,omitempty"` // tie-breaker (stable_id) for files; empty for datasets + PageSize int `json:"s"` // original pageSize (reject if changed) + QueryHash string `json:"q"` // sha256 fingerprint of (datasetId, filters) + Exp int64 `json:"e"` // Unix timestamp expiry +} + +// paginationSecret holds the HMAC key. Set at startup. +var paginationSecret []byte + +// SetPaginationSecret sets the HMAC signing key for page tokens. +func SetPaginationSecret(secret []byte) { + paginationSecret = secret +} + +// parsePageSize validates the "pageSize" query parameter. +// Returns a value in [1, 1000], defaulting to 100 if absent. +func parsePageSize(c *gin.Context) (int, error) { + raw := c.Query("pageSize") + if raw == "" { + return 100, nil + } + + size, err := strconv.Atoi(raw) + if err != nil { + return 0, errors.New("pageSize must be an integer") + } + + if size < 1 || size > 1000 { + return 0, errors.New("pageSize must be between 1 and 1000") + } + + return size, nil +} + +// encodePageToken creates a signed, base64-encoded page token. +// Wire format: base64(json) + "." + base64(hmac-sha256(json)). +func encodePageToken(cursor, cursorID string, pageSize int, queryHash string) string { + tok := pageToken{ + Cursor: cursor, + CursorID: cursorID, + PageSize: pageSize, + QueryHash: queryHash, + Exp: time.Now().Add(1 * time.Hour).Unix(), + } + + payload, _ := json.Marshal(tok) + sig := computeHMAC(payload) + + return base64.RawURLEncoding.EncodeToString(payload) + "." + base64.RawURLEncoding.EncodeToString(sig) +} + +// decodePageToken verifies the HMAC signature, checks expiry, and returns the parsed token. +func decodePageToken(token string) (*pageToken, error) { + parts := strings.SplitN(token, ".", 2) + if len(parts) != 2 { + return nil, errors.New("malformed page token") + } + + payload, err := base64.RawURLEncoding.DecodeString(parts[0]) + if err != nil { + return nil, errors.New("malformed page token") + } + + sig, err := base64.RawURLEncoding.DecodeString(parts[1]) + if err != nil { + return nil, errors.New("malformed page token") + } + + if !hmac.Equal(computeHMAC(payload), sig) { + return nil, errors.New("invalid page token signature") + } + + var tok pageToken + if err := json.Unmarshal(payload, &tok); err != nil { + return nil, errors.New("malformed page token") + } + + if time.Now().Unix() > tok.Exp { + return nil, errors.New("page token expired") + } + + return &tok, nil +} + +// queryFingerprint returns a deterministic hex-encoded prefix of SHA-256 +// over the given parts joined by null bytes. +func queryFingerprint(parts ...string) string { + h := sha256.New() + for i, p := range parts { + if i > 0 { + h.Write([]byte{0x00}) + } + h.Write([]byte(p)) + } + + return fmt.Sprintf("%x", h.Sum(nil))[:16] +} + +// computeHMAC returns the HMAC-SHA256 of data using paginationSecret. +func computeHMAC(data []byte) []byte { + mac := hmac.New(sha256.New, paginationSecret) + mac.Write(data) + + return mac.Sum(nil) +} diff --git a/sda/cmd/download/handlers/pagination_test.go b/sda/cmd/download/handlers/pagination_test.go new file mode 100644 index 0000000000..4cf95a08be --- /dev/null +++ b/sda/cmd/download/handlers/pagination_test.go @@ -0,0 +1,197 @@ +package handlers + +import ( + "encoding/base64" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func init() { + SetPaginationSecret([]byte("test-secret-key-for-pagination")) +} + +func TestParsePageSize_DefaultWhenAbsent(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/test", nil) + + size, err := parsePageSize(c) + + require.NoError(t, err) + assert.Equal(t, 100, size) +} + +func TestParsePageSize_AcceptsValidRange(t *testing.T) { + tests := []struct { + param string + expected int + }{ + {"1", 1}, + {"50", 50}, + {"100", 100}, + {"500", 500}, + {"1000", 1000}, + } + + for _, tt := range tests { + t.Run(tt.param, func(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/test?pageSize="+tt.param, nil) + + size, err := parsePageSize(c) + + require.NoError(t, err) + assert.Equal(t, tt.expected, size) + }) + } +} + +func TestParsePageSize_RejectsInvalid(t *testing.T) { + tests := []struct { + name string + param string + }{ + {"zero", "0"}, + {"negative", "-1"}, + {"over_max", "1001"}, + {"non_integer", "abc"}, + {"float", "10.5"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = httptest.NewRequest(http.MethodGet, "/test?pageSize="+tt.param, nil) + + _, err := parsePageSize(c) + + assert.Error(t, err) + }) + } +} + +func TestEncodeDecodePageToken_RoundTrip(t *testing.T) { + token := encodePageToken("dataset-042", "EGAF00000000099", 50, "abcdef1234567890") + + decoded, err := decodePageToken(token) + + require.NoError(t, err) + assert.Equal(t, "dataset-042", decoded.Cursor) + assert.Equal(t, "EGAF00000000099", decoded.CursorID) + assert.Equal(t, 50, decoded.PageSize) + assert.Equal(t, "abcdef1234567890", decoded.QueryHash) + assert.True(t, decoded.Exp > time.Now().Unix()) +} + +func TestEncodeDecodePageToken_EmptyCursorID(t *testing.T) { + token := encodePageToken("EGAD00000000010", "", 100, "abc123") + + decoded, err := decodePageToken(token) + + require.NoError(t, err) + assert.Equal(t, "EGAD00000000010", decoded.Cursor) + assert.Empty(t, decoded.CursorID) +} + +func TestDecodePageToken_TamperedPayload(t *testing.T) { + token := encodePageToken("cursor", "", 100, "hash") + + // Tamper with the payload portion (before the dot) + parts := splitToken(t, token) + payload, err := base64.RawURLEncoding.DecodeString(parts[0]) + require.NoError(t, err) + + var tok pageToken + require.NoError(t, json.Unmarshal(payload, &tok)) + tok.Cursor = "tampered" + tampered, _ := json.Marshal(tok) + + tamperedToken := base64.RawURLEncoding.EncodeToString(tampered) + "." + parts[1] + + _, err = decodePageToken(tamperedToken) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "signature") +} + +func TestDecodePageToken_ExpiredToken(t *testing.T) { + // Create a token, then manually re-encode with expired timestamp + tok := pageToken{ + Cursor: "some-cursor", + PageSize: 100, + QueryHash: "hash", + Exp: time.Now().Add(-2 * time.Hour).Unix(), + } + + payload, _ := json.Marshal(tok) + sig := computeHMAC(payload) + token := base64.RawURLEncoding.EncodeToString(payload) + "." + base64.RawURLEncoding.EncodeToString(sig) + + _, err := decodePageToken(token) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "page token expired") +} + +func TestDecodePageToken_MalformedNoDot(t *testing.T) { + _, err := decodePageToken("nodothere") + + assert.Error(t, err) + assert.Contains(t, err.Error(), "malformed") +} + +func TestDecodePageToken_MalformedBadBase64(t *testing.T) { + _, err := decodePageToken("!!!.!!!") + + assert.Error(t, err) + assert.Contains(t, err.Error(), "malformed") +} + +func TestQueryFingerprint_Deterministic(t *testing.T) { + fp1 := queryFingerprint("EGAD00000000001", "filePath", "/data/file.txt") + fp2 := queryFingerprint("EGAD00000000001", "filePath", "/data/file.txt") + + assert.Equal(t, fp1, fp2) + assert.Len(t, fp1, 16) +} + +func TestQueryFingerprint_DifferentInputs(t *testing.T) { + fp1 := queryFingerprint("EGAD00000000001") + fp2 := queryFingerprint("EGAD00000000002") + + assert.NotEqual(t, fp1, fp2) +} + +func TestQueryFingerprint_OrderMatters(t *testing.T) { + fp1 := queryFingerprint("a", "b") + fp2 := queryFingerprint("b", "a") + + assert.NotEqual(t, fp1, fp2) +} + +// splitToken is a test helper that splits a page token on "." and asserts two parts. +func splitToken(t *testing.T, token string) [2]string { + t.Helper() + + idx := -1 + for i, c := range token { + if c == '.' { + idx = i + + break + } + } + + require.NotEqual(t, -1, idx, "token must contain a dot separator") + + return [2]string{token[:idx], token[idx+1:]} +} diff --git a/sda/cmd/download/handlers/problem.go b/sda/cmd/download/handlers/problem.go new file mode 100644 index 0000000000..8cba59edb7 --- /dev/null +++ b/sda/cmd/download/handlers/problem.go @@ -0,0 +1,54 @@ +package handlers + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/google/uuid" +) + +// ProblemDetails represents an RFC 9457 Problem Details response. +type ProblemDetails struct { + Type string `json:"type,omitempty"` + Title string `json:"title"` + Status int `json:"status"` + Detail string `json:"detail,omitempty"` + Instance string `json:"instance,omitempty"` + ErrorCode string `json:"errorCode,omitempty"` + CorrelationID string `json:"correlationId,omitempty"` +} + +// problemJSON sends an RFC 9457 Problem Details JSON response. +// Title is derived from the HTTP status code. CorrelationID is read from the gin context. +func problemJSON(c *gin.Context, status int, detail string) { + c.Header("Content-Type", "application/problem+json") + c.JSON(status, ProblemDetails{ + Title: http.StatusText(status), + Status: status, + Detail: detail, + CorrelationID: c.GetString("correlationId"), + }) +} + +// problemJSONWithCode sends an RFC 9457 Problem Details JSON response with an error code. +func problemJSONWithCode(c *gin.Context, status int, detail, errorCode string) { + c.Header("Content-Type", "application/problem+json") + c.JSON(status, ProblemDetails{ + Title: http.StatusText(status), + Status: status, + Detail: detail, + ErrorCode: errorCode, + CorrelationID: c.GetString("correlationId"), + }) +} + +// correlationIDMiddleware generates a UUID per request, stores it in the gin context +// as "correlationId", and sets the X-Correlation-ID response header. +func correlationIDMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + id := uuid.New().String() + c.Set("correlationId", id) + c.Header("X-Correlation-ID", id) + c.Next() + } +} diff --git a/sda/cmd/download/handlers/problem_test.go b/sda/cmd/download/handlers/problem_test.go new file mode 100644 index 0000000000..2506bcf100 --- /dev/null +++ b/sda/cmd/download/handlers/problem_test.go @@ -0,0 +1,108 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestProblemJSON_ContentType(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + + problemJSON(c, http.StatusBadRequest, "something went wrong") + + assert.Equal(t, "application/problem+json", w.Header().Get("Content-Type")) +} + +func TestProblemJSON_ResponseBody(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Set("correlationId", "test-correlation-id") + + problemJSON(c, http.StatusNotFound, "resource not found") + + assert.Equal(t, http.StatusNotFound, w.Code) + + var pd ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &pd) + require.NoError(t, err) + assert.Equal(t, "Not Found", pd.Title) + assert.Equal(t, http.StatusNotFound, pd.Status) + assert.Equal(t, "resource not found", pd.Detail) + assert.Equal(t, "test-correlation-id", pd.CorrelationID) + assert.Empty(t, pd.ErrorCode) +} + +func TestProblemJSONWithCode_IncludesErrorCode(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Set("correlationId", "abc-123") + + problemJSONWithCode(c, http.StatusBadRequest, "invalid range", "RANGE_INVALID") + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Equal(t, "application/problem+json", w.Header().Get("Content-Type")) + + var pd ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &pd) + require.NoError(t, err) + assert.Equal(t, "Bad Request", pd.Title) + assert.Equal(t, http.StatusBadRequest, pd.Status) + assert.Equal(t, "invalid range", pd.Detail) + assert.Equal(t, "RANGE_INVALID", pd.ErrorCode) + assert.Equal(t, "abc-123", pd.CorrelationID) +} + +func TestCorrelationIDMiddleware_GeneratesUUID(t *testing.T) { + w := httptest.NewRecorder() + c, router := gin.CreateTestContext(w) + + router.Use(correlationIDMiddleware()) + router.GET("/test", func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"id": c.GetString("correlationId")}) + }) + + c.Request = httptest.NewRequest(http.MethodGet, "/test", nil) + router.ServeHTTP(w, c.Request) + + // Check the response header is set + headerID := w.Header().Get("X-Correlation-ID") + assert.NotEmpty(t, headerID) + assert.Len(t, headerID, 36) // UUID format: 8-4-4-4-12 + + // Check the context value matches the header + var body map[string]string + err := json.Unmarshal(w.Body.Bytes(), &body) + require.NoError(t, err) + assert.Equal(t, headerID, body["id"]) +} + +func TestCorrelationIDMiddleware_PropagatesToProblemResponse(t *testing.T) { + w := httptest.NewRecorder() + c, router := gin.CreateTestContext(w) + + router.Use(correlationIDMiddleware()) + router.GET("/test", func(c *gin.Context) { + problemJSON(c, http.StatusInternalServerError, "something broke") + }) + + c.Request = httptest.NewRequest(http.MethodGet, "/test", nil) + router.ServeHTTP(w, c.Request) + + headerID := w.Header().Get("X-Correlation-ID") + assert.NotEmpty(t, headerID) + + var pd ProblemDetails + err := json.Unmarshal(w.Body.Bytes(), &pd) + require.NoError(t, err) + assert.Equal(t, headerID, pd.CorrelationID) + assert.Equal(t, http.StatusInternalServerError, pd.Status) + assert.Equal(t, "Internal Server Error", pd.Title) + assert.Equal(t, "something broke", pd.Detail) +} diff --git a/sda/cmd/download/handlers/publickey.go b/sda/cmd/download/handlers/publickey.go new file mode 100644 index 0000000000..0f04355d1c --- /dev/null +++ b/sda/cmd/download/handlers/publickey.go @@ -0,0 +1,22 @@ +package handlers + +import "github.com/gin-gonic/gin" + +// extractPublicKey reads the client's crypt4gh public key from request headers. +// Exactly one of X-C4GH-Public-Key or Htsget-Context-Public-Key must be present. +// Returns the raw header value (base64-encoded string) or ("", errorCode, detail). +func extractPublicKey(c *gin.Context) (string, string, string) { + primary := c.GetHeader("X-C4GH-Public-Key") + secondary := c.GetHeader("Htsget-Context-Public-Key") + + switch { + case primary != "" && secondary != "": + return "", "KEY_CONFLICT", "only one of X-C4GH-Public-Key or Htsget-Context-Public-Key may be provided" + case primary != "": + return primary, "", "" + case secondary != "": + return secondary, "", "" + default: + return "", "KEY_MISSING", "X-C4GH-Public-Key or Htsget-Context-Public-Key header is required" + } +} diff --git a/sda/cmd/download/handlers/publickey_test.go b/sda/cmd/download/handlers/publickey_test.go new file mode 100644 index 0000000000..a429187541 --- /dev/null +++ b/sda/cmd/download/handlers/publickey_test.go @@ -0,0 +1,58 @@ +package handlers + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" +) + +func TestExtractPublicKey_PrimaryOnly(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request, _ = http.NewRequest(http.MethodGet, "/", nil) + c.Request.Header.Set("X-C4GH-Public-Key", "dGVzdC1rZXk=") + + key, errCode, detail := extractPublicKey(c) + assert.Equal(t, "dGVzdC1rZXk=", key) + assert.Empty(t, errCode) + assert.Empty(t, detail) +} + +func TestExtractPublicKey_SecondaryOnly(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request, _ = http.NewRequest(http.MethodGet, "/", nil) + c.Request.Header.Set("Htsget-Context-Public-Key", "c2Vjb25kYXJ5LWtleQ==") + + key, errCode, detail := extractPublicKey(c) + assert.Equal(t, "c2Vjb25kYXJ5LWtleQ==", key) + assert.Empty(t, errCode) + assert.Empty(t, detail) +} + +func TestExtractPublicKey_BothPresent(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request, _ = http.NewRequest(http.MethodGet, "/", nil) + c.Request.Header.Set("X-C4GH-Public-Key", "key1") + c.Request.Header.Set("Htsget-Context-Public-Key", "key2") + + key, errCode, detail := extractPublicKey(c) + assert.Empty(t, key) + assert.Equal(t, "KEY_CONFLICT", errCode) + assert.Contains(t, detail, "only one of") +} + +func TestExtractPublicKey_NeitherPresent(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request, _ = http.NewRequest(http.MethodGet, "/", nil) + + key, errCode, detail := extractPublicKey(c) + assert.Empty(t, key) + assert.Equal(t, "KEY_MISSING", errCode) + assert.Contains(t, detail, "header is required") +} diff --git a/sda/cmd/download/handlers/serviceinfo.go b/sda/cmd/download/handlers/serviceinfo.go new file mode 100644 index 0000000000..8487c3bffd --- /dev/null +++ b/sda/cmd/download/handlers/serviceinfo.go @@ -0,0 +1,47 @@ +package handlers + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +// serviceInfoResponse represents a GA4GH Service Info response. +type serviceInfoResponse struct { + ID string `json:"id"` + Name string `json:"name"` + Type serviceInfoType `json:"type"` + Organization serviceInfoOrg `json:"organization"` + Version string `json:"version"` +} + +type serviceInfoType struct { + Group string `json:"group"` + Artifact string `json:"artifact"` + Version string `json:"version"` +} + +type serviceInfoOrg struct { + Name string `json:"name"` + URL string `json:"url"` +} + +// ServiceInfo returns GA4GH service-info metadata. +// GET /service-info +func (h *Handlers) ServiceInfo(c *gin.Context) { + c.Header("Cache-Control", "public, max-age=300") + c.JSON(http.StatusOK, serviceInfoResponse{ + ID: "se.nbis.sda.download", + Name: "SDA Download", + Type: serviceInfoType{ + Group: "org.ga4gh", + Artifact: "drs", + Version: "1.0.0", + }, + Organization: serviceInfoOrg{ + Name: "NBIS", + URL: "https://nbis.se", + }, + Version: "2.0.0", + }) +} diff --git a/sda/cmd/download/handlers/serviceinfo_test.go b/sda/cmd/download/handlers/serviceinfo_test.go new file mode 100644 index 0000000000..f68b09deb9 --- /dev/null +++ b/sda/cmd/download/handlers/serviceinfo_test.go @@ -0,0 +1,38 @@ +package handlers + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestServiceInfo_Response(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request, _ = http.NewRequest(http.MethodGet, "/service-info", nil) + + h := newTestHandlers(t) + h.ServiceInfo(c) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "application/json; charset=utf-8", w.Header().Get("Content-Type")) + assert.Equal(t, "public, max-age=300", w.Header().Get("Cache-Control")) + + var response serviceInfoResponse + err := json.Unmarshal(w.Body.Bytes(), &response) + require.NoError(t, err) + + assert.Equal(t, "se.nbis.sda.download", response.ID) + assert.Equal(t, "SDA Download", response.Name) + assert.Equal(t, "org.ga4gh", response.Type.Group) + assert.Equal(t, "drs", response.Type.Artifact) + assert.Equal(t, "1.0.0", response.Type.Version) + assert.Equal(t, "NBIS", response.Organization.Name) + assert.Equal(t, "https://nbis.se", response.Organization.URL) + assert.Equal(t, "2.0.0", response.Version) +} From 3cb7a7df8f5225f717584fc38675e9c104e9cebb Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:08:13 +0100 Subject: [PATCH 376/530] feat(download): add audit logging and production guards Audit logging: - audit.Logger interface with StdoutLogger (JSON lines) and NoopLogger - download.complete/content/header events on successful operations - download.denied events on 401/403 (middleware + handlers) - download.failed events on storage/streaming errors with ErrorReason - Correlation ID propagated to all audit events Production guards (app.environment=production): - Fail startup if jwt.allow-all-data is enabled - Fail startup if pagination.hmac-secret is empty - Fail startup if gRPC client TLS certs are missing - validateProductionConfig extracted for unit testing --- sda/cmd/download/audit/audit.go | 37 +++++ sda/cmd/download/audit/audit_test.go | 206 ++++++++++++++++++++++++ sda/cmd/download/audit/stdout_logger.go | 46 ++++++ sda/cmd/download/main_test.go | 62 +++++++ 4 files changed, 351 insertions(+) create mode 100644 sda/cmd/download/audit/audit.go create mode 100644 sda/cmd/download/audit/audit_test.go create mode 100644 sda/cmd/download/audit/stdout_logger.go create mode 100644 sda/cmd/download/main_test.go diff --git a/sda/cmd/download/audit/audit.go b/sda/cmd/download/audit/audit.go new file mode 100644 index 0000000000..b964b6b273 --- /dev/null +++ b/sda/cmd/download/audit/audit.go @@ -0,0 +1,37 @@ +// Package audit provides audit logging for the download service. +// It defines the Logger interface and event types used to record +// download operations for compliance and monitoring. +package audit + +import ( + "context" + "time" +) + +// Event represents an audit event for download operations. +type Event struct { + Type string `json:"type"` // routing tag: always "audit" + Event string `json:"event"` // "download.completed", "download.denied", "download.failed" + Timestamp time.Time `json:"timestamp"` + UserID string `json:"userId"` + FileID string `json:"fileId,omitempty"` + DatasetID string `json:"datasetId,omitempty"` + CorrelationID string `json:"correlationId"` + Endpoint string `json:"endpoint"` + HTTPStatus int `json:"httpStatus"` + BytesTransferred int64 `json:"bytesTransferred,omitempty"` + AuthType string `json:"authType,omitempty"` + ErrorReason string `json:"errorReason,omitempty"` +} + +// Logger defines the interface for audit logging. +type Logger interface { + Log(ctx context.Context, event Event) + IsNoop() bool +} + +// NoopLogger is a no-op implementation of Logger for testing and development. +type NoopLogger struct{} + +func (NoopLogger) Log(context.Context, Event) {} +func (NoopLogger) IsNoop() bool { return true } diff --git a/sda/cmd/download/audit/audit_test.go b/sda/cmd/download/audit/audit_test.go new file mode 100644 index 0000000000..7be4af2363 --- /dev/null +++ b/sda/cmd/download/audit/audit_test.go @@ -0,0 +1,206 @@ +package audit + +import ( + "bytes" + "context" + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNoopLogger_IsNoop(t *testing.T) { + var logger Logger = NoopLogger{} + assert.True(t, logger.IsNoop()) +} + +func TestNoopLogger_LogDoesNotPanic(t *testing.T) { + var logger Logger = NoopLogger{} + assert.NotPanics(t, func() { + logger.Log(context.Background(), Event{ + Event: "download.completed", + CorrelationID: "abc-123", + Endpoint: "/files/test", + HTTPStatus: 200, + }) + }) +} + +func TestStdoutLogger_IsNoop(t *testing.T) { + var buf bytes.Buffer + var logger Logger = newStdoutLoggerWithWriter(&buf) + assert.False(t, logger.IsNoop()) +} + +func TestStdoutLogger_OutputsValidJSON(t *testing.T) { + var buf bytes.Buffer + logger := newStdoutLoggerWithWriter(&buf) + + logger.Log(context.Background(), Event{ + Event: "download.completed", + UserID: "user@example.org", + FileID: "file-001", + DatasetID: "EGAD00000000001", + CorrelationID: "corr-123", + Endpoint: "/files/file-001", + HTTPStatus: 200, + }) + + var decoded Event + err := json.Unmarshal(buf.Bytes(), &decoded) + require.NoError(t, err, "output must be valid JSON") + assert.Equal(t, "download.completed", decoded.Event) + assert.Equal(t, "user@example.org", decoded.UserID) + assert.Equal(t, "file-001", decoded.FileID) + assert.Equal(t, "EGAD00000000001", decoded.DatasetID) + assert.Equal(t, "corr-123", decoded.CorrelationID) + assert.Equal(t, "/files/file-001", decoded.Endpoint) + assert.Equal(t, 200, decoded.HTTPStatus) +} + +func TestStdoutLogger_AlwaysSetsTypeToAudit(t *testing.T) { + var buf bytes.Buffer + logger := newStdoutLoggerWithWriter(&buf) + + logger.Log(context.Background(), Event{ + Type: "something-else", + Event: "download.denied", + CorrelationID: "corr-456", + Endpoint: "/files/file-002", + HTTPStatus: 403, + }) + + var decoded Event + err := json.Unmarshal(buf.Bytes(), &decoded) + require.NoError(t, err) + assert.Equal(t, "audit", decoded.Type, "Type must always be 'audit' regardless of caller input") +} + +func TestStdoutLogger_TimestampIsUTC(t *testing.T) { + var buf bytes.Buffer + logger := newStdoutLoggerWithWriter(&buf) + + // Provide a non-UTC timestamp + eastern := time.FixedZone("EST", -5*60*60) + ts := time.Date(2025, 6, 15, 10, 30, 0, 0, eastern) + + logger.Log(context.Background(), Event{ + Event: "download.completed", + Timestamp: ts, + CorrelationID: "corr-789", + Endpoint: "/files/file-003", + HTTPStatus: 200, + }) + + var decoded Event + err := json.Unmarshal(buf.Bytes(), &decoded) + require.NoError(t, err) + assert.Equal(t, time.UTC, decoded.Timestamp.Location(), "timestamp must be in UTC") + assert.Equal(t, ts.UTC(), decoded.Timestamp) +} + +func TestStdoutLogger_ZeroTimestampFilled(t *testing.T) { + var buf bytes.Buffer + logger := newStdoutLoggerWithWriter(&buf) + + before := time.Now().UTC() + logger.Log(context.Background(), Event{ + Event: "download.failed", + CorrelationID: "corr-000", + Endpoint: "/files/file-004", + HTTPStatus: 500, + }) + after := time.Now().UTC() + + var decoded Event + err := json.Unmarshal(buf.Bytes(), &decoded) + require.NoError(t, err) + assert.False(t, decoded.Timestamp.IsZero(), "zero timestamp must be filled with current time") + assert.True(t, !decoded.Timestamp.Before(before) && !decoded.Timestamp.After(after), + "auto-filled timestamp must be between before and after test boundaries") +} + +func TestStdoutLogger_OmitsEmptyOptionalFields(t *testing.T) { + var buf bytes.Buffer + logger := newStdoutLoggerWithWriter(&buf) + + logger.Log(context.Background(), Event{ + Event: "download.denied", + CorrelationID: "corr-omit", + Endpoint: "/files/file-005", + HTTPStatus: 401, + }) + + var raw map[string]any + err := json.Unmarshal(buf.Bytes(), &raw) + require.NoError(t, err) + + // Fields with omitempty should be absent when zero-valued + assert.NotContains(t, raw, "fileId", "empty fileId should be omitted") + assert.NotContains(t, raw, "datasetId", "empty datasetId should be omitted") + assert.NotContains(t, raw, "bytesTransferred", "zero bytesTransferred should be omitted") + assert.NotContains(t, raw, "authType", "empty authType should be omitted") + assert.NotContains(t, raw, "errorReason", "empty errorReason should be omitted") + + // Required fields should always be present + assert.Contains(t, raw, "type") + assert.Contains(t, raw, "event") + assert.Contains(t, raw, "timestamp") + assert.Contains(t, raw, "correlationId") + assert.Contains(t, raw, "endpoint") + assert.Contains(t, raw, "httpStatus") +} + +func TestStdoutLogger_MultipleEvents(t *testing.T) { + var buf bytes.Buffer + logger := newStdoutLoggerWithWriter(&buf) + + logger.Log(context.Background(), Event{ + Event: "download.completed", + CorrelationID: "corr-a", + Endpoint: "/files/a", + HTTPStatus: 200, + }) + logger.Log(context.Background(), Event{ + Event: "download.denied", + CorrelationID: "corr-b", + Endpoint: "/files/b", + HTTPStatus: 403, + }) + + decoder := json.NewDecoder(&buf) + + var first Event + require.NoError(t, decoder.Decode(&first)) + assert.Equal(t, "corr-a", first.CorrelationID) + + var second Event + require.NoError(t, decoder.Decode(&second)) + assert.Equal(t, "corr-b", second.CorrelationID) +} + +func TestAuditContract_RequiredFieldsPopulated(t *testing.T) { + var buf bytes.Buffer + logger := newStdoutLoggerWithWriter(&buf) + + logger.Log(context.Background(), Event{ + Event: "download.completed", + UserID: "user@example.org", + CorrelationID: "corr-contract", + Endpoint: "/files/file-006", + HTTPStatus: 200, + }) + + var decoded Event + err := json.Unmarshal(buf.Bytes(), &decoded) + require.NoError(t, err) + + assert.Equal(t, "audit", decoded.Type, "Type must be 'audit'") + assert.NotEmpty(t, decoded.Event, "Event must be populated") + assert.False(t, decoded.Timestamp.IsZero(), "Timestamp must be populated") + assert.NotEmpty(t, decoded.CorrelationID, "CorrelationID must be populated") + assert.NotEmpty(t, decoded.Endpoint, "Endpoint must be populated") + assert.NotZero(t, decoded.HTTPStatus, "HTTPStatus must be populated") +} diff --git a/sda/cmd/download/audit/stdout_logger.go b/sda/cmd/download/audit/stdout_logger.go new file mode 100644 index 0000000000..3b1e8b0a0b --- /dev/null +++ b/sda/cmd/download/audit/stdout_logger.go @@ -0,0 +1,46 @@ +package audit + +import ( + "context" + "encoding/json" + "io" + "os" + "sync" + "time" +) + +// StdoutLogger outputs audit events as JSON lines to stdout. +type StdoutLogger struct { + encoder *json.Encoder + mu sync.Mutex +} + +// NewStdoutLogger creates a StdoutLogger that writes to os.Stdout. +func NewStdoutLogger() *StdoutLogger { + return newStdoutLoggerWithWriter(os.Stdout) +} + +// newStdoutLoggerWithWriter creates a StdoutLogger that writes to w. +// Used in tests to capture output without redirecting os.Stdout. +func newStdoutLoggerWithWriter(w io.Writer) *StdoutLogger { + return &StdoutLogger{ + encoder: json.NewEncoder(w), + } +} + +func (l *StdoutLogger) Log(_ context.Context, event Event) { + // Enforce Type is always "audit" + event.Type = "audit" + // Ensure timestamp is UTC + if event.Timestamp.IsZero() { + event.Timestamp = time.Now().UTC() + } else { + event.Timestamp = event.Timestamp.UTC() + } + + l.mu.Lock() + defer l.mu.Unlock() + _ = l.encoder.Encode(event) // best-effort, don't block HTTP response +} + +func (l *StdoutLogger) IsNoop() bool { return false } diff --git a/sda/cmd/download/main_test.go b/sda/cmd/download/main_test.go new file mode 100644 index 0000000000..445ee9f764 --- /dev/null +++ b/sda/cmd/download/main_test.go @@ -0,0 +1,62 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateProductionConfig_AllValid(t *testing.T) { + err := validateProductionConfig(productionConfig{ + AllowAllData: false, + HMACSecret: "some-secret", + GRPCClientCert: "/path/to/cert", + GRPCClientKey: "/path/to/key", + }) + require.NoError(t, err) +} + +func TestValidateProductionConfig_AllowAllDataBlocked(t *testing.T) { + err := validateProductionConfig(productionConfig{ + AllowAllData: true, + HMACSecret: "some-secret", + GRPCClientCert: "/path/to/cert", + GRPCClientKey: "/path/to/key", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "allow-all-data") +} + +func TestValidateProductionConfig_EmptyHMACSecret(t *testing.T) { + err := validateProductionConfig(productionConfig{ + AllowAllData: false, + HMACSecret: "", + GRPCClientCert: "/path/to/cert", + GRPCClientKey: "/path/to/key", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "pagination.hmac-secret") +} + +func TestValidateProductionConfig_MissingGRPCCert(t *testing.T) { + err := validateProductionConfig(productionConfig{ + AllowAllData: false, + HMACSecret: "some-secret", + GRPCClientCert: "", + GRPCClientKey: "/path/to/key", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "grpc.client-cert") +} + +func TestValidateProductionConfig_MissingGRPCKey(t *testing.T) { + err := validateProductionConfig(productionConfig{ + AllowAllData: false, + HMACSecret: "some-secret", + GRPCClientCert: "/path/to/cert", + GRPCClientKey: "", + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "grpc.client-cert") +} From 33d1193169f6a7765668521d1fd8ba9c662c3903 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:08:45 +0100 Subject: [PATCH 377/530] test(download): add integration tests with capability probes Docker Compose integration test environment: - postgres, minio (S3), mock-aai, mockoidc (OIDC + visa JWTs), reencrypt (gRPC), download service under test - database_seed with test dataset + file - make_download_credentials.sh: RSA keypair, JWT token, trusted issuers - mockoidc.py: OIDC discovery, JWKS, userinfo with visa datasets 33 integration tests covering: - Health, auth (JWT + opaque), session cookies, service-info - Dataset listing, file listing, pagination, invalid pageSize/pageToken - Encoded-slash dataset IDs (UseRawPath verification) - Range requests, multi-range rejection, If-Range ETag contract - Content-Disposition, pathPrefix filter with SQL wildcard escaping - Expired token rejection, long-transfer resume scenario - Problem Details format, access control (no existence leakage) Environment capability probes (SetupSuite): - REQUIRES_REENCRYPT, REQUIRES_STORAGE_FILE, REQUIRES_SESSION_CACHE - Tests skip on missing prerequisites, hard-fail on regressions --- .../scripts/make_download_credentials.sh | 89 ++ .../scripts/make_sda_credentials.sh | 19 +- .github/integration/scripts/mockoidc.py | 181 +++ .../sda-cmd-download-integration.yml | 264 +++++ .github/integration/sda-s3-integration.yml | 30 + .../integration/sda/cmd-download-config.yaml | 86 ++ .github/integration/tests/download/go.mod | 14 + .github/integration/tests/download/go.sum | 12 + .../integration/tests/download/main_test.go | 1048 +++++++++++++++++ .github/integration/tests/sda/11_api_test.sh | 6 +- .../tests/sda/50_reencrypt_test.sh | 6 + .../integration/tests/sda/55_download_test.sh | 162 +++ .../workflows/pr_build_and_test_images.yml | 10 + 13 files changed, 1915 insertions(+), 12 deletions(-) create mode 100644 .github/integration/scripts/make_download_credentials.sh create mode 100644 .github/integration/scripts/mockoidc.py create mode 100644 .github/integration/sda-cmd-download-integration.yml create mode 100644 .github/integration/sda/cmd-download-config.yaml create mode 100644 .github/integration/tests/download/go.mod create mode 100644 .github/integration/tests/download/go.sum create mode 100644 .github/integration/tests/download/main_test.go create mode 100755 .github/integration/tests/sda/55_download_test.sh diff --git a/.github/integration/scripts/make_download_credentials.sh b/.github/integration/scripts/make_download_credentials.sh new file mode 100644 index 0000000000..48dac59ae9 --- /dev/null +++ b/.github/integration/scripts/make_download_credentials.sh @@ -0,0 +1,89 @@ +#!/bin/sh +# Simplified credentials script for download service integration tests +# This version doesn't require RabbitMQ +set -e + +apt-get -o DPkg::Lock::Timeout=60 update > /dev/null +apt-get -o DPkg::Lock::Timeout=60 install -y curl jq openssh-client openssl postgresql-client xxd >/dev/null + +pip install --upgrade pip > /dev/null +pip install aiohttp Authlib joserfc requests > /dev/null + +# Create database credentials for download service +for n in api download; do + echo "creating credentials for: $n" + psql -U postgres -h postgres -d sda -c "ALTER ROLE $n LOGIN PASSWORD '$n';" || true + psql -U postgres -h postgres -d sda -c "GRANT base TO $n;" || true +done + +# create RSA key for signing the JWT tokens +mkdir -p /shared/keys/pub +if [ ! -f "/shared/keys/jwt.key" ]; then + echo "creating jwt key" + cat << 'EOF' > /shared/keys/jwt.key +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDhuZjxPmOGUIW1 +LhxzKfxkN+1aTbvI5w+AptqT33X+bWuzfjvhEodiNz0bBfQgJJpQ3TZ8J1IZpM2F +Tnzox+FGxKPe5T9Mgngzd4N6eByWVPXoNMk7IdmBXMdPZBFSyjMW4ba1MELCpiKV +05de4J5opRDwmHmyMqYJxBk78e3iiYYixVk+j1Ku+yFl4d2R29y2+O9PlZegJloe +8FGnKIGZApS/8t9iyCkXg8WbjSPzgYCTQKxn/E4lcGdTrAt/McKrWmAuppcr+rpP ++BInm3l5Zu/QiRSZcMb5O460ojP9eKnaUlDpGZv9CY5j4x4lq8vjU2kK77YXBO8I +2oxse5a5AgMBAAECggEABbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKP +RushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF+eqipSQXKghdKZCV +AsY8io0MmpXB1omDSygp7h3j52yHdayE2muav+VTAPOYn5QwG0/gGgVqYrR9x7CM +iTuyOIuGNO4Wlly4/5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI +3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq/77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsn +mccF9plVTzem7N/vo+T+hFRPUO9TZUao91mMV8iV5QKBgQD1nZbQW3NHdol0fXA8 +nw5JRkTLZx1zcZ5l36WVPkwCjJOyXQ2vWHm4lz7F81Rr8dQnMKLWMDKjrBT9Dbfs +xYK2bYxENS1W/n+0jOIaX/792DY9tfX7vvHU9yGSdoJE5os6DGCHYInOD0xnRmnl +3vS7gKv8miDwDzFsbjtDg6WfSwKBgQDrRLkmmfZCMcmLA02YSrErAlUseuyad7lY +HEJApXKfn262iHELlQa2zOBZpJGXIcHsNf1XGpMeU5pH+ILKE4Y5qbclq+AzFCcZ +nBFUfDeawmWdV5FJqNDd1L8Mb8aE+6q0Y5rNb3RL7A2ypH2ZeYKSGpHz3C7Rn5KW +voWAXRWriwKBgQCH4bxK3x0ivxiCgtcyIojDzwVGRnDLqmMIVzeDHqjsjBs2BTcJ +9/e3QK1w1BKzeWF2oPilaJrLY+tkqE9FxWtwQ6DjJ0xDIZ9DIuH/13X5t8EiWOWS +devSdzpyje+58JW78pcArk7u2hXZ2OHDU5qvlRsRL6/jP3SHWWCeFFnviwKBgGov +M02r0YygwfEfBYeFtp7Nx7lypZU2Eg4levWIdsp6f9KclEEA+u3IXD25XAiVMNw2 +pegJU3stioWPMSCZXUxrQAEdqOwE3XzehqfWBJaxxIEWQ7m2Gsb0PWIUlMnyeGJA +Tl8IPboCiVAmk5WQVREyMsuYhf0Qg23MAZ8k5CHvAoGBAJm55NQZVKAEDGd4a21q +TDcRddtPwwL2oP3qa0gbGk4YFRUCrX99hIejOTvQW1xf6vGxTd7E1QizvFse4yRz +ZRKyXIc7DCcdzOnpMrSd1+aXwZtRHLSw0EDS6PWeJZdjJYHxl2YpAmMdURdcGTrH +b6b/6vhU90+xL14CX7Awofp/ +-----END PRIVATE KEY----- +EOF + cat << 'EOF' > /shared/keys/pub/jwt.pub +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4bmY8T5jhlCFtS4ccyn8 +ZDftWk27yOcPgKbak991/m1rs3474RKHYjc9GwX0ICSaUN02fCdSGaTNhU586Mfh +RsSj3uU/TIJ4M3eDengcllT16DTJOyHZgVzHT2QRUsozFuG2tTBCwqYildOXXuCe +aKUQ8Jh5sjKmCcQZO/Ht4omGIsVZPo9SrvshZeHdkdvctvjvT5WXoCZaHvBRpyiB +mQKUv/LfYsgpF4PFm40j84GAk0CsZ/xOJXBnU6wLfzHCq1pgLqaXK/q6T/gSJ5t5 +eWbv0IkUmXDG+TuOtKIz/Xip2lJQ6Rmb/QmOY+MeJavL41NpCu+2FwTvCNqMbHuW +uQIDAQAB +-----END PUBLIC KEY----- +EOF + chmod 644 /shared/keys/pub/jwt.pub /shared/keys/jwt.key +fi + +echo "creating token" +python /scripts/sign_jwt.py integration_test@example.org > "/shared/token" + +# Create trusted issuers file for visa validation +# The issuer and JKU must match what mockoidc.py uses when signing visas +cat > "/shared/trusted-issuers.json" <<'EOF' +[ + { + "iss": "https://demo.example", + "jku": "http://mockauth:8000/jwks" + } +] +EOF + +## create crypt4gh key +if [ ! -f "/shared/c4gh.pub.pem" ]; then + echo "downloading crypt4gh" + latest_c4gh=$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name') + curl --retry 100 -sL "https://github.com/neicnordic/crypt4gh/releases/download/${latest_c4gh}/crypt4gh_linux_x86_64.tar.gz" | tar -xz -C /shared/ + /shared/crypt4gh generate -n /shared/c4gh -p c4gh +fi + +echo "download credentials setup complete" diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 397afb3634..3f1e295444 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -135,16 +135,17 @@ if [ ! -f "/shared/rotatekey.sec.pem" ]; then /shared/crypt4gh generate -n /shared/rotatekey -p rotatekeyPass fi -# register the rotation key in the db -resp=$(psql -U postgres -h postgres -d sda -At -c "SELECT description FROM sda.encryption_keys;") -if ! echo "$resp" | grep -q 'this is the new key to rotate to'; then - rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256) - resp=$(psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.encryption_keys(key_hash, description) VALUES('$rotateKeyHash', 'this is the new key to rotate to');") - if [ "$(echo "$resp" | tr -d '\n')" != "INSERT 0 1" ]; then - echo "insert keyhash failed" +# register the rotation key in the db (idempotent) +rotateKeyHash=$(cat /shared/rotatekey.pub.pem | awk 'NR==2' | base64 -d | xxd -p -c256) +resp=$(psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.encryption_keys(key_hash, description) VALUES('$rotateKeyHash', 'this is the new key to rotate to') ON CONFLICT (key_hash) DO UPDATE SET description = EXCLUDED.description;") +# psql prints e.g. "INSERT 0 1" or "INSERT 0 0" depending on conflict; accept both. +case "$(echo "$resp" | tr -d '\n')" in + "INSERT 0 1"|"INSERT 0 0") : ;; + *) + echo "insert/upsert keyhash failed" exit 1 - fi -fi + ;; +esac if [ ! -f "/shared/keys/ssh" ]; then ssh-keygen -o -a 256 -t ed25519 -f /shared/keys/ssh -N "" diff --git a/.github/integration/scripts/mockoidc.py b/.github/integration/scripts/mockoidc.py new file mode 100644 index 0000000000..97a3a92c8e --- /dev/null +++ b/.github/integration/scripts/mockoidc.py @@ -0,0 +1,181 @@ +"""Mock OIDC server for benchmark testing. + +This mock uses the same RSA key as make_sda_credentials.sh so both the +OLD and NEW download services can use the same token. +""" + +from aiohttp import web +from authlib.jose import jwt, RSAKey +import json +import base64 + +# RSA private key - same as in make_sda_credentials.sh +PRIVATE_KEY_PEM = """-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDhuZjxPmOGUIW1 +LhxzKfxkN+1aTbvI5w+AptqT33X+bWuzfjvhEodiNz0bBfQgJJpQ3TZ8J1IZpM2F +Tnzox+FGxKPe5T9Mgngzd4N6eByWVPXoNMk7IdmBXMdPZBFSyjMW4ba1MELCpiKV +05de4J5opRDwmHmyMqYJxBk78e3iiYYixVk+j1Ku+yFl4d2R29y2+O9PlZegJloe +8FGnKIGZApS/8t9iyCkXg8WbjSPzgYCTQKxn/E4lcGdTrAt/McKrWmAuppcr+rpP ++BInm3l5Zu/QiRSZcMb5O460ojP9eKnaUlDpGZv9CY5j4x4lq8vjU2kK77YXBO8I +2oxse5a5AgMBAAECggEABbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKP +RushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF+eqipSQXKghdKZCV +AsY8io0MmpXB1omDSygp7h3j52yHdayE2muav+VTAPOYn5QwG0/gGgVqYrR9x7CM +iTuyOIuGNO4Wlly4/5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI +3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq/77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsn +mccF9plVTzem7N/vo+T+hFRPUO9TZUao91mMV8iV5QKBgQD1nZbQW3NHdol0fXA8 +nw5JRkTLZx1zcZ5l36WVPkwCjJOyXQ2vWHm4lz7F81Rr8dQnMKLWMDKjrBT9Dbfs +xYK2bYxENS1W/n+0jOIaX/792DY9tfX7vvHU9yGSdoJE5os6DGCHYInOD0xnRmnl +3vS7gKv8miDwDzFsbjtDg6WfSwKBgQDrRLkmmfZCMcmLA02YSrErAlUseuyad7lY +HEJApXKfn262iHELlQa2zOBZpJGXIcHsNf1XGpMeU5pH+ILKE4Y5qbclq+AzFCcZ +nBFUfDeawmWdV5FJqNDd1L8Mb8aE+6q0Y5rNb3RL7A2ypH2ZeYKSGpHz3C7Rn5KW +voWAXRWriwKBgQCH4bxK3x0ivxiCgtcyIojDzwVGRnDLqmMIVzeDHqjsjBs2BTcJ +9/e3QK1w1BKzeWF2oPilaJrLY+tkqE9FxWtwQ6DjJ0xDIZ9DIuH/13X5t8EiWOWS +devSdzpyje+58JW78pcArk7u2hXZ2OHDU5qvlRsRL6/jP3SHWWCeFFnviwKBgGov +M02r0YygwfEfBYeFtp7Nx7lypZU2Eg4levWIdsp6f9KclEEA+u3IXD25XAiVMNw2 +pegJU3stioWPMSCZXUxrQAEdqOwE3XzehqfWBJaxxIEWQ7m2Gsb0PWIUlMnyeGJA +Tl8IPboCiVAmk5WQVREyMsuYhf0Qg23MAZ8k5CHvAoGBAJm55NQZVKAEDGd4a21q +TDcRddtPwwL2oP3qa0gbGk4YFRUCrX99hIejOTvQW1xf6vGxTd7E1QizvFse4yRz +ZRKyXIc7DCcdzOnpMrSd1+aXwZtRHLSw0EDS6PWeJZdjJYHxl2YpAmMdURdcGTrH +b6b/6vhU90+xL14CX7Awofp/ +-----END PRIVATE KEY-----""" + +# Load RSA key +KEY = RSAKey.import_key(PRIVATE_KEY_PEM) +PUBLIC_JWK = KEY.as_dict(is_private=False) +PUBLIC_JWK["kid"] = "rsa1" +PUBLIC_JWK["use"] = "sig" +PUBLIC_JWK["alg"] = "RS256" +PRIVATE_JWK = dict(KEY) + +# Generate visa JWTs for test datasets +def generate_visa(dataset: str, sub: str) -> str: + """Generate a signed visa JWT for a dataset.""" + header = { + "alg": "RS256", + "typ": "JWT", + "kid": "rsa1", + "jku": "http://mockauth:8000/jwks", + } + payload = { + "iss": "https://demo.example", + "sub": sub, + "ga4gh_visa_v1": { + "type": "ControlledAccessGrants", + "value": dataset, + "source": "https://doi.example/no_org", + "by": "self", + "asserted": 1568699331, + }, + "iat": 1571144438, + "exp": 9999999999, + "jti": f"visa-{dataset}", + } + return jwt.encode(header, payload, PRIVATE_JWK).decode("utf-8") + + +# Pre-generate visas for test datasets +VISAS = {} + + +def get_visas_for_user(sub: str) -> list: + """Get visas for a user, generating them if needed.""" + if sub not in VISAS: + VISAS[sub] = [ + generate_visa("SYNC-001-12345", sub), + generate_visa("EGAD74900000101", sub), + generate_visa("BENCHMARK-DATASET-001", sub), + generate_visa("https://doi.example/ty009.sfrrss/600.45asasga", sub), + ] + return VISAS[sub] + + +async def well_known(request: web.Request) -> web.Response: + """Return OIDC discovery document.""" + config = { + "issuer": "http://mockauth:8000", + "authorization_endpoint": "http://mockauth:8000/authorize", + "token_endpoint": "http://mockauth:8000/token", + "userinfo_endpoint": "http://mockauth:8000/userinfo", + "jwks_uri": "http://mockauth:8000/jwks", + "response_types_supported": ["code", "token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + "scopes_supported": ["openid", "ga4gh_passport_v1"], + "claims_supported": ["sub", "iss", "aud", "exp", "iat", "ga4gh_passport_v1"], + } + return web.json_response(config) + + +async def jwks(request: web.Request) -> web.Response: + """Return JSON Web Key Set.""" + return web.json_response({"keys": [PUBLIC_JWK]}) + + +async def userinfo(request: web.Request) -> web.Response: + """Return user info with GA4GH visas.""" + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + return web.json_response( + {"error": "invalid_token", "error_description": "Missing bearer token"}, + status=401 + ) + + # Extract token and decode payload to get subject + token = auth_header.split(" ")[1] + try: + # Decode JWT payload (base64url encoded, middle part) + parts = token.split(".") + if len(parts) != 3: + raise ValueError("Not a JWT") + + # Add padding if needed + payload_b64 = parts[1].replace("-", "+").replace("_", "/") + padding = 4 - len(payload_b64) % 4 + if padding != 4: + payload_b64 += "=" * padding + + payload = json.loads(base64.b64decode(payload_b64)) + sub = payload.get("sub", "unknown@example.org") + print(f"[mockoidc] userinfo request for JWT sub={sub}") + except Exception as e: + # Opaque token: use the raw token value as the subject. + # This allows integration tests to pass the desired subject + # directly as an opaque token (e.g., "Bearer user@example.org"). + sub = token + print(f"[mockoidc] opaque token, using token value as sub={sub}") + + # Special token value to simulate userinfo failure (for negative tests) + if token == "__fail__": + print("[mockoidc] __fail__ token, returning 401") + return web.json_response( + {"error": "invalid_token", "error_description": "Token rejected"}, + status=401 + ) + + # Return visas for the user + visas = get_visas_for_user(sub) + response = { + "sub": sub, + "ga4gh_passport_v1": visas, + } + + print(f"[mockoidc] returning {len(visas)} visas for {sub}") + return web.json_response(response) + + +def init() -> web.Application: + """Initialize the web application.""" + app = web.Application() + app.router.add_get("/.well-known/openid-configuration", well_known) + app.router.add_get("/jwks", jwks) + app.router.add_get("/userinfo", userinfo) + # Also support alternate paths + app.router.add_get("/idp/profile/oidc/keyset", jwks) + app.router.add_get("/idp/profile/oidc/userinfo", userinfo) + return app + + +if __name__ == "__main__": + print("[mockoidc] Starting mock OIDC server on port 8000") + print(f"[mockoidc] Using RSA key with kid=rsa1") + web.run_app(init(), port=8000) diff --git a/.github/integration/sda-cmd-download-integration.yml b/.github/integration/sda-cmd-download-integration.yml new file mode 100644 index 0000000000..7089977b0a --- /dev/null +++ b/.github/integration/sda-cmd-download-integration.yml @@ -0,0 +1,264 @@ +# Docker Compose for download service integration tests +# Based on validator-orchestrator pattern - Go-based tests running in container + +services: + # PostgreSQL with SDA schema + postgres: + build: + context: ../../postgresql + container_name: postgres + environment: + - POSTGRES_PASSWORD=rootpasswd + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 2s + retries: 6 + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + extra_hosts: + - "localhost:host-gateway" + ports: + - "15432:5432" + restart: always + volumes: + - postgres_data:/var/lib/postgresql/data + + # MinIO for archive storage + minio: + image: minio/minio + command: server /data --console-address ":9001" + container_name: s3 + environment: + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretKey + - MINIO_SERVER_URL=http://127.0.0.1:9000 + healthcheck: + test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 2s + retries: 6 + ports: + - "19000:9000" + - "19001:9001" + volumes: + - minio_data:/data + + # Mock AAI for JWT validation + mock-aai: + container_name: ls-aai-mock + depends_on: + aai-db: + condition: service_healthy + environment: + - DOCKERHOST=localhost + healthcheck: + test: ["CMD", "/bin/true"] + interval: 10s + timeout: 2s + retries: 6 + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.3-broker2.1.13-tomcat9.0-jdk21 + ports: + - "8800:8080" + volumes: + - "./sda/aai-mock:/etc/lsaai-mock" + + aai-db: + container_name: ls-aai-db + environment: + MYSQL_ROOT_PASSWORD: "aaiPass" + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: "aai" + MYSQL_USER: "aai" + MYSQL_PASSWORD: "aaiPass" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 2s + retries: 6 + image: mysql/mysql-server:latest + volumes: + - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql + + # Credentials generator - creates JWT keys and test tokens + credentials: + container_name: credentials + command: + - "/bin/sh" + - "-c" + - | + sh /scripts/make_download_credentials.sh + depends_on: + postgres: + condition: service_healthy + environment: + - PGPASSWORD=rootpasswd + image: python:3.11-slim-bookworm + volumes: + - ./scripts:/scripts + - shared:/shared + + # Initialize MinIO with archive bucket and test data + minio_init: + container_name: minio_init + image: minio/mc + depends_on: + minio: + condition: service_healthy + credentials: + condition: service_completed_successfully + entrypoint: > + /bin/sh -c " + mc alias set myminio http://s3:9000 access secretKey; + mc mb myminio/archive || true; + dd if=/dev/urandom bs=1000 count=1 2>/dev/null | mc pipe myminio/archive/test-file.c4gh; + echo 'MinIO initialized with archive bucket and test file'; + " + + # Seed database with test data (files and datasets) + database_seed: + container_name: database_seed + image: postgres:15.4-alpine + depends_on: + credentials: + condition: service_completed_successfully + minio_init: + condition: service_completed_successfully + environment: + - PGHOST=postgres + - PGPORT=5432 + - PGUSER=postgres + - PGPASSWORD=rootpasswd + - PGDATABASE=sda + command: + - "/bin/sh" + - "-c" + - | + # Wait for postgres to be ready + pg_isready -h postgres -p 5432 -U postgres + + # Insert test file + psql -c "INSERT INTO sda.files ( + id, stable_id, submission_user, submission_file_path, + archive_file_path, archive_location, archive_file_size, decrypted_file_size, + header, encryption_method + ) VALUES ( + 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + 'EGAF00000000001', + 'integration_test@example.org', + 'test-file.c4gh', + 'test-file.c4gh', + 's3:9000/archive', + 1000, + 500, + '63727970743467680100000000000000', + 'CRYPT4GH' + ) ON CONFLICT DO NOTHING;" + + # Insert test dataset (id is SERIAL, so let postgres generate it) + psql -c "INSERT INTO sda.datasets (stable_id, title) + VALUES ('EGAD00000000001', 'Test Dataset') + ON CONFLICT DO NOTHING;" + + # Get the dataset id and link file to dataset + psql -c "INSERT INTO sda.file_dataset (file_id, dataset_id) + SELECT 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'::uuid, d.id + FROM sda.datasets d + WHERE d.stable_id = 'EGAD00000000001' + ON CONFLICT DO NOTHING;" + + echo 'Database seeded with test data'; + + # Mock OIDC server for opaque token auth and visa validation + mockauth: + command: + - /bin/sh + - -c + - | + pip install --upgrade pip > /dev/null + pip install aiohttp Authlib joserfc > /dev/null + python -u /mockoidc.py + container_name: mockauth + image: python:3.10-slim + volumes: + - ./scripts/mockoidc.py:/mockoidc.py + ports: + - "8000:8000" + restart: always + + # Re-encrypt gRPC service + reencrypt: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [sda-reencrypt] + container_name: reencrypt + depends_on: + credentials: + condition: service_completed_successfully + ports: + - "50051:50051" + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + # Download service under test + download: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: + - sda-download + - "--config-file=/config.yaml" + container_name: download + depends_on: + credentials: + condition: service_completed_successfully + database_seed: + condition: service_completed_successfully + postgres: + condition: service_healthy + reencrypt: + condition: service_started + mock-aai: + condition: service_healthy + mockauth: + condition: service_started + environment: + - CONFIG_FILE=/config.yaml + extra_hosts: + - "localhost:host-gateway" + ports: + - "8085:8080" + - "8086:8081" + restart: always + volumes: + - ./sda/cmd-download-config.yaml:/config.yaml + - shared:/shared + + # Go-based integration test runner + integration_test: + container_name: download_tester + command: + - "/bin/sh" + - "-c" + - | + cd /tests + go mod download + go test -v ./... + depends_on: + download: + condition: service_started + environment: + - DOWNLOAD_URL=http://download:8080 + - JWT_KEY_FILE=/shared/keys/jwt.key + - C4GH_PUBKEY_FILE=/shared/c4gh.pub.pem + extra_hosts: + - "localhost:host-gateway" + image: golang:1.25-alpine + profiles: + - tests + volumes: + - shared:/shared + - ./tests/download:/tests + +volumes: + minio_data: + postgres_data: + shared: diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 0ac7df74bd..7cfc8767a1 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -253,6 +253,36 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared + # New download service (sda/cmd/download) - feature of this branch + download: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: + - sda-download + - "--config-file=/config.yaml" + container_name: download + depends_on: + credentials: + condition: service_completed_successfully + postgres: + condition: service_healthy + reencrypt: + condition: service_started + mock-aai: + condition: service_healthy + minio: + condition: service_healthy + environment: + - CONFIG_FILE=/config.yaml + extra_hosts: + - "localhost:host-gateway" + ports: + - "8085:8080" + - "8086:8081" + restart: always + volumes: + - ./sda/cmd-download-config.yaml:/config.yaml + - shared:/shared + rotatekey: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} command: [sda-rotatekey] diff --git a/.github/integration/sda/cmd-download-config.yaml b/.github/integration/sda/cmd-download-config.yaml new file mode 100644 index 0000000000..0ca6bd1d3d --- /dev/null +++ b/.github/integration/sda/cmd-download-config.yaml @@ -0,0 +1,86 @@ +# Configuration for sda/cmd/download service in integration tests + +# API Server +api: + host: "0.0.0.0" + port: 8080 + +# gRPC Health check +health: + port: 8081 + +# Database +db: + host: "postgres" + port: 5432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +# gRPC reencrypt service +grpc: + host: "reencrypt" + port: 50051 + timeout: 30 + +# JWT configuration - uses keys created by credentials script +jwt: + pubkey-path: "/shared/keys/pub/" + allow-all-data: true # For testing: allow all authenticated users access to all datasets + +# Permission model: combined (ownership + visa) +permission: + model: "combined" + +# Auth: allow opaque tokens (userinfo-based authentication) +auth: + allow-opaque: true + +# GA4GH visa configuration +visa: + enabled: true + source: "userinfo" + userinfo-url: "http://mockauth:8000/userinfo" + trusted-issuers-path: "/shared/trusted-issuers.json" + dataset-id-mode: "raw" + identity: + mode: "broker-bound" + validate-asserted: true + allow-insecure-jku: true # mockauth uses HTTP (testing only) + limits: + max-visas: 200 + max-jwks-per-request: 10 + max-visa-size: 16384 + cache: + token-ttl: 3600 + jwk-ttl: 300 + validation-ttl: 120 + userinfo-ttl: 60 + +# Session +session: + expiration: 3600 + secure: false + http-only: true + name: "sda_session" + +# Database query cache +cache: + enabled: true + file-ttl: 300 # 5 minutes + permission-ttl: 120 # 2 minutes + dataset-ttl: 300 # 5 minutes + +# Storage configuration (S3 backend) +storage: + backend: "archive" + archive: + s3: + # NOTE: include scheme to keep storage/v2 location parsing consistent (archive_location values are URL-parsed). + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true diff --git a/.github/integration/tests/download/go.mod b/.github/integration/tests/download/go.mod new file mode 100644 index 0000000000..4669d5b214 --- /dev/null +++ b/.github/integration/tests/download/go.mod @@ -0,0 +1,14 @@ +module download-tester + +go 1.25.0 + +require ( + github.com/golang-jwt/jwt/v5 v5.3.0 + github.com/stretchr/testify v1.11.1 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/.github/integration/tests/download/go.sum b/.github/integration/tests/download/go.sum new file mode 100644 index 0000000000..5154654a88 --- /dev/null +++ b/.github/integration/tests/download/go.sum @@ -0,0 +1,12 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/.github/integration/tests/download/main_test.go b/.github/integration/tests/download/main_test.go new file mode 100644 index 0000000000..03b805a0ba --- /dev/null +++ b/.github/integration/tests/download/main_test.go @@ -0,0 +1,1048 @@ +package main + +import ( + "context" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "fmt" + "io" + "net/http" + "os" + "testing" + "time" + + "github.com/golang-jwt/jwt/v5" + "github.com/stretchr/testify/suite" +) + +// TestSuite defines the download service integration test suite +type TestSuite struct { + suite.Suite + + // Configuration + downloadURL string + jwtKeyFilePath string + + // Generated tokens + token string + + // Environment capabilities (probed once in SetupSuite) + hasReencrypt bool // true if re-encryption works with seed data + hasStorageFile bool // true if the test file is accessible in storage + hasSessionCache bool // true if session cookies are being set +} + +func TestDownloadTestSuite(t *testing.T) { + suite.Run(t, new(TestSuite)) +} + +func (ts *TestSuite) SetupSuite() { + // Configuration - matches docker compose service names + ts.downloadURL = getEnv("DOWNLOAD_URL", "http://download:8080") + ts.jwtKeyFilePath = getEnv("JWT_KEY_FILE", "/shared/keys/jwt.key") + + // Generate JWT token for authenticated requests + var err error + ts.token, err = ts.generateToken("integration_test@example.org") + if err != nil { + ts.FailNow("failed to generate token", err.Error()) + } + + // Wait for download service to be ready + ts.waitForService() + + // Probe environment capabilities + ts.probeCapabilities() +} + +// probeCapabilities tests environment-specific features once and caches results. +// This avoids skipping on generic 500 — tests skip on known missing prerequisites. +func (ts *TestSuite) probeCapabilities() { + fileID, err := ts.getFirstFileID() + if err != nil { + return + } + + // Probe re-encryption: HEAD /files/:fileId with public key + pubkey := ts.readPublicKeyBase64() + if pubkey != "" { + headers := ts.authHeaders() + headers["X-C4GH-Public-Key"] = pubkey + resp, _, reqErr := ts.doRequest("HEAD", "/files/"+fileID, nil, headers) + ts.hasReencrypt = reqErr == nil && resp.StatusCode == http.StatusOK + } + + // Probe storage access: GET /files/:fileId/content + resp, _, reqErr := ts.doRequest("GET", "/files/"+fileID+"/content", nil, ts.authHeaders()) + ts.hasStorageFile = reqErr == nil && resp.StatusCode == http.StatusOK + + // Probe session cookie: check if first auth request sets sda_session + req, _ := http.NewRequestWithContext(context.Background(), "GET", ts.downloadURL+"/datasets", nil) + req.Header.Set("Authorization", "Bearer "+ts.token) + cookieResp, cookieErr := http.DefaultClient.Do(req) + if cookieErr == nil { + for _, c := range cookieResp.Cookies() { + if c.Name == "sda_session" { + ts.hasSessionCache = true + + break + } + } + cookieResp.Body.Close() + } + + ts.T().Logf("Environment capabilities: reencrypt=%v, storageFile=%v, sessionCache=%v", + ts.hasReencrypt, ts.hasStorageFile, ts.hasSessionCache) +} + +func (ts *TestSuite) waitForService() { + client := &http.Client{Timeout: 2 * time.Second} + maxAttempts := 30 + + for range maxAttempts { + resp, err := client.Get(ts.downloadURL + "/health/live") + if err == nil && resp.StatusCode == http.StatusOK { + _ = resp.Body.Close() + return + } + if resp != nil { + _ = resp.Body.Close() + } + time.Sleep(1 * time.Second) + } + ts.FailNow("download service not ready after waiting") +} + +func (ts *TestSuite) generateToken(sub string) (string, error) { + keyPem, err := os.ReadFile(ts.jwtKeyFilePath) + if err != nil { + return "", fmt.Errorf("failed to read key file from path %s: %w", ts.jwtKeyFilePath, err) + } + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, &jwt.RegisteredClaims{ + Subject: sub, + Issuer: "http://integration.test", + ExpiresAt: jwt.NewNumericDate(time.Now().Add(30 * time.Minute)), + IssuedAt: jwt.NewNumericDate(time.Now()), + }) + + token.Header["kid"] = "rsa1" + block, _ := pem.Decode(keyPem) + if block == nil { + return "", fmt.Errorf("failed to decode PEM block containing private key") + } + keyRaw, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("failed to parse private key: %w", err) + } + + return token.SignedString(keyRaw) +} + +func (ts *TestSuite) doRequest(method, path string, body io.Reader, headers map[string]string) (*http.Response, []byte, error) { + req, err := http.NewRequestWithContext(context.Background(), method, ts.downloadURL+path, body) + if err != nil { + return nil, nil, err + } + + for k, v := range headers { + req.Header.Set(k, v) + } + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return nil, nil, err + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return resp, nil, err + } + + return resp, respBody, nil +} + +func (ts *TestSuite) authHeaders() map[string]string { + return map[string]string{ + "Authorization": "Bearer " + ts.token, + } +} + +// Test01_HealthLive tests the liveness probe endpoint +func (ts *TestSuite) Test01_HealthLive() { + resp, _, err := ts.doRequest("GET", "/health/live", nil, nil) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "health/live should return 200") +} + +// Test02_HealthReady tests the readiness probe endpoint +func (ts *TestSuite) Test02_HealthReady() { + resp, _, err := ts.doRequest("GET", "/health/ready", nil, nil) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "health/ready should return 200") +} + +// Test03_UnauthenticatedRequest tests that unauthenticated requests are rejected +func (ts *TestSuite) Test03_UnauthenticatedRequest() { + resp, _, err := ts.doRequest("GET", "/datasets", nil, nil) + ts.Require().NoError(err) + ts.Equal(http.StatusUnauthorized, resp.StatusCode, "unauthenticated request should return 401") +} + +// Test04_ListDatasets tests listing available datasets +func (ts *TestSuite) Test04_ListDatasets() { + resp, body, err := ts.doRequest("GET", "/datasets", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "list datasets should return 200") + + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + err = json.Unmarshal(body, &datasetsResp) + ts.Require().NoError(err, "response should be valid JSON") + + if len(datasetsResp.Datasets) > 0 { + ts.T().Logf("Found %d dataset(s)", len(datasetsResp.Datasets)) + } else { + ts.T().Log("No datasets found (expected if pipeline hasn't run)") + } +} + +// Test05_GetDatasetInfo tests getting info for a specific dataset +func (ts *TestSuite) Test05_GetDatasetInfo() { + // First get list of datasets + resp, body, err := ts.doRequest("GET", "/datasets", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Require().Equal(http.StatusOK, resp.StatusCode) + + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + err = json.Unmarshal(body, &datasetsResp) + ts.Require().NoError(err) + + if len(datasetsResp.Datasets) == 0 { + ts.T().Skip("No datasets available - skipping dataset info test") + return + } + + datasetID := datasetsResp.Datasets[0] + + // Get dataset info + resp, body, err = ts.doRequest("GET", "/datasets/"+datasetID, nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "get dataset info should return 200") + + var datasetInfo map[string]any + err = json.Unmarshal(body, &datasetInfo) + ts.Require().NoError(err, "response should be valid JSON") + ts.Contains(datasetInfo, "datasetId", "dataset info should contain 'datasetId'") + ts.Contains(datasetInfo, "date", "dataset info should contain 'date'") + ts.Contains(datasetInfo, "files", "dataset info should contain 'files'") + ts.Contains(datasetInfo, "size", "dataset info should contain 'size'") +} + +// Test06_ListFilesInDataset tests listing files in a dataset +func (ts *TestSuite) Test06_ListFilesInDataset() { + // First get list of datasets + resp, body, err := ts.doRequest("GET", "/datasets", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Require().Equal(http.StatusOK, resp.StatusCode) + + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + err = json.Unmarshal(body, &datasetsResp) + ts.Require().NoError(err) + + if len(datasetsResp.Datasets) == 0 { + ts.T().Skip("No datasets available - skipping files list test") + return + } + + datasetID := datasetsResp.Datasets[0] + + // List files in dataset + resp, body, err = ts.doRequest("GET", "/datasets/"+datasetID+"/files", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "list files should return 200") + + var filesResp struct { + Files []map[string]any `json:"files"` + } + err = json.Unmarshal(body, &filesResp) + ts.Require().NoError(err, "response should be valid JSON") + + ts.T().Logf("Found %d file(s) in dataset %s", len(filesResp.Files), datasetID) +} + +// Test07_DownloadWithReencryption tests file download with re-encryption +func (ts *TestSuite) Test07_DownloadWithReencryption() { + if !ts.hasReencrypt { + ts.T().Skip("REQUIRES_REENCRYPT: seed header cannot be re-encrypted") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available - skipping download test: " + err.Error()) + return + } + + pubkeyBase64 := ts.readPublicKeyBase64() + headers := ts.authHeaders() + headers["X-C4GH-Public-Key"] = pubkeyBase64 + + resp, body, err := ts.doRequest("GET", "/files/"+fileID, nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "download should return 200") + + if len(body) >= 8 { + magic := string(body[:8]) + ts.Equal("crypt4gh", magic, "file should have crypt4gh magic bytes") + } +} + +// Test08_RangeRequest tests partial file download with Range header +func (ts *TestSuite) Test08_RangeRequest() { + if !ts.hasReencrypt { + ts.T().Skip("REQUIRES_REENCRYPT: seed header cannot be re-encrypted") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available - skipping range request test") + return + } + + pubkeyBase64 := ts.readPublicKeyBase64() + headers := ts.authHeaders() + headers["X-C4GH-Public-Key"] = pubkeyBase64 + headers["Range"] = "bytes=0-99" + + resp, _, err := ts.doRequest("GET", "/files/"+fileID, nil, headers) + ts.Require().NoError(err) + + // Accept either 206 (Partial Content) or 200 (full content if small file) + ts.True(resp.StatusCode == http.StatusOK || + resp.StatusCode == http.StatusPartialContent, + "range request should return 200 or 206, got %d", resp.StatusCode) +} + +// Test09_AccessControlNonExistentFile tests that non-existent files return 403 +func (ts *TestSuite) Test09_AccessControlNonExistentFile() { + headers := ts.authHeaders() + headers["X-C4GH-Public-Key"] = base64.StdEncoding.EncodeToString([]byte("dummy")) + + resp, _, err := ts.doRequest("GET", "/files/EGAF00000000000", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusForbidden, resp.StatusCode, + "non-existent file should return 403, got %d", resp.StatusCode) +} + +// Test10_InvalidToken tests that invalid tokens are rejected +func (ts *TestSuite) Test10_InvalidToken() { + // Use a JWT-shaped token with invalid signature (not just a random string, + // since opaque tokens now authenticate via userinfo). + // This is a valid JWT structure but with a bogus signature that won't verify. + invalidJWT := "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJpbnZhbGlkIiwiZXhwIjo5OTk5OTk5OTk5fQ.invalidsignatureinvalidsignatureinvalidsignature" + + headers := map[string]string{ + "Authorization": "Bearer " + invalidJWT, + } + + resp, _, err := ts.doRequest("GET", "/datasets", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusUnauthorized, resp.StatusCode, "invalid JWT signature should return 401") +} + +// Test12_OpaqueTokenListDatasets tests that an opaque token authenticates via userinfo +func (ts *TestSuite) Test12_OpaqueTokenListDatasets() { + // Use the submission user as the opaque token value. + // mockoidc.py returns this as the "sub" claim in the userinfo response. + opaqueToken := "integration_test@example.org" + headers := map[string]string{ + "Authorization": "Bearer " + opaqueToken, + } + + resp, body, err := ts.doRequest("GET", "/datasets", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "opaque token should authenticate via userinfo") + + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + err = json.Unmarshal(body, &datasetsResp) + ts.Require().NoError(err, "response should be valid JSON") + ts.T().Logf("Opaque token: found %d dataset(s)", len(datasetsResp.Datasets)) +} + +// Test13_OpaqueTokenDownloadFile tests file download using an opaque token +func (ts *TestSuite) Test13_OpaqueTokenDownloadFile() { + if !ts.hasReencrypt { + ts.T().Skip("REQUIRES_REENCRYPT: seed header cannot be re-encrypted") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available - skipping opaque token download test: " + err.Error()) + return + } + + pubkeyBase64 := ts.readPublicKeyBase64() + opaqueToken := "integration_test@example.org" + headers := map[string]string{ + "Authorization": "Bearer " + opaqueToken, + "X-C4GH-Public-Key": pubkeyBase64, + } + + resp, body, err := ts.doRequest("GET", "/files/"+fileID, nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "opaque token download should return 200") + + if len(body) >= 8 { + magic := string(body[:8]) + ts.Equal("crypt4gh", magic, "file should have crypt4gh magic bytes") + } +} + +// Test14_OpaqueTokenArbitrarySubject tests that any opaque token with a valid +// userinfo response authenticates successfully (userinfo returns the token +// value as the subject, and allow-all-data grants access to all datasets) +func (ts *TestSuite) Test14_OpaqueTokenArbitrarySubject() { + headers := map[string]string{ + "Authorization": "Bearer random-opaque-token-12345", + } + + resp, _, err := ts.doRequest("GET", "/datasets", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, + "opaque token with valid userinfo response should authenticate") +} + +// Test16_OpaqueTokenUserinfoFailure tests that an opaque token is rejected +// when the userinfo endpoint returns an error +func (ts *TestSuite) Test16_OpaqueTokenUserinfoFailure() { + // The special "__fail__" token causes mockoidc.py to return 401 + headers := map[string]string{ + "Authorization": "Bearer __fail__", + } + + resp, _, err := ts.doRequest("GET", "/datasets", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusUnauthorized, resp.StatusCode, + "opaque token should return 401 when userinfo fails") +} + +// Test15_OpaqueTokenNotJWTShaped tests tokens with dots that aren't JWTs +func (ts *TestSuite) Test15_OpaqueTokenNotJWTShaped() { + // Azure AD-style opaque token with dots (not a valid JWT structure) + opaqueWithDots := "at.integration_test@example.org.suffix" + headers := map[string]string{ + "Authorization": "Bearer " + opaqueWithDots, + } + + resp, _, err := ts.doRequest("GET", "/datasets", nil, headers) + ts.Require().NoError(err) + // This should be treated as opaque (dots don't make it a JWT) and succeed via userinfo + ts.Equal(http.StatusOK, resp.StatusCode, + "dotted opaque token should be treated as opaque, not JWT") +} + +// Test17_SessionCookieReuse tests that a session cookie from a previous request +// can be reused to authenticate without sending the token again +func (ts *TestSuite) Test17_SessionCookieReuse() { + // First request with JWT to get a session cookie + req, err := http.NewRequestWithContext(context.Background(), "GET", ts.downloadURL+"/datasets", nil) + ts.Require().NoError(err) + req.Header.Set("Authorization", "Bearer "+ts.token) + + // Use a jar-less client so we can inspect cookies manually + resp, err := http.DefaultClient.Do(req) + ts.Require().NoError(err) + defer resp.Body.Close() + ts.Require().Equal(http.StatusOK, resp.StatusCode) + + // Extract session cookie from response + var sessionCookie *http.Cookie + for _, c := range resp.Cookies() { + if c.Name == "sda_session" { + sessionCookie = c + break + } + } + if !ts.hasSessionCache { + ts.T().Skip("REQUIRES_SESSION_CACHE: visa processing may prevent caching in combined mode") + return + } + ts.Require().NotNil(sessionCookie, "response should set sda_session cookie") + + // Second request with only the session cookie (no Authorization header) + req2, err := http.NewRequestWithContext(context.Background(), "GET", ts.downloadURL+"/datasets", nil) + ts.Require().NoError(err) + req2.AddCookie(sessionCookie) + + resp2, err := http.DefaultClient.Do(req2) + ts.Require().NoError(err) + defer resp2.Body.Close() + ts.Equal(http.StatusOK, resp2.StatusCode, + "session cookie should authenticate without access token") +} + +// Test18_ServiceInfo tests the GA4GH service-info endpoint (no auth required) +func (ts *TestSuite) Test18_ServiceInfo() { + resp, body, err := ts.doRequest("GET", "/service-info", nil, nil) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "service-info should return 200 without auth") + + var info map[string]any + err = json.Unmarshal(body, &info) + ts.Require().NoError(err, "response should be valid JSON") + ts.Contains(info, "id", "service-info should have 'id' field") + ts.Contains(info, "type", "service-info should have 'type' field") +} + +// Test19_HeadFileEndpoint tests HEAD /files/:fileId returns metadata without body +func (ts *TestSuite) Test19_HeadFileEndpoint() { + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available - skipping HEAD file test: " + err.Error()) + return + } + + pubkeyBase64 := ts.readPublicKeyBase64() + if pubkeyBase64 == "" { + ts.T().Skip("No public key available - skipping HEAD file test") + return + } + + headers := ts.authHeaders() + headers["X-C4GH-Public-Key"] = pubkeyBase64 + + if !ts.hasReencrypt { + ts.T().Skip("REQUIRES_REENCRYPT: seed header cannot be re-encrypted") + return + } + + resp, body, err := ts.doRequest("HEAD", "/files/"+fileID, nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "HEAD /files/:fileId should return 200") + ts.Empty(body, "HEAD response should have no body") + ts.NotEmpty(resp.Header.Get("Content-Length"), "HEAD response should have Content-Length header") + ts.NotEmpty(resp.Header.Get("Content-Type"), "HEAD response should have Content-Type header") +} + +// Test20_SplitContentEndpoint tests GET /files/:fileId/content returns raw archive bytes +func (ts *TestSuite) Test20_SplitContentEndpoint() { + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available - skipping content endpoint test: " + err.Error()) + return + } + + if !ts.hasStorageFile { + ts.T().Skip("REQUIRES_STORAGE_FILE: test file not accessible in storage") + return + } + + resp, body, err := ts.doRequest("GET", "/files/"+fileID+"/content", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "GET /files/:fileId/content should return 200") + ts.NotEmpty(body, "content response should have a body") + ts.NotEmpty(resp.Header.Get("ETag"), "content response should have ETag header") + ts.Equal("bytes", resp.Header.Get("Accept-Ranges"), "content response should have Accept-Ranges: bytes") +} + +// Test21_SplitHeaderEndpoint tests GET /files/:fileId/header returns re-encrypted header +func (ts *TestSuite) Test21_SplitHeaderEndpoint() { + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available - skipping header endpoint test: " + err.Error()) + return + } + + pubkeyPath := getEnv("C4GH_PUBKEY_FILE", "/shared/c4gh.pub.pem") + pubkeyBytes, err := os.ReadFile(pubkeyPath) + if err != nil { + ts.T().Skipf("No public key available at %s - skipping header endpoint test", pubkeyPath) + return + } + pubkeyBase64 := base64.StdEncoding.EncodeToString(pubkeyBytes) + + headers := ts.authHeaders() + headers["X-C4GH-Public-Key"] = pubkeyBase64 + + if !ts.hasReencrypt { + ts.T().Skip("REQUIRES_REENCRYPT: seed header cannot be re-encrypted") + return + } + + resp, body, err := ts.doRequest("GET", "/files/"+fileID+"/header", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "GET /files/:fileId/header should return 200") + ts.NotEmpty(body, "header response should have a body") + ts.Equal("application/octet-stream", resp.Header.Get("Content-Type"), + "header response should have Content-Type: application/octet-stream") +} + +// Test22_ProblemDetailsFormat tests that error responses use application/problem+json +func (ts *TestSuite) Test22_ProblemDetailsFormat() { + resp, body, err := ts.doRequest("GET", "/datasets", nil, nil) + ts.Require().NoError(err) + ts.Equal(http.StatusUnauthorized, resp.StatusCode, "unauthenticated request should return 401") + ts.Equal("application/problem+json", resp.Header.Get("Content-Type"), + "error response should have Content-Type: application/problem+json") + + var problem map[string]any + err = json.Unmarshal(body, &problem) + ts.Require().NoError(err, "response should be valid JSON") + ts.Contains(problem, "title", "problem response should have 'title' field") + ts.Contains(problem, "status", "problem response should have 'status' field") + ts.Contains(problem, "detail", "problem response should have 'detail' field") +} + +// Test23_PaginationPageSize tests the pageSize query parameter on datasets and files +func (ts *TestSuite) Test23_PaginationPageSize() { + resp, body, err := ts.doRequest("GET", "/datasets?pageSize=1", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "pageSize=1 should return 200") + + var result struct { + Datasets []string `json:"datasets"` + NextPageToken *string `json:"nextPageToken"` + } + err = json.Unmarshal(body, &result) + ts.Require().NoError(err, "response should be valid JSON with datasets and nextPageToken fields") + ts.NotNil(result.Datasets, "response should have 'datasets' field") + + // Also test pageSize on file listing + if len(result.Datasets) > 0 { + datasetID := result.Datasets[0] + resp, body, err = ts.doRequest("GET", + "/datasets/"+datasetID+"/files?pageSize=1", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "file list with pageSize=1 should return 200") + + var filesResult struct { + Files []map[string]any `json:"files"` + NextPageToken *string `json:"nextPageToken"` + } + err = json.Unmarshal(body, &filesResult) + ts.Require().NoError(err, "file list response should have files and nextPageToken fields") + } +} + +// Test24_PaginationInvalidPageSize tests that invalid pageSize values return 400 +func (ts *TestSuite) Test24_PaginationInvalidPageSize() { + invalidSizes := []string{"0", "-1", "1001", "abc"} + for _, size := range invalidSizes { + resp, _, err := ts.doRequest("GET", "/datasets?pageSize="+size, nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusBadRequest, resp.StatusCode, + "pageSize=%s should return 400", size) + } +} + +// Test25_EncodedSlashDatasetID tests that URL-encoded slashes in dataset IDs route correctly +func (ts *TestSuite) Test25_EncodedSlashDatasetID() { + // URL-encode a dataset ID containing slashes. + // With UseRawPath=true, %2F should not be treated as path separators. + encodedID := "https%3A%2F%2Fdoi.example%2Fty009.sfrrss%2F600.45asasga" + + // GetDataset: route matches, auth passes, but dataset not in DB → 403 (no existence leakage) + // A 404 would indicate routing failure (slashes interpreted as path separators) + resp, _, err := ts.doRequest("GET", "/datasets/"+encodedID, nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusForbidden, resp.StatusCode, + "encoded-slash dataset ID should route correctly (expect 403, not 404)") + + // ListDatasetFiles: route matches, auth passes, but dataset not in DB → 403 + resp, _, err = ts.doRequest("GET", "/datasets/"+encodedID+"/files", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusForbidden, resp.StatusCode, + "encoded-slash dataset files should route correctly (expect 403, not 404)") +} + +// Test26_InvalidRangeHeader tests that malformed Range headers return 400 +func (ts *TestSuite) Test26_InvalidRangeHeader() { + if !ts.hasStorageFile { + ts.T().Skip("REQUIRES_STORAGE_FILE: test file not accessible in storage") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available: " + err.Error()) + return + } + + headers := ts.authHeaders() + headers["Range"] = "invalid" + + resp, _, err := ts.doRequest("GET", "/files/"+fileID+"/content", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusBadRequest, resp.StatusCode, "malformed Range header should return 400") +} + +// Test27_MultiRangeRejected tests that multi-range requests return 400 +func (ts *TestSuite) Test27_MultiRangeRejected() { + if !ts.hasStorageFile { + ts.T().Skip("REQUIRES_STORAGE_FILE: test file not accessible in storage") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available: " + err.Error()) + return + } + + headers := ts.authHeaders() + headers["Range"] = "bytes=0-10,20-30" + + resp, _, err := ts.doRequest("GET", "/files/"+fileID+"/content", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusBadRequest, resp.StatusCode, "multi-range request should return 400") +} + +// Test28_IfRangeETagContract tests the If-Range header with valid and stale ETags +func (ts *TestSuite) Test28_IfRangeETagContract() { + if !ts.hasStorageFile { + ts.T().Skip("REQUIRES_STORAGE_FILE: test file not accessible in storage") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available: " + err.Error()) + return + } + + etag := ts.getContentETag(fileID) + ts.Require().NotEmpty(etag, "HEAD /files/:fileId/content should return ETag") + + // Valid ETag in If-Range + Range → 206 Partial Content + headers := ts.authHeaders() + headers["If-Range"] = etag + headers["Range"] = "bytes=0-99" + + resp, _, err := ts.doRequest("GET", "/files/"+fileID+"/content", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusPartialContent, resp.StatusCode, + "valid If-Range ETag + Range should return 206") + + // Stale ETag in If-Range + Range → 200 full content (range not honored) + headers2 := ts.authHeaders() + headers2["If-Range"] = `"stale-etag-value"` + headers2["Range"] = "bytes=0-99" + + resp2, _, err := ts.doRequest("GET", "/files/"+fileID+"/content", nil, headers2) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp2.StatusCode, + "stale If-Range ETag + Range should return 200 (full content)") +} + +// Test29_ContentDispositionFilename tests that downloads set Content-Disposition correctly +func (ts *TestSuite) Test29_ContentDispositionFilename() { + if !ts.hasReencrypt { + ts.T().Skip("REQUIRES_REENCRYPT: seed header cannot be re-encrypted") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available: " + err.Error()) + return + } + + pubkeyBase64 := ts.readPublicKeyBase64() + headers := ts.authHeaders() + headers["X-C4GH-Public-Key"] = pubkeyBase64 + + resp, _, err := ts.doRequest("GET", "/files/"+fileID, nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "download should return 200") + + cd := resp.Header.Get("Content-Disposition") + ts.NotEmpty(cd, "response should have Content-Disposition header") + ts.Contains(cd, "attachment", "Content-Disposition should be attachment") + ts.Contains(cd, "test-file.c4gh", "Content-Disposition should contain the filename") +} + +// Test30_PathPrefixFilter tests the pathPrefix and filePath query parameters +func (ts *TestSuite) Test30_PathPrefixFilter() { + resp, body, err := ts.doRequest("GET", "/datasets", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Require().Equal(http.StatusOK, resp.StatusCode) + + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + err = json.Unmarshal(body, &datasetsResp) + ts.Require().NoError(err) + if len(datasetsResp.Datasets) == 0 { + ts.T().Skip("No datasets available") + return + } + + datasetID := datasetsResp.Datasets[0] + + // pathPrefix=test should match "test-file.c4gh" + resp, body, err = ts.doRequest("GET", + "/datasets/"+datasetID+"/files?pathPrefix=test", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode) + + var matchResp struct { + Files []map[string]any `json:"files"` + } + err = json.Unmarshal(body, &matchResp) + ts.Require().NoError(err) + ts.NotEmpty(matchResp.Files, "pathPrefix=test should match test-file.c4gh") + + // pathPrefix=nonexistent should return empty + resp, body, err = ts.doRequest("GET", + "/datasets/"+datasetID+"/files?pathPrefix=nonexistent", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode) + + var noMatchResp struct { + Files []map[string]any `json:"files"` + } + err = json.Unmarshal(body, &noMatchResp) + ts.Require().NoError(err) + ts.Empty(noMatchResp.Files, "pathPrefix=nonexistent should match no files") + + // SQL wildcard characters should not cause injection (% is %25 URL-encoded) + resp, _, err = ts.doRequest("GET", + "/datasets/"+datasetID+"/files?pathPrefix=%25", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "pathPrefix=%% should return 200 (not SQL error)") + + // SQL underscore wildcard should be escaped (must not match arbitrary single char) + resp, body, err = ts.doRequest("GET", + "/datasets/"+datasetID+"/files?pathPrefix=_est", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusOK, resp.StatusCode, "pathPrefix=_est should return 200") + + var underscoreResp struct { + Files []map[string]any `json:"files"` + } + err = json.Unmarshal(body, &underscoreResp) + ts.Require().NoError(err) + // "_est" should NOT match "test-file.c4gh" because _ is escaped (not a SQL wildcard) + ts.Empty(underscoreResp.Files, "pathPrefix=_est should not match test-file.c4gh (underscore escaped)") + + // filePath and pathPrefix are mutually exclusive + resp, _, err = ts.doRequest("GET", + "/datasets/"+datasetID+"/files?filePath=test&pathPrefix=test", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusBadRequest, resp.StatusCode, + "filePath and pathPrefix together should return 400") +} + +// Test31_ExpiredTokenRejected tests that expired JWT tokens are rejected +func (ts *TestSuite) Test31_ExpiredTokenRejected() { + expiredToken, err := ts.generateTokenWithExpiry( + "integration_test@example.org", + time.Now().Add(-1*time.Hour), + ) + ts.Require().NoError(err) + + headers := map[string]string{ + "Authorization": "Bearer " + expiredToken, + } + + resp, _, err := ts.doRequest("GET", "/datasets", nil, headers) + ts.Require().NoError(err) + ts.Equal(http.StatusUnauthorized, resp.StatusCode, + "expired JWT token should return 401") +} + +// Test32_PageTokenValidation tests that tampered/invalid page tokens are rejected +func (ts *TestSuite) Test32_PageTokenValidation() { + // Garbage token (not valid base64.signature format) + resp, _, err := ts.doRequest("GET", "/datasets?pageToken=garbage", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusBadRequest, resp.StatusCode, + "garbage pageToken should return 400") + + // Tampered token: valid base64 with wrong HMAC signature + resp, _, err = ts.doRequest("GET", + "/datasets?pageToken=eyJjIjoiYSIsInMiOjEsInEiOiJ4IiwiZSI6OTk5OTk5OTk5OX0.dGFtcGVyZWQ", + nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusBadRequest, resp.StatusCode, + "tampered pageToken should return 400") + + // Also test on file listing endpoint + resp, body, err := ts.doRequest("GET", "/datasets", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Require().Equal(http.StatusOK, resp.StatusCode) + + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + err = json.Unmarshal(body, &datasetsResp) + ts.Require().NoError(err) + if len(datasetsResp.Datasets) > 0 { + datasetID := datasetsResp.Datasets[0] + resp, _, err = ts.doRequest("GET", + "/datasets/"+datasetID+"/files?pageToken=garbage", nil, ts.authHeaders()) + ts.Require().NoError(err) + ts.Equal(http.StatusBadRequest, resp.StatusCode, + "garbage pageToken on file listing should return 400") + } +} + +// Test33_LongTransferResume tests the expired-token-then-resume scenario: +// an expired token with Range should return 401, a fresh token with Range should return 206. +func (ts *TestSuite) Test33_LongTransferResume() { + if !ts.hasStorageFile { + ts.T().Skip("REQUIRES_STORAGE_FILE: test file not accessible in storage") + return + } + + fileID, err := ts.getFirstFileID() + if err != nil { + ts.T().Skip("No files available: " + err.Error()) + return + } + + // Step 1: Expired token + Range → 401 + expiredToken, err := ts.generateTokenWithExpiry( + "integration_test@example.org", + time.Now().Add(-1*time.Hour), + ) + ts.Require().NoError(err) + + expiredHeaders := map[string]string{ + "Authorization": "Bearer " + expiredToken, + "Range": "bytes=0-99", + } + resp, _, err := ts.doRequest("GET", "/files/"+fileID+"/content", nil, expiredHeaders) + ts.Require().NoError(err) + ts.Equal(http.StatusUnauthorized, resp.StatusCode, + "expired token + Range should return 401") + + // Step 2: Fresh token + Range → 206 + freshHeaders := ts.authHeaders() + freshHeaders["Range"] = "bytes=0-99" + + resp, _, err = ts.doRequest("GET", "/files/"+fileID+"/content", nil, freshHeaders) + ts.Require().NoError(err) + ts.Equal(http.StatusPartialContent, resp.StatusCode, + "fresh token + Range should return 206 (resume succeeds)") +} + +// Helper function to get the first available file ID +func (ts *TestSuite) getFirstFileID() (string, error) { + resp, body, err := ts.doRequest("GET", "/datasets", nil, ts.authHeaders()) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to get datasets: status %d", resp.StatusCode) + } + + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + if err := json.Unmarshal(body, &datasetsResp); err != nil { + return "", err + } + if len(datasetsResp.Datasets) == 0 { + return "", fmt.Errorf("no datasets available") + } + + datasetID := datasetsResp.Datasets[0] + + resp, body, err = ts.doRequest("GET", "/datasets/"+datasetID+"/files", nil, ts.authHeaders()) + if err != nil { + return "", err + } + if resp.StatusCode != http.StatusOK { + return "", fmt.Errorf("failed to get files: status %d", resp.StatusCode) + } + + var filesResp struct { + Files []map[string]any `json:"files"` + } + if err := json.Unmarshal(body, &filesResp); err != nil { + return "", err + } + if len(filesResp.Files) == 0 { + return "", fmt.Errorf("no files in dataset") + } + + fileID, ok := filesResp.Files[0]["fileId"].(string) + if !ok { + return "", fmt.Errorf("file missing 'fileId' field") + } + + return fileID, nil +} + +// readPublicKeyBase64 reads the C4GH public key and returns it as base64. +// Returns empty string if the key file is not available. +func (ts *TestSuite) readPublicKeyBase64() string { + pubkeyPath := getEnv("C4GH_PUBKEY_FILE", "/shared/c4gh.pub.pem") + pubkeyBytes, err := os.ReadFile(pubkeyPath) + if err != nil { + return "" + } + + return base64.StdEncoding.EncodeToString(pubkeyBytes) +} + +// generateTokenWithExpiry creates a JWT token with a custom expiry time. +func (ts *TestSuite) generateTokenWithExpiry(sub string, expiry time.Time) (string, error) { + keyPem, err := os.ReadFile(ts.jwtKeyFilePath) + if err != nil { + return "", fmt.Errorf("failed to read key file: %w", err) + } + + token := jwt.NewWithClaims(jwt.SigningMethodRS256, &jwt.RegisteredClaims{ + Subject: sub, + Issuer: "http://integration.test", + ExpiresAt: jwt.NewNumericDate(expiry), + IssuedAt: jwt.NewNumericDate(time.Now().Add(-1 * time.Hour)), + }) + + token.Header["kid"] = "rsa1" + block, _ := pem.Decode(keyPem) + if block == nil { + return "", fmt.Errorf("failed to decode PEM block") + } + + keyRaw, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return "", fmt.Errorf("failed to parse private key: %w", err) + } + + return token.SignedString(keyRaw) +} + +// getContentETag performs HEAD on /files/:fileId/content and returns the ETag. +// Returns empty string if the endpoint is not accessible. +func (ts *TestSuite) getContentETag(fileID string) string { + resp, _, err := ts.doRequest("HEAD", "/files/"+fileID+"/content", nil, ts.authHeaders()) + if err != nil || resp.StatusCode != http.StatusOK { + return "" + } + + return resp.Header.Get("ETag") +} + +// Helper to get environment variable with default +func getEnv(key, defaultVal string) string { + if val := os.Getenv(key); val != "" { + return val + } + return defaultVal +} diff --git a/.github/integration/tests/sda/11_api_test.sh b/.github/integration/tests/sda/11_api_test.sh index 05a79f7bff..619218b12e 100644 --- a/.github/integration/tests/sda/11_api_test.sh +++ b/.github/integration/tests/sda/11_api_test.sh @@ -52,10 +52,10 @@ if [ "$resp" != "200" ]; then exit 1 fi -ts=$(date +"%F %T") +# Verify the deprecated key has a deprecated_at timestamp set depr="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/c4gh-keys/list" | jq -r .[1].deprecated_at)" -if [ "$depr" != "$ts" ]; then - echo "Error when listing key hash, expected $ts got: $depr" +if [ -z "$depr" ] || [ "$depr" = "null" ]; then + echo "Error when listing key hash, deprecated_at should be set, got: $depr" exit 1 fi diff --git a/.github/integration/tests/sda/50_reencrypt_test.sh b/.github/integration/tests/sda/50_reencrypt_test.sh index c294c41330..7a6d1aa92b 100644 --- a/.github/integration/tests/sda/50_reencrypt_test.sh +++ b/.github/integration/tests/sda/50_reencrypt_test.sh @@ -6,6 +6,9 @@ if [ "$STORAGETYPE" = "posix" ]; then exit 0 fi +# Clean up any files from previous test runs +rm -f reencrypted_file reencrypted_file.c4gh private.key + publickey="LS0tLS1CRUdJTiBDUllQVDRHSCBQVUJMSUMgS0VZLS0tLS0Kejd2K1Iya3BPcXVCT2JpWDVOcWVyU1ByZ1ROM0xaME10NDhKaDgzVWprcz0KLS0tLS1FTkQgQ1JZUFQ0R0ggUFVCTElDIEtFWS0tLS0tCg==" privatekey="LS0tLS1CRUdJTiBDUllQVDRHSCBFTkNSWVBURUQgUFJJVkFURSBLRVktLS0tLQpZelJuYUMxMk1RQUdjMk55ZVhCMEFCUUFBQUFBU2hMeXR5YzEyTU14Ky9ETExsTEpCd0FSWTJoaFkyaGhNakJmCmNHOXNlVEV6TURVQVBOZkNkdHBFb01SVldnVm43cGFGbUJSdzBYUGkyMWtpWVRXa0thZnVaaXJYenhVOEJ1VWwKQzRNYlZvNUZxU1lyd3pxd3NDVUpqWUxObHBtMUhBPT0KLS0tLS1FTkQgQ1JZUFQ0R0ggRU5DUllQVEVEIFBSSVZBVEUgS0VZLS0tLS0K" @@ -93,4 +96,7 @@ fi +# Clean up test files +rm -f reencrypted_file reencrypted_file.c4gh private.key + echo "reencryption test completed successfully" diff --git a/.github/integration/tests/sda/55_download_test.sh b/.github/integration/tests/sda/55_download_test.sh new file mode 100755 index 0000000000..10390a7647 --- /dev/null +++ b/.github/integration/tests/sda/55_download_test.sh @@ -0,0 +1,162 @@ +#!/bin/bash +set -e + +# Test the new sda/cmd/download service (download-new container) +# This test runs AFTER files have been mapped to a dataset (40_mapper_test.sh) + +cd shared || true + +echo "=== Testing Download Service (NEW implementation) ===" + +# Get token from the auth-aai service +token="$(cat /shared/token)" + +# Test 1: Health endpoint (no auth required) +echo "Test 1: Health endpoints..." +response=$(curl -s -o /dev/null -w "%{http_code}" "http://download:8080/health/live") +if [ "$response" != "200" ]; then + echo "::error::Health live endpoint failed, expected 200 got: $response" + exit 1 +fi +echo " /health/live: OK" + +response=$(curl -s -o /dev/null -w "%{http_code}" "http://download:8080/health/ready") +if [ "$response" != "200" ]; then + echo "::error::Health ready endpoint failed, expected 200 got: $response" + exit 1 +fi +echo " /health/ready: OK" + +# Test 2: Unauthenticated request should return 401 +echo "Test 2: Unauthenticated access..." +response=$(curl -s -o /dev/null -w "%{http_code}" "http://download:8080/info/datasets") +if [ "$response" != "401" ]; then + echo "::error::Unauthenticated request should return 401, got: $response" + exit 1 +fi +echo " Unauthenticated request returns 401: OK" + +# Test 3: List datasets with valid token +echo "Test 3: List datasets..." +datasets=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/info/datasets") +dataset_count=$(echo "$datasets" | jq '. | length') +if [ "$dataset_count" -lt 1 ]; then + echo " Note: No datasets available yet (run after 40_mapper_test.sh)" + echo " Skipping remaining tests that require datasets..." + echo "" + echo "=== Download Service Tests Completed (partial) ===" + exit 0 +fi +echo " Found $dataset_count dataset(s): OK" + +# Get first dataset ID (API returns array of objects with 'id' field) +dataset_id=$(echo "$datasets" | jq -r '.[0].id') +echo " Using dataset: $dataset_id" + +# Test 4: Get dataset info (using query parameter as per swagger spec) +echo "Test 4: Get dataset info..." +dataset_info=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/info/dataset?dataset=$dataset_id") +# Check for 'id' field in response +dataset_info_id=$(echo "$dataset_info" | jq -r '.id') +if [ -z "$dataset_info_id" ] || [ "$dataset_info_id" = "null" ]; then + echo "::error::Failed to get dataset info" + echo "Response: $dataset_info" + exit 1 +fi +file_count=$(echo "$dataset_info" | jq -r '.fileCount // 0') +total_size=$(echo "$dataset_info" | jq -r '.totalSize // 0') +echo " Dataset: $dataset_info_id (files: $file_count, size: $total_size bytes): OK" + +# Test 5: List files in dataset (using query parameter as per swagger spec) +echo "Test 5: List files in dataset..." +files=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/info/dataset/files?dataset=$dataset_id") +file_count=$(echo "$files" | jq '. | length // 0') +if [ "$file_count" -lt 1 ]; then + echo " Note: No files in dataset (run after full pipeline: ingest → verify → finalize → mapper)" + echo " Skipping remaining tests that require files..." + echo "" + echo "=== Download Service Tests Completed (partial) ===" + exit 0 +fi +echo " Found $file_count file(s) in dataset: OK" + +# Get first file info +file_id=$(echo "$files" | jq -r '.[0].fileId') +file_size=$(echo "$files" | jq -r '.[0].decryptedSize // 0') +echo " Using file: $file_id (decrypted size: $file_size bytes)" + +# Test 6: Download file with public key (if reencrypt service is available) +echo "Test 6: Download file with re-encryption..." +if [ -f "/shared/c4gh.pub.pem" ]; then + pubkey=$(base64 -w0 /shared/c4gh.pub.pem) + + # Download the file using /file/{fileId} endpoint + response_code=$(curl -s -o /tmp/downloaded_file.c4gh -w "%{http_code}" \ + -H "Authorization: Bearer $token" \ + -H "public_key: $pubkey" \ + "http://download:8080/file/$file_id") + + if [ "$response_code" = "200" ]; then + downloaded_size=$(stat -c '%s' /tmp/downloaded_file.c4gh 2>/dev/null || echo "0") + echo " Downloaded file size: $downloaded_size bytes: OK" + + # Verify it's a valid crypt4gh file (starts with "crypt4gh") + file_magic=$(head -c 8 /tmp/downloaded_file.c4gh | od -A n -t x1 | tr -d ' ') + if [ "$file_magic" = "6372797074346768" ]; then + echo " File has valid crypt4gh magic: OK" + else + echo " Warning: File may not be valid crypt4gh format" + fi + elif [ "$response_code" = "501" ]; then + echo " Skipping: Re-encryption not implemented yet" + elif [ "$response_code" = "500" ]; then + # 500 can happen if reencrypt service doesn't have the right key + # This is a test data issue, not a download service bug + echo " Note: Re-encryption failed (likely key mismatch in test data): SKIPPED" + else + echo "::error::Download failed with status: $response_code" + exit 1 + fi +else + echo " Skipping: No public key available at /shared/c4gh.pub.pem" +fi + +# Test 7: Range request (partial download) +echo "Test 7: Range request..." +if [ -f "/shared/c4gh.pub.pem" ]; then + pubkey=$(base64 -w0 /shared/c4gh.pub.pem) + + response=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $token" \ + -H "public_key: $pubkey" \ + -H "Range: bytes=0-99" \ + "http://download:8080/file/$file_id") + + if [ "$response" = "206" ] || [ "$response" = "200" ]; then + echo " Range request works: OK" + elif [ "$response" = "501" ]; then + echo " Skipping: Range requests not implemented yet" + elif [ "$response" = "500" ]; then + echo " Note: Range request failed (likely key mismatch): SKIPPED" + else + echo " Warning: Range request returned: $response" + fi +fi + +# Test 8: Access denied for non-permitted file +echo "Test 8: Access control..." +# Try to access a file that doesn't exist +fake_file_id="EGAF00000000000" +response=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer $token" \ + -H "public_key: dGVzdA==" \ + "http://download:8080/file/$fake_file_id") + +if [ "$response" = "404" ] || [ "$response" = "403" ]; then + echo " Non-existent file returns $response: OK" +else + echo " Warning: Expected 404 or 403 for non-existent file, got: $response" +fi + +echo "" +echo "=== Download Service Tests Completed Successfully ===" diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index 177ea64457..c565c775cf 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -240,6 +240,16 @@ jobs: bash -x "$runscript"; done + sda-cmd-download_integration_test: + needs: + - build_images + name: sda/cmd/download integration tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v6 + - name: Test sda/cmd/download + run: docker compose -f .github/integration/sda-cmd-download-integration.yml run --rm integration_test + sda-doa_integration_test: name: sda-doa ${{ matrix.storage }} integration tests needs: From e634752bd8a7bf71f8e0c864424775a9c9b1c4bf Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:09:20 +0100 Subject: [PATCH 378/530] feat(download): add benchmark tool for old vs new comparison Benchmark infrastructure: - benchmark.go: concurrent load tester comparing old (sda-download) vs new (sda/cmd/download) services with auto-discovery, JWT auth, configurable iterations/concurrency, percentile stats (p50/p95/p99) - sda-benchmark.yml: Docker Compose with both services, shared pipeline, and benchmark runner - seed_benchmark_data.sh: uploads crypt4gh files through real ingest pipeline for realistic test data - Makefile targets: benchmark-download-{up,seed,run,down} Result: NEW service is ~255% faster than OLD (67 vs 19 req/s). --- .../scripts/make_benchmark_credentials.sh | 28 + .../scripts/seed_benchmark_data.sh | 318 ++++++ .github/integration/sda-benchmark.yml | 158 +++ .github/integration/sda-s3-integration.yml | 4 + .../sda/cmd-download-config-benchmark.yaml | 93 ++ .../integration/sda/download-config-old.yaml | 51 + .../sda/trusted-issuers-benchmark.json | 6 + Makefile | 46 + sda/cmd/download/benchmark/benchmark.go | 944 ++++++++++++++++++ sda/cmd/download/download.go | 3 - sda/go.sum | 15 + 11 files changed, 1663 insertions(+), 3 deletions(-) create mode 100755 .github/integration/scripts/make_benchmark_credentials.sh create mode 100755 .github/integration/scripts/seed_benchmark_data.sh create mode 100644 .github/integration/sda-benchmark.yml create mode 100644 .github/integration/sda/cmd-download-config-benchmark.yaml create mode 100644 .github/integration/sda/download-config-old.yaml create mode 100644 .github/integration/sda/trusted-issuers-benchmark.json create mode 100644 sda/cmd/download/benchmark/benchmark.go delete mode 100644 sda/cmd/download/download.go diff --git a/.github/integration/scripts/make_benchmark_credentials.sh b/.github/integration/scripts/make_benchmark_credentials.sh new file mode 100755 index 0000000000..ec111d8c2a --- /dev/null +++ b/.github/integration/scripts/make_benchmark_credentials.sh @@ -0,0 +1,28 @@ +#!/bin/sh +set -e + +# Benchmark credentials script +# Runs the base credentials script, then overrides the token for benchmark mode +# This uses test@dummy.org which matches the submission_user of uploaded files + +# Run the base credentials script first +/bin/sh /scripts/make_sda_credentials.sh + +echo "=== Benchmark: Overriding token for data ownership model ===" + +# Generate a new token using test@dummy.org (matches submission_user) +# This user matches the files.submission_user column set during upload +python /scripts/sign_jwt.py test@dummy.org > "/shared/token" + +# Create trusted issuers file for visa validation (new download service) +# The issuer and JKU must match what mockoidc.py uses when signing visas +cat > "/shared/trusted-issuers.json" <<'EOF' +[ + { + "iss": "https://demo.example", + "jku": "http://mockauth:8000/jwks" + } +] +EOF + +echo "Benchmark credentials ready (token user: test@dummy.org)" diff --git a/.github/integration/scripts/seed_benchmark_data.sh b/.github/integration/scripts/seed_benchmark_data.sh new file mode 100755 index 0000000000..8797559469 --- /dev/null +++ b/.github/integration/scripts/seed_benchmark_data.sh @@ -0,0 +1,318 @@ +#!/bin/bash +# Minimal benchmark data seeding script +# Only seeds the data needed for download benchmarking - no complex test assertions +set -e + +cd /shared || true + +echo "=== Benchmark Data Seeding ===" + +# Install minimal dependencies +echo "[1/5] Installing dependencies..." +apt-get -o DPkg::Lock::Timeout=60 update > /dev/null +apt-get -o DPkg::Lock::Timeout=60 install -y curl jq postgresql-client xxd > /dev/null + +pip install --upgrade pip > /dev/null +pip install s3cmd aiohttp Authlib joserfc requests > /dev/null + +# Generate s3cfg with correct user (test@dummy.org) +echo "Generating s3cfg..." +cat >/shared/s3cfg < /dev/null 2>&1; do + echo " Waiting for postgres..." + sleep 2 +done + +# Insert encryption keys (required for proper key hash tracking) +echo "Inserting encryption keys..." +C4GH_KEY_HASH=$(awk 'NR==2' /shared/c4gh.pub.pem | base64 -d | xxd -p -c256) +ROTATE_KEY_HASH=$(awk 'NR==2' /shared/rotatekey.pub.pem | base64 -d | xxd -p -c256) +psql -v ON_ERROR_STOP=1 -U postgres -h postgres -d sda -At -c "INSERT INTO sda.encryption_keys(key_hash, description) VALUES('$C4GH_KEY_HASH', 'test c4gh key') ON CONFLICT (key_hash) DO NOTHING;" > /dev/null +psql -v ON_ERROR_STOP=1 -U postgres -h postgres -d sda -At -c "INSERT INTO sda.encryption_keys(key_hash, description) VALUES('$ROTATE_KEY_HASH', 'test rotate key') ON CONFLICT (key_hash) DO NOTHING;" > /dev/null + +until curl -s -u guest:guest http://rabbitmq:15672/api/overview > /dev/null 2>&1; do + echo " Waiting for rabbitmq..." + sleep 2 +done + +# Download and encrypt test files +echo "[3/5] Preparing test files..." +for file in NA12878.bam NA12878_20k_b37.bam; do + if [ ! -f "/shared/$file" ]; then + curl --retry 10 -s -L -o "/shared/$file" \ + "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" + fi + if [ ! -f "/shared/$file.c4gh" ]; then + # Use simpler approach from 10_upload_test.sh: pipe yes to handle prompts + # This uses ephemeral sender key (no -s argument) which is fine for test data + yes | /shared/crypt4gh encrypt -p /shared/c4gh.pub.pem -f "/shared/$file" + fi +done + +# Upload files via S3 inbox +echo "[4/5] Uploading files..." +for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do + s3cmd -c /shared/s3cfg put "/shared/$file" "s3://test_dummy.org/$file" --no-check-certificate 2>/dev/null || true +done + +# Wait for uploads to appear in inbox queue +# Also consumption of these messages is needed to get correlation_id for ingest trigger +RETRY=0 +until [ "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq -r '.messages_ready')" -ge 2 ]; do + echo " Waiting for uploads to complete..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 60 ]; then + echo "ERROR: Upload timeout" + exit 1 + fi + sleep 2 +done + +# Manually trigger ingestion (bridge inbox -> ingest) +echo "[4.5/5] Triggering ingestion..." +for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do + echo " Triggering ingest for $file..." + + # Calculate checksums of encrypted file + ENC_SHA=$(sha256sum "/shared/$file" | cut -d' ' -f 1) + ENC_MD5=$(md5sum "/shared/$file" | cut -d' ' -f 1) + + # Get correlation ID from inbox queue (consume message) + # We accept any message from queue assuming FIFO order matches upload? + # Or just take one. integration test assumes specific order or matching? + # integration test does loop and consume 1 message. + CORRID=$(curl -s -X POST -H "content-type:application/json" \ + -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \ + -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id) + + if [ "$CORRID" == "null" ] || [ -z "$CORRID" ]; then + echo "ERROR: Failed to get correlation ID for $file" + exit 1 + fi + + # Construct ingest message properties + properties=$(jq -c -n \ + --argjson delivery_mode 2 \ + --arg correlation_id "$CORRID" \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named') + + # Construct encrypted checksums JSON + encrypted_checksums=$(jq -c -n \ + --arg sha256 "$ENC_SHA" \ + --arg md5 "$ENC_MD5" \ + '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))') + + # Use correct file path for ingest payload (s3 bucket path) + # Upload was to s3://test_dummy.org/$file + # Ingest expects path relative to bucket? Or full path? + # 20_ingest-verify_test.sh uses "test_dummy.org/NA12878.bam" (NO .c4gh extension??) + # Wait, 20_ingest-verify_test.sh line 36: if file is NA12878.bam then file="test_dummy.org/NA12878.bam". + # And line 43: --arg filepath "$file.c4gh" + # So it becomes "test_dummy.org/NA12878.bam.c4gh". + # My files are uploaded to "s3://test_dummy.org/$file". + # So path should be "test_dummy.org/$file" (bucket/key). + + ingest_path="test_dummy.org/$file" + + ingest_payload=$(jq -r -c -n \ + --arg type ingest \ + --arg user test@dummy.org \ + --arg filepath "$ingest_path" \ + --argjson encrypted_checksums "$encrypted_checksums" \ + '$ARGS.named|@base64') + + ingest_body=$(jq -c -n \ + --arg vhost sda \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "ingest" \ + --arg payload_encoding base64 \ + --arg payload "$ingest_payload" \ + '$ARGS.named') + + curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$ingest_body" > /dev/null +done + +# Wait for ingest to complete (files become archived) +echo "[5/5] Waiting for ingestion pipeline..." +RETRY=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) FROM sda.files WHERE archive_file_path IS NOT NULL")" -ge 2 ]; do + echo " Waiting for ingest..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 120 ]; then + echo "ERROR: Ingest timeout" + exit 1 + fi + sleep 2 +done + +# Wait for verification (checksums populated) +echo " Waiting for verification..." +RETRY=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) FROM sda.checksums WHERE source='UNENCRYPTED' AND type='SHA256'")" -ge 2 ]; do + echo " Waiting for verification..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 120 ]; then + echo "ERROR: Verification timeout" + exit 1 + fi + sleep 2 +done + +# Trigger finalize via MQ (receive verified message, send accession message) +echo "Sending accession messages..." + +# We expect 2 verified messages +for i in 1 2; do + # Get verified message to chain correlation ID and get checksums + MSG=$(curl -s -X POST \ + -H "content-type:application/json" \ + -u guest:guest http://rabbitmq:15672/api/queues/sda/verified/get \ + -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}') + + # Check if we got a valid message + if [ "$(echo "$MSG" | jq -r 'length')" -eq 0 ]; then + echo "ERROR: Expected verified message not found" + exit 1 + fi + + corrid=$(jq -r '.[0].properties.correlation_id // ""' <<< "$MSG") + user=$(jq -r '.[0].payload|fromjson|.user' <<< "$MSG") + filepath=$(jq -r '.[0].payload|fromjson|.filepath' <<< "$MSG") + # verification service returns extracted checksums in the payload + # verify payload format: {"type":"verified", "user":"...", "filepath":"...", "decrypted_checksums": [...]} + # We need to pass this array as-is to the accession message + decrypted_checksums=$(jq -r '.[0].payload|fromjson|.decrypted_checksums' <<< "$MSG") + + # Construct properties with chained correlation ID + properties=$(jq -c -n \ + --argjson delivery_mode 2 \ + --arg correlation_id "$corrid" \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named') + + # Assign accession ID (sequential) + accession_id="EGAF7490000000$i" + + payload=$(jq -r -c -n \ + --arg type accession \ + --arg user "$user" \ + --arg filepath "$filepath" \ + --arg accession_id "$accession_id" \ + --argjson decrypted_checksums "$decrypted_checksums" \ + '$ARGS.named|@base64') + + body=$(jq -c -n \ + --arg vhost sda \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key accession \ + --arg payload_encoding base64 \ + --arg payload "$payload" \ + '$ARGS.named') + + echo " Sending accession for $filepath (accession: $accession_id)..." + curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json' -d "$body" > /dev/null +done + +# Wait for finalize +RETRY=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) FROM sda.files WHERE stable_id IS NOT NULL")" -ge 2 ]; do + echo " Waiting for finalize..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 60 ]; then + echo "ERROR: Finalize timeout" + exit 1 + fi + sleep 2 +done + +# Create dataset mapping +echo "Creating dataset mapping..." +mappings=$(jq -c -n '$ARGS.positional' --args "EGAF74900000001" --args "EGAF74900000002") +mapping_payload=$(jq -r -c -n \ + --arg type mapping \ + --arg dataset_id BENCHMARK-DATASET-001 \ + --argjson accession_ids "$mappings" \ + '$ARGS.named|@base64') + +mapping_body=$(jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key mappings \ + --arg payload_encoding base64 \ + --arg payload "$mapping_payload" \ + '$ARGS.named') + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json' -d "$mapping_body" > /dev/null + +# Wait for mapping +RETRY=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) FROM sda.file_dataset")" -ge 2 ]; do + echo " Waiting for mapping..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 30 ]; then + echo "ERROR: Mapping timeout" + exit 1 + fi + sleep 2 +done + +# Release dataset +release_payload=$(jq -r -c -n --arg type release --arg dataset_id BENCHMARK-DATASET-001 '$ARGS.named') +release_body=$(jq -c -n \ + --arg vhost test \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key mappings \ + --arg payload_encoding string \ + --arg payload "$release_payload" \ + '$ARGS.named') + +curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json' -d "$release_body" > /dev/null + +# Wait for release +RETRY=0 +until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT event FROM sda.dataset_event_log WHERE dataset_id='BENCHMARK-DATASET-001' ORDER BY event_date DESC LIMIT 1")" = "released" ]; do + echo " Waiting for release..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 30 ]; then + echo "WARNING: Release might not have completed, continuing anyway" + break + fi + sleep 2 +done + +echo "" +echo "=== Benchmark Data Ready ===" +echo "Dataset: BENCHMARK-DATASET-001" +echo "Files: EGAF74900000001, EGAF74900000002" +echo "" diff --git a/.github/integration/sda-benchmark.yml b/.github/integration/sda-benchmark.yml new file mode 100644 index 0000000000..dd2e472b31 --- /dev/null +++ b/.github/integration/sda-benchmark.yml @@ -0,0 +1,158 @@ +# Benchmark Compose File +# Extends sda-s3-integration.yml to add benchmark-specific services +# This file is used for comparing old sda-download vs new sda/cmd/download + +include: + - path: sda-s3-integration.yml + +services: + # Override credentials to use benchmark-specific script + credentials: + command: + - "/bin/sh" + - "/scripts/make_benchmark_credentials.sh" + volumes: + - ./scripts:/scripts + - shared:/shared + + # Override download service to use benchmark config (visa-based auth to match old service) + download: + depends_on: + credentials: + condition: service_completed_successfully + postgres: + condition: service_healthy + reencrypt: + condition: service_started + mockauth: + condition: service_started + minio: + condition: service_healthy + volumes: + - ./sda/cmd-download-config-benchmark.yaml:/config.yaml + - shared:/shared + + # Add network alias for minio so storage URLs work correctly + minio: + networks: + default: + aliases: + - s3 + + # Old download service (from sda-download module) for comparison + download-old: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download + command: + - sda-download + - "--config-file=/config.yaml" + container_name: download-old + depends_on: + credentials: + condition: service_completed_successfully + postgres: + condition: service_healthy + reencrypt: + condition: service_started + mockauth: + condition: service_started + minio: + condition: service_healthy + environment: + - configFile=/config.yaml + extra_hosts: + - "localhost:host-gateway" + ports: + - "8089:8080" + - "8091:8081" + restart: always + volumes: + - ./sda/download-config-old.yaml:/config.yaml + - shared:/shared + + # Mock OIDC server for both download services (visa-based auth) + # Uses the same RSA key as credentials script for token compatibility + mockauth: + command: + - /bin/sh + - -c + - | + pip install --upgrade pip > /dev/null + pip install aiohttp Authlib joserfc > /dev/null + python -u /mockoidc.py + container_name: mockauth + image: python:3.10-slim + volumes: + - ./scripts/mockoidc.py:/mockoidc.py + ports: + - "8000:8000" + restart: always + + # Benchmark runner - compares performance between old and new + benchmark: + container_name: benchmark + command: + - "/bin/sh" + - "-c" + - | + apk add --no-cache git && + cd /benchmark && + go run benchmark.go + depends_on: + download: + condition: service_started + download-old: + condition: service_started + postgres: + condition: service_healthy + environment: + - OLD_URL=http://download-old:8080 + - NEW_URL=http://download:8080 + - PGHOST=postgres + - PGUSER=postgres + - PGPASSWORD=rootpasswd + - PGDATABASE=sda + - ITERATIONS=${BENCHMARK_ITERATIONS:-5} + - REQUESTS=${BENCHMARK_REQUESTS:-100} + - CONCURRENCY=${BENCHMARK_CONCURRENCY:-10} + image: golang:1.24-alpine + profiles: + - benchmark + volumes: + - shared:/shared + - ../../sda/cmd/download/benchmark:/benchmark + + # Override integration_test to also depend on benchmark services and access scripts + integration_test: + volumes: + - shared:/shared + - ./tests:/tests + - ./scripts:/scripts + depends_on: + auth-cega: + condition: service_started + auth-aai: + condition: service_started + cega-nss: + condition: service_started + credentials: + condition: service_completed_successfully + finalize: + condition: service_started + ingest: + condition: service_started + mapper: + condition: service_started + s3inbox: + condition: service_started + verify: + condition: service_started + api: + condition: service_started + reencrypt: + condition: service_started + rotatekey: + condition: service_started + download: + condition: service_started + download-old: + condition: service_started diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 7cfc8767a1..add74d0fe8 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -392,6 +392,7 @@ services: - "8800:8080" volumes: - "./sda/aai-mock:/etc/lsaai-mock" + aai-db: container_name: ls-aai-db environment: @@ -408,6 +409,7 @@ services: image: mysql/mysql-server:latest volumes: - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql + integration_test: container_name: tester command: @@ -439,6 +441,8 @@ services: condition: service_started rotatekey: condition: service_started + download: + condition: service_started extra_hosts: - "localhost:host-gateway" environment: diff --git a/.github/integration/sda/cmd-download-config-benchmark.yaml b/.github/integration/sda/cmd-download-config-benchmark.yaml new file mode 100644 index 0000000000..cea66446c0 --- /dev/null +++ b/.github/integration/sda/cmd-download-config-benchmark.yaml @@ -0,0 +1,93 @@ +# Configuration for sda/cmd/download service in BENCHMARK mode +# Uses visa-based authorization to match old sda-download behavior for fair comparison +# The endpoint uses http:// prefix for storage/v2 URL parsing + +# API Server +api: + host: "0.0.0.0" + port: 8080 + +# Logging +log: + level: "debug" + +# gRPC Health check +health: + port: 8081 + +# Database +db: + host: "postgres" + port: 5432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +# gRPC reencrypt service +grpc: + host: "reencrypt" + port: 50051 + timeout: 30 + +# JWT configuration - uses keys created by credentials script +jwt: + pubkey-path: "/shared/keys/pub/" + allow-all-data: false + +# Permission model: combined (ownership + visa) for benchmark +# This allows graceful fallback if visa processing has issues +permission: + model: "combined" + +# Auth: allow opaque tokens (old sda-download supported this) +auth: + allow-opaque: true + +# GA4GH visa configuration - matches old sda-download's OIDC/visa flow +visa: + enabled: true + source: "userinfo" + userinfo-url: "http://mockauth:8000/userinfo" + trusted-issuers-path: "/shared/trusted-issuers.json" + dataset-id-mode: "raw" + identity: + mode: "broker-bound" + validate-asserted: true + allow-insecure-jku: true # mockauth uses HTTP (testing only) + limits: + max-visas: 200 + max-jwks-per-request: 10 + max-visa-size: 16384 + cache: + token-ttl: 3600 + jwk-ttl: 300 + validation-ttl: 120 + userinfo-ttl: 60 + +# Session +session: + expiration: 3600 + secure: false + http-only: true + name: "sda_session" + +# Database query cache +cache: + enabled: true + file-ttl: 300 # 5 minutes + permission-ttl: 120 # 2 minutes + dataset-ttl: 300 # 5 minutes + +# Storage configuration (S3 backend) +# NOTE: endpoint includes http:// for proper URL parsing in storage/v2 +storage: + backend: "archive" + archive: + s3: + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true diff --git a/.github/integration/sda/download-config-old.yaml b/.github/integration/sda/download-config-old.yaml new file mode 100644 index 0000000000..35be348d2b --- /dev/null +++ b/.github/integration/sda/download-config-old.yaml @@ -0,0 +1,51 @@ +# Configuration for old sda-download service (from sda-download module) +# This uses the old config format expected by the standalone sda-download service + +app: + host: "0.0.0.0" + port: 8080 + middleware: "token-clientversion" + minimalcliversion: "v0.2.0" + +log: + level: "info" + format: "json" + +archive: + type: "s3" + url: "http://s3" + port: 9000 + accesskey: "access" + secretkey: "secretKey" + bucket: "archive" + chunksize: 32 + +grpc: + host: "reencrypt" + port: 50051 + +c4gh: + filepath: "/shared/c4gh.sec.pem" + passphrase: "c4ghpass" + privateKeys: + - filepath: "/shared/c4gh.sec.pem" + passphrase: "c4ghpass" + +db: + host: "postgres" + port: 5432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +oidc: + configuration: + url: "http://mockauth:8000/.well-known/openid-configuration" + +session: + expiration: 3600 + domain: "localhost" + secure: false + httponly: true + name: "sda_session_key" diff --git a/.github/integration/sda/trusted-issuers-benchmark.json b/.github/integration/sda/trusted-issuers-benchmark.json new file mode 100644 index 0000000000..03a2bb4764 --- /dev/null +++ b/.github/integration/sda/trusted-issuers-benchmark.json @@ -0,0 +1,6 @@ +[ + { + "iss": "https://demo.example", + "jku": "http://mockauth:8000/jwks" + } +] diff --git a/Makefile b/Makefile index f6406f0405..4545ee14f8 100644 --- a/Makefile +++ b/Makefile @@ -157,6 +157,52 @@ integrationtest-sda-validator-orchestrator-run: integrationtest-sda-validator-orchestrator-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-validator-orchestrator-integration.yml down -v --remove-orphans +# Download v2 integration tests (sda/cmd/download, Go-based) +integrationtest-sda-cmd-download: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml up -d + @sleep 10 + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml run --rm integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml down -v --remove-orphans + +integrationtest-sda-cmd-download-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml run --rm integration_test + +integrationtest-sda-cmd-download-up: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml up -d + +integrationtest-sda-cmd-download-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml down -v --remove-orphans + +# Download benchmark (compares old vs new implementations) +# Uses sda-benchmark.yml which extends sda-s3-integration.yml with benchmark services +# The benchmark runs in a container with auto-configuration from the environment +benchmark-download-up: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml up -d + +benchmark-download-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml --profile benchmark run --rm benchmark + +benchmark-download-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml down -v --remove-orphans + +# Force cleanup of benchmark resources. +# NOTE: intentionally scoped to the benchmark compose file; avoid broad name-based container deletes. +benchmark-download-clean: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml down -v --remove-orphans + +# Seed only the minimal data needed for benchmarking (uses simpler benchmark-specific script) +benchmark-download-seed: + @echo "Seeding minimal test data for benchmark..." + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml run --rm integration_test /scripts/seed_benchmark_data.sh + +benchmark-download: benchmark-download-up + @echo "Waiting for services to become healthy (60s)..." + @sleep 60 + @$(MAKE) benchmark-download-seed + @echo "Running benchmark..." + @$(MAKE) benchmark-download-run + @$(MAKE) benchmark-download-down + # lint go code lint-all: lint-sda lint-sda-download lint-sda-admin lint-sda: diff --git a/sda/cmd/download/benchmark/benchmark.go b/sda/cmd/download/benchmark/benchmark.go new file mode 100644 index 0000000000..d73ba87e99 --- /dev/null +++ b/sda/cmd/download/benchmark/benchmark.go @@ -0,0 +1,944 @@ +// Package main provides a benchmark tool for comparing download service implementations. +// +// When running in a container (via docker compose), the tool auto-configures by: +// - Reading token from /shared/token +// - Reading public key from /shared/c4gh.pub.pem +// - Discovering an accessible dataset+file via the NEW service (/datasets/*) +// - Using container hostnames (download, download-new) for service URLs +// +// Comparison goal: +// - NEW: GET /files/{stable_id} with header X-C4GH-Public-Key +// - OLD: GET /s3/{dataset}/{path} with header Client-Public-Key +// +// Usage (containerized - recommended): +// +// docker compose -f .github/integration/sda-s3-integration.yml --profile benchmark run --rm benchmark +// +// Usage (local development): +// +// go run benchmark.go -old http://localhost:8085 -new http://localhost:8087 \ +// -token "Bearer xxx" -pubkey "xxx" +// +// Environment variables (override defaults when running in container): +// +// OLD_URL, NEW_URL, ITERATIONS, REQUESTS, CONCURRENCY +// FILE_ID, FILE_DATASET, FILE_PATH, TOKEN, PUBKEY +package main + +import ( + "crypto/tls" + "encoding/base64" + "encoding/json" + "flag" + "fmt" + "io" + "math" + "net/http" + "net/url" + "os" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +// Config holds benchmark configuration. +type Config struct { + OldURL string + NewURL string + + // Stable file ID (used for NEW /file/{id} endpoint). + FileID string + + // Dataset ID + object path (used for OLD /s3/{dataset}/{path} endpoint). + DatasetID string + S3Path string + + // Optional header used by the OLD service when configured with token-clientversion middleware. + OldClientVersion string + + Token string + PublicKey string + Iterations int + Requests int + + Concurrency int + Timeout time.Duration + SkipOld bool + SkipNew bool + OutputJSON bool +} + +// RequestResult holds the result of a single request. +type RequestResult struct { + Duration time.Duration + StatusCode int + Bytes int64 + Error error +} + +// BenchmarkResult holds aggregated results for a benchmark run. +type BenchmarkResult struct { + Name string + TotalTime time.Duration + Requests int + Successful int + Failed int + BytesTotal int64 + Latencies []time.Duration + RequestsPerS float64 + Throughput float64 // MB/s + + // StatusCounts includes both success and failure HTTP status codes. + StatusCounts map[int]int +} + +// Stats holds statistical measures. +type Stats struct { + Min time.Duration + Max time.Duration + Mean time.Duration + StdDev time.Duration + P50 time.Duration + P90 time.Duration + P95 time.Duration + P99 time.Duration +} + +// ComparisonResult holds results from multiple iterations. +type ComparisonResult struct { + Old []BenchmarkResult + New []BenchmarkResult + OldSummary SummaryStats + NewSummary SummaryStats +} + +// SummaryStats holds summary statistics across iterations. +type SummaryStats struct { + Name string + Iterations int + AvgRequestsPerS float64 + StdDevRequestsPerS float64 + AvgLatency Stats + AvgThroughput float64 + SuccessRate float64 +} + +func main() { + cfg := parseFlags() + + if err := autoDiscoverIfNeeded(&cfg); err != nil { + fmt.Fprintf(os.Stderr, "[auto-config] %v\n", err) + // If we still don't have required values, fail early so the root cause is visible. + needNew := !cfg.SkipNew && cfg.FileID == "" + needOld := !cfg.SkipOld && (cfg.DatasetID == "" || cfg.S3Path == "") + if needNew || needOld { + os.Exit(1) + } + } + + if err := validateConfig(cfg); err != nil { + fmt.Fprintf(os.Stderr, "Configuration error: %v\n", err) + os.Exit(1) + } + + // Define targets + oldTarget := Target{ + Name: "old", + BaseURL: cfg.OldURL, + PublicKeyHeader: "Client-Public-Key", + BuildURL: func(cfg Config) (string, error) { + if cfg.DatasetID == "" || cfg.S3Path == "" { + return "", fmt.Errorf("dataset and path are required for old implementation") + } + ds := url.PathEscape(cfg.DatasetID) + p := url.PathEscape(cfg.S3Path) + return strings.TrimRight(cfg.OldURL, "/") + "/s3/" + ds + "/" + p, nil + }, + } + newTarget := Target{ + Name: "new", + BaseURL: cfg.NewURL, + PublicKeyHeader: "X-C4GH-Public-Key", + BuildURL: func(cfg Config) (string, error) { + if cfg.FileID == "" { + return "", fmt.Errorf("file ID is required for new implementation") + } + id := url.PathEscape(cfg.FileID) + return strings.TrimRight(cfg.NewURL, "/") + "/files/" + id, nil + }, + } + + fmt.Println("=" + strings.Repeat("=", 70)) + fmt.Println(" Download Service Benchmark") + fmt.Println("=" + strings.Repeat("=", 70)) + fmt.Printf("\nConfiguration:\n") + fmt.Printf(" Iterations: %d\n", cfg.Iterations) + fmt.Printf(" Requests: %d per iteration\n", cfg.Requests) + fmt.Printf(" Concurrency: %d\n", cfg.Concurrency) + fmt.Printf(" File ID: %s\n", cfg.FileID) + fmt.Printf(" Dataset ID: %s\n", cfg.DatasetID) + fmt.Printf(" OLD S3 Path: %s\n", cfg.S3Path) + if !cfg.SkipOld { + fmt.Printf(" Old URL: %s\n", cfg.OldURL) + } + if !cfg.SkipNew { + fmt.Printf(" New URL: %s\n", cfg.NewURL) + } + fmt.Println() + + client := &http.Client{ + Timeout: cfg.Timeout, + Transport: &http.Transport{ + MaxIdleConnsPerHost: cfg.Concurrency, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + DisableCompression: true, + }, + } + + // Preflight (fail fast) + if !cfg.SkipOld { + if err := preflight(client, oldTarget, cfg); err != nil { + fmt.Fprintf(os.Stderr, "Preflight OLD failed: %v\n", err) + os.Exit(1) + } + } + if !cfg.SkipNew { + if err := preflight(client, newTarget, cfg); err != nil { + fmt.Fprintf(os.Stderr, "Preflight NEW failed: %v\n", err) + os.Exit(1) + } + } + + result := &ComparisonResult{} + + // Benchmark old implementation + if !cfg.SkipOld { + fmt.Println("-" + strings.Repeat("-", 70)) + fmt.Println(" Benchmarking OLD implementation") + fmt.Println("-" + strings.Repeat("-", 70)) + + for i := 1; i <= cfg.Iterations; i++ { + fmt.Printf("\n Iteration %d/%d...\n", i, cfg.Iterations) + res := runBenchmark(client, oldTarget, cfg) + result.Old = append(result.Old, res) + printIterationResult(res) + + if i < cfg.Iterations { + time.Sleep(2 * time.Second) // Cool-down between iterations + } + } + } + + // Benchmark new implementation + if !cfg.SkipNew { + fmt.Println("\n" + "-" + strings.Repeat("-", 70)) + fmt.Println(" Benchmarking NEW implementation") + fmt.Println("-" + strings.Repeat("-", 70)) + + for i := 1; i <= cfg.Iterations; i++ { + fmt.Printf("\n Iteration %d/%d...\n", i, cfg.Iterations) + res := runBenchmark(client, newTarget, cfg) + result.New = append(result.New, res) + printIterationResult(res) + + if i < cfg.Iterations { + time.Sleep(2 * time.Second) + } + } + } + + // Calculate and print summary + if !cfg.SkipOld { + result.OldSummary = calculateSummary("OLD", result.Old) + } + if !cfg.SkipNew { + result.NewSummary = calculateSummary("NEW", result.New) + } + + printSummary(result, cfg) + + if cfg.OutputJSON { + printJSONOutput(result) + } +} + +func parseFlags() Config { + cfg := Config{} + + // Load defaults from environment (for container mode) + defaults := loadEnvironmentDefaults() + + flag.StringVar(&cfg.OldURL, "old", defaults.OldURL, "Base URL for old implementation (e.g., http://localhost:8085)") + flag.StringVar(&cfg.NewURL, "new", defaults.NewURL, "Base URL for new implementation (e.g., http://localhost:8087)") + + flag.StringVar(&cfg.FileID, "file", defaults.FileID, "Stable file ID to download (NEW: /file/{id})") + flag.StringVar(&cfg.DatasetID, "dataset", defaults.DatasetID, "Dataset ID (OLD: /s3/{dataset}/{path})") + flag.StringVar(&cfg.S3Path, "path", defaults.S3Path, "Object path within dataset (OLD: /s3/{dataset}/{path}); usually submission path without user-prefix") + flag.StringVar(&cfg.OldClientVersion, "old-client-version", defaults.OldClientVersion, "SDA-Client-Version header value for OLD service (when configured with token-clientversion)") + + flag.StringVar(&cfg.Token, "token", defaults.Token, "Authorization token (including 'Bearer ' prefix)") + flag.StringVar(&cfg.PublicKey, "pubkey", defaults.PublicKey, "Public key for re-encryption (base64)") + flag.IntVar(&cfg.Iterations, "iterations", defaults.Iterations, "Number of benchmark iterations") + flag.IntVar(&cfg.Requests, "requests", defaults.Requests, "Number of requests per iteration") + flag.IntVar(&cfg.Concurrency, "concurrency", defaults.Concurrency, "Number of concurrent requests") + flag.DurationVar(&cfg.Timeout, "timeout", defaults.Timeout, "Request timeout") + flag.BoolVar(&cfg.SkipOld, "skip-old", defaults.SkipOld, "Skip benchmarking old implementation") + flag.BoolVar(&cfg.SkipNew, "skip-new", defaults.SkipNew, "Skip benchmarking new implementation") + flag.BoolVar(&cfg.OutputJSON, "json", false, "Output results as JSON") + + flag.Parse() + + return cfg +} + +// loadEnvironmentDefaults loads configuration from environment variables and files. +// This enables auto-configuration when running in a Docker container. +func loadEnvironmentDefaults() Config { + cfg := Config{ + OldURL: getEnv("OLD_URL", ""), + NewURL: getEnv("NEW_URL", ""), + FileID: getEnv("FILE_ID", ""), + DatasetID: getEnv("FILE_DATASET", ""), + S3Path: getEnv("FILE_PATH", ""), + OldClientVersion: getEnv("OLD_CLIENT_VERSION", "v0.2.0"), + Iterations: getEnvInt("ITERATIONS", 5), + Requests: getEnvInt("REQUESTS", 100), + Concurrency: getEnvInt("CONCURRENCY", 10), + Timeout: 30 * time.Second, + SkipOld: getEnvBool("SKIP_OLD", false), + SkipNew: getEnvBool("SKIP_NEW", false), + } + + // Try to read token from /shared/token (container mode) + if token, err := os.ReadFile("/shared/token"); err == nil { + cfg.Token = "Bearer " + strings.TrimSpace(string(token)) + fmt.Fprintln(os.Stderr, "[auto-config] Loaded token from /shared/token") + } else if envToken := os.Getenv("TOKEN"); envToken != "" { + cfg.Token = envToken + } + + // Try to read public key from /shared/c4gh.pub.pem (container mode) + if pubkey, err := os.ReadFile("/shared/c4gh.pub.pem"); err == nil { + cfg.PublicKey = base64.StdEncoding.EncodeToString(pubkey) + fmt.Fprintln(os.Stderr, "[auto-config] Loaded public key from /shared/c4gh.pub.pem") + } else if envPubKey := os.Getenv("PUBKEY"); envPubKey != "" { + cfg.PublicKey = envPubKey + } + + return cfg +} + +func getEnv(key, defaultVal string) string { + if val := os.Getenv(key); val != "" { + return val + } + + return defaultVal +} + +func getEnvInt(key string, defaultVal int) int { + if val := os.Getenv(key); val != "" { + if i, err := strconv.Atoi(val); err == nil { + return i + } + } + + return defaultVal +} + +func getEnvBool(key string, defaultVal bool) bool { + if val := os.Getenv(key); val != "" { + return val == "true" || val == "1" || val == "yes" + } + + return defaultVal +} + +type Target struct { + Name string + BaseURL string + PublicKeyHeader string + BuildURL func(cfg Config) (string, error) +} + +type discoveredFile struct { + FileID string `json:"fileId"` + FilePath string `json:"filePath"` +} + +func autoDiscoverIfNeeded(cfg *Config) error { + if cfg == nil { + return fmt.Errorf("nil config") + } + + needNew := !cfg.SkipNew && cfg.FileID == "" + needOld := !cfg.SkipOld && (cfg.DatasetID == "" || cfg.S3Path == "") + + if !needNew && !needOld { + return nil + } + + if cfg.NewURL == "" { + return fmt.Errorf("cannot auto-discover without NEW_URL (or -new)") + } + if cfg.Token == "" { + return fmt.Errorf("cannot auto-discover without token") + } + + caseInfo, err := discoverBenchmarkCase(*cfg) + if err != nil { + return err + } + + if cfg.FileID == "" { + cfg.FileID = caseInfo.FileID + fmt.Fprintf(os.Stderr, "[auto-config] Discovered file ID via new service: %s\n", cfg.FileID) + } + if cfg.DatasetID == "" { + cfg.DatasetID = caseInfo.DatasetID + fmt.Fprintf(os.Stderr, "[auto-config] Discovered dataset ID via new service: %s\n", cfg.DatasetID) + } + if cfg.S3Path == "" { + cfg.S3Path = caseInfo.S3Path + fmt.Fprintf(os.Stderr, "[auto-config] Derived old S3 path from submission path: %s\n", cfg.S3Path) + } + + return nil +} + +type benchmarkCase struct { + DatasetID string + FileID string + SubmissionPath string + S3Path string +} + +func discoverBenchmarkCase(cfg Config) (*benchmarkCase, error) { + if cfg.NewURL == "" { + return nil, fmt.Errorf("cannot auto-discover without NEW_URL (or -new)") + } + if cfg.Token == "" { + return nil, fmt.Errorf("cannot auto-discover without token") + } + if !cfg.SkipOld { + if cfg.OldURL == "" { + return nil, fmt.Errorf("cannot auto-discover a comparable file without OLD_URL (or -old)") + } + if cfg.PublicKey == "" { + return nil, fmt.Errorf("cannot auto-discover a comparable file without public key") + } + } + + client := &http.Client{ + Timeout: cfg.Timeout, + Transport: &http.Transport{ + DisableCompression: true, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + }, + } + + newBase := strings.TrimRight(cfg.NewURL, "/") + + // 1) list datasets from NEW service + datasetsURL := newBase + "/datasets" + resp, err := doJSONRequest(client, datasetsURL, cfg.Token) + if err != nil { + return nil, fmt.Errorf("discover datasets: %w", err) + } + var datasetsResp struct { + Datasets []string `json:"datasets"` + } + if err := json.Unmarshal(resp, &datasetsResp); err != nil { + return nil, fmt.Errorf("discover datasets: invalid JSON: %w", err) + } + if len(datasetsResp.Datasets) == 0 { + return nil, fmt.Errorf("discover datasets: no datasets returned (DB likely not seeded yet; run the integration tests to create/map datasets before benchmarking)") + } + + // 2) try datasets until we find one that also works against OLD + var lastOldStatus int + for _, d := range datasetsResp.Datasets { + if d == "" { + continue + } + filesURL := newBase + "/datasets/" + url.PathEscape(d) + "/files" + resp, err := doJSONRequest(client, filesURL, cfg.Token) + if err != nil { + continue + } + var filesResp struct { + Files []discoveredFile `json:"files"` + } + if err := json.Unmarshal(resp, &filesResp); err != nil { + continue + } + if len(filesResp.Files) == 0 { + continue + } + + // Try first few files in the dataset + maxFiles := 5 + if len(filesResp.Files) < maxFiles { + maxFiles = len(filesResp.Files) + } + for i := 0; i < maxFiles; i++ { + f := filesResp.Files[i] + if f.FileID == "" { + continue + } + s3Path, err := deriveOldS3Path(f.FilePath) + if err != nil { + continue + } + candidate := &benchmarkCase{DatasetID: d, FileID: f.FileID, SubmissionPath: f.FilePath, S3Path: s3Path} + + if cfg.SkipOld { + return candidate, nil + } + + status, err := tryOldPreflight(client, cfg, candidate) + lastOldStatus = status + if err == nil { + return candidate, nil + } + } + } + + if cfg.SkipOld { + return nil, fmt.Errorf("failed to discover any file from NEW datasets") + } + return nil, fmt.Errorf("no discovered NEW dataset/file was accessible via OLD (last status=%d). This usually means the OLD service authorisation (visas) does not grant access to the datasets returned by the NEW service", lastOldStatus) +} + +func tryOldPreflight(client *http.Client, cfg Config, c *benchmarkCase) (int, error) { + ds := url.PathEscape(c.DatasetID) + p := url.PathEscape(c.S3Path) + u := strings.TrimRight(cfg.OldURL, "/") + "/s3/" + ds + "/" + p + + extra := map[string]string{ + "Range": "bytes=0-0", + } + if cfg.OldClientVersion != "" { + extra["SDA-Client-Version"] = cfg.OldClientVersion + } + + res := makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, "Client-Public-Key", extra) + if res.Error != nil { + return 0, res.Error + } + if res.StatusCode >= 400 { + return res.StatusCode, fmt.Errorf("old returned %d", res.StatusCode) + } + return res.StatusCode, nil +} + +func doJSONRequest(client *http.Client, urlStr, token string) ([]byte, error) { + req, err := http.NewRequest("GET", urlStr, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", token) + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1024*1024)) + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("%s returned %d: %s", urlStr, resp.StatusCode, strings.TrimSpace(string(body))) + } + + return body, nil +} + +func deriveOldS3Path(submittedPath string) (string, error) { + p := strings.TrimPrefix(submittedPath, "/") + p = strings.TrimSpace(p) + if p == "" { + return "", fmt.Errorf("empty submission path") + } + parts := strings.SplitN(p, "/", 2) + if len(parts) == 2 { + p = parts[1] + } else { + p = parts[0] + } + p = strings.TrimPrefix(p, "/") + p = strings.TrimSpace(p) + if p == "" { + return "", fmt.Errorf("could not derive old S3 path from submission path %q", submittedPath) + } + if !strings.HasSuffix(p, ".c4gh") { + p += ".c4gh" + } + return p, nil +} + +func validateConfig(cfg Config) error { + if cfg.SkipOld && cfg.SkipNew { + return fmt.Errorf("cannot skip both old and new implementations") + } + if cfg.Token == "" { + return fmt.Errorf("-token is required (or set TOKEN or mount /shared/token)") + } + if cfg.PublicKey == "" { + return fmt.Errorf("-pubkey is required (or set PUBKEY or mount /shared/c4gh.pub.pem)") + } + if !cfg.SkipOld && cfg.OldURL == "" { + return fmt.Errorf("-old URL is required (or use -skip-old)") + } + if !cfg.SkipNew && cfg.NewURL == "" { + return fmt.Errorf("-new URL is required (or use -skip-new)") + } + if !cfg.SkipNew && cfg.FileID == "" { + return fmt.Errorf("file ID is required for new implementation (use -file or set FILE_ID)") + } + if !cfg.SkipOld { + if cfg.DatasetID == "" { + return fmt.Errorf("dataset ID is required for old implementation (use -dataset or set FILE_DATASET)") + } + if cfg.S3Path == "" { + return fmt.Errorf("path is required for old implementation (use -path or set FILE_PATH)") + } + } + if cfg.Iterations < 1 { + return fmt.Errorf("-iterations must be at least 1") + } + if cfg.Requests < 1 { + return fmt.Errorf("-requests must be at least 1") + } + if cfg.Concurrency < 1 { + return fmt.Errorf("-concurrency must be at least 1") + } + return nil +} + +func preflight(client *http.Client, target Target, cfg Config) error { + u, err := target.BuildURL(cfg) + if err != nil { + return err + } + extra := map[string]string{ + "Range": "bytes=0-0", + } + if target.Name == "old" && cfg.OldClientVersion != "" { + extra["SDA-Client-Version"] = cfg.OldClientVersion + } + res := makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) + if res.Error != nil { + return res.Error + } + if res.StatusCode >= 400 { + return fmt.Errorf("%s preflight returned %d", target.Name, res.StatusCode) + } + if res.Bytes <= 0 { + // not necessarily fatal, but almost always indicates we’re not benchmarking a real file + fmt.Fprintf(os.Stderr, "[warn] %s preflight read %d bytes\n", target.Name, res.Bytes) + } + fmt.Fprintf(os.Stderr, "[preflight] %s OK (status=%d, bytes=%d)\n", strings.ToUpper(target.Name), res.StatusCode, res.Bytes) + return nil +} + +func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResult { + u, err := target.BuildURL(cfg) + if err != nil { + return BenchmarkResult{Name: target.Name, Requests: cfg.Requests, Failed: cfg.Requests, StatusCounts: map[int]int{0: cfg.Requests}} + } + + results := make([]RequestResult, cfg.Requests) + var wg sync.WaitGroup + semaphore := make(chan struct{}, cfg.Concurrency) + + startTime := time.Now() + + for i := 0; i < cfg.Requests; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + semaphore <- struct{}{} + defer func() { <-semaphore }() + + var extra map[string]string + if target.Name == "old" && cfg.OldClientVersion != "" { + extra = map[string]string{"SDA-Client-Version": cfg.OldClientVersion} + } + results[idx] = makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) + }(i) + } + + wg.Wait() + totalTime := time.Since(startTime) + + return aggregateResults(target.Name, results, totalTime) +} + +func makeRequest(client *http.Client, urlStr, token, publicKey, publicKeyHeader string) RequestResult { + return makeRequestWithExtras(client, urlStr, token, publicKey, publicKeyHeader, nil) +} + +func makeRequestWithExtras(client *http.Client, urlStr, token, publicKey, publicKeyHeader string, extraHeaders map[string]string) RequestResult { + req, err := http.NewRequest("GET", urlStr, nil) + if err != nil { + return RequestResult{Error: err} + } + req.Header.Set("Authorization", token) + req.Header.Set(publicKeyHeader, publicKey) + for k, v := range extraHeaders { + req.Header.Set(k, v) + } + + start := time.Now() + resp, err := client.Do(req) + if err != nil { + return RequestResult{Duration: time.Since(start), Error: err} + } + defer resp.Body.Close() + + bytesRead, _ := io.Copy(io.Discard, resp.Body) + duration := time.Since(start) + + return RequestResult{ + Duration: duration, + StatusCode: resp.StatusCode, + Bytes: bytesRead, + } +} + +func aggregateResults(name string, results []RequestResult, totalTime time.Duration) BenchmarkResult { + br := BenchmarkResult{ + Name: name, + TotalTime: totalTime, + Requests: len(results), + StatusCounts: make(map[int]int), + } + + for _, r := range results { + code := r.StatusCode + if r.Error != nil { + code = 0 + } + br.StatusCounts[code]++ + + if r.Error != nil || r.StatusCode >= 400 { + br.Failed++ + continue + } + + br.Successful++ + br.Latencies = append(br.Latencies, r.Duration) + br.BytesTotal += r.Bytes + } + + if totalTime > 0 { + br.RequestsPerS = float64(br.Successful) / totalTime.Seconds() + br.Throughput = float64(br.BytesTotal) / totalTime.Seconds() / 1024 / 1024 // MB/s + } + + return br +} + +func calculateStats(latencies []time.Duration) Stats { + if len(latencies) == 0 { + return Stats{} + } + + sorted := make([]time.Duration, len(latencies)) + copy(sorted, latencies) + sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + + var sum time.Duration + for _, l := range sorted { + sum += l + } + mean := time.Duration(int64(sum) / int64(len(sorted))) + + var variance float64 + for _, l := range sorted { + diff := float64(l - mean) + variance += diff * diff + } + stdDev := time.Duration(math.Sqrt(variance / float64(len(sorted)))) + + return Stats{ + Min: sorted[0], + Max: sorted[len(sorted)-1], + Mean: mean, + StdDev: stdDev, + P50: percentile(sorted, 50), + P90: percentile(sorted, 90), + P95: percentile(sorted, 95), + P99: percentile(sorted, 99), + } +} + +func percentile(sorted []time.Duration, p int) time.Duration { + if len(sorted) == 0 { + return 0 + } + idx := int(float64(len(sorted)-1) * float64(p) / 100) + + return sorted[idx] +} + +func calculateSummary(name string, results []BenchmarkResult) SummaryStats { + summary := SummaryStats{ + Name: name, + Iterations: len(results), + } + + if len(results) == 0 { + return summary + } + + var totalRPS, totalThroughput float64 + var totalSuccessful, totalRequests int + var allLatencies []time.Duration + + for _, r := range results { + totalRPS += r.RequestsPerS + totalThroughput += r.Throughput + totalSuccessful += r.Successful + totalRequests += r.Requests + allLatencies = append(allLatencies, r.Latencies...) + } + + n := float64(len(results)) + summary.AvgRequestsPerS = totalRPS / n + summary.AvgThroughput = totalThroughput / n + summary.SuccessRate = float64(totalSuccessful) / float64(totalRequests) * 100 + summary.AvgLatency = calculateStats(allLatencies) + + // Calculate RPS standard deviation + var variance float64 + for _, r := range results { + diff := r.RequestsPerS - summary.AvgRequestsPerS + variance += diff * diff + } + summary.StdDevRequestsPerS = math.Sqrt(variance / n) + + return summary +} + +func printIterationResult(r BenchmarkResult) { + stats := calculateStats(r.Latencies) + fmt.Printf(" Requests: %d successful, %d failed\n", r.Successful, r.Failed) + if r.Failed > 0 { + fmt.Printf(" Failures: %s\n", formatStatusCounts(r.StatusCounts)) + } + fmt.Printf(" RPS: %.2f req/s\n", r.RequestsPerS) + fmt.Printf(" Latency: mean=%v, p95=%v, p99=%v\n", stats.Mean.Round(time.Millisecond), stats.P95.Round(time.Millisecond), stats.P99.Round(time.Millisecond)) + fmt.Printf(" Throughput: %.2f MB/s\n", r.Throughput) +} + +func formatStatusCounts(m map[int]int) string { + if len(m) == 0 { + return "(none)" + } + type kv struct { + Code int + Count int + } + pairs := make([]kv, 0, len(m)) + for k, v := range m { + pairs = append(pairs, kv{Code: k, Count: v}) + } + sort.Slice(pairs, func(i, j int) bool { return pairs[i].Count > pairs[j].Count }) + + parts := make([]string, 0, len(pairs)) + for _, p := range pairs { + label := fmt.Sprintf("%d", p.Code) + if p.Code == 0 { + label = "err" + } + parts = append(parts, fmt.Sprintf("%s=%d", label, p.Count)) + } + return strings.Join(parts, ", ") +} + +func printSummary(result *ComparisonResult, cfg Config) { + fmt.Println("\n" + "=" + strings.Repeat("=", 70)) + fmt.Println(" SUMMARY") + fmt.Println("=" + strings.Repeat("=", 70)) + + if !cfg.SkipOld { + printSummaryStats(result.OldSummary) + } + + if !cfg.SkipNew { + printSummaryStats(result.NewSummary) + } + + // Print comparison if both were run + if !cfg.SkipOld && !cfg.SkipNew { + fmt.Println("\n" + "-" + strings.Repeat("-", 70)) + fmt.Println(" COMPARISON (NEW vs OLD)") + fmt.Println("-" + strings.Repeat("-", 70)) + + rpsChange := (result.NewSummary.AvgRequestsPerS - result.OldSummary.AvgRequestsPerS) / result.OldSummary.AvgRequestsPerS * 100 + latencyChange := float64(result.NewSummary.AvgLatency.Mean-result.OldSummary.AvgLatency.Mean) / float64(result.OldSummary.AvgLatency.Mean) * 100 + throughputChange := (result.NewSummary.AvgThroughput - result.OldSummary.AvgThroughput) / result.OldSummary.AvgThroughput * 100 + + fmt.Printf("\n Requests/sec: %+.1f%% ", rpsChange) + printChangeIndicator(rpsChange, true) + + fmt.Printf(" Mean Latency: %+.1f%% ", latencyChange) + printChangeIndicator(latencyChange, false) + + fmt.Printf(" Throughput: %+.1f%% ", throughputChange) + printChangeIndicator(throughputChange, true) + + fmt.Println() + + // Verdict + fmt.Println("\n Verdict:") + if rpsChange > 5 { + fmt.Println(" NEW implementation is FASTER") + } else if rpsChange < -5 { + fmt.Println(" OLD implementation is FASTER") + } else { + fmt.Println(" Performance is SIMILAR (within 5%)") + } + } + + fmt.Println() +} + +func printSummaryStats(s SummaryStats) { + fmt.Printf("\n %s Implementation (%d iterations):\n", s.Name, s.Iterations) + fmt.Printf(" Requests/sec: %.2f (+/- %.2f)\n", s.AvgRequestsPerS, s.StdDevRequestsPerS) + fmt.Printf(" Success rate: %.1f%%\n", s.SuccessRate) + fmt.Printf(" Throughput: %.2f MB/s\n", s.AvgThroughput) + fmt.Println(" Latency:") + fmt.Printf(" Mean: %v\n", s.AvgLatency.Mean.Round(time.Millisecond)) + fmt.Printf(" StdDev: %v\n", s.AvgLatency.StdDev.Round(time.Millisecond)) + fmt.Printf(" Min: %v\n", s.AvgLatency.Min.Round(time.Millisecond)) + fmt.Printf(" P50: %v\n", s.AvgLatency.P50.Round(time.Millisecond)) + fmt.Printf(" P90: %v\n", s.AvgLatency.P90.Round(time.Millisecond)) + fmt.Printf(" P95: %v\n", s.AvgLatency.P95.Round(time.Millisecond)) + fmt.Printf(" P99: %v\n", s.AvgLatency.P99.Round(time.Millisecond)) + fmt.Printf(" Max: %v\n", s.AvgLatency.Max.Round(time.Millisecond)) +} + +func printChangeIndicator(change float64, higherIsBetter bool) { + good := (change > 0) == higherIsBetter + if math.Abs(change) < 5 { + fmt.Println("(~)") + } else if good { + fmt.Println("(better)") + } else { + fmt.Println("(worse)") + } +} + +func printJSONOutput(result *ComparisonResult) { + fmt.Println("\n" + "-" + strings.Repeat("-", 70)) + fmt.Println(" JSON Output") + fmt.Println("-" + strings.Repeat("-", 70)) + + output, _ := json.MarshalIndent(result, "", " ") + fmt.Println(string(output)) +} diff --git a/sda/cmd/download/download.go b/sda/cmd/download/download.go deleted file mode 100644 index 38dd16da61..0000000000 --- a/sda/cmd/download/download.go +++ /dev/null @@ -1,3 +0,0 @@ -package main - -func main() {} diff --git a/sda/go.sum b/sda/go.sum index 99f583566b..941e24199d 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -11,6 +11,8 @@ github.com/CloudyKit/fastprinter v0.0.0-20251202014920-1725d2651bd4 h1:DQ1+lDdBv github.com/CloudyKit/fastprinter v0.0.0-20251202014920-1725d2651bd4/go.mod h1:eauGmjfZG874MOAEPVeqg21mZCbTOLW+tFe8F7NpfnY= github.com/CloudyKit/jet/v6 v6.3.1 h1:6IAo5Cx21xrHVaR8zzXN5gJatKV/wO7Nf6bfCnCSbUw= github.com/CloudyKit/jet/v6 v6.3.1/go.mod h1:lf8ksdNsxZt7/yH/3n4vJQWA9RUq4wpaHtArHhGVMOw= +github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= +github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk= @@ -19,6 +21,8 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 h1:dG7/gLroJGht/jSQtHiLvT48Hxn+crbmvyItZC8cWOs= github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= @@ -87,6 +91,8 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= @@ -110,6 +116,10 @@ github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFn github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM= @@ -221,6 +231,7 @@ github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA= github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= +github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -336,6 +347,8 @@ github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -354,6 +367,7 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -513,6 +527,7 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From ebccf5015fb864d72c3a7c301d84ce1ec9871924 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:46:54 +0100 Subject: [PATCH 379/530] fix(download): update S3 integration test to v2 paths, fix CodeQL overflow hint - 55_download_test.sh: update from v1 API paths (/info/datasets, /file/{id}, public_key header) to v2 (/datasets, /files/{id}, X-C4GH-Public-Key). Fixes CI failure in sda (s3) integration job. - middleware/auth.go: use len(a) instead of len(a)+len(b) as map capacity hint in mergeDatasets to silence CodeQL overflow warning. --- .../integration/tests/sda/55_download_test.sh | 100 ++++++------------ sda/cmd/download/middleware/auth.go | 2 +- 2 files changed, 34 insertions(+), 68 deletions(-) diff --git a/.github/integration/tests/sda/55_download_test.sh b/.github/integration/tests/sda/55_download_test.sh index 10390a7647..20223aa2b4 100755 --- a/.github/integration/tests/sda/55_download_test.sh +++ b/.github/integration/tests/sda/55_download_test.sh @@ -1,17 +1,16 @@ #!/bin/bash set -e -# Test the new sda/cmd/download service (download-new container) +# Test the new sda/cmd/download service (v2 API) # This test runs AFTER files have been mapped to a dataset (40_mapper_test.sh) cd shared || true -echo "=== Testing Download Service (NEW implementation) ===" +echo "=== Testing Download Service (v2 API) ===" -# Get token from the auth-aai service token="$(cat /shared/token)" -# Test 1: Health endpoint (no auth required) +# Test 1: Health endpoints echo "Test 1: Health endpoints..." response=$(curl -s -o /dev/null -w "%{http_code}" "http://download:8080/health/live") if [ "$response" != "200" ]; then @@ -29,7 +28,7 @@ echo " /health/ready: OK" # Test 2: Unauthenticated request should return 401 echo "Test 2: Unauthenticated access..." -response=$(curl -s -o /dev/null -w "%{http_code}" "http://download:8080/info/datasets") +response=$(curl -s -o /dev/null -w "%{http_code}" "http://download:8080/datasets") if [ "$response" != "401" ]; then echo "::error::Unauthenticated request should return 401, got: $response" exit 1 @@ -38,10 +37,10 @@ echo " Unauthenticated request returns 401: OK" # Test 3: List datasets with valid token echo "Test 3: List datasets..." -datasets=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/info/datasets") -dataset_count=$(echo "$datasets" | jq '. | length') +body=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/datasets") +dataset_count=$(echo "$body" | jq '.datasets | length') if [ "$dataset_count" -lt 1 ]; then - echo " Note: No datasets available yet (run after 40_mapper_test.sh)" + echo " Note: No datasets available yet" echo " Skipping remaining tests that require datasets..." echo "" echo "=== Download Service Tests Completed (partial) ===" @@ -49,30 +48,28 @@ if [ "$dataset_count" -lt 1 ]; then fi echo " Found $dataset_count dataset(s): OK" -# Get first dataset ID (API returns array of objects with 'id' field) -dataset_id=$(echo "$datasets" | jq -r '.[0].id') +dataset_id=$(echo "$body" | jq -r '.datasets[0]') echo " Using dataset: $dataset_id" -# Test 4: Get dataset info (using query parameter as per swagger spec) +# Test 4: Get dataset info echo "Test 4: Get dataset info..." -dataset_info=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/info/dataset?dataset=$dataset_id") -# Check for 'id' field in response -dataset_info_id=$(echo "$dataset_info" | jq -r '.id') +dataset_info=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/datasets/$dataset_id") +dataset_info_id=$(echo "$dataset_info" | jq -r '.datasetId') if [ -z "$dataset_info_id" ] || [ "$dataset_info_id" = "null" ]; then echo "::error::Failed to get dataset info" echo "Response: $dataset_info" exit 1 fi -file_count=$(echo "$dataset_info" | jq -r '.fileCount // 0') -total_size=$(echo "$dataset_info" | jq -r '.totalSize // 0') +file_count=$(echo "$dataset_info" | jq -r '.files // 0') +total_size=$(echo "$dataset_info" | jq -r '.size // 0') echo " Dataset: $dataset_info_id (files: $file_count, size: $total_size bytes): OK" -# Test 5: List files in dataset (using query parameter as per swagger spec) +# Test 5: List files in dataset echo "Test 5: List files in dataset..." -files=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/info/dataset/files?dataset=$dataset_id") -file_count=$(echo "$files" | jq '. | length // 0') +files_body=$(curl -s -H "Authorization: Bearer $token" "http://download:8080/datasets/$dataset_id/files") +file_count=$(echo "$files_body" | jq '.files | length // 0') if [ "$file_count" -lt 1 ]; then - echo " Note: No files in dataset (run after full pipeline: ingest → verify → finalize → mapper)" + echo " Note: No files in dataset" echo " Skipping remaining tests that require files..." echo "" echo "=== Download Service Tests Completed (partial) ===" @@ -80,38 +77,30 @@ if [ "$file_count" -lt 1 ]; then fi echo " Found $file_count file(s) in dataset: OK" -# Get first file info -file_id=$(echo "$files" | jq -r '.[0].fileId') -file_size=$(echo "$files" | jq -r '.[0].decryptedSize // 0') -echo " Using file: $file_id (decrypted size: $file_size bytes)" +file_id=$(echo "$files_body" | jq -r '.files[0].fileId') +echo " Using file: $file_id" -# Test 6: Download file with public key (if reencrypt service is available) +# Test 6: Download file with public key echo "Test 6: Download file with re-encryption..." if [ -f "/shared/c4gh.pub.pem" ]; then pubkey=$(base64 -w0 /shared/c4gh.pub.pem) - - # Download the file using /file/{fileId} endpoint + response_code=$(curl -s -o /tmp/downloaded_file.c4gh -w "%{http_code}" \ -H "Authorization: Bearer $token" \ - -H "public_key: $pubkey" \ - "http://download:8080/file/$file_id") - + -H "X-C4GH-Public-Key: $pubkey" \ + "http://download:8080/files/$file_id") + if [ "$response_code" = "200" ]; then downloaded_size=$(stat -c '%s' /tmp/downloaded_file.c4gh 2>/dev/null || echo "0") echo " Downloaded file size: $downloaded_size bytes: OK" - - # Verify it's a valid crypt4gh file (starts with "crypt4gh") + file_magic=$(head -c 8 /tmp/downloaded_file.c4gh | od -A n -t x1 | tr -d ' ') if [ "$file_magic" = "6372797074346768" ]; then echo " File has valid crypt4gh magic: OK" else echo " Warning: File may not be valid crypt4gh format" fi - elif [ "$response_code" = "501" ]; then - echo " Skipping: Re-encryption not implemented yet" elif [ "$response_code" = "500" ]; then - # 500 can happen if reencrypt service doesn't have the right key - # This is a test data issue, not a download service bug echo " Note: Re-encryption failed (likely key mismatch in test data): SKIPPED" else echo "::error::Download failed with status: $response_code" @@ -121,41 +110,18 @@ else echo " Skipping: No public key available at /shared/c4gh.pub.pem" fi -# Test 7: Range request (partial download) -echo "Test 7: Range request..." -if [ -f "/shared/c4gh.pub.pem" ]; then - pubkey=$(base64 -w0 /shared/c4gh.pub.pem) - - response=$(curl -s -o /dev/null -w "%{http_code}" \ - -H "Authorization: Bearer $token" \ - -H "public_key: $pubkey" \ - -H "Range: bytes=0-99" \ - "http://download:8080/file/$file_id") - - if [ "$response" = "206" ] || [ "$response" = "200" ]; then - echo " Range request works: OK" - elif [ "$response" = "501" ]; then - echo " Skipping: Range requests not implemented yet" - elif [ "$response" = "500" ]; then - echo " Note: Range request failed (likely key mismatch): SKIPPED" - else - echo " Warning: Range request returned: $response" - fi -fi - -# Test 8: Access denied for non-permitted file -echo "Test 8: Access control..." -# Try to access a file that doesn't exist -fake_file_id="EGAF00000000000" +# Test 7: Access denied for non-existent file (no existence leakage) +echo "Test 7: Access control..." response=$(curl -s -o /dev/null -w "%{http_code}" \ -H "Authorization: Bearer $token" \ - -H "public_key: dGVzdA==" \ - "http://download:8080/file/$fake_file_id") + -H "X-C4GH-Public-Key: dGVzdA==" \ + "http://download:8080/files/EGAF00000000000") -if [ "$response" = "404" ] || [ "$response" = "403" ]; then - echo " Non-existent file returns $response: OK" +if [ "$response" = "403" ]; then + echo " Non-existent file returns 403: OK" else - echo " Warning: Expected 404 or 403 for non-existent file, got: $response" + echo "::error::Expected 403 for non-existent file, got: $response" + exit 1 fi echo "" diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index 2150d79ddb..120bccf4a2 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -666,7 +666,7 @@ func (sc *SessionCache) Set(key string, value AuthContext, ttl time.Duration) { // mergeDatasets returns a deduplicated union of two string slices. func mergeDatasets(a, b []string) []string { - seen := make(map[string]bool, len(a)+len(b)) + seen := make(map[string]bool, len(a)) var result []string for _, s := range a { From a0a6f13fbdff25098b8605b3601ee29844286384 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 13:35:42 +0100 Subject: [PATCH 380/530] fix(download): resolve golangci-lint warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - benchmark.go: fmt.Errorf → errors.New where no format verbs, nlreturn blank lines, ifElseChain → switch, nolint for gosec G402 (TLS skip in benchmark tool) and revive deep-exit - handlers/files.go: remove duplicate ArchivePath check - middleware/auth.go: extract tokenExpiry() to reduce nestif - visa/: nolint:gosec for SSRF (URLs validated against allowlist) --- sda/cmd/download/benchmark/benchmark.go | 83 ++++++++++++++----------- sda/cmd/download/handlers/files.go | 7 --- sda/cmd/download/middleware/auth.go | 25 +++++--- sda/cmd/download/visa/jwks_cache.go | 4 +- sda/cmd/download/visa/userinfo.go | 4 +- 5 files changed, 68 insertions(+), 55 deletions(-) diff --git a/sda/cmd/download/benchmark/benchmark.go b/sda/cmd/download/benchmark/benchmark.go index d73ba87e99..a9885868f7 100644 --- a/sda/cmd/download/benchmark/benchmark.go +++ b/sda/cmd/download/benchmark/benchmark.go @@ -29,6 +29,7 @@ import ( "crypto/tls" "encoding/base64" "encoding/json" + "errors" "flag" "fmt" "io" @@ -150,10 +151,11 @@ func main() { PublicKeyHeader: "Client-Public-Key", BuildURL: func(cfg Config) (string, error) { if cfg.DatasetID == "" || cfg.S3Path == "" { - return "", fmt.Errorf("dataset and path are required for old implementation") + return "", errors.New("dataset and path are required for old implementation") } ds := url.PathEscape(cfg.DatasetID) p := url.PathEscape(cfg.S3Path) + return strings.TrimRight(cfg.OldURL, "/") + "/s3/" + ds + "/" + p, nil }, } @@ -163,9 +165,10 @@ func main() { PublicKeyHeader: "X-C4GH-Public-Key", BuildURL: func(cfg Config) (string, error) { if cfg.FileID == "" { - return "", fmt.Errorf("file ID is required for new implementation") + return "", errors.New("file ID is required for new implementation") } id := url.PathEscape(cfg.FileID) + return strings.TrimRight(cfg.NewURL, "/") + "/files/" + id, nil }, } @@ -173,7 +176,7 @@ func main() { fmt.Println("=" + strings.Repeat("=", 70)) fmt.Println(" Download Service Benchmark") fmt.Println("=" + strings.Repeat("=", 70)) - fmt.Printf("\nConfiguration:\n") + fmt.Println("\nConfiguration:") fmt.Printf(" Iterations: %d\n", cfg.Iterations) fmt.Printf(" Requests: %d per iteration\n", cfg.Requests) fmt.Printf(" Concurrency: %d\n", cfg.Concurrency) @@ -192,7 +195,7 @@ func main() { Timeout: cfg.Timeout, Transport: &http.Transport{ MaxIdleConnsPerHost: cfg.Concurrency, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // benchmark tool, not production DisableCompression: true, }, } @@ -288,7 +291,7 @@ func parseFlags() Config { flag.BoolVar(&cfg.SkipNew, "skip-new", defaults.SkipNew, "Skip benchmarking new implementation") flag.BoolVar(&cfg.OutputJSON, "json", false, "Output results as JSON") - flag.Parse() + flag.Parse() //nolint:revive // deep-exit: called only from main() return cfg } @@ -370,7 +373,7 @@ type discoveredFile struct { func autoDiscoverIfNeeded(cfg *Config) error { if cfg == nil { - return fmt.Errorf("nil config") + return errors.New("nil config") } needNew := !cfg.SkipNew && cfg.FileID == "" @@ -381,10 +384,10 @@ func autoDiscoverIfNeeded(cfg *Config) error { } if cfg.NewURL == "" { - return fmt.Errorf("cannot auto-discover without NEW_URL (or -new)") + return errors.New("cannot auto-discover without NEW_URL (or -new)") } if cfg.Token == "" { - return fmt.Errorf("cannot auto-discover without token") + return errors.New("cannot auto-discover without token") } caseInfo, err := discoverBenchmarkCase(*cfg) @@ -417,17 +420,17 @@ type benchmarkCase struct { func discoverBenchmarkCase(cfg Config) (*benchmarkCase, error) { if cfg.NewURL == "" { - return nil, fmt.Errorf("cannot auto-discover without NEW_URL (or -new)") + return nil, errors.New("cannot auto-discover without NEW_URL (or -new)") } if cfg.Token == "" { - return nil, fmt.Errorf("cannot auto-discover without token") + return nil, errors.New("cannot auto-discover without token") } if !cfg.SkipOld { if cfg.OldURL == "" { - return nil, fmt.Errorf("cannot auto-discover a comparable file without OLD_URL (or -old)") + return nil, errors.New("cannot auto-discover a comparable file without OLD_URL (or -old)") } if cfg.PublicKey == "" { - return nil, fmt.Errorf("cannot auto-discover a comparable file without public key") + return nil, errors.New("cannot auto-discover a comparable file without public key") } } @@ -435,7 +438,7 @@ func discoverBenchmarkCase(cfg Config) (*benchmarkCase, error) { Timeout: cfg.Timeout, Transport: &http.Transport{ DisableCompression: true, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // benchmark tool, not production }, } @@ -454,7 +457,7 @@ func discoverBenchmarkCase(cfg Config) (*benchmarkCase, error) { return nil, fmt.Errorf("discover datasets: invalid JSON: %w", err) } if len(datasetsResp.Datasets) == 0 { - return nil, fmt.Errorf("discover datasets: no datasets returned (DB likely not seeded yet; run the integration tests to create/map datasets before benchmarking)") + return nil, errors.New("discover datasets: no datasets returned (DB likely not seeded yet; run the integration tests to create/map datasets before benchmarking)") } // 2) try datasets until we find one that also works against OLD @@ -507,8 +510,9 @@ func discoverBenchmarkCase(cfg Config) (*benchmarkCase, error) { } if cfg.SkipOld { - return nil, fmt.Errorf("failed to discover any file from NEW datasets") + return nil, errors.New("failed to discover any file from NEW datasets") } + return nil, fmt.Errorf("no discovered NEW dataset/file was accessible via OLD (last status=%d). This usually means the OLD service authorisation (visas) does not grant access to the datasets returned by the NEW service", lastOldStatus) } @@ -531,6 +535,7 @@ func tryOldPreflight(client *http.Client, cfg Config, c *benchmarkCase) (int, er if res.StatusCode >= 400 { return res.StatusCode, fmt.Errorf("old returned %d", res.StatusCode) } + return res.StatusCode, nil } @@ -541,7 +546,7 @@ func doJSONRequest(client *http.Client, urlStr, token string) ([]byte, error) { } req.Header.Set("Authorization", token) - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:gosec // benchmark tool, URLs from CLI flags if err != nil { return nil, err } @@ -559,7 +564,7 @@ func deriveOldS3Path(submittedPath string) (string, error) { p := strings.TrimPrefix(submittedPath, "/") p = strings.TrimSpace(p) if p == "" { - return "", fmt.Errorf("empty submission path") + return "", errors.New("empty submission path") } parts := strings.SplitN(p, "/", 2) if len(parts) == 2 { @@ -575,45 +580,47 @@ func deriveOldS3Path(submittedPath string) (string, error) { if !strings.HasSuffix(p, ".c4gh") { p += ".c4gh" } + return p, nil } func validateConfig(cfg Config) error { if cfg.SkipOld && cfg.SkipNew { - return fmt.Errorf("cannot skip both old and new implementations") + return errors.New("cannot skip both old and new implementations") } if cfg.Token == "" { - return fmt.Errorf("-token is required (or set TOKEN or mount /shared/token)") + return errors.New("-token is required (or set TOKEN or mount /shared/token)") } if cfg.PublicKey == "" { - return fmt.Errorf("-pubkey is required (or set PUBKEY or mount /shared/c4gh.pub.pem)") + return errors.New("-pubkey is required (or set PUBKEY or mount /shared/c4gh.pub.pem)") } if !cfg.SkipOld && cfg.OldURL == "" { - return fmt.Errorf("-old URL is required (or use -skip-old)") + return errors.New("-old URL is required (or use -skip-old)") } if !cfg.SkipNew && cfg.NewURL == "" { - return fmt.Errorf("-new URL is required (or use -skip-new)") + return errors.New("-new URL is required (or use -skip-new)") } if !cfg.SkipNew && cfg.FileID == "" { - return fmt.Errorf("file ID is required for new implementation (use -file or set FILE_ID)") + return errors.New("file ID is required for new implementation (use -file or set FILE_ID)") } if !cfg.SkipOld { if cfg.DatasetID == "" { - return fmt.Errorf("dataset ID is required for old implementation (use -dataset or set FILE_DATASET)") + return errors.New("dataset ID is required for old implementation (use -dataset or set FILE_DATASET)") } if cfg.S3Path == "" { - return fmt.Errorf("path is required for old implementation (use -path or set FILE_PATH)") + return errors.New("path is required for old implementation (use -path or set FILE_PATH)") } } if cfg.Iterations < 1 { - return fmt.Errorf("-iterations must be at least 1") + return errors.New("-iterations must be at least 1") } if cfg.Requests < 1 { - return fmt.Errorf("-requests must be at least 1") + return errors.New("-requests must be at least 1") } if cfg.Concurrency < 1 { - return fmt.Errorf("-concurrency must be at least 1") + return errors.New("-concurrency must be at least 1") } + return nil } @@ -640,6 +647,7 @@ func preflight(client *http.Client, target Target, cfg Config) error { fmt.Fprintf(os.Stderr, "[warn] %s preflight read %d bytes\n", target.Name, res.Bytes) } fmt.Fprintf(os.Stderr, "[preflight] %s OK (status=%d, bytes=%d)\n", strings.ToUpper(target.Name), res.StatusCode, res.Bytes) + return nil } @@ -692,7 +700,7 @@ func makeRequestWithExtras(client *http.Client, urlStr, token, publicKey, public } start := time.Now() - resp, err := client.Do(req) + resp, err := client.Do(req) //nolint:gosec // benchmark tool, URLs from CLI flags if err != nil { return RequestResult{Duration: time.Since(start), Error: err} } @@ -725,6 +733,7 @@ func aggregateResults(name string, results []RequestResult, totalTime time.Durat if r.Error != nil || r.StatusCode >= 400 { br.Failed++ + continue } @@ -856,6 +865,7 @@ func formatStatusCounts(m map[int]int) string { } parts = append(parts, fmt.Sprintf("%s=%d", label, p.Count)) } + return strings.Join(parts, ", ") } @@ -895,11 +905,12 @@ func printSummary(result *ComparisonResult, cfg Config) { // Verdict fmt.Println("\n Verdict:") - if rpsChange > 5 { + switch { + case rpsChange > 5: fmt.Println(" NEW implementation is FASTER") - } else if rpsChange < -5 { + case rpsChange < -5: fmt.Println(" OLD implementation is FASTER") - } else { + default: fmt.Println(" Performance is SIMILAR (within 5%)") } } @@ -925,11 +936,13 @@ func printSummaryStats(s SummaryStats) { func printChangeIndicator(change float64, higherIsBetter bool) { good := (change > 0) == higherIsBetter - if math.Abs(change) < 5 { + + switch { + case math.Abs(change) < 5: fmt.Println("(~)") - } else if good { + case good: fmt.Println("(better)") - } else { + default: fmt.Println("(worse)") } } diff --git a/sda/cmd/download/handlers/files.go b/sda/cmd/download/handlers/files.go index 1bf9667904..691b3ddc8a 100644 --- a/sda/cmd/download/handlers/files.go +++ b/sda/cmd/download/handlers/files.go @@ -91,13 +91,6 @@ func (h *Handlers) resolveFileBase(c *gin.Context) (*resolvedBase, bool) { return nil, false } - if file.ArchivePath == "" { - log.Errorf("file %s has no archive path", file.ID) - problemJSON(c, http.StatusInternalServerError, "file not in archive") - - return nil, false - } - // Determine storage location var location string if file.ArchiveLocation != "" { diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index 120bccf4a2..8971f8e285 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -558,15 +558,7 @@ func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogge // Create session and token cache entries if !skipCache { - // Extract JWT token expiry for cache TTL bounding - var tokenExp *time.Time - if authCtx.Token != nil { - exp := authCtx.Token.Expiration() - if !exp.IsZero() { - tokenExp = &exp - } - } - + tokenExp := tokenExpiry(authCtx) cacheTTL := computeCacheTTL(tokenExp, visaMinExpiry) // Guard against zero or negative TTL (expired token/visa) @@ -726,6 +718,21 @@ func sha256Hex(s string) string { // JWT + no accepted visas: min(configTTL, token.exp - now) // Opaque + accepted visas: min(configTTL, min(visa.exp) - now) // Opaque + no accepted visas: configTTL + +// tokenExpiry extracts the JWT expiry from the auth context, or nil for opaque tokens. +func tokenExpiry(authCtx AuthContext) *time.Time { + if authCtx.Token == nil { + return nil + } + + exp := authCtx.Token.Expiration() + if exp.IsZero() { + return nil + } + + return &exp +} + func computeCacheTTL(tokenExp *time.Time, visaMinExpiry *time.Time) time.Duration { configTTL := time.Duration(config.VisaCacheTokenTTL()) * time.Second if configTTL == 0 { diff --git a/sda/cmd/download/visa/jwks_cache.go b/sda/cmd/download/visa/jwks_cache.go index 213b001efe..158ea6cbe1 100644 --- a/sda/cmd/download/visa/jwks_cache.go +++ b/sda/cmd/download/visa/jwks_cache.go @@ -106,13 +106,13 @@ func (jc *JWKSCache) fetchJWKS(jkuURL string) (jwk.Set, error) { log.Debugf("fetching JWKS from %s", jkuURL) - req, err := http.NewRequestWithContext(context.Background(), "GET", jkuURL, nil) + req, err := http.NewRequestWithContext(context.Background(), "GET", jkuURL, nil) //nolint:gosec // jkuURL is validated against trusted issuers allowlist before fetch if err != nil { return nil, fmt.Errorf("failed to create JWKS request: %w", err) } req.Header.Set("Accept", "application/json") - resp, err := jc.httpClient.Do(req) + resp, err := jc.httpClient.Do(req) //nolint:gosec // jkuURL validated against trusted issuers allowlist if err != nil { return nil, fmt.Errorf("JWKS fetch failed for %s: %w", jkuURL, err) } diff --git a/sda/cmd/download/visa/userinfo.go b/sda/cmd/download/visa/userinfo.go index 9e159e902f..46790bf966 100644 --- a/sda/cmd/download/visa/userinfo.go +++ b/sda/cmd/download/visa/userinfo.go @@ -87,14 +87,14 @@ func (uc *UserinfoClient) FetchUserinfo(accessToken string) (*UserinfoResponse, return nil, errors.New("userinfo endpoint URL not configured") } - req, err := http.NewRequest("GET", endpoint, nil) + req, err := http.NewRequest("GET", endpoint, nil) //nolint:gosec // endpoint is from OIDC discovery or config, not user input if err != nil { return nil, fmt.Errorf("failed to create userinfo request: %w", err) } req.Header.Set("Authorization", "Bearer "+accessToken) req.Header.Set("Accept", "application/json") - resp, err := uc.httpClient.Do(req) + resp, err := uc.httpClient.Do(req) //nolint:gosec // endpoint from OIDC discovery or config if err != nil { return nil, fmt.Errorf("userinfo request failed: %w", err) } From db788c504c436daf9aea716734f2b600a52206ed Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 22:18:42 +0100 Subject: [PATCH 381/530] docs(download): rewrite download.md for v2 service Rewrites the service documentation to match the actual v2 API routes, authentication model, GA4GH visa support, and full configuration reference. The previous version documented draft/old routes that did not match the implementation. --- sda/cmd/download/download.md | 599 ++++++++++++++++++++++------------- 1 file changed, 376 insertions(+), 223 deletions(-) diff --git a/sda/cmd/download/download.md b/sda/cmd/download/download.md index 5fbe4f0adc..10a2155070 100644 --- a/sda/cmd/download/download.md +++ b/sda/cmd/download/download.md @@ -1,264 +1,423 @@ # Download -The Download service provides secure file download functionality for archived data. -Users are authenticated with a JWT token, and files are re-encrypted with the user's -public key before download. +The download service provides the Data Out API for the NeIC Sensitive Data Archive. +It replaces both the standalone [sda-download](https://github.com/neicnordic/sda-download) +Go service and the Java-based [sda-doa](https://github.com/neicnordic/sda-doa), unifying +data retrieval into a single service within the `sda` binary. + +Key improvements over the previous implementations: + +- **GA4GH Passport/Visa support** with trusted issuer allowlists and JWKS validation +- **Three permission models**: ownership-only, visa-only, or combined +- **Split file endpoints** — separate header and content downloads for efficient Crypt4GH operations +- **Keyset pagination** with HMAC-signed page tokens +- **Multi-layer caching** — database queries, sessions, tokens, JWK sets, and visa validation results +- **Audit logging** for compliance tracking +- **Production safety guards** that prevent dangerous configurations in production ## Service Description -This service implements the Data Out API for the NeIC Sensitive Data Archive. It allows -authenticated users to browse datasets and download files that have been archived and -verified. +The service is a standalone HTTP server built with Gin. It connects to: + +- **PostgreSQL** — file/dataset metadata and permissions +- **S3/POSIX storage** — archived file data +- **gRPC reencrypt service** — Crypt4GH re-encryption for downloads +- **OIDC provider** (optional) — userinfo endpoint for opaque tokens and GA4GH passports + +On startup the service validates configuration, initializes caches, and runs +production safety guards when `app.environment` is set to `production`. + +## Authentication + +All endpoints except `/health/*` and `/service-info` require authentication. +Tokens are extracted from the `Authorization: Bearer ` header or the +`X-Amz-Security-Token` header. + +The service uses structure-based detection to classify tokens: + +### JWT Tokens + +Tokens with three dot-separated base64url segments (valid JSON header and payload) +are treated as JWTs. They are validated locally against public keys loaded from +`jwt.pubkey-path` or fetched from `jwt.pubkey-url`. If `oidc.issuer` is configured, +the JWT `iss` claim must match. JWT validation failures are final — there is no +userinfo fallback. + +### Opaque Tokens + +When `auth.allow-opaque` is enabled (default: `true`), tokens that do not look like +JWTs are sent to the OIDC userinfo endpoint. The `sub` claim from the userinfo +response is used as the user identity. The issuer is taken from `oidc.issuer`. + +### Session Caching + +Authenticated sessions are cached in-memory keyed by `sha256(token)`. The cache TTL +is bounded by `min(token.exp, min(visa.exp), configured TTL)`. A session cookie +(configurable via `session.name`, default `sda_session`) provides fast lookups for +repeat requests. The legacy cookie name `sda_session_key` is also checked for +backwards compatibility. + +## Permission Model + +The permission model is configured via `permission.model` (default: `combined`). + +| Model | Description | +|-------------|---------------------------------------------------------------------| +| `ownership` | Users access datasets containing files where they are the `submission_user` | +| `visa` | Access determined solely by GA4GH ControlledAccessGrants visas | +| `combined` | Union of ownership and visa datasets. Visa failures are non-fatal — ownership results are still served | + +When `jwt.allow-all-data` is `true`, all authenticated users can access all +datasets regardless of the permission model. **This flag is blocked by production +safety guards.** + +## GA4GH Visa Support + +When `visa.enabled` is `true`, the service validates +[GA4GH Passport v1](https://github.com/ga4gh/data-security/blob/master/AAI/AAIConnectProfile.md) +visas to determine dataset access. + +### Visa Validation Flow + +1. After authentication, the userinfo endpoint is called to retrieve the `ga4gh_passport_v1` claim +2. Each visa JWT in the passport is validated: + - The `(iss, jku)` pair must appear in the trusted issuers allowlist + - The JWKS at the `jku` URL is fetched (cached) and the visa signature is verified + - The visa must not be expired +3. Only `ControlledAccessGrants` type visas are processed: + - `by` must be non-empty + - `value` and `source` must be ≤ 255 characters + - `conditions` must be empty + - `asserted` must not be in the future (when `visa.validate-asserted` is `true`) +4. The `value` field determines dataset access. Unknown visa types are silently ignored per the GA4GH spec. + +### Trusted Issuers -### Authentication +A JSON file at `visa.trusted-issuers-path` defines the allowed `(iss, jku)` pairs: -All endpoints except health checks require JWT authentication. The token must be: +```json +[ + {"iss": "https://login.example.org", "jku": "https://login.example.org/jwks"} +] +``` + +Only visas from issuers in this allowlist are accepted. JKU URLs must use HTTPS +unless `visa.allow-insecure-jku` is enabled (testing only). + +### Dataset ID Matching -- Provided in the `Authorization: Bearer ` header -- Signed with a key matching the configured JWKS (via `jwt.pubkey-path` or `jwt.pubkey-url`) +Configured via `visa.dataset-id-mode`: -### Permission Model +| Mode | Description | +|----------|----------------------------------------------------------| +| `raw` | The visa `value` is used as-is as the dataset identifier | +| `suffix` | The last segment of the URL or URN path is extracted | -The service supports two permission modes (configured via `jwt.allow-all-data`): +### Identity Binding -| Mode | Description | -| ----------------------- | --------------------------------------------------------- | -| `allow-all-data: true` | All authenticated users can access all datasets (testing) | -| `allow-all-data: false` | Users can only access datasets they submitted | +Configured via `visa.identity.mode`: -In production, the data ownership model is used: users have access to datasets -containing files where they are the `submission_user`. +| Mode | Description | +|-------------------|-----------------------------------------------------------------------| +| `broker-bound` | Default. Visa identity is not checked against the authenticated user | +| `strict-sub` | Visa `sub` must match the authenticated user's `sub` | +| `strict-iss-sub` | Visa `iss`+`sub` must match the authenticated user's `iss`+`sub` | + +### Safety Limits + +| Config | Default | Description | +|-----------------------------------|---------|-----------------------------------------| +| `visa.limits.max-visas` | 200 | Maximum visas to process per passport | +| `visa.limits.max-visa-size` | 16384 | Maximum size per visa JWT (bytes) | +| `visa.limits.max-jwks-per-request`| 10 | Maximum distinct JWKS fetches per request | ## Endpoints ### Health Endpoints -#### `/health/ready` +#### `GET /health/ready` + +Returns readiness status with dependency checks for database, storage, gRPC, and OIDC. + +Example: + +```bash +curl https://HOSTNAME/health/ready +{"status":"ok","services":{"database":"ok","storage":"ok","grpc":"ok","oidc":"ok"}} +``` + +#### `GET /health/live` -- accepts `GET` requests -- Returns readiness status with dependency checks +Returns simple liveness status. - Example: +Example: - ```bash - curl https://HOSTNAME/health/ready - {"status":"ok","services":{"database":"ok","storage":"ok","grpc":"ok","oidc":"ok"}} - ``` +```bash +curl https://HOSTNAME/health/live +{"status":"ok"} +``` + +### Service Info -#### `/health/live` +#### `GET /service-info` -- accepts `GET` requests -- Returns simple liveness status +Returns service metadata following the +[GA4GH service-info specification](https://github.com/ga4gh-discovery/ga4gh-service-info). +No authentication required. - Example: +Example: + +```bash +curl https://HOSTNAME/service-info +``` - ```bash - curl https://HOSTNAME/health/live - {"status":"ok"} - ``` +### Dataset Endpoints -### Info Endpoints +All dataset endpoints require authentication. -#### `/info/datasets` +#### `GET /datasets` -- accepts `GET` requests -- Returns all datasets the authenticated user has access to +Returns a paginated list of datasets accessible to the authenticated user. +- Query Parameters + - `page_size` (optional): Number of results per page + - `page_token` (optional): Opaque token for the next page + +- Response: JSON array of dataset objects - Error codes - - `200` Query executed successfully + - `200` Success - `401` Invalid or missing token - - `500` Internal error due to DB failures - Example: +Example: - ```bash - curl -H "Authorization: Bearer $token" https://HOSTNAME/info/datasets - [{"id":"EGAD74900000101","title":"Example Dataset"}] - ``` +```bash +curl -H "Authorization: Bearer $token" https://HOSTNAME/datasets +``` -#### `/info/dataset` +#### `GET /datasets/:datasetId` -- accepts `GET` requests with query parameter `dataset` -- Returns metadata for a specific dataset +Returns metadata for a specific dataset. - Error codes - - `200` Query executed successfully - - `400` Missing dataset parameter + - `200` Success - `401` Invalid or missing token - - `403` Access denied to dataset - - `404` Dataset not found - - `500` Internal error due to DB failures + - `403` Access denied or dataset does not exist - Example: +Example: - ```bash - curl -H "Authorization: Bearer $token" "https://HOSTNAME/info/dataset?dataset=EGAD74900000101" - {"id":"EGAD74900000101","title":"Example Dataset","fileCount":2,"totalSize":15242998} - ``` +```bash +curl -H "Authorization: Bearer $token" https://HOSTNAME/datasets/EGAD00000000001 +``` + +#### `GET /datasets/:datasetId/files` -#### `/info/dataset/files` +Returns a paginated list of files in the specified dataset. -- accepts `GET` requests with query parameter `dataset` -- Returns list of files in the specified dataset +- Query Parameters + - `page_size` (optional): Number of results per page + - `page_token` (optional): Opaque token for the next page - Error codes - - `200` Query executed successfully - - `400` Missing dataset parameter + - `200` Success - `401` Invalid or missing token - - `403` Access denied to dataset - - `500` Internal error due to DB failures + - `403` Access denied or dataset does not exist + +Example: + +```bash +curl -H "Authorization: Bearer $token" https://HOSTNAME/datasets/EGAD00000000001/files +``` + +### File Endpoints + +All file endpoints require authentication. Download endpoints also require a Crypt4GH +public key in the `X-C4GH-Public-Key` header (base64-encoded). + +Files are served through three tiers: + +| Tier | Endpoint | Content | Range Support | +|-------------|------------------------------|---------------------------------|---------------| +| Combined | `GET /files/:fileId` | Re-encrypted header + data | Yes | +| Header only | `GET /files/:fileId/header` | Re-encrypted Crypt4GH header | No | +| Content only| `GET /files/:fileId/content` | Encrypted data segments | Yes | - Example: +All three tiers support `HEAD` requests for metadata without a response body. - ```bash - curl -H "Authorization: Bearer $token" "https://HOSTNAME/info/dataset/files?dataset=EGAD74900000101" - [{"id":"EGAF74900000001","path":"data/file1.c4gh","size":1024000,"checksum":"abc123...","checksumType":"SHA256"}] - ``` +#### `HEAD /files/:fileId` -### File Download Endpoints +Returns file metadata headers without downloading the file. -File downloads require a `public_key` header containing the user's Crypt4GH public key -(base64-encoded). The archived file is re-encrypted with this key before streaming to -the client. +- Response Headers + - `Content-Length`: total size (header + content) + - `Accept-Ranges: bytes` + - `ETag`: entity tag for caching -#### `/file/:fileId` +#### `GET /files/:fileId` -- accepts `GET` requests with file stable ID in path -- Downloads a file by its stable ID (e.g., `EGAF74900000001`) -- Supports HTTP Range requests for partial downloads +Downloads the complete re-encrypted file (header + data segments). - Request Headers - `Authorization: Bearer ` (required) - - `public_key: ` (required) - - `Range: bytes=START-END` (optional, for partial downloads) + - `X-C4GH-Public-Key: ` (required) + - `Range: bytes=START-END` (optional) - Response Headers - `Content-Type: application/octet-stream` - `Content-Disposition: attachment; filename=".c4gh"` - `Accept-Ranges: bytes` - - `X-File-Id: ` - - `X-Decrypted-Size: ` - - `X-Decrypted-Checksum: ` (if available) - - `X-Decrypted-Checksum-Type: ` (if available) - Error codes - `200` Download successful - `206` Partial content (Range request) - - `400` Missing fileId or public_key header + - `400` Missing public key header - `401` Invalid or missing token - - `403` Access denied to file - - `404` File not found - - `500` Internal error (storage, reencrypt, or streaming failures) - - Example: + - `403` Access denied or file does not exist + - `500` Internal error (storage, reencrypt, or streaming failure) - ```bash - curl -H "Authorization: Bearer $token" \ - -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ - https://HOSTNAME/file/EGAF74900000001 \ - -o downloaded_file.c4gh - ``` +Example: - Example with Range header: +```bash +curl -H "Authorization: Bearer $token" \ + -H "X-C4GH-Public-Key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ + https://HOSTNAME/files/EGAF00000000001 \ + -o downloaded_file.c4gh +``` - ```bash - curl -H "Authorization: Bearer $token" \ - -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ - -H "Range: bytes=0-1023" \ - https://HOSTNAME/file/EGAF74900000001 - ``` +#### `HEAD /files/:fileId/header` and `GET /files/:fileId/header` -#### `/file` +Returns only the Crypt4GH header re-encrypted to the recipient's public key. +Useful when the client needs to process the header separately. Does not support +Range requests. -- accepts `GET` requests with query parameters -- Downloads a file by dataset and either fileId or filePath -- Supports HTTP Range requests for partial downloads +#### `HEAD /files/:fileId/content` and `GET /files/:fileId/content` -- Query Parameters - - `dataset` (required): Dataset stable ID - - `fileId` (optional): File stable ID within the dataset - - `filePath` (optional): File path within the dataset - - Note: Either `fileId` or `filePath` must be provided, but not both +Returns only the encrypted data segments (without the Crypt4GH header). +Range byte offsets refer to the data segments only. The ETag is stable for a +given file and independent of the recipient key. -- Request Headers: Same as `/file/:fileId` -- Response Headers: Same as `/file/:fileId` +Example with Range: -- Error codes - - `200` Download successful - - `206` Partial content (Range request) - - `400` Missing required parameters or invalid combination - - `401` Invalid or missing token - - `403` Access denied to file - - `404` File not found - - `500` Internal error +```bash +curl -H "Authorization: Bearer $token" \ + -H "X-C4GH-Public-Key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ + -H "Range: bytes=0-65535" \ + https://HOSTNAME/files/EGAF00000000001/content +``` - Example by fileId: +### Error Format - ```bash - curl -H "Authorization: Bearer $token" \ - -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ - "https://HOSTNAME/file?dataset=EGAD74900000101&fileId=EGAF74900000001" \ - -o downloaded_file.c4gh - ``` +All error responses use [RFC 9457 Problem Details](https://www.rfc-editor.org/rfc/rfc9457): - Example by filePath: +```json +{ + "type": "about:blank", + "title": "Forbidden", + "status": 403, + "detail": "access denied" +} +``` - ```bash - curl -H "Authorization: Bearer $token" \ - -H "public_key: $(base64 -w0 /path/to/c4gh.pub.pem)" \ - "https://HOSTNAME/file?dataset=EGAD74900000101&filePath=data/sample.c4gh" \ - -o downloaded_file.c4gh - ``` +Resource-by-ID endpoints (`/datasets/:datasetId`, `/files/:fileId`) return `403` +for both "access denied" and "does not exist" to prevent existence leakage. ## Configuration The service is configured via YAML config file or environment variables. +Environment variables use uppercase with underscores replacing dots +(e.g., `api.host` → `API_HOST`). -### API Server Settings - -| Variable | Config Key | Description | Default | -| ---------- | ---------- | ----------------- | --------- | -| `API_HOST` | `api.host` | Host to bind to | `0.0.0.0` | -| `API_PORT` | `api.port` | Port to listen on | `8080` | - -### gRPC Health Check Settings - -| Variable | Config Key | Description | Default | -| ------------- | ------------- | ---------------------- | ------- | -| `HEALTH_PORT` | `health.port` | gRPC health check port | `8081` | - -### Database Settings +Example: -| Variable | Config Key | Description | -| ------------- | ------------- | ------------------------------ | -| `DB_HOST` | `db.host` | PostgreSQL hostname | -| `DB_PORT` | `db.port` | PostgreSQL port | -| `DB_USER` | `db.user` | Database username | -| `DB_PASSWORD` | `db.password` | Database password | -| `DB_DATABASE` | `db.database` | Database name | -| `DB_SSLMODE` | `db.sslmode` | SSL mode (disable/require/etc) | - -### JWT/Authentication Settings +```yaml +api: + host: "0.0.0.0" + port: 8080 +db: + host: "postgres" + port: 5432 +``` -| Variable | Config Key | Description | -| -------------------- | -------------------- | ------------------------------------ | -| `JWT_PUBKEY_PATH` | `jwt.pubkey-path` | Path to directory with public keys | -| `JWT_PUBKEY_URL` | `jwt.pubkey-url` | JWKS URL for key fetching | -| `JWT_ALLOW_ALL_DATA` | `jwt.allow-all-data` | Allow all authenticated users access | +### API Server + +| Variable | Config Key | Description | Default | +|------------------|------------------|--------------------------------|-----------| +| `API_HOST` | `api.host` | Host address to bind to | `0.0.0.0` | +| `API_PORT` | `api.port` | Port to listen on | `8080` | +| `API_SERVER_CERT`| `api.server-cert`| Path to TLS certificate | | +| `API_SERVER_KEY` | `api.server-key` | Path to TLS private key | | +| `HEALTH_PORT` | `health.port` | gRPC health check port | `8081` | + +### Database + +| Variable | Config Key | Description | Default | +|---------------|---------------|-----------------------------------------------------|-------------| +| `DB_HOST` | `db.host` | PostgreSQL hostname | `localhost` | +| `DB_PORT` | `db.port` | PostgreSQL port | `5432` | +| `DB_USER` | `db.user` | Database username | *required* | +| `DB_PASSWORD` | `db.password` | Database password | *required* | +| `DB_DATABASE` | `db.database` | Database name | `sda` | +| `DB_SSLMODE` | `db.sslmode` | SSL mode (disable, allow, prefer, require, verify-ca, verify-full) | `prefer` | +| `DB_CACERT` | `db.cacert` | Path to CA certificate for database TLS | | + +### JWT / Authentication + +| Variable | Config Key | Description | Default | +|----------------------|----------------------|--------------------------------------------------|---------| +| `JWT_PUBKEY_PATH` | `jwt.pubkey-path` | Path to directory with PEM public keys | | +| `JWT_PUBKEY_URL` | `jwt.pubkey-url` | JWKS URL for key fetching | | +| `JWT_ALLOW_ALL_DATA` | `jwt.allow-all-data` | Allow all authenticated users access (testing only) | `false` | +| `AUTH_ALLOW_OPAQUE` | `auth.allow-opaque` | Allow opaque tokens via userinfo | `true` | At least one of `jwt.pubkey-path` or `jwt.pubkey-url` must be configured. -### gRPC Reencrypt Service Settings - -| Variable | Config Key | Description | -| -------------- | -------------- | -------------------------- | -| `GRPC_HOST` | `grpc.host` | Reencrypt service hostname | -| `GRPC_PORT` | `grpc.port` | Reencrypt service port | -| `GRPC_TIMEOUT` | `grpc.timeout` | Request timeout in seconds | - -### Storage Settings - -Storage is configured using the `storage/v2` package. See [storage/v2 README](../../internal/storage/v2/README.md) for details. +### OIDC + +| Variable | Config Key | Description | Default | +|-----------------|-----------------|-----------------------------------|---------| +| `OIDC_ISSUER` | `oidc.issuer` | Expected JWT issuer (optional) | | +| `OIDC_AUDIENCE` | `oidc.audience` | Expected JWT audience (optional) | | + +### Permission + +| Variable | Config Key | Description | Default | +|--------------------|--------------------|------------------------------------------|------------| +| `PERMISSION_MODEL` | `permission.model` | Permission model: ownership, visa, or combined | `combined` | + +### GA4GH Visa + +| Variable | Config Key | Description | Default | +|-----------------------------------|----------------------------------|------------------------------------------------|----------------| +| `VISA_ENABLED` | `visa.enabled` | Enable GA4GH visa support | `false` | +| `VISA_SOURCE` | `visa.source` | Visa source: `userinfo` or `token` | `userinfo` | +| `VISA_USERINFO_URL` | `visa.userinfo-url` | Userinfo endpoint (discovered from OIDC if not set) | | +| `VISA_TRUSTED_ISSUERS_PATH` | `visa.trusted-issuers-path` | Path to JSON trusted issuers file | | +| `VISA_ALLOW_INSECURE_JKU` | `visa.allow-insecure-jku` | Allow HTTP JKU URLs (testing only) | `false` | +| `VISA_DATASET_ID_MODE` | `visa.dataset-id-mode` | Dataset ID mode: `raw` or `suffix` | `raw` | +| `VISA_IDENTITY_MODE` | `visa.identity.mode` | Identity binding: `broker-bound`, `strict-sub`, `strict-iss-sub` | `broker-bound` | +| `VISA_VALIDATE_ASSERTED` | `visa.validate-asserted` | Reject visas with future asserted timestamps | `true` | +| `VISA_LIMITS_MAX_VISAS` | `visa.limits.max-visas` | Max visas per passport | `200` | +| `VISA_LIMITS_MAX_JWKS_PER_REQUEST`| `visa.limits.max-jwks-per-request`| Max distinct JWKS fetches per request | `10` | +| `VISA_LIMITS_MAX_VISA_SIZE` | `visa.limits.max-visa-size` | Max visa JWT size (bytes) | `16384` | +| `VISA_CACHE_TOKEN_TTL` | `visa.cache.token-ttl` | Token cache TTL (seconds) | `3600` | +| `VISA_CACHE_JWK_TTL` | `visa.cache.jwk-ttl` | JWK cache TTL (seconds) | `300` | +| `VISA_CACHE_VALIDATION_TTL` | `visa.cache.validation-ttl` | Visa validation cache TTL (seconds) | `120` | +| `VISA_CACHE_USERINFO_TTL` | `visa.cache.userinfo-ttl` | Userinfo cache TTL (seconds) | `60` | + +### gRPC Reencrypt Service + +| Variable | Config Key | Description | Default | +|-------------------|-------------------|------------------------------------|---------| +| `GRPC_HOST` | `grpc.host` | Reencrypt service hostname | *required* | +| `GRPC_PORT` | `grpc.port` | Reencrypt service port | `50051` | +| `GRPC_TIMEOUT` | `grpc.timeout` | Request timeout (seconds) | `10` | +| `GRPC_CACERT` | `grpc.cacert` | Path to CA certificate for gRPC TLS | | +| `GRPC_CLIENT_CERT`| `grpc.client-cert` | Path to client certificate for mTLS | | +| `GRPC_CLIENT_KEY` | `grpc.client-key` | Path to client key for mTLS | | + +### Storage + +Storage is configured using the `storage/v2` package. The `storage.backend` flag +(default: `archive`) selects the named storage backend. Example S3 configuration: @@ -274,80 +433,74 @@ storage: bucket_prefix: "archive" ``` -### Session Settings - -| Variable | Config Key | Description | -| -------------------- | -------------------- | ----------------------------- | -| `SESSION_EXPIRATION` | `session.expiration` | Session expiration in seconds | -| `SESSION_SECURE` | `session.secure` | Use secure cookies | -| `SESSION_HTTP_ONLY` | `session.http-only` | HTTP-only cookies | -| `SESSION_NAME` | `session.name` | Session cookie name | +### Session -### Cache Settings +| Variable | Config Key | Description | Default | +|----------------------|----------------------|----------------------------------|---------------| +| `SESSION_EXPIRATION` | `session.expiration` | Session expiration (seconds) | `3600` | +| `SESSION_DOMAIN` | `session.domain` | Cookie domain | | +| `SESSION_SECURE` | `session.secure` | Use secure cookies (HTTPS only) | `true` | +| `SESSION_HTTP_ONLY` | `session.http-only` | HTTP-only cookies | `true` | +| `SESSION_NAME` | `session.name` | Session cookie name | `sda_session` | -Database query results are cached in-memory to reduce database roundtrips, which is -particularly beneficial for streaming use cases where the same file metadata may be -requested multiple times (e.g., HTTP Range requests). +### Database Cache | Variable | Config Key | Description | Default | -| ---------------------- | ---------------------- | ----------------------------------- | ------- | +|------------------------|------------------------|-------------------------------------|---------| | `CACHE_ENABLED` | `cache.enabled` | Enable database query caching | `true` | | `CACHE_FILE_TTL` | `cache.file-ttl` | TTL for file queries (seconds) | `300` | | `CACHE_PERMISSION_TTL` | `cache.permission-ttl` | TTL for permission checks (seconds) | `120` | | `CACHE_DATASET_TTL` | `cache.dataset-ttl` | TTL for dataset queries (seconds) | `300` | -The cache uses [ristretto](https://github.com/dgraph-io/ristretto), a high-performance -concurrent cache. Cached queries include: - -- File lookups by ID and path -- Permission checks (user-scoped via dataset IDs) -- Dataset listings and metadata -- Dataset file listings - -Cache keys for permission checks are scoped by the user's accessible datasets, ensuring -that users with different permissions get appropriately cached results. - -#### Cache Behavior and Considerations +### Pagination -**Cache Invalidation:** The cache uses time-based expiration (TTL) but does not support -explicit invalidation. This means: +| Variable | Config Key | Description | Default | +|---------------------------|-------------------------|--------------------------------------------------|---------| +| `PAGINATION_HMAC_SECRET` | `pagination.hmac-secret`| HMAC secret for page tokens (must match across replicas) | | -- If a file's permissions change, the cached permission check will be stale until the - TTL expires (default: 2 minutes for permissions, 5 minutes for file/dataset metadata) -- For most use cases this is acceptable, as permission changes are infrequent -- If immediate permission revocation is required, the cache can be disabled or the - service restarted +If not configured, a random secret is generated at startup. Page tokens will not +survive restarts or work across replicas without a configured secret. -**Memory Usage:** The ristretto cache is configured with: +### Audit -- `MaxCost: 100,000` - Maximum number of items to cache -- `NumCounters: 1,000,000` - Counters for admission policy +| Variable | Config Key | Description | Default | +|------------------|------------------|-------------------------------------------|---------| +| `AUDIT_REQUIRED` | `audit.required` | Require a real audit logger (fail if noop) | `false` | -Estimated memory usage: +### Application Environment -- Base overhead: ~8 MB for counters -- Per-item overhead: varies by cached data size -- Typical total: 50-200 MB depending on active dataset/file counts +| Variable | Config Key | Description | Default | +|-------------------|-------------------|------------------------------------------------------|---------| +| `APP_ENVIRONMENT` | `app.environment` | Set to `production` to enable production safety guards | | -For production deployments, monitor memory usage and adjust `MaxCost` if needed. The -cache automatically evicts least-recently-used items when capacity is reached. +Production safety guards enforce: +- `jwt.allow-all-data` must be `false` +- `pagination.hmac-secret` must be configured +- `grpc.client-cert` and `grpc.client-key` must be configured ## Testing -### Integration Tests +### Unit Tests -Run the standalone Go-based integration tests: +```bash +cd sda && go test ./cmd/download/... +``` + +Visa-related tests use a build tag: ```bash -docker compose -f .github/integration/sda-download-integration.yml run integration_test +cd sda && go test -tags visas -count=1 ./cmd/download/... ``` -Or run as part of the full SDA pipeline: +### Integration Tests ```bash -make integrationtest-sda-s3-run +make integrationtest-sda-cmd-download-up # Start test environment +make integrationtest-sda-cmd-download-run # Run tests +make integrationtest-sda-cmd-download-down # Tear down ``` ### Local Development -See [TESTING.md](TESTING.md) for detailed local testing instructions. +See [TESTING.md](TESTING.md) for detailed local testing instructions including +Docker Compose setup, database seeding, and example curl commands. From 64fa3ece36c521b8dfc279b8e7a774e48782dd6b Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 22:33:04 +0100 Subject: [PATCH 382/530] fix(download): update stale routes and headers in TESTING.md Update curl examples to use v2 API routes (/datasets, /files) instead of old /info/ and /file/ routes. Fix public key header name from public_key to X-C4GH-Public-Key. --- sda/cmd/download/TESTING.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sda/cmd/download/TESTING.md b/sda/cmd/download/TESTING.md index 9e56ccc7f3..dffb4dfc8b 100644 --- a/sda/cmd/download/TESTING.md +++ b/sda/cmd/download/TESTING.md @@ -12,7 +12,7 @@ The download service has its own Docker Compose-based Go integration tests: ```bash # From repository root -docker compose -f .github/integration/sda-download-integration.yml run integration_test +docker compose -f .github/integration/sda-cmd-download-integration.yml run integration_test ``` This runs a minimal environment with: @@ -152,21 +152,21 @@ curl http://localhost:8080/health/ready ```bash curl -H "Authorization: Bearer $TOKEN" \ - http://localhost:8080/info/datasets + http://localhost:8080/datasets ``` #### Get Dataset Info ```bash curl -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/info/dataset?dataset=EGAD74900000101" + http://localhost:8080/datasets/EGAD74900000101 ``` #### List Files in Dataset ```bash curl -H "Authorization: Bearer $TOKEN" \ - "http://localhost:8080/info/dataset/files?dataset=EGAD74900000101" + http://localhost:8080/datasets/EGAD74900000101/files ``` #### Download a File @@ -177,15 +177,15 @@ curl -H "Authorization: Bearer $TOKEN" \ # Full file download curl -H "Authorization: Bearer $TOKEN" \ - -H "public_key: $(base64 -w0 mykey.pub.pem)" \ - http://localhost:8080/file/EGAF74900000001 \ + -H "X-C4GH-Public-Key: $(base64 -w0 mykey.pub.pem)" \ + http://localhost:8080/files/EGAF74900000001 \ -o downloaded_file.c4gh # Partial download (Range header) curl -H "Authorization: Bearer $TOKEN" \ - -H "public_key: $(base64 -w0 mykey.pub.pem)" \ + -H "X-C4GH-Public-Key: $(base64 -w0 mykey.pub.pem)" \ -H "Range: bytes=0-1023" \ - http://localhost:8080/file/EGAF74900000001 + http://localhost:8080/files/EGAF74900000001 ``` ## Database Queries for Debugging @@ -259,11 +259,11 @@ Check that storage configuration in `dev_config.yaml` matches the running Docker 1. Ensure ls-aai-mock is running: `docker logs ls-aai-mock` 2. Check JWKS endpoint: `curl http://localhost:8800/oidc/jwk` -3. Verify token is valid: `curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/info/datasets` +3. Verify token is valid: `curl -H "Authorization: Bearer $TOKEN" http://localhost:8080/datasets` ### No Datasets Returned -If `/info/datasets` returns empty: +If `/datasets` returns empty: 1. Check integration tests completed successfully 2. Verify files have stable IDs: `SELECT stable_id FROM sda.files WHERE stable_id IS NOT NULL;` From d7a7b3e273423ec3bb00fa380efca6f44d9565c4 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 9 Mar 2026 14:31:55 +0100 Subject: [PATCH 383/530] fix(download): update benchmark config for storage v2 and replace sleep with health poll - Update download-config-old.yaml to use storage v2 config format (sda-download on main migrated to storage/v2) - Replace sleep 60 with health endpoint polling for both services - Keep shared volume between runs to avoid re-downloading binaries --- .../integration/sda/download-config-old.yaml | 17 +++++++++-------- Makefile | 11 ++++++++--- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/.github/integration/sda/download-config-old.yaml b/.github/integration/sda/download-config-old.yaml index 35be348d2b..8142a7c1f3 100644 --- a/.github/integration/sda/download-config-old.yaml +++ b/.github/integration/sda/download-config-old.yaml @@ -11,14 +11,15 @@ log: level: "info" format: "json" -archive: - type: "s3" - url: "http://s3" - port: 9000 - accesskey: "access" - secretkey: "secretKey" - bucket: "archive" - chunksize: 32 +storage: + archive: + s3: + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true grpc: host: "reencrypt" diff --git a/Makefile b/Makefile index 4545ee14f8..71592e189e 100644 --- a/Makefile +++ b/Makefile @@ -183,7 +183,7 @@ benchmark-download-run: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml --profile benchmark run --rm benchmark benchmark-download-down: - @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml down -v --remove-orphans + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml down --remove-orphans # Force cleanup of benchmark resources. # NOTE: intentionally scoped to the benchmark compose file; avoid broad name-based container deletes. @@ -196,8 +196,13 @@ benchmark-download-seed: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml run --rm integration_test /scripts/seed_benchmark_data.sh benchmark-download: benchmark-download-up - @echo "Waiting for services to become healthy (60s)..." - @sleep 60 + @echo "Waiting for services to become healthy..." + @printf " waiting for download..."; retries=0; \ + until curl -sf http://localhost:$$(docker port download 8080 | head -1 | cut -d: -f2)/health/ready > /dev/null 2>&1; do \ + retries=$$((retries + 1)); if [ $$retries -ge 60 ]; then echo " TIMEOUT"; exit 1; fi; sleep 1; done; echo " ok" + @printf " waiting for download-old..."; retries=0; \ + until curl -sf http://localhost:$$(docker port download-old 8080 | head -1 | cut -d: -f2)/health > /dev/null 2>&1; do \ + retries=$$((retries + 1)); if [ $$retries -ge 60 ]; then echo " TIMEOUT"; exit 1; fi; sleep 1; done; echo " ok" @$(MAKE) benchmark-download-seed @echo "Running benchmark..." @$(MAKE) benchmark-download-run From e1964907980917cd80d9bd445a7cdb45b61945c0 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 10 Mar 2026 09:14:47 +0100 Subject: [PATCH 384/530] fix(download): fix opaque auth, config error handling, and visa cache expiry - Create standalone userinfo client when auth.allow-opaque=true but visa.enabled=false, so opaque tokens work without visa support - Return config file read/parse errors when --config-file is explicitly set, instead of silently falling through to defaults - Preserve visa expiry on validation-cache hits so session TTL remains bounded by the visa exp claim --- sda/cmd/download/middleware/auth.go | 25 ++++++++++++++++++++++--- sda/cmd/download/visa/validator.go | 15 ++++++++++++--- sda/internal/config/v2/config.go | 8 +++++++- 3 files changed, 41 insertions(+), 7 deletions(-) diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index 8971f8e285..9287e2445f 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -408,10 +408,29 @@ func authenticateStructureBased(rawToken string, userinfoClient *visa.UserinfoCl // If visaValidator is provided, GA4GH visa-based access is also computed based on permission.model. func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogger audit.Logger) gin.HandlerFunc { // Determine userinfo client for opaque token support. - // The visa validator's userinfo client can be reused for opaque token auth. var userinfoClient *visa.UserinfoClient - if visaValidator != nil && config.AuthAllowOpaque() { - userinfoClient = visaValidator.UserinfoClient() + if config.AuthAllowOpaque() { + if visaValidator != nil { + userinfoClient = visaValidator.UserinfoClient() + } else { + // No visa validator — create a standalone userinfo client. + userinfoURL := config.VisaUserinfoURL() + if userinfoURL == "" && config.OIDCIssuer() != "" { + discovered, err := visa.DiscoverUserinfoURL(config.OIDCIssuer()) + if err == nil { + userinfoURL = discovered + } + } + if userinfoURL != "" { + uc, err := visa.NewUserinfoClient(userinfoURL, time.Duration(config.VisaCacheUserinfoTTL())*time.Second) + if err == nil { + userinfoClient = uc + } + } + if userinfoClient == nil { + log.Warn("auth.allow-opaque is enabled but no userinfo client could be created (configure visa.userinfo-url or oidc.issuer)") + } + } } auditDenied := func(c *gin.Context, status int) { diff --git a/sda/cmd/download/visa/validator.go b/sda/cmd/download/visa/validator.go index 3320befeb9..1c95530268 100644 --- a/sda/cmd/download/visa/validator.go +++ b/sda/cmd/download/visa/validator.go @@ -18,6 +18,12 @@ import ( const maxURLClaimLength = 255 +// cachedVisaResult stores validated visa datasets alongside expiry for cache TTL bounding. +type cachedVisaResult struct { + Datasets []string + Expiry time.Time +} + // DatasetChecker checks whether a dataset exists in the local database. type DatasetChecker interface { CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) @@ -178,13 +184,16 @@ func (v *Validator) processVisas(ctx context.Context, identity Identity, passpor // Check validation cache visaHash := hashToken(visaJWT) if cached, found := v.validCache.Get(visaHash); found { - if datasets, ok := cached.([]string); ok { - for _, ds := range datasets { + if cr, ok := cached.(cachedVisaResult); ok { + for _, ds := range cr.Datasets { if !seen[ds] { seen[ds] = true result.Datasets = append(result.Datasets, ds) } } + if !cr.Expiry.IsZero() && (result.MinExpiry.IsZero() || cr.Expiry.Before(result.MinExpiry)) { + result.MinExpiry = cr.Expiry + } continue } @@ -211,7 +220,7 @@ func (v *Validator) processVisas(ctx context.Context, identity Identity, passpor } } if cacheTTL > 0 { - v.validCache.SetWithTTL(visaHash, datasets, 1, cacheTTL) + v.validCache.SetWithTTL(visaHash, cachedVisaResult{Datasets: datasets, Expiry: expiry}, 1, cacheTTL) } for _, ds := range datasets { diff --git a/sda/internal/config/v2/config.go b/sda/internal/config/v2/config.go index c670456062..7d89aa0c56 100644 --- a/sda/internal/config/v2/config.go +++ b/sda/internal/config/v2/config.go @@ -122,7 +122,13 @@ func Load() error { viper.SetConfigFile(viper.GetString("config-file")) } - _ = viper.ReadInConfig() + if err := viper.ReadInConfig(); err != nil { + // If a config file was explicitly specified, any error is fatal. + // If not, a "not found" error is expected and can be ignored. + if viper.IsSet("config-file") { + return fmt.Errorf("failed to read config file: %w", err) + } + } var missingFlags error for _, flag := range registeredFlags { From 03a0a22b656d33f9025693bed2887cb947e8c821 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 12 Mar 2026 19:43:38 +0100 Subject: [PATCH 385/530] refactor(download): remove IsNoop from audit Logger interface Move the audit logger selection to main.go: StdoutLogger when audit is required, NoopLogger otherwise. This removes the IsNoop type-check method from the interface and clarifies that NoopLogger is the "logging disabled" implementation, not a test helper. --- sda/cmd/download/audit/audit.go | 4 +--- sda/cmd/download/audit/audit_test.go | 11 ----------- sda/cmd/download/audit/stdout_logger.go | 2 -- sda/cmd/download/handlers/mock_database_test.go | 2 -- sda/cmd/download/main.go | 8 +++++--- sda/cmd/download/middleware/auth_test.go | 2 -- 6 files changed, 6 insertions(+), 23 deletions(-) diff --git a/sda/cmd/download/audit/audit.go b/sda/cmd/download/audit/audit.go index b964b6b273..f5736a4fac 100644 --- a/sda/cmd/download/audit/audit.go +++ b/sda/cmd/download/audit/audit.go @@ -27,11 +27,9 @@ type Event struct { // Logger defines the interface for audit logging. type Logger interface { Log(ctx context.Context, event Event) - IsNoop() bool } -// NoopLogger is a no-op implementation of Logger for testing and development. +// NoopLogger discards all audit events. Used when audit logging is not required. type NoopLogger struct{} func (NoopLogger) Log(context.Context, Event) {} -func (NoopLogger) IsNoop() bool { return true } diff --git a/sda/cmd/download/audit/audit_test.go b/sda/cmd/download/audit/audit_test.go index 7be4af2363..8c4e9c078f 100644 --- a/sda/cmd/download/audit/audit_test.go +++ b/sda/cmd/download/audit/audit_test.go @@ -11,11 +11,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestNoopLogger_IsNoop(t *testing.T) { - var logger Logger = NoopLogger{} - assert.True(t, logger.IsNoop()) -} - func TestNoopLogger_LogDoesNotPanic(t *testing.T) { var logger Logger = NoopLogger{} assert.NotPanics(t, func() { @@ -28,12 +23,6 @@ func TestNoopLogger_LogDoesNotPanic(t *testing.T) { }) } -func TestStdoutLogger_IsNoop(t *testing.T) { - var buf bytes.Buffer - var logger Logger = newStdoutLoggerWithWriter(&buf) - assert.False(t, logger.IsNoop()) -} - func TestStdoutLogger_OutputsValidJSON(t *testing.T) { var buf bytes.Buffer logger := newStdoutLoggerWithWriter(&buf) diff --git a/sda/cmd/download/audit/stdout_logger.go b/sda/cmd/download/audit/stdout_logger.go index 3b1e8b0a0b..3decff4f5b 100644 --- a/sda/cmd/download/audit/stdout_logger.go +++ b/sda/cmd/download/audit/stdout_logger.go @@ -42,5 +42,3 @@ func (l *StdoutLogger) Log(_ context.Context, event Event) { defer l.mu.Unlock() _ = l.encoder.Encode(event) // best-effort, don't block HTTP response } - -func (l *StdoutLogger) IsNoop() bool { return false } diff --git a/sda/cmd/download/handlers/mock_database_test.go b/sda/cmd/download/handlers/mock_database_test.go index ac64feb891..c936a8d05c 100644 --- a/sda/cmd/download/handlers/mock_database_test.go +++ b/sda/cmd/download/handlers/mock_database_test.go @@ -20,8 +20,6 @@ func (l *capturingLogger) Log(_ context.Context, event audit.Event) { l.events = append(l.events, event) } -func (l *capturingLogger) IsNoop() bool { return false } - // mockDatabase is a mock implementation of the database.Database interface for testing. type mockDatabase struct { datasets []database.Dataset diff --git a/sda/cmd/download/main.go b/sda/cmd/download/main.go index 063a2f9c35..b14f1ab584 100644 --- a/sda/cmd/download/main.go +++ b/sda/cmd/download/main.go @@ -100,9 +100,11 @@ func run() error { } // Initialize audit logger - auditLogger := audit.NewStdoutLogger() - if config.AuditRequired() && auditLogger.IsNoop() { - return errors.New("audit.required is true but audit logger is noop") + var auditLogger audit.Logger + if config.AuditRequired() { + auditLogger = audit.NewStdoutLogger() + } else { + auditLogger = audit.NoopLogger{} } // Initialize pagination HMAC secret diff --git a/sda/cmd/download/middleware/auth_test.go b/sda/cmd/download/middleware/auth_test.go index 2a4e667827..02256f0471 100644 --- a/sda/cmd/download/middleware/auth_test.go +++ b/sda/cmd/download/middleware/auth_test.go @@ -202,8 +202,6 @@ func (l *capturingLogger) Log(_ context.Context, event audit.Event) { l.events = append(l.events, event) } -func (l *capturingLogger) IsNoop() bool { return false } - func TestTokenMiddleware_NoToken_EmitsAuditDenied(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) From 8a54e7230cd1457b037cad8ca64ac90b18538abd Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 07:32:26 +0100 Subject: [PATCH 386/530] fix(download): resolve golangci-lint v2.11 warnings - Replace sort.Slice/sort.Strings with slices.SortFunc/slices.Sort - Replace wg.Add/go/Done pattern with wg.Go - Use os.OpenRoot to scope file reads in filepath.Walk (gosec G122) - Extract helpers to reduce nesting complexity (nestif) --- sda/cmd/download/benchmark/benchmark.go | 14 ++-- sda/cmd/download/database/cache.go | 4 +- sda/cmd/download/handlers/datasets.go | 4 +- sda/cmd/download/middleware/auth.go | 102 ++++++++++++++---------- sda/cmd/download/visa/validator.go | 34 +++++--- 5 files changed, 92 insertions(+), 66 deletions(-) diff --git a/sda/cmd/download/benchmark/benchmark.go b/sda/cmd/download/benchmark/benchmark.go index a9885868f7..5637cc6c85 100644 --- a/sda/cmd/download/benchmark/benchmark.go +++ b/sda/cmd/download/benchmark/benchmark.go @@ -37,7 +37,7 @@ import ( "net/http" "net/url" "os" - "sort" + "slices" "strconv" "strings" "sync" @@ -664,9 +664,7 @@ func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResul startTime := time.Now() for i := 0; i < cfg.Requests; i++ { - wg.Add(1) - go func(idx int) { - defer wg.Done() + wg.Go(func() { semaphore <- struct{}{} defer func() { <-semaphore }() @@ -674,8 +672,8 @@ func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResul if target.Name == "old" && cfg.OldClientVersion != "" { extra = map[string]string{"SDA-Client-Version": cfg.OldClientVersion} } - results[idx] = makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) - }(i) + results[i] = makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) + }) } wg.Wait() @@ -757,7 +755,7 @@ func calculateStats(latencies []time.Duration) Stats { sorted := make([]time.Duration, len(latencies)) copy(sorted, latencies) - sort.Slice(sorted, func(i, j int) bool { return sorted[i] < sorted[j] }) + slices.SortFunc(sorted, func(a, b time.Duration) int { return int(a - b) }) var sum time.Duration for _, l := range sorted { @@ -855,7 +853,7 @@ func formatStatusCounts(m map[int]int) string { for k, v := range m { pairs = append(pairs, kv{Code: k, Count: v}) } - sort.Slice(pairs, func(i, j int) bool { return pairs[i].Count > pairs[j].Count }) + slices.SortFunc(pairs, func(a, b kv) int { return b.Count - a.Count }) parts := make([]string, 0, len(pairs)) for _, p := range pairs { diff --git a/sda/cmd/download/database/cache.go b/sda/cmd/download/database/cache.go index 19010ad5ed..22e5110e1c 100644 --- a/sda/cmd/download/database/cache.go +++ b/sda/cmd/download/database/cache.go @@ -6,7 +6,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" - "sort" + "slices" "time" "github.com/dgraph-io/ristretto" @@ -323,7 +323,7 @@ func hashStrings(strs []string) string { // Sort a copy to avoid modifying the input sorted := make([]string, len(strs)) copy(sorted, strs) - sort.Strings(sorted) + slices.Sort(sorted) h := sha256.New() for _, s := range sorted { diff --git a/sda/cmd/download/handlers/datasets.go b/sda/cmd/download/handlers/datasets.go index 6c18e0e5a3..6139abc83f 100644 --- a/sda/cmd/download/handlers/datasets.go +++ b/sda/cmd/download/handlers/datasets.go @@ -2,7 +2,7 @@ package handlers import ( "net/http" - "sort" + "slices" "github.com/gin-gonic/gin" "github.com/neicnordic/sensitive-data-archive/cmd/download/config" @@ -68,7 +68,7 @@ func (h *Handlers) ListDatasets(c *gin.Context) { for i, d := range datasets { ids[i] = d.ID } - sort.Strings(ids) + slices.Sort(ids) // Parse pagination parameters pageSize, err := parsePageSize(c) diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index 9287e2445f..368de8f0c1 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -184,37 +184,50 @@ func (a *Authenticator) loadKeysFromPath(keyPath string) error { a.mu.Lock() defer a.mu.Unlock() - return filepath.Walk(keyPath, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err + root, err := os.OpenRoot(keyPath) + if err != nil { + return fmt.Errorf("failed to open key directory %s: %w", keyPath, err) + } + defer root.Close() + + return filepath.Walk(keyPath, func(path string, info os.FileInfo, walkErr error) error { + if walkErr != nil { + return walkErr } - if info.Mode().IsRegular() { - log.Debugf("Loading key from file: %s", path) + if !info.Mode().IsRegular() { + return nil + } - keyData, err := os.ReadFile(path) - if err != nil { - return fmt.Errorf("failed to read key file %s: %w", path, err) - } + relPath, err := filepath.Rel(keyPath, path) + if err != nil { + return fmt.Errorf("failed to compute relative path for %s: %w", path, err) + } - key, err := jwk.ParseKey(keyData, jwk.WithPEM(true)) - if err != nil { - log.Warnf("failed to parse key from %s: %v (skipping)", path, err) + log.Debugf("Loading key from file: %s", path) - return nil - } + keyData, err := root.ReadFile(relPath) + if err != nil { + return fmt.Errorf("failed to read key file %s: %w", path, err) + } - if err := jwk.AssignKeyID(key); err != nil { - return fmt.Errorf("failed to assign key ID: %w", err) - } + key, err := jwk.ParseKey(keyData, jwk.WithPEM(true)) + if err != nil { + log.Warnf("failed to parse key from %s: %v (skipping)", path, err) - if err := a.Keyset.AddKey(key); err != nil { - return fmt.Errorf("failed to add key to set: %w", err) - } + return nil + } - log.Debugf("Loaded key with ID: %s", key.KeyID()) + if err := jwk.AssignKeyID(key); err != nil { + return fmt.Errorf("failed to assign key ID: %w", err) } + if err := a.Keyset.AddKey(key); err != nil { + return fmt.Errorf("failed to add key to set: %w", err) + } + + log.Debugf("Loaded key with ID: %s", key.KeyID()) + return nil }) } @@ -402,6 +415,31 @@ func authenticateStructureBased(rawToken string, userinfoClient *visa.UserinfoCl }, nil } +// resolveUserinfoClient returns a UserinfoClient for opaque token exchange. +// It prefers the visa validator's client, falling back to standalone discovery. +func resolveUserinfoClient(visaValidator *visa.Validator) *visa.UserinfoClient { + if visaValidator != nil { + return visaValidator.UserinfoClient() + } + + userinfoURL := config.VisaUserinfoURL() + if userinfoURL == "" && config.OIDCIssuer() != "" { + if discovered, err := visa.DiscoverUserinfoURL(config.OIDCIssuer()); err == nil { + userinfoURL = discovered + } + } + + if userinfoURL != "" { + if uc, err := visa.NewUserinfoClient(userinfoURL, time.Duration(config.VisaCacheUserinfoTTL())*time.Second); err == nil { + return uc + } + } + + log.Warn("auth.allow-opaque is enabled but no userinfo client could be created (configure visa.userinfo-url or oidc.issuer)") + + return nil +} + // TokenMiddleware performs access token verification and validation. // The authenticated user's subject is stored in the request context. // If db is provided and allow-all-data is disabled, the user's datasets are populated from the database. @@ -410,27 +448,7 @@ func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogge // Determine userinfo client for opaque token support. var userinfoClient *visa.UserinfoClient if config.AuthAllowOpaque() { - if visaValidator != nil { - userinfoClient = visaValidator.UserinfoClient() - } else { - // No visa validator — create a standalone userinfo client. - userinfoURL := config.VisaUserinfoURL() - if userinfoURL == "" && config.OIDCIssuer() != "" { - discovered, err := visa.DiscoverUserinfoURL(config.OIDCIssuer()) - if err == nil { - userinfoURL = discovered - } - } - if userinfoURL != "" { - uc, err := visa.NewUserinfoClient(userinfoURL, time.Duration(config.VisaCacheUserinfoTTL())*time.Second) - if err == nil { - userinfoClient = uc - } - } - if userinfoClient == nil { - log.Warn("auth.allow-opaque is enabled but no userinfo client could be created (configure visa.userinfo-url or oidc.issuer)") - } - } + userinfoClient = resolveUserinfoClient(visaValidator) } auditDenied := func(c *gin.Context, status int) { diff --git a/sda/cmd/download/visa/validator.go b/sda/cmd/download/visa/validator.go index 1c95530268..3a5b96cc68 100644 --- a/sda/cmd/download/visa/validator.go +++ b/sda/cmd/download/visa/validator.go @@ -183,20 +183,18 @@ func (v *Validator) processVisas(ctx context.Context, identity Identity, passpor // Check validation cache visaHash := hashToken(visaJWT) - if cached, found := v.validCache.Get(visaHash); found { - if cr, ok := cached.(cachedVisaResult); ok { - for _, ds := range cr.Datasets { - if !seen[ds] { - seen[ds] = true - result.Datasets = append(result.Datasets, ds) - } + if cr, ok := v.getCachedVisa(visaHash); ok { + for _, ds := range cr.Datasets { + if !seen[ds] { + seen[ds] = true + result.Datasets = append(result.Datasets, ds) } - if !cr.Expiry.IsZero() && (result.MinExpiry.IsZero() || cr.Expiry.Before(result.MinExpiry)) { - result.MinExpiry = cr.Expiry - } - - continue } + if !cr.Expiry.IsZero() && (result.MinExpiry.IsZero() || cr.Expiry.Before(result.MinExpiry)) { + result.MinExpiry = cr.Expiry + } + + continue } datasets, expiry, err := v.validateSingleVisa(ctx, identity, visaJWT, jkuTracker, identities) @@ -241,6 +239,18 @@ func (v *Validator) processVisas(ctx context.Context, identity Identity, passpor return result, nil } +// getCachedVisa retrieves a cached visa validation result by hash. +func (v *Validator) getCachedVisa(visaHash string) (cachedVisaResult, bool) { + cached, found := v.validCache.Get(visaHash) + if !found { + return cachedVisaResult{}, false + } + + cr, ok := cached.(cachedVisaResult) + + return cr, ok +} + // validateSingleVisa validates one visa JWT and returns granted datasets and expiry. func (v *Validator) validateSingleVisa(ctx context.Context, identity Identity, visaJWT string, jkuTracker map[string]bool, identities map[string]bool) ([]string, time.Time, error) { var zeroTime time.Time From 5713e933147bac372b1b303c88e95ca6e4d39792 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 13:19:50 +0100 Subject: [PATCH 387/530] fix(download): guard mergeDatasets allocation against int overflow Check that len(a)+len(b) won't overflow before using it as a map capacity hint. Falls back to zero hint (runtime-grown) otherwise. --- sda/cmd/download/middleware/auth.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index 368de8f0c1..51aeac0131 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -9,6 +9,7 @@ import ( "encoding/json" "errors" "fmt" + "math" "net/http" "net/url" "os" @@ -695,7 +696,12 @@ func (sc *SessionCache) Set(key string, value AuthContext, ttl time.Duration) { // mergeDatasets returns a deduplicated union of two string slices. func mergeDatasets(a, b []string) []string { - seen := make(map[string]bool, len(a)) + hint := 0 + if len(a) <= math.MaxInt-len(b) { + hint = len(a) + len(b) + } + + seen := make(map[string]bool, hint) var result []string for _, s := range a { From 953052f90bfa0d5daeca7990806cdceec3ecaf69 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 13:19:57 +0100 Subject: [PATCH 388/530] fix(download): normalize root-path URLs consistently in NormalizeURL path.Clean("/") returns "/" not ".", so https://issuer.example/ did not match https://issuer.example. Treat u.Path == "/" the same as "." by clearing it to empty string. Adds test coverage. --- sda/cmd/download/visa/trust.go | 2 +- sda/cmd/download/visa/trust_test.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sda/cmd/download/visa/trust.go b/sda/cmd/download/visa/trust.go index e4ef87899f..1a7bbf6b11 100644 --- a/sda/cmd/download/visa/trust.go +++ b/sda/cmd/download/visa/trust.go @@ -68,7 +68,7 @@ func NormalizeURL(rawURL string) string { return rawURL } u.Path = path.Clean(u.Path) - if u.Path == "." { + if u.Path == "." || u.Path == "/" { u.Path = "" } diff --git a/sda/cmd/download/visa/trust_test.go b/sda/cmd/download/visa/trust_test.go index ca554e4a1e..fac4ccf359 100644 --- a/sda/cmd/download/visa/trust_test.go +++ b/sda/cmd/download/visa/trust_test.go @@ -29,6 +29,12 @@ func TestLoadTrustedIssuers_AllowsHTTPJKUWhenEnabled(t *testing.T) { assert.Equal(t, "http://example.org/jwks", issuers[0].JKU) } +func TestNormalizeURL_TrailingSlash(t *testing.T) { + assert.Equal(t, "https://issuer.example", NormalizeURL("https://issuer.example/")) + assert.Equal(t, "https://issuer.example", NormalizeURL("https://issuer.example")) + assert.Equal(t, "https://issuer.example/path", NormalizeURL("https://issuer.example/path/")) +} + func writeTrustedIssuersFile(t *testing.T, issuers []TrustedIssuer) string { t.Helper() From 965c6cc2dba76ef2a8fc46be0ef6f700ae611ccc Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 13:20:02 +0100 Subject: [PATCH 389/530] perf(download): compile range header regexp once at package level Move regexp.MustCompile from per-call in ParseRangeHeader to a package-level var, avoiding repeated compilation on the hot download path. --- sda/cmd/download/streaming/streaming.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sda/cmd/download/streaming/streaming.go b/sda/cmd/download/streaming/streaming.go index ecbeb30d75..9a00bc2800 100644 --- a/sda/cmd/download/streaming/streaming.go +++ b/sda/cmd/download/streaming/streaming.go @@ -30,6 +30,9 @@ var ErrRangeNotSatisfiable = errors.New("range not satisfiable") // The handler should return HTTP 400 or reject the range. var ErrRangeInvalid = errors.New("range invalid") +// rangeRe matches a single byte range: bytes=START-END, bytes=START-, or bytes=-SUFFIX. +var rangeRe = regexp.MustCompile(`^bytes=(\d*)-(\d*)$`) + // ParseRangeHeader parses an RFC 7233 Range header. // Returns: // - (nil, nil) if no range is specified - serve full file with 200 @@ -50,9 +53,7 @@ func ParseRangeHeader(rangeHeader string, fileSize int64) (*RangeSpec, error) { } } - // Match: bytes=START-END or bytes=START- or bytes=-SUFFIX - re := regexp.MustCompile(`^bytes=(\d*)-(\d*)$`) - matches := re.FindStringSubmatch(rangeHeader) + matches := rangeRe.FindStringSubmatch(rangeHeader) if matches == nil { log.Warnf("invalid range header format: %s", rangeHeader) From 2e5384ed617226af96ba40c713fe6ee9259c5d5d Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 13:20:08 +0100 Subject: [PATCH 390/530] fix(download): fix config v2 path handling, error surfacing, and panic - Use filepath.Clean instead of TrimLeft+path.Join to preserve absolute config paths like /etc/sda - Surface ReadInConfig errors (only ignore ConfigFileNotFoundError) - Replace BindPFlags panic with returned error --- sda/internal/config/v2/config.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sda/internal/config/v2/config.go b/sda/internal/config/v2/config.go index 7d89aa0c56..d739612a98 100644 --- a/sda/internal/config/v2/config.go +++ b/sda/internal/config/v2/config.go @@ -29,7 +29,7 @@ import ( "errors" "fmt" "os" - "path" + "path/filepath" "strings" "text/tabwriter" @@ -105,7 +105,7 @@ func Load() error { } if err := viper.BindPFlags(command.Flags()); err != nil { - panic(err) + return fmt.Errorf("failed to bind flags: %w", err) } if err := command.Execute(); err != nil { @@ -113,9 +113,7 @@ func Load() error { } if viper.IsSet("config-path") { - configPath := viper.GetString("config-path") - splitPath := strings.Split(strings.TrimLeft(configPath, "/"), "/") - viper.AddConfigPath(path.Join(splitPath...)) + viper.AddConfigPath(filepath.Clean(viper.GetString("config-path"))) } if viper.IsSet("config-file") { @@ -123,9 +121,7 @@ func Load() error { } if err := viper.ReadInConfig(); err != nil { - // If a config file was explicitly specified, any error is fatal. - // If not, a "not found" error is expected and can be ignored. - if viper.IsSet("config-file") { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { return fmt.Errorf("failed to read config file: %w", err) } } From 7007b09c21a33e8563eaa27600e68638ab43b4dc Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 13:20:15 +0100 Subject: [PATCH 391/530] docs(download): fix env var name, Go version, and Makefile comment - CONFIGFILE -> CONFIG_FILE in TESTING.md and dev_config.yaml - Go 1.20+ -> Go 1.25+ to match go.mod - Update stale config error troubleshooting text - Document build-all prerequisite for integration test target --- Makefile | 1 + sda/cmd/download/TESTING.md | 7 ++++--- sda/cmd/download/dev_config.yaml | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 71592e189e..8178cc28e5 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,7 @@ integrationtest-sda-validator-orchestrator-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-validator-orchestrator-integration.yml down -v --remove-orphans # Download v2 integration tests (sda/cmd/download, Go-based) +# Prerequisite: run `make build-all` first if container images are not already built. integrationtest-sda-cmd-download: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml up -d @sleep 10 diff --git a/sda/cmd/download/TESTING.md b/sda/cmd/download/TESTING.md index dffb4dfc8b..4797ffd9b7 100644 --- a/sda/cmd/download/TESTING.md +++ b/sda/cmd/download/TESTING.md @@ -47,7 +47,7 @@ This tests the download service with real data that has been processed through t ### Prerequisites - Docker and Docker Compose v2+ -- Go 1.20+ +- Go 1.25+ - `make` utility - `curl` and `jq` for testing @@ -102,7 +102,7 @@ After tests complete, you'll have: ```bash cd sda -CONFIGFILE=cmd/download/dev_config.yaml go run cmd/download/main.go +CONFIG_FILE=cmd/download/dev_config.yaml go run cmd/download/main.go ``` The service starts on: @@ -241,7 +241,8 @@ make sda-s3-down ### "Config File Not Found" -This is a viper warning, not an error. The service loads config from `CONFIGFILE` environment variable. +If `CONFIG_FILE` is set and the file is missing, the service exits with an error. +If no `CONFIG_FILE` is set, the service searches default paths and ignores missing files. ### Database Connection Issues diff --git a/sda/cmd/download/dev_config.yaml b/sda/cmd/download/dev_config.yaml index 7288d9e328..51505c31ab 100644 --- a/sda/cmd/download/dev_config.yaml +++ b/sda/cmd/download/dev_config.yaml @@ -1,5 +1,5 @@ # Development configuration for sda/cmd/download -# Use with: CONFIGFILE=cmd/download/dev_config.yaml go run cmd/download/main.go +# Use with: CONFIG_FILE=cmd/download/dev_config.yaml go run cmd/download/main.go # API Server configuration api: From 54abc9e855d762566285b737bc132c3e3482af85 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 21:30:07 +0100 Subject: [PATCH 392/530] =?UTF-8?q?fix(download):=20fix=20benchmark=20fair?= =?UTF-8?q?ness=20=E2=80=94=20disable=20cache,=20fix=20session=20domain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Disable query cache in benchmark config so the new service doesn't get an unfair caching advantage over the old service - Fix old service session domain from "localhost" to "download-old" so cookies are actually sent back in the Docker network --- .../integration/sda/cmd-download-config-benchmark.yaml | 10 ++++++---- .github/integration/sda/download-config-old.yaml | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/integration/sda/cmd-download-config-benchmark.yaml b/.github/integration/sda/cmd-download-config-benchmark.yaml index cea66446c0..f2312489e2 100644 --- a/.github/integration/sda/cmd-download-config-benchmark.yaml +++ b/.github/integration/sda/cmd-download-config-benchmark.yaml @@ -1,5 +1,5 @@ # Configuration for sda/cmd/download service in BENCHMARK mode -# Uses visa-based authorization to match old sda-download behavior for fair comparison +# This benchmark compares public endpoints, not just internal implementation details. # The endpoint uses http:// prefix for storage/v2 URL parsing # API Server @@ -9,7 +9,7 @@ api: # Logging log: - level: "debug" + level: "info" # gRPC Health check health: @@ -36,7 +36,7 @@ jwt: allow-all-data: false # Permission model: combined (ownership + visa) for benchmark -# This allows graceful fallback if visa processing has issues +# This keeps the benchmark environment resilient if visa processing has issues. permission: model: "combined" @@ -73,8 +73,10 @@ session: name: "sda_session" # Database query cache +# Disabled here so the benchmark does not compare a cached new endpoint +# against an uncached old endpoint. cache: - enabled: true + enabled: false file-ttl: 300 # 5 minutes permission-ttl: 120 # 2 minutes dataset-ttl: 300 # 5 minutes diff --git a/.github/integration/sda/download-config-old.yaml b/.github/integration/sda/download-config-old.yaml index 8142a7c1f3..0c0e19c506 100644 --- a/.github/integration/sda/download-config-old.yaml +++ b/.github/integration/sda/download-config-old.yaml @@ -46,7 +46,9 @@ oidc: session: expiration: 3600 - domain: "localhost" + # The benchmark runner reaches this service via the Docker hostname + # "download-old", so the session cookie must be scoped to that host. + domain: "download-old" secure: false httponly: true name: "sda_session_key" From f63ac203f7e01032f34e8132cc593b41a261f78f Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 21:30:12 +0100 Subject: [PATCH 393/530] fix(download): fix benchmark seed race with verified queue wait Wait for RabbitMQ verified queue to have messages before consuming, preventing a race where the DB checksums appear before the verify worker has pushed messages. --- .../scripts/seed_benchmark_data.sh | 64 +++++++++++++------ 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/.github/integration/scripts/seed_benchmark_data.sh b/.github/integration/scripts/seed_benchmark_data.sh index 8797559469..12061abbfb 100755 --- a/.github/integration/scripts/seed_benchmark_data.sh +++ b/.github/integration/scripts/seed_benchmark_data.sh @@ -74,17 +74,29 @@ for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do s3cmd -c /shared/s3cfg put "/shared/$file" "s3://test_dummy.org/$file" --no-check-certificate 2>/dev/null || true done -# Wait for uploads to appear in inbox queue -# Also consumption of these messages is needed to get correlation_id for ingest trigger -RETRY=0 -until [ "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq -r '.messages_ready')" -ge 2 ]; do - echo " Waiting for uploads to complete..." - RETRY=$((RETRY + 1)) - if [ "$RETRY" -ge 60 ]; then - echo "ERROR: Upload timeout" - exit 1 - fi - sleep 2 +# Resolve the latest uploaded file rows directly from Postgres. +# The inbox queue is not safe to use for this because s3inbox emits multiple +# messages per upload, so FIFO consumption can pair the wrong file with the +# wrong correlation ID on repeated runs. +declare -A FILE_IDS +echo " Resolving uploaded file IDs..." +for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do + RETRY=0 + FILE_ID="" + until [ -n "$FILE_ID" ]; do + FILE_ID=$(psql -U postgres -h postgres -d sda -At -c "SELECT id FROM sda.files WHERE submission_user='test@dummy.org' AND submission_file_path='$file' ORDER BY created_at DESC LIMIT 1") + if [ -n "$FILE_ID" ]; then + FILE_IDS["$file"]="$FILE_ID" + break + fi + echo " Waiting for database registration for $file..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 60 ]; then + echo "ERROR: Database registration timeout for $file" + exit 1 + fi + sleep 2 + done done # Manually trigger ingestion (bridge inbox -> ingest) @@ -96,16 +108,10 @@ for file in NA12878.bam.c4gh NA12878_20k_b37.bam.c4gh; do ENC_SHA=$(sha256sum "/shared/$file" | cut -d' ' -f 1) ENC_MD5=$(md5sum "/shared/$file" | cut -d' ' -f 1) - # Get correlation ID from inbox queue (consume message) - # We accept any message from queue assuming FIFO order matches upload? - # Or just take one. integration test assumes specific order or matching? - # integration test does loop and consume 1 message. - CORRID=$(curl -s -X POST -H "content-type:application/json" \ - -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox/get \ - -d '{"count":1,"encoding":"auto","ackmode":"ack_requeue_false"}' | jq -r .[0].properties.correlation_id) - - if [ "$CORRID" == "null" ] || [ -z "$CORRID" ]; then - echo "ERROR: Failed to get correlation ID for $file" + CORRID="${FILE_IDS[$file]}" + + if [ -z "$CORRID" ]; then + echo "ERROR: Failed to resolve file ID for $file" exit 1 fi @@ -182,6 +188,22 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) FROM sda. sleep 2 done +# Wait for verified messages to be published before consuming them. +# The DB checksums can appear slightly before the verify worker has pushed +# messages into RabbitMQ, which caused a race in the benchmark seeding flow. +echo " Waiting for verified queue..." +RETRY=0 +until [ "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/verified | jq -r '.messages_ready')" -ge 2 ]; do + echo " Waiting for verified queue..." + RETRY=$((RETRY + 1)) + if [ "$RETRY" -ge 120 ]; then + echo "ERROR: Verified queue timeout" + curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/verified | jq . + exit 1 + fi + sleep 2 +done + # Trigger finalize via MQ (receive verified message, send accession message) echo "Sending accession messages..." From e0269ad7b6b3a0af5293ec6d35701d35c7469148 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 21:30:20 +0100 Subject: [PATCH 394/530] feat(download): add paired iterations and validated-payload benchmark mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Alternate old→new / new→old order between iterations to cancel warm-up bias - Add cookie jar to HTTP client so session cookies work properly - Add validated-payload mode that downloads from both endpoints, decrypts with crypt4gh, and compares plaintext SHA256 - Extract buildRequest helper shared between benchmark and validation - Add percentChange helper to handle division-by-zero safely --- sda/cmd/download/benchmark/benchmark.go | 425 ++++++++++++++++++++---- 1 file changed, 357 insertions(+), 68 deletions(-) diff --git a/sda/cmd/download/benchmark/benchmark.go b/sda/cmd/download/benchmark/benchmark.go index 5637cc6c85..02c5d0fd96 100644 --- a/sda/cmd/download/benchmark/benchmark.go +++ b/sda/cmd/download/benchmark/benchmark.go @@ -1,4 +1,4 @@ -// Package main provides a benchmark tool for comparing download service implementations. +// Package main provides benchmark modes for comparing download service endpoints. // // When running in a container (via docker compose), the tool auto-configures by: // - Reading token from /shared/token @@ -6,9 +6,12 @@ // - Discovering an accessible dataset+file via the NEW service (/datasets/*) // - Using container hostnames (download, download-new) for service URLs // -// Comparison goal: -// - NEW: GET /files/{stable_id} with header X-C4GH-Public-Key -// - OLD: GET /s3/{dataset}/{path} with header Client-Public-Key +// Supported modes: +// - endpoint-e2e: compare the public download endpoints as exposed today +// NEW: GET /files/{stable_id} with header X-C4GH-Public-Key +// OLD: GET /s3/{dataset}/{path} with header Client-Public-Key +// - validated-payload: run the same endpoint comparison, but first verify that +// both responses decrypt to the same plaintext using the benchmark private key // // Usage (containerized - recommended): // @@ -23,11 +26,14 @@ // // OLD_URL, NEW_URL, ITERATIONS, REQUESTS, CONCURRENCY // FILE_ID, FILE_DATASET, FILE_PATH, TOKEN, PUBKEY +// BENCHMARK_MODE, VERIFY_PRIVATE_KEY_PATH, VERIFY_PRIVATE_KEY_PASSPHRASE package main import ( + "crypto/sha256" "crypto/tls" "encoding/base64" + "encoding/hex" "encoding/json" "errors" "flag" @@ -35,6 +41,7 @@ import ( "io" "math" "net/http" + "net/http/cookiejar" "net/url" "os" "slices" @@ -42,6 +49,16 @@ import ( "strings" "sync" "time" + + "github.com/neicnordic/crypt4gh/keys" + crypt4ghstreaming "github.com/neicnordic/crypt4gh/streaming" +) + +type BenchmarkMode string + +const ( + ModeEndpointE2E BenchmarkMode = "endpoint-e2e" + ModeValidatedPayload BenchmarkMode = "validated-payload" ) // Config holds benchmark configuration. @@ -69,6 +86,10 @@ type Config struct { SkipOld bool SkipNew bool OutputJSON bool + Mode BenchmarkMode + + VerifyPrivateKeyPath string + VerifyPrivateKeyPassphrase string } // RequestResult holds the result of a single request. @@ -109,12 +130,28 @@ type Stats struct { // ComparisonResult holds results from multiple iterations. type ComparisonResult struct { + Mode BenchmarkMode Old []BenchmarkResult New []BenchmarkResult OldSummary SummaryStats NewSummary SummaryStats } +type payloadDigest struct { + EncryptedBytes int64 + PlaintextBytes int64 + PlaintextSHA256 string +} + +type payloadValidationResult struct { + Old payloadDigest + New payloadDigest +} + +type payloadValidator struct { + privateKey [32]byte +} + // SummaryStats holds summary statistics across iterations. type SummaryStats struct { Name string @@ -174,9 +211,10 @@ func main() { } fmt.Println("=" + strings.Repeat("=", 70)) - fmt.Println(" Download Service Benchmark") + fmt.Printf(" Download Service Benchmark (%s)\n", cfg.Mode) fmt.Println("=" + strings.Repeat("=", 70)) fmt.Println("\nConfiguration:") + fmt.Printf(" Mode: %s\n", cfg.Mode) fmt.Printf(" Iterations: %d\n", cfg.Iterations) fmt.Printf(" Requests: %d per iteration\n", cfg.Requests) fmt.Printf(" Concurrency: %d\n", cfg.Concurrency) @@ -191,13 +229,10 @@ func main() { } fmt.Println() - client := &http.Client{ - Timeout: cfg.Timeout, - Transport: &http.Transport{ - MaxIdleConnsPerHost: cfg.Concurrency, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // benchmark tool, not production - DisableCompression: true, - }, + client, err := newHTTPClient(cfg) + if err != nil { + fmt.Fprintf(os.Stderr, "Client setup error: %v\n", err) + os.Exit(1) } // Preflight (fail fast) @@ -214,43 +249,22 @@ func main() { } } - result := &ComparisonResult{} - - // Benchmark old implementation - if !cfg.SkipOld { - fmt.Println("-" + strings.Repeat("-", 70)) - fmt.Println(" Benchmarking OLD implementation") - fmt.Println("-" + strings.Repeat("-", 70)) - - for i := 1; i <= cfg.Iterations; i++ { - fmt.Printf("\n Iteration %d/%d...\n", i, cfg.Iterations) - res := runBenchmark(client, oldTarget, cfg) - result.Old = append(result.Old, res) - printIterationResult(res) - - if i < cfg.Iterations { - time.Sleep(2 * time.Second) // Cool-down between iterations - } + if cfg.Mode == ModeValidatedPayload { + validation, err := validatePayloadEquivalence(client, cfg, oldTarget, newTarget) + if err != nil { + fmt.Fprintf(os.Stderr, "Payload validation failed: %v\n", err) + os.Exit(1) } + fmt.Fprintf(os.Stderr, + "[validation] plaintext match OK (bytes=%d, sha256=%s, encrypted-bytes old=%d new=%d)\n", + validation.Old.PlaintextBytes, + validation.Old.PlaintextSHA256, + validation.Old.EncryptedBytes, + validation.New.EncryptedBytes, + ) } - // Benchmark new implementation - if !cfg.SkipNew { - fmt.Println("\n" + "-" + strings.Repeat("-", 70)) - fmt.Println(" Benchmarking NEW implementation") - fmt.Println("-" + strings.Repeat("-", 70)) - - for i := 1; i <= cfg.Iterations; i++ { - fmt.Printf("\n Iteration %d/%d...\n", i, cfg.Iterations) - res := runBenchmark(client, newTarget, cfg) - result.New = append(result.New, res) - printIterationResult(res) - - if i < cfg.Iterations { - time.Sleep(2 * time.Second) - } - } - } + result := runBenchmarks(client, cfg, oldTarget, newTarget) // Calculate and print summary if !cfg.SkipOld { @@ -290,9 +304,23 @@ func parseFlags() Config { flag.BoolVar(&cfg.SkipOld, "skip-old", defaults.SkipOld, "Skip benchmarking old implementation") flag.BoolVar(&cfg.SkipNew, "skip-new", defaults.SkipNew, "Skip benchmarking new implementation") flag.BoolVar(&cfg.OutputJSON, "json", false, "Output results as JSON") + flag.Func("mode", "Benchmark mode: endpoint-e2e or validated-payload", func(val string) error { + mode, err := parseBenchmarkMode(val) + if err != nil { + return err + } + cfg.Mode = mode + return nil + }) + flag.StringVar(&cfg.VerifyPrivateKeyPath, "verify-private-key", defaults.VerifyPrivateKeyPath, "Private key path used to decrypt responses in validated-payload mode") + flag.StringVar(&cfg.VerifyPrivateKeyPassphrase, "verify-private-key-passphrase", defaults.VerifyPrivateKeyPassphrase, "Private key passphrase used in validated-payload mode") flag.Parse() //nolint:revive // deep-exit: called only from main() + if cfg.Mode == "" { + cfg.Mode = defaults.Mode + } + return cfg } @@ -300,18 +328,21 @@ func parseFlags() Config { // This enables auto-configuration when running in a Docker container. func loadEnvironmentDefaults() Config { cfg := Config{ - OldURL: getEnv("OLD_URL", ""), - NewURL: getEnv("NEW_URL", ""), - FileID: getEnv("FILE_ID", ""), - DatasetID: getEnv("FILE_DATASET", ""), - S3Path: getEnv("FILE_PATH", ""), - OldClientVersion: getEnv("OLD_CLIENT_VERSION", "v0.2.0"), - Iterations: getEnvInt("ITERATIONS", 5), - Requests: getEnvInt("REQUESTS", 100), - Concurrency: getEnvInt("CONCURRENCY", 10), - Timeout: 30 * time.Second, - SkipOld: getEnvBool("SKIP_OLD", false), - SkipNew: getEnvBool("SKIP_NEW", false), + OldURL: getEnv("OLD_URL", ""), + NewURL: getEnv("NEW_URL", ""), + FileID: getEnv("FILE_ID", ""), + DatasetID: getEnv("FILE_DATASET", ""), + S3Path: getEnv("FILE_PATH", ""), + OldClientVersion: getEnv("OLD_CLIENT_VERSION", "v0.2.0"), + Iterations: getEnvInt("ITERATIONS", 5), + Requests: getEnvInt("REQUESTS", 100), + Concurrency: getEnvInt("CONCURRENCY", 10), + Timeout: 30 * time.Second, + SkipOld: getEnvBool("SKIP_OLD", false), + SkipNew: getEnvBool("SKIP_NEW", false), + Mode: mustParseBenchmarkMode(getEnv("BENCHMARK_MODE", string(ModeEndpointE2E))), + VerifyPrivateKeyPath: getEnv("VERIFY_PRIVATE_KEY_PATH", "/shared/c4gh.sec.pem"), + VerifyPrivateKeyPassphrase: getEnv("VERIFY_PRIVATE_KEY_PASSPHRASE", "c4ghpass"), } // Try to read token from /shared/token (container mode) @@ -333,6 +364,26 @@ func loadEnvironmentDefaults() Config { return cfg } +func parseBenchmarkMode(val string) (BenchmarkMode, error) { + switch BenchmarkMode(strings.TrimSpace(val)) { + case ModeEndpointE2E: + return ModeEndpointE2E, nil + case ModeValidatedPayload: + return ModeValidatedPayload, nil + default: + return "", fmt.Errorf("unsupported benchmark mode %q", val) + } +} + +func mustParseBenchmarkMode(val string) BenchmarkMode { + mode, err := parseBenchmarkMode(val) + if err != nil { + return ModeEndpointE2E + } + + return mode +} + func getEnv(key, defaultVal string) string { if val := os.Getenv(key); val != "" { return val @@ -620,10 +671,38 @@ func validateConfig(cfg Config) error { if cfg.Concurrency < 1 { return errors.New("-concurrency must be at least 1") } + if _, err := parseBenchmarkMode(string(cfg.Mode)); err != nil { + return err + } + if cfg.Mode == ModeValidatedPayload { + if cfg.SkipOld || cfg.SkipNew { + return errors.New("validated-payload mode requires both old and new implementations") + } + if cfg.VerifyPrivateKeyPath == "" { + return errors.New("-verify-private-key is required in validated-payload mode") + } + } return nil } +func newHTTPClient(cfg Config) (*http.Client, error) { + jar, err := cookiejar.New(nil) + if err != nil { + return nil, fmt.Errorf("create cookie jar: %w", err) + } + + return &http.Client{ + Timeout: cfg.Timeout, + Jar: jar, + Transport: &http.Transport{ + MaxIdleConnsPerHost: cfg.Concurrency, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, //nolint:gosec // benchmark tool, not production + DisableCompression: true, + }, + }, nil +} + func preflight(client *http.Client, target Target, cfg Config) error { u, err := target.BuildURL(cfg) if err != nil { @@ -664,6 +743,7 @@ func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResul startTime := time.Now() for i := 0; i < cfg.Requests; i++ { + idx := i wg.Go(func() { semaphore <- struct{}{} defer func() { <-semaphore }() @@ -672,7 +752,7 @@ func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResul if target.Name == "old" && cfg.OldClientVersion != "" { extra = map[string]string{"SDA-Client-Version": cfg.OldClientVersion} } - results[i] = makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) + results[idx] = makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) }) } @@ -686,10 +766,10 @@ func makeRequest(client *http.Client, urlStr, token, publicKey, publicKeyHeader return makeRequestWithExtras(client, urlStr, token, publicKey, publicKeyHeader, nil) } -func makeRequestWithExtras(client *http.Client, urlStr, token, publicKey, publicKeyHeader string, extraHeaders map[string]string) RequestResult { +func buildRequest(urlStr, token, publicKey, publicKeyHeader string, extraHeaders map[string]string) (*http.Request, error) { req, err := http.NewRequest("GET", urlStr, nil) if err != nil { - return RequestResult{Error: err} + return nil, err } req.Header.Set("Authorization", token) req.Header.Set(publicKeyHeader, publicKey) @@ -697,6 +777,15 @@ func makeRequestWithExtras(client *http.Client, urlStr, token, publicKey, public req.Header.Set(k, v) } + return req, nil +} + +func makeRequestWithExtras(client *http.Client, urlStr, token, publicKey, publicKeyHeader string, extraHeaders map[string]string) RequestResult { + req, err := buildRequest(urlStr, token, publicKey, publicKeyHeader, extraHeaders) + if err != nil { + return RequestResult{Error: err} + } + start := time.Now() resp, err := client.Do(req) //nolint:gosec // benchmark tool, URLs from CLI flags if err != nil { @@ -867,9 +956,209 @@ func formatStatusCounts(m map[int]int) string { return strings.Join(parts, ", ") } +func runBenchmarks(client *http.Client, cfg Config, oldTarget, newTarget Target) *ComparisonResult { + result := &ComparisonResult{Mode: cfg.Mode} + + switch { + case !cfg.SkipOld && !cfg.SkipNew: + fmt.Println("-" + strings.Repeat("-", 70)) + fmt.Printf(" Benchmarking paired endpoint runs (%s)\n", cfg.Mode) + fmt.Println("-" + strings.Repeat("-", 70)) + + for i := 1; i <= cfg.Iterations; i++ { + order := pairedIterationTargets(i, oldTarget, newTarget) + fmt.Printf("\n Paired iteration %d/%d (%s -> %s)\n", + i, cfg.Iterations, strings.ToUpper(order[0].Name), strings.ToUpper(order[1].Name)) + for _, target := range order { + res := runBenchmark(client, target, cfg) + appendBenchmarkResult(result, target.Name, res) + fmt.Printf(" %s:\n", strings.ToUpper(target.Name)) + printIterationResult(res) + } + + if i < cfg.Iterations { + time.Sleep(2 * time.Second) + } + } + case !cfg.SkipOld: + fmt.Println("-" + strings.Repeat("-", 70)) + fmt.Println(" Benchmarking OLD endpoint") + fmt.Println("-" + strings.Repeat("-", 70)) + for i := 1; i <= cfg.Iterations; i++ { + fmt.Printf("\n Iteration %d/%d...\n", i, cfg.Iterations) + res := runBenchmark(client, oldTarget, cfg) + result.Old = append(result.Old, res) + printIterationResult(res) + if i < cfg.Iterations { + time.Sleep(2 * time.Second) + } + } + case !cfg.SkipNew: + fmt.Println("-" + strings.Repeat("-", 70)) + fmt.Println(" Benchmarking NEW endpoint") + fmt.Println("-" + strings.Repeat("-", 70)) + for i := 1; i <= cfg.Iterations; i++ { + fmt.Printf("\n Iteration %d/%d...\n", i, cfg.Iterations) + res := runBenchmark(client, newTarget, cfg) + result.New = append(result.New, res) + printIterationResult(res) + if i < cfg.Iterations { + time.Sleep(2 * time.Second) + } + } + } + + return result +} + +func pairedIterationTargets(iteration int, oldTarget, newTarget Target) []Target { + if iteration%2 == 1 { + return []Target{oldTarget, newTarget} + } + + return []Target{newTarget, oldTarget} +} + +func appendBenchmarkResult(result *ComparisonResult, targetName string, res BenchmarkResult) { + switch targetName { + case "old": + result.Old = append(result.Old, res) + case "new": + result.New = append(result.New, res) + } +} + +func validatePayloadEquivalence(client *http.Client, cfg Config, oldTarget, newTarget Target) (*payloadValidationResult, error) { + validator, err := newPayloadValidator(cfg.VerifyPrivateKeyPath, cfg.VerifyPrivateKeyPassphrase) + if err != nil { + return nil, err + } + + oldDigest, err := fetchPayloadDigest(client, oldTarget, cfg, validator) + if err != nil { + return nil, fmt.Errorf("old: %w", err) + } + newDigest, err := fetchPayloadDigest(client, newTarget, cfg, validator) + if err != nil { + return nil, fmt.Errorf("new: %w", err) + } + if err := comparePayloadDigests(oldDigest, newDigest); err != nil { + return nil, err + } + + return &payloadValidationResult{Old: oldDigest, New: newDigest}, nil +} + +func newPayloadValidator(path, passphrase string) (*payloadValidator, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("open verification private key %q: %w", path, err) + } + defer file.Close() + + var password []byte + if passphrase != "" { + password = []byte(passphrase) + } + + privateKey, err := keys.ReadPrivateKey(file, password) + if err != nil { + return nil, fmt.Errorf("read verification private key %q: %w", path, err) + } + + return &payloadValidator{privateKey: privateKey}, nil +} + +func fetchPayloadDigest(client *http.Client, target Target, cfg Config, validator *payloadValidator) (payloadDigest, error) { + u, err := target.BuildURL(cfg) + if err != nil { + return payloadDigest{}, err + } + + var extraHeaders map[string]string + if target.Name == "old" && cfg.OldClientVersion != "" { + extraHeaders = map[string]string{"SDA-Client-Version": cfg.OldClientVersion} + } + + req, err := buildRequest(u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extraHeaders) + if err != nil { + return payloadDigest{}, err + } + + resp, err := client.Do(req) //nolint:gosec // benchmark tool, URLs from CLI flags + if err != nil { + return payloadDigest{}, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4*1024)) + return payloadDigest{}, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) + } + + return validator.digest(resp.Body) +} + +func (v *payloadValidator) digest(body io.Reader) (payloadDigest, error) { + counted := &countingReader{reader: body} + reader, err := crypt4ghstreaming.NewCrypt4GHReader(counted, v.privateKey, nil) + if err != nil { + return payloadDigest{}, err + } + defer reader.Close() + + hash := sha256.New() + plaintextBytes, err := io.Copy(hash, reader) + if err != nil { + return payloadDigest{}, err + } + + return payloadDigest{ + EncryptedBytes: counted.bytesRead, + PlaintextBytes: plaintextBytes, + PlaintextSHA256: hex.EncodeToString(hash.Sum(nil)), + }, nil +} + +func comparePayloadDigests(oldDigest, newDigest payloadDigest) error { + if oldDigest.PlaintextBytes != newDigest.PlaintextBytes { + return fmt.Errorf("plaintext byte length mismatch: old=%d new=%d", oldDigest.PlaintextBytes, newDigest.PlaintextBytes) + } + if oldDigest.PlaintextSHA256 != newDigest.PlaintextSHA256 { + return fmt.Errorf("plaintext sha256 mismatch: old=%s new=%s", oldDigest.PlaintextSHA256, newDigest.PlaintextSHA256) + } + + return nil +} + +func percentChange(newValue, oldValue float64) float64 { + switch { + case oldValue == 0 && newValue == 0: + return 0 + case oldValue == 0 && newValue > 0: + return math.Inf(1) + case oldValue == 0: + return math.Inf(-1) + default: + return (newValue - oldValue) / oldValue * 100 + } +} + +type countingReader struct { + reader io.Reader + bytesRead int64 +} + +func (r *countingReader) Read(p []byte) (int, error) { + n, err := r.reader.Read(p) + r.bytesRead += int64(n) + + return n, err +} + func printSummary(result *ComparisonResult, cfg Config) { fmt.Println("\n" + "=" + strings.Repeat("=", 70)) - fmt.Println(" SUMMARY") + fmt.Printf(" SUMMARY (%s)\n", cfg.Mode) fmt.Println("=" + strings.Repeat("=", 70)) if !cfg.SkipOld { @@ -886,9 +1175,9 @@ func printSummary(result *ComparisonResult, cfg Config) { fmt.Println(" COMPARISON (NEW vs OLD)") fmt.Println("-" + strings.Repeat("-", 70)) - rpsChange := (result.NewSummary.AvgRequestsPerS - result.OldSummary.AvgRequestsPerS) / result.OldSummary.AvgRequestsPerS * 100 - latencyChange := float64(result.NewSummary.AvgLatency.Mean-result.OldSummary.AvgLatency.Mean) / float64(result.OldSummary.AvgLatency.Mean) * 100 - throughputChange := (result.NewSummary.AvgThroughput - result.OldSummary.AvgThroughput) / result.OldSummary.AvgThroughput * 100 + rpsChange := percentChange(result.NewSummary.AvgRequestsPerS, result.OldSummary.AvgRequestsPerS) + latencyChange := percentChange(float64(result.NewSummary.AvgLatency.Mean), float64(result.OldSummary.AvgLatency.Mean)) + throughputChange := percentChange(result.NewSummary.AvgThroughput, result.OldSummary.AvgThroughput) fmt.Printf("\n Requests/sec: %+.1f%% ", rpsChange) printChangeIndicator(rpsChange, true) @@ -905,9 +1194,9 @@ func printSummary(result *ComparisonResult, cfg Config) { fmt.Println("\n Verdict:") switch { case rpsChange > 5: - fmt.Println(" NEW implementation is FASTER") + fmt.Println(" NEW endpoint is FASTER") case rpsChange < -5: - fmt.Println(" OLD implementation is FASTER") + fmt.Println(" OLD endpoint is FASTER") default: fmt.Println(" Performance is SIMILAR (within 5%)") } @@ -917,7 +1206,7 @@ func printSummary(result *ComparisonResult, cfg Config) { } func printSummaryStats(s SummaryStats) { - fmt.Printf("\n %s Implementation (%d iterations):\n", s.Name, s.Iterations) + fmt.Printf("\n %s Endpoint (%d iterations):\n", s.Name, s.Iterations) fmt.Printf(" Requests/sec: %.2f (+/- %.2f)\n", s.AvgRequestsPerS, s.StdDevRequestsPerS) fmt.Printf(" Success rate: %.1f%%\n", s.SuccessRate) fmt.Printf(" Throughput: %.2f MB/s\n", s.AvgThroughput) From e66affc8053adb72a8b15fb84809a07763a11200 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 21:30:27 +0100 Subject: [PATCH 395/530] fix(download): fix benchmark compose to mount sda module, add validated target - Mount full sda/ module instead of benchmark dir alone so go.mod is available for crypt4gh dependency resolution - Bump benchmark image to golang:1.25-alpine to match go.mod - Add BENCHMARK_MODE and VERIFY_PRIVATE_KEY env vars - Add benchmark-download-validated Makefile target --- .github/integration/sda-benchmark.yml | 13 ++++++++----- Makefile | 26 +++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/.github/integration/sda-benchmark.yml b/.github/integration/sda-benchmark.yml index dd2e472b31..b5b9d3aaa0 100644 --- a/.github/integration/sda-benchmark.yml +++ b/.github/integration/sda-benchmark.yml @@ -87,7 +87,7 @@ services: - "8000:8000" restart: always - # Benchmark runner - compares performance between old and new + # Benchmark runner - compares public endpoint performance between old and new benchmark: container_name: benchmark command: @@ -95,8 +95,8 @@ services: - "-c" - | apk add --no-cache git && - cd /benchmark && - go run benchmark.go + cd /sda && + go run ./cmd/download/benchmark depends_on: download: condition: service_started @@ -114,12 +114,15 @@ services: - ITERATIONS=${BENCHMARK_ITERATIONS:-5} - REQUESTS=${BENCHMARK_REQUESTS:-100} - CONCURRENCY=${BENCHMARK_CONCURRENCY:-10} - image: golang:1.24-alpine + - BENCHMARK_MODE=${BENCHMARK_MODE:-endpoint-e2e} + - VERIFY_PRIVATE_KEY_PATH=${VERIFY_PRIVATE_KEY_PATH:-/shared/c4gh.sec.pem} + - VERIFY_PRIVATE_KEY_PASSPHRASE=${VERIFY_PRIVATE_KEY_PASSPHRASE:-c4ghpass} + image: golang:1.25-alpine profiles: - benchmark volumes: - shared:/shared - - ../../sda/cmd/download/benchmark:/benchmark + - ../../sda:/sda # Override integration_test to also depend on benchmark services and access scripts integration_test: diff --git a/Makefile b/Makefile index 8178cc28e5..07dcc3445d 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,7 @@ integrationtest-sda-cmd-download-up: build-all integrationtest-sda-cmd-download-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml down -v --remove-orphans -# Download benchmark (compares old vs new implementations) +# Download benchmark (compares old vs new public endpoints) # Uses sda-benchmark.yml which extends sda-s3-integration.yml with benchmark services # The benchmark runs in a container with auto-configuration from the environment benchmark-download-up: build-all @@ -183,6 +183,9 @@ benchmark-download-up: build-all benchmark-download-run: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml --profile benchmark run --rm benchmark +benchmark-download-run-validated: + @PR_NUMBER=$$(date +%F) BENCHMARK_MODE=validated-payload docker compose -f .github/integration/sda-benchmark.yml --profile benchmark run --rm benchmark + benchmark-download-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml down --remove-orphans @@ -196,7 +199,9 @@ benchmark-download-seed: @echo "Seeding minimal test data for benchmark..." @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-benchmark.yml run --rm integration_test /scripts/seed_benchmark_data.sh -benchmark-download: benchmark-download-up +benchmark-download: + @$(MAKE) benchmark-download-clean + @$(MAKE) benchmark-download-up @echo "Waiting for services to become healthy..." @printf " waiting for download..."; retries=0; \ until curl -sf http://localhost:$$(docker port download 8080 | head -1 | cut -d: -f2)/health/ready > /dev/null 2>&1; do \ @@ -209,6 +214,21 @@ benchmark-download: benchmark-download-up @$(MAKE) benchmark-download-run @$(MAKE) benchmark-download-down +benchmark-download-validated: + @$(MAKE) benchmark-download-clean + @$(MAKE) benchmark-download-up + @echo "Waiting for services to become healthy..." + @printf " waiting for download..."; retries=0; \ + until curl -sf http://localhost:$$(docker port download 8080 | head -1 | cut -d: -f2)/health/ready > /dev/null 2>&1; do \ + retries=$$((retries + 1)); if [ $$retries -ge 60 ]; then echo " TIMEOUT"; exit 1; fi; sleep 1; done; echo " ok" + @printf " waiting for download-old..."; retries=0; \ + until curl -sf http://localhost:$$(docker port download-old 8080 | head -1 | cut -d: -f2)/health > /dev/null 2>&1; do \ + retries=$$((retries + 1)); if [ $$retries -ge 60 ]; then echo " TIMEOUT"; exit 1; fi; sleep 1; done; echo " ok" + @$(MAKE) benchmark-download-seed + @echo "Running validated benchmark..." + @$(MAKE) benchmark-download-run-validated + @$(MAKE) benchmark-download-down + # lint go code lint-all: lint-sda lint-sda-download lint-sda-admin lint-sda: @@ -285,4 +305,4 @@ k3d-deploy-sda-s3-isolated-tls: k3d-deploy-sda-posix: @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false posix k3d-cleanup-all-deployments: - @bash .github/integration/scripts/charts/cleanup.sh \ No newline at end of file + @bash .github/integration/scripts/charts/cleanup.sh From 8aab67960693c368166e26ba7b8c2e28c278f78e Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 13 Mar 2026 21:31:57 +0100 Subject: [PATCH 396/530] test(download): add benchmark unit tests Tests for parseBenchmarkMode, pairedIterationTargets, comparePayloadDigests, and percentChange. --- sda/cmd/download/benchmark/benchmark_test.go | 104 +++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 sda/cmd/download/benchmark/benchmark_test.go diff --git a/sda/cmd/download/benchmark/benchmark_test.go b/sda/cmd/download/benchmark/benchmark_test.go new file mode 100644 index 0000000000..c4cde3018e --- /dev/null +++ b/sda/cmd/download/benchmark/benchmark_test.go @@ -0,0 +1,104 @@ +package main + +import ( + "math" + "testing" +) + +func TestParseBenchmarkMode(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + want BenchmarkMode + wantErr bool + }{ + {name: "endpoint-e2e", input: "endpoint-e2e", want: ModeEndpointE2E}, + {name: "validated-payload", input: "validated-payload", want: ModeValidatedPayload}, + {name: "invalid", input: "nope", wantErr: true}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := parseBenchmarkMode(tt.input) + if tt.wantErr { + if err == nil { + t.Fatalf("expected error for %q", tt.input) + } + + return + } + if err != nil { + t.Fatalf("unexpected error for %q: %v", tt.input, err) + } + if got != tt.want { + t.Fatalf("got %q, want %q", got, tt.want) + } + }) + } +} + +func TestPairedIterationTargets(t *testing.T) { + t.Parallel() + + oldTarget := Target{Name: "old"} + newTarget := Target{Name: "new"} + + if got := pairedIterationTargets(1, oldTarget, newTarget); got[0].Name != "old" || got[1].Name != "new" { + t.Fatalf("odd iteration order mismatch: %#v", got) + } + if got := pairedIterationTargets(2, oldTarget, newTarget); got[0].Name != "new" || got[1].Name != "old" { + t.Fatalf("even iteration order mismatch: %#v", got) + } +} + +func TestComparePayloadDigests(t *testing.T) { + t.Parallel() + + base := payloadDigest{ + EncryptedBytes: 10, + PlaintextBytes: 8, + PlaintextSHA256: "abc", + } + + if err := comparePayloadDigests(base, base); err != nil { + t.Fatalf("expected equal digests, got error: %v", err) + } + + if err := comparePayloadDigests(base, payloadDigest{ + EncryptedBytes: 11, + PlaintextBytes: 8, + PlaintextSHA256: "abc", + }); err != nil { + t.Fatalf("expected encrypted byte mismatch to be tolerated, got error: %v", err) + } + + mismatch := []payloadDigest{ + {EncryptedBytes: 10, PlaintextBytes: 9, PlaintextSHA256: "abc"}, + {EncryptedBytes: 10, PlaintextBytes: 8, PlaintextSHA256: "def"}, + } + + for _, other := range mismatch { + if err := comparePayloadDigests(base, other); err == nil { + t.Fatalf("expected mismatch error for %#v", other) + } + } +} + +func TestPercentChange(t *testing.T) { + t.Parallel() + + if got := percentChange(110, 100); got != 10 { + t.Fatalf("got %v, want 10", got) + } + if got := percentChange(0, 0); got != 0 { + t.Fatalf("got %v, want 0", got) + } + if got := percentChange(5, 0); !math.IsInf(got, 1) { + t.Fatalf("got %v, want +Inf", got) + } +} From c2b0e49dbb0c32c53f36ab8cba9aa04d2da46427 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Sat, 14 Mar 2026 09:02:44 +0100 Subject: [PATCH 397/530] fix(download): fix benchmark sort overflow, remove dead code - Replace int(a-b) duration sort with cmp.Compare to prevent overflow on 32-bit platforms - Remove unused makeRequest function - Remove redundant idx := i and tt := tt copies (Go 1.25 has per-iteration loop variables) --- sda/cmd/download/benchmark/benchmark.go | 16 +++++++++------- sda/cmd/download/benchmark/benchmark_test.go | 1 - 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sda/cmd/download/benchmark/benchmark.go b/sda/cmd/download/benchmark/benchmark.go index 02c5d0fd96..33c432c2d0 100644 --- a/sda/cmd/download/benchmark/benchmark.go +++ b/sda/cmd/download/benchmark/benchmark.go @@ -30,6 +30,7 @@ package main import ( + "cmp" "crypto/sha256" "crypto/tls" "encoding/base64" @@ -310,6 +311,7 @@ func parseFlags() Config { return err } cfg.Mode = mode + return nil }) flag.StringVar(&cfg.VerifyPrivateKeyPath, "verify-private-key", defaults.VerifyPrivateKeyPath, "Private key path used to decrypt responses in validated-payload mode") @@ -743,7 +745,6 @@ func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResul startTime := time.Now() for i := 0; i < cfg.Requests; i++ { - idx := i wg.Go(func() { semaphore <- struct{}{} defer func() { <-semaphore }() @@ -752,7 +753,7 @@ func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResul if target.Name == "old" && cfg.OldClientVersion != "" { extra = map[string]string{"SDA-Client-Version": cfg.OldClientVersion} } - results[idx] = makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) + results[i] = makeRequestWithExtras(client, u, cfg.Token, cfg.PublicKey, target.PublicKeyHeader, extra) }) } @@ -762,10 +763,6 @@ func runBenchmark(client *http.Client, target Target, cfg Config) BenchmarkResul return aggregateResults(target.Name, results, totalTime) } -func makeRequest(client *http.Client, urlStr, token, publicKey, publicKeyHeader string) RequestResult { - return makeRequestWithExtras(client, urlStr, token, publicKey, publicKeyHeader, nil) -} - func buildRequest(urlStr, token, publicKey, publicKeyHeader string, extraHeaders map[string]string) (*http.Request, error) { req, err := http.NewRequest("GET", urlStr, nil) if err != nil { @@ -844,7 +841,7 @@ func calculateStats(latencies []time.Duration) Stats { sorted := make([]time.Duration, len(latencies)) copy(sorted, latencies) - slices.SortFunc(sorted, func(a, b time.Duration) int { return int(a - b) }) + slices.SortFunc(sorted, cmp.Compare) var sum time.Duration for _, l := range sorted { @@ -1006,6 +1003,8 @@ func runBenchmarks(client *http.Client, cfg Config, oldTarget, newTarget Target) time.Sleep(2 * time.Second) } } + default: + // Both benchmarks are being skipped; nothing to do } return result @@ -1025,6 +1024,8 @@ func appendBenchmarkResult(result *ComparisonResult, targetName string, res Benc result.Old = append(result.Old, res) case "new": result.New = append(result.New, res) + default: + // Unknown target name; skip appending } } @@ -1093,6 +1094,7 @@ func fetchPayloadDigest(client *http.Client, target Target, cfg Config, validato if resp.StatusCode >= 400 { body, _ := io.ReadAll(io.LimitReader(resp.Body, 4*1024)) + return payloadDigest{}, fmt.Errorf("unexpected status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) } diff --git a/sda/cmd/download/benchmark/benchmark_test.go b/sda/cmd/download/benchmark/benchmark_test.go index c4cde3018e..e02fc8f972 100644 --- a/sda/cmd/download/benchmark/benchmark_test.go +++ b/sda/cmd/download/benchmark/benchmark_test.go @@ -20,7 +20,6 @@ func TestParseBenchmarkMode(t *testing.T) { } for _, tt := range tests { - tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() From c9d37629d7fa89a355adfdb81e51c7abad1f10a1 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Sat, 14 Mar 2026 09:39:56 +0100 Subject: [PATCH 398/530] fix(download): use consistent audit event name download.completed The handler emitted "download.complete" but the audit package documents and tests "download.completed". Align with the documented contract. --- sda/cmd/download/handlers/files.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/download/handlers/files.go b/sda/cmd/download/handlers/files.go index 691b3ddc8a..8132b2d118 100644 --- a/sda/cmd/download/handlers/files.go +++ b/sda/cmd/download/handlers/files.go @@ -265,7 +265,7 @@ func (h *Handlers) DownloadFile(c *gin.Context) { // Audit event on completion h.auditLogger.Log(c.Request.Context(), audit.Event{ - Event: "download.complete", + Event: "download.completed", UserID: resolved.authCtx.Subject, FileID: file.ID, DatasetID: file.DatasetID, From 69200a4c500b007ea9564c550895faeb92def4f8 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 17 Mar 2026 21:05:36 +0100 Subject: [PATCH 399/530] fix(download): include identity in visa cache key, reject bytes=- range Two fixes from 3-model code review (Gemini + Claude + GPT-5.4): 1. Visa validation cache keyed only by visa hash allowed identity binding bypass on cache hit. Include (issuer, subject) in cache key so strict binding modes are enforced even for cached visas. 2. Range header "bytes=-" (both groups empty) fell through to a full-file RangeSpec served as 206. Per RFC 9110 this is syntactically invalid; now rejected as ErrRangeInvalid. --- sda/cmd/download/streaming/streaming.go | 5 +++++ sda/cmd/download/streaming/streaming_test.go | 7 +++++++ sda/cmd/download/visa/validator.go | 5 +++-- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/sda/cmd/download/streaming/streaming.go b/sda/cmd/download/streaming/streaming.go index 9a00bc2800..a49a38751b 100644 --- a/sda/cmd/download/streaming/streaming.go +++ b/sda/cmd/download/streaming/streaming.go @@ -60,6 +60,11 @@ func ParseRangeHeader(rangeHeader string, fileSize int64) (*RangeSpec, error) { return nil, ErrRangeInvalid } + // Reject "bytes=-" (both groups empty) — syntactically invalid per RFC 9110 + if matches[1] == "" && matches[2] == "" { + return nil, ErrRangeInvalid + } + var start, end int64 // bytes=-SUFFIX (last N bytes) diff --git a/sda/cmd/download/streaming/streaming_test.go b/sda/cmd/download/streaming/streaming_test.go index ab6b9c0f97..e93ea58382 100644 --- a/sda/cmd/download/streaming/streaming_test.go +++ b/sda/cmd/download/streaming/streaming_test.go @@ -101,6 +101,13 @@ func TestParseRangeHeader_MultipleRanges(t *testing.T) { assert.Nil(t, result) } +func TestParseRangeHeader_BareHyphen(t *testing.T) { + // "bytes=-" (empty start and empty suffix) is syntactically invalid per RFC 9110 + result, err := ParseRangeHeader("bytes=-", 1000) + assert.ErrorIs(t, err, ErrRangeInvalid) + assert.Nil(t, result) +} + func TestParseRangeHeader_SuffixZero(t *testing.T) { // bytes=-0 requests 0 bytes, which is unsatisfiable result, err := ParseRangeHeader("bytes=-0", 1000) diff --git a/sda/cmd/download/visa/validator.go b/sda/cmd/download/visa/validator.go index 3a5b96cc68..51b8ad5f39 100644 --- a/sda/cmd/download/visa/validator.go +++ b/sda/cmd/download/visa/validator.go @@ -181,8 +181,9 @@ func (v *Validator) processVisas(ctx context.Context, identity Identity, passpor continue } - // Check validation cache - visaHash := hashToken(visaJWT) + // Check validation cache — key includes identity so that strict + // binding modes cannot be bypassed via a cache hit from another user. + visaHash := hashToken(visaJWT + "\x00" + identity.Issuer + "\x00" + identity.Subject) if cr, ok := v.getCachedVisa(visaHash); ok { for _, ds := range cr.Datasets { if !seen[ds] { From 5e8f633e5b608dad9b5ca65a1bc897abe35f325c Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:08:45 +0100 Subject: [PATCH 400/530] test(download): add integration tests with capability probes Docker Compose integration test environment: - postgres, minio (S3), mock-aai, mockoidc (OIDC + visa JWTs), reencrypt (gRPC), download service under test - database_seed with test dataset + file - make_download_credentials.sh: RSA keypair, JWT token, trusted issuers - mockoidc.py: OIDC discovery, JWKS, userinfo with visa datasets 33 integration tests covering: - Health, auth (JWT + opaque), session cookies, service-info - Dataset listing, file listing, pagination, invalid pageSize/pageToken - Encoded-slash dataset IDs (UseRawPath verification) - Range requests, multi-range rejection, If-Range ETag contract - Content-Disposition, pathPrefix filter with SQL wildcard escaping - Expired token rejection, long-transfer resume scenario - Problem Details format, access control (no existence leakage) Environment capability probes (SetupSuite): - REQUIRES_REENCRYPT, REQUIRES_STORAGE_FILE, REQUIRES_SESSION_CACHE - Tests skip on missing prerequisites, hard-fail on regressions --- .../integration/sda-download-integration.yml | 264 ++++++++++++++++++ .github/integration/sda/download-config.yaml | 86 ++++++ 2 files changed, 350 insertions(+) create mode 100644 .github/integration/sda-download-integration.yml create mode 100644 .github/integration/sda/download-config.yaml diff --git a/.github/integration/sda-download-integration.yml b/.github/integration/sda-download-integration.yml new file mode 100644 index 0000000000..718571a5f3 --- /dev/null +++ b/.github/integration/sda-download-integration.yml @@ -0,0 +1,264 @@ +# Docker Compose for download service integration tests +# Based on validator-orchestrator pattern - Go-based tests running in container + +services: + # PostgreSQL with SDA schema + postgres: + build: + context: ../../postgresql + container_name: postgres + environment: + - POSTGRES_PASSWORD=rootpasswd + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 2s + retries: 6 + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + extra_hosts: + - "localhost:host-gateway" + ports: + - "15432:5432" + restart: always + volumes: + - postgres_data:/var/lib/postgresql/data + + # MinIO for archive storage + minio: + image: minio/minio + command: server /data --console-address ":9001" + container_name: s3 + environment: + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretKey + - MINIO_SERVER_URL=http://127.0.0.1:9000 + healthcheck: + test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 2s + retries: 6 + ports: + - "19000:9000" + - "19001:9001" + volumes: + - minio_data:/data + + # Mock AAI for JWT validation + mock-aai: + container_name: ls-aai-mock + depends_on: + aai-db: + condition: service_healthy + environment: + - DOCKERHOST=localhost + healthcheck: + test: ["CMD", "/bin/true"] + interval: 10s + timeout: 2s + retries: 6 + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.3-broker2.1.13-tomcat9.0-jdk21 + ports: + - "8800:8080" + volumes: + - "./sda/aai-mock:/etc/lsaai-mock" + + aai-db: + container_name: ls-aai-db + environment: + MYSQL_ROOT_PASSWORD: "aaiPass" + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: "aai" + MYSQL_USER: "aai" + MYSQL_PASSWORD: "aaiPass" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 2s + retries: 6 + image: mysql/mysql-server:latest + volumes: + - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql + + # Credentials generator - creates JWT keys and test tokens + credentials: + container_name: credentials + command: + - "/bin/sh" + - "-c" + - | + sh /scripts/make_download_credentials.sh + depends_on: + postgres: + condition: service_healthy + environment: + - PGPASSWORD=rootpasswd + image: python:3.11-slim-bookworm + volumes: + - ./scripts:/scripts + - shared:/shared + + # Initialize MinIO with archive bucket and test data + minio_init: + container_name: minio_init + image: minio/mc + depends_on: + minio: + condition: service_healthy + credentials: + condition: service_completed_successfully + entrypoint: > + /bin/sh -c " + mc alias set myminio http://s3:9000 access secretKey; + mc mb myminio/archive || true; + dd if=/dev/urandom bs=1000 count=1 2>/dev/null | mc pipe myminio/archive/test-file.c4gh; + echo 'MinIO initialized with archive bucket and test file'; + " + + # Seed database with test data (files and datasets) + database_seed: + container_name: database_seed + image: postgres:15.4-alpine + depends_on: + credentials: + condition: service_completed_successfully + minio_init: + condition: service_completed_successfully + environment: + - PGHOST=postgres + - PGPORT=5432 + - PGUSER=postgres + - PGPASSWORD=rootpasswd + - PGDATABASE=sda + command: + - "/bin/sh" + - "-c" + - | + # Wait for postgres to be ready + pg_isready -h postgres -p 5432 -U postgres + + # Insert test file + psql -c "INSERT INTO sda.files ( + id, stable_id, submission_user, submission_file_path, + archive_file_path, archive_location, archive_file_size, decrypted_file_size, + header, encryption_method + ) VALUES ( + 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee', + 'EGAF00000000001', + 'integration_test@example.org', + 'test-file.c4gh', + 'test-file.c4gh', + 's3:9000/archive', + 1000, + 500, + '63727970743467680100000000000000', + 'CRYPT4GH' + ) ON CONFLICT DO NOTHING;" + + # Insert test dataset (id is SERIAL, so let postgres generate it) + psql -c "INSERT INTO sda.datasets (stable_id, title) + VALUES ('EGAD00000000001', 'Test Dataset') + ON CONFLICT DO NOTHING;" + + # Get the dataset id and link file to dataset + psql -c "INSERT INTO sda.file_dataset (file_id, dataset_id) + SELECT 'aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee'::uuid, d.id + FROM sda.datasets d + WHERE d.stable_id = 'EGAD00000000001' + ON CONFLICT DO NOTHING;" + + echo 'Database seeded with test data'; + + # Mock OIDC server for opaque token auth and visa validation + mockauth: + command: + - /bin/sh + - -c + - | + pip install --upgrade pip > /dev/null + pip install aiohttp Authlib joserfc > /dev/null + python -u /mockoidc.py + container_name: mockauth + image: python:3.10-slim + volumes: + - ./scripts/mockoidc.py:/mockoidc.py + ports: + - "8000:8000" + restart: always + + # Re-encrypt gRPC service + reencrypt: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: [sda-reencrypt] + container_name: reencrypt + depends_on: + credentials: + condition: service_completed_successfully + ports: + - "50051:50051" + restart: always + volumes: + - ./sda/config.yaml:/config.yaml + - shared:/shared + + # Download service under test + download: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + command: + - sda-download + - "--config-file=/config.yaml" + container_name: download + depends_on: + credentials: + condition: service_completed_successfully + database_seed: + condition: service_completed_successfully + postgres: + condition: service_healthy + reencrypt: + condition: service_started + mock-aai: + condition: service_healthy + mockauth: + condition: service_started + environment: + - CONFIG_FILE=/config.yaml + extra_hosts: + - "localhost:host-gateway" + ports: + - "8085:8080" + - "8086:8081" + restart: always + volumes: + - ./sda/download-config.yaml:/config.yaml + - shared:/shared + + # Go-based integration test runner + integration_test: + container_name: download_tester + command: + - "/bin/sh" + - "-c" + - | + cd /tests + go mod download + go test -v ./... + depends_on: + download: + condition: service_started + environment: + - DOWNLOAD_URL=http://download:8080 + - JWT_KEY_FILE=/shared/keys/jwt.key + - C4GH_PUBKEY_FILE=/shared/c4gh.pub.pem + extra_hosts: + - "localhost:host-gateway" + image: golang:1.25-alpine + profiles: + - tests + volumes: + - shared:/shared + - ./tests/download:/tests + +volumes: + minio_data: + postgres_data: + shared: diff --git a/.github/integration/sda/download-config.yaml b/.github/integration/sda/download-config.yaml new file mode 100644 index 0000000000..0ca6bd1d3d --- /dev/null +++ b/.github/integration/sda/download-config.yaml @@ -0,0 +1,86 @@ +# Configuration for sda/cmd/download service in integration tests + +# API Server +api: + host: "0.0.0.0" + port: 8080 + +# gRPC Health check +health: + port: 8081 + +# Database +db: + host: "postgres" + port: 5432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +# gRPC reencrypt service +grpc: + host: "reencrypt" + port: 50051 + timeout: 30 + +# JWT configuration - uses keys created by credentials script +jwt: + pubkey-path: "/shared/keys/pub/" + allow-all-data: true # For testing: allow all authenticated users access to all datasets + +# Permission model: combined (ownership + visa) +permission: + model: "combined" + +# Auth: allow opaque tokens (userinfo-based authentication) +auth: + allow-opaque: true + +# GA4GH visa configuration +visa: + enabled: true + source: "userinfo" + userinfo-url: "http://mockauth:8000/userinfo" + trusted-issuers-path: "/shared/trusted-issuers.json" + dataset-id-mode: "raw" + identity: + mode: "broker-bound" + validate-asserted: true + allow-insecure-jku: true # mockauth uses HTTP (testing only) + limits: + max-visas: 200 + max-jwks-per-request: 10 + max-visa-size: 16384 + cache: + token-ttl: 3600 + jwk-ttl: 300 + validation-ttl: 120 + userinfo-ttl: 60 + +# Session +session: + expiration: 3600 + secure: false + http-only: true + name: "sda_session" + +# Database query cache +cache: + enabled: true + file-ttl: 300 # 5 minutes + permission-ttl: 120 # 2 minutes + dataset-ttl: 300 # 5 minutes + +# Storage configuration (S3 backend) +storage: + backend: "archive" + archive: + s3: + # NOTE: include scheme to keep storage/v2 location parsing consistent (archive_location values are URL-parsed). + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true From 8b84add182d895fc7597da9727a2ad5add182917 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 23 Feb 2026 11:09:20 +0100 Subject: [PATCH 401/530] feat(download): add benchmark tool for old vs new comparison Benchmark infrastructure: - benchmark.go: concurrent load tester comparing old (sda-download) vs new (sda/cmd/download) services with auto-discovery, JWT auth, configurable iterations/concurrency, percentile stats (p50/p95/p99) - sda-benchmark.yml: Docker Compose with both services, shared pipeline, and benchmark runner - seed_benchmark_data.sh: uploads crypt4gh files through real ingest pipeline for realistic test data - Makefile targets: benchmark-download-{up,seed,run,down} Result: NEW service is ~255% faster than OLD (67 vs 19 req/s). --- .../sda/download-config-benchmark.yaml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/integration/sda/download-config-benchmark.yaml diff --git a/.github/integration/sda/download-config-benchmark.yaml b/.github/integration/sda/download-config-benchmark.yaml new file mode 100644 index 0000000000..cea66446c0 --- /dev/null +++ b/.github/integration/sda/download-config-benchmark.yaml @@ -0,0 +1,93 @@ +# Configuration for sda/cmd/download service in BENCHMARK mode +# Uses visa-based authorization to match old sda-download behavior for fair comparison +# The endpoint uses http:// prefix for storage/v2 URL parsing + +# API Server +api: + host: "0.0.0.0" + port: 8080 + +# Logging +log: + level: "debug" + +# gRPC Health check +health: + port: 8081 + +# Database +db: + host: "postgres" + port: 5432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +# gRPC reencrypt service +grpc: + host: "reencrypt" + port: 50051 + timeout: 30 + +# JWT configuration - uses keys created by credentials script +jwt: + pubkey-path: "/shared/keys/pub/" + allow-all-data: false + +# Permission model: combined (ownership + visa) for benchmark +# This allows graceful fallback if visa processing has issues +permission: + model: "combined" + +# Auth: allow opaque tokens (old sda-download supported this) +auth: + allow-opaque: true + +# GA4GH visa configuration - matches old sda-download's OIDC/visa flow +visa: + enabled: true + source: "userinfo" + userinfo-url: "http://mockauth:8000/userinfo" + trusted-issuers-path: "/shared/trusted-issuers.json" + dataset-id-mode: "raw" + identity: + mode: "broker-bound" + validate-asserted: true + allow-insecure-jku: true # mockauth uses HTTP (testing only) + limits: + max-visas: 200 + max-jwks-per-request: 10 + max-visa-size: 16384 + cache: + token-ttl: 3600 + jwk-ttl: 300 + validation-ttl: 120 + userinfo-ttl: 60 + +# Session +session: + expiration: 3600 + secure: false + http-only: true + name: "sda_session" + +# Database query cache +cache: + enabled: true + file-ttl: 300 # 5 minutes + permission-ttl: 120 # 2 minutes + dataset-ttl: 300 # 5 minutes + +# Storage configuration (S3 backend) +# NOTE: endpoint includes http:// for proper URL parsing in storage/v2 +storage: + backend: "archive" + archive: + s3: + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true From 642b4cbb36d4018927d0e4cf84a02fed300131b4 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 3 Mar 2026 16:01:37 +0100 Subject: [PATCH 402/530] feat(download): add CreatedAt to File struct and SQL queries --- sda/cmd/download/database/database.go | 9 +++++-- sda/cmd/download/database/database_test.go | 30 +++++++++++----------- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/sda/cmd/download/database/database.go b/sda/cmd/download/database/database.go index 768ff72312..d46dc050f0 100644 --- a/sda/cmd/download/database/database.go +++ b/sda/cmd/download/database/database.go @@ -117,7 +117,8 @@ var queries = map[string]string{ f.decrypted_file_size, c.checksum as decrypted_checksum, c.type as decrypted_checksum_type, - f.header + f.header, + f.created_at FROM sda.files f INNER JOIN sda.file_dataset fd ON f.id = fd.file_id INNER JOIN sda.datasets d ON fd.dataset_id = d.id @@ -135,7 +136,8 @@ var queries = map[string]string{ f.decrypted_file_size, c.checksum as decrypted_checksum, c.type as decrypted_checksum_type, - f.header + f.header, + f.created_at FROM sda.files f INNER JOIN sda.file_dataset fd ON f.id = fd.file_id INNER JOIN sda.datasets d ON fd.dataset_id = d.id @@ -270,6 +272,7 @@ type File struct { DecryptedChecksum string `json:"decryptedChecksum"` DecryptedChecksumType string `json:"decryptedChecksumType"` Header []byte `json:"-"` + CreatedAt time.Time `json:"-"` Checksums []Checksum `json:"checksums,omitempty"` // Aggregated checksums (from paginated queries) } @@ -546,6 +549,7 @@ func (p *PostgresDB) GetFileByID(ctx context.Context, fileID string) (*File, err &decryptedChecksum, &decryptedChecksumType, &headerHex, + &f.CreatedAt, ) if err == sql.ErrNoRows { return nil, nil @@ -604,6 +608,7 @@ func (p *PostgresDB) GetFileByPath(ctx context.Context, datasetID, filePath stri &decryptedChecksum, &decryptedChecksumType, &headerHex, + &f.CreatedAt, ) if err == sql.ErrNoRows { return nil, nil diff --git a/sda/cmd/download/database/database_test.go b/sda/cmd/download/database/database_test.go index 324305dfb7..805898cdb2 100644 --- a/sda/cmd/download/database/database_test.go +++ b/sda/cmd/download/database/database_test.go @@ -247,10 +247,10 @@ func TestGetFileByID(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "decrypted_file_checksum_type", "header", "created_at", }). AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", - int64(1024), int64(900), "abc123", "SHA256", headerHex) + int64(1024), int64(900), "abc123", "SHA256", headerHex, time.Now()) mock.ExpectQuery(queries[getFileByIDQuery]). WithArgs("file-1"). @@ -274,7 +274,7 @@ func TestGetFileByID_NotFound(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "decrypted_file_checksum_type", "header", "created_at", }) mock.ExpectQuery(queries[getFileByIDQuery]). @@ -297,10 +297,10 @@ func TestGetFileByID_HeaderTooSmall(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "decrypted_file_checksum_type", "header", "created_at", }). AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", - int64(1024), int64(900), "abc123", "SHA256", headerHex) + int64(1024), int64(900), "abc123", "SHA256", headerHex, time.Now()) mock.ExpectQuery(queries[getFileByIDQuery]). WithArgs("file-1"). @@ -323,10 +323,10 @@ func TestGetFileByID_InvalidHexHeader(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "decrypted_file_checksum_type", "header", "created_at", }). AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", - int64(1024), int64(900), "abc123", "SHA256", headerHex) + int64(1024), int64(900), "abc123", "SHA256", headerHex, time.Now()) mock.ExpectQuery(queries[getFileByIDQuery]). WithArgs("file-1"). @@ -350,10 +350,10 @@ func TestGetFileByPath(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "decrypted_file_checksum_type", "header", "created_at", }). AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", - int64(1024), int64(900), "abc123", "SHA256", headerHex) + int64(1024), int64(900), "abc123", "SHA256", headerHex, time.Now()) mock.ExpectQuery(queries[getFileByPathQuery]). WithArgs("dataset-1", "/path/to/file.txt"). @@ -378,10 +378,10 @@ func TestGetFileByPath_HeaderTooSmall(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "decrypted_file_checksum_type", "header", "created_at", }). AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", - int64(1024), int64(900), "abc123", "SHA256", headerHex) + int64(1024), int64(900), "abc123", "SHA256", headerHex, time.Now()) mock.ExpectQuery(queries[getFileByPathQuery]). WithArgs("dataset-1", "/path/to/file.txt"). @@ -404,10 +404,10 @@ func TestGetFileByPath_InvalidHexHeader(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "decrypted_file_checksum_type", "header", "created_at", }). AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", - int64(1024), int64(900), "abc123", "SHA256", headerHex) + int64(1024), int64(900), "abc123", "SHA256", headerHex, time.Now()) mock.ExpectQuery(queries[getFileByPathQuery]). WithArgs("dataset-1", "/path/to/file.txt"). @@ -558,8 +558,8 @@ func TestGetFileByPath_NotFound(t *testing.T) { rows := sqlmock.NewRows([]string{ "stable_id", "dataset_id", "submission_file_path", "archive_file_path", - "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", - "decrypted_file_checksum_type", "header", + "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", + "decrypted_file_checksum_type", "header", "created_at", }) mock.ExpectQuery(queries[getFileByPathQuery]). From 53b3fa440df40a429de828045d7d351452109eea Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 3 Mar 2026 16:03:59 +0100 Subject: [PATCH 403/530] docs(download): add DRS object endpoint to v2 API spec --- sda/cmd/download/swagger_v2.yml | 137 ++++++++++++++++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/sda/cmd/download/swagger_v2.yml b/sda/cmd/download/swagger_v2.yml index 2669453fa9..4abdeac852 100644 --- a/sda/cmd/download/swagger_v2.yml +++ b/sda/cmd/download/swagger_v2.yml @@ -64,6 +64,8 @@ tags: description: File metadata and download operations. - name: Health description: Service health endpoints. + - name: DRS + description: GA4GH Data Repository Service (DRS) 1.5 compatible endpoints. paths: /service-info: @@ -632,6 +634,71 @@ paths: # 3) Concatenate into a valid Crypt4GH file cat header.bin content.bin > file.c4gh + /objects/{path}: + get: + operationId: getDrsObject + summary: Resolve a dataset file to a DRS 1.5 object + description: | + Resolves a `{datasetId}/{filePath}` to a minimal GA4GH DRS 1.5 `DrsObject` + with a pre-resolved `access_url` pointing to the file content endpoint. + + This endpoint enables htsget-rs and other DRS-aware clients to discover + download URLs without knowing the internal file ID. + + The `{path}` parameter is composite: `{datasetId}/{filePath}`, + e.g. `EGAD00001000001/samples/sample1.bam.c4gh`. Everything before the + first `/` is the dataset ID; everything after is the file path. + + **Note:** Some OpenAPI code generators may not handle slash-containing + path parameters. Clients should URL-encode slashes in filePath segments + if their HTTP library requires it. The server accepts both encoded and + unencoded paths (`UseRawPath` is enabled). + tags: + - DRS + security: + - bearerAuth: [] + parameters: + - name: path + in: path + required: true + description: | + Composite path: `{datasetId}/{filePath}`. Everything before the first `/` + is the dataset ID; everything after is the file path within the dataset. + schema: + type: string + example: "EGAD00001000001/samples/controls/sample1.bam.c4gh" + responses: + '200': + description: DRS object with resolved access URL + content: + application/json: + schema: + $ref: '#/components/schemas/DrsObject' + '400': + description: Malformed path (missing dataset or file component) + content: + application/problem+json: + schema: + $ref: '#/components/schemas/ProblemDetails' + '401': + description: Missing or invalid authentication token + content: + application/problem+json: + schema: + $ref: '#/components/schemas/ProblemDetails' + '403': + description: Access denied (also returned when the resource does not exist) + content: + application/problem+json: + schema: + $ref: '#/components/schemas/ProblemDetails' + '500': + description: Internal server error + content: + application/problem+json: + schema: + $ref: '#/components/schemas/ProblemDetails' + /health/ready: get: tags: [Health] @@ -876,6 +943,76 @@ components: storage: ok required: [status] + DrsObject: + type: object + description: Minimal GA4GH DRS 1.5 object response. + required: + - id + - self_uri + - size + - created_time + - checksums + - access_methods + properties: + id: + type: string + description: DRS object identifier (file stable ID). + example: "urn:neic:001-002-003" + self_uri: + type: string + description: Self-referential DRS URI. + example: "drs://download.example.org/urn:neic:001-002-003" + size: + type: integer + format: int64 + description: Decrypted file size in bytes. + created_time: + type: string + format: date-time + description: File creation timestamp (RFC 3339). + example: "2026-01-15T10:30:00Z" + checksums: + type: array + items: + $ref: '#/components/schemas/DrsChecksum' + access_methods: + type: array + items: + $ref: '#/components/schemas/DrsAccessMethod' + DrsChecksum: + type: object + required: + - checksum + - type + properties: + checksum: + type: string + description: Checksum value. + type: + type: string + description: Checksum algorithm (e.g. "sha-256", "md5"). + DrsAccessMethod: + type: object + required: + - type + - access_url + properties: + type: + type: string + description: Access type (always "https"). + example: "https" + access_url: + $ref: '#/components/schemas/DrsAccessURL' + DrsAccessURL: + type: object + required: + - url + properties: + url: + type: string + description: Pre-resolved download URL for the file content. + example: "https://download.example.org/files/urn:neic:001-002-003/content" + parameters: DatasetIdPath: name: datasetId From 8130c483389f21e2c204372f6800bc28910fa63c Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 3 Mar 2026 16:44:06 +0100 Subject: [PATCH 404/530] feat(download): add DRS object endpoint handler and tests --- sda/cmd/download/handlers/drs.go | 124 +++++++++++ sda/cmd/download/handlers/drs_test.go | 243 +++++++++++++++++++++ sda/cmd/download/handlers/handlers.go | 7 + sda/cmd/download/handlers/handlers_test.go | 1 + 4 files changed, 375 insertions(+) create mode 100644 sda/cmd/download/handlers/drs.go create mode 100644 sda/cmd/download/handlers/drs_test.go diff --git a/sda/cmd/download/handlers/drs.go b/sda/cmd/download/handlers/drs.go new file mode 100644 index 0000000000..10b4e28666 --- /dev/null +++ b/sda/cmd/download/handlers/drs.go @@ -0,0 +1,124 @@ +package handlers + +import ( + "fmt" + "net/http" + "strings" + + "github.com/gin-gonic/gin" + "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + log "github.com/sirupsen/logrus" +) + +// DrsObject represents a GA4GH DRS object response. +type DrsObject struct { + ID string `json:"id"` + SelfURI string `json:"self_uri"` + Size int64 `json:"size"` + CreatedTime string `json:"created_time"` + Checksums []DrsChecksum `json:"checksums"` + AccessMethods []DrsAccessMethod `json:"access_methods"` +} + +// DrsChecksum represents a checksum in a DRS object. +type DrsChecksum struct { + Checksum string `json:"checksum"` + Type string `json:"type"` +} + +// DrsAccessMethod represents an access method in a DRS object. +type DrsAccessMethod struct { + Type string `json:"type"` + AccessURL DrsAccessURL `json:"access_url"` +} + +// DrsAccessURL represents an access URL in a DRS access method. +type DrsAccessURL struct { + URL string `json:"url"` +} + +// drsChecksumType normalises an SDA checksum type to the DRS/GA4GH lowercase form. +func drsChecksumType(sdaType string) string { + switch strings.ToLower(sdaType) { + case "sha256", "sha-256": + return "sha-256" + case "sha512", "sha-512": + return "sha-512" + default: + return strings.ToLower(sdaType) + } +} + +// GetDrsObject returns a GA4GH DRS object for a file identified by dataset and path. +// GET /objects/{datasetId}/{filePath} +func (h *Handlers) GetDrsObject(c *gin.Context) { + rawPath := strings.TrimPrefix(c.Param("path"), "/") + + idx := strings.Index(rawPath, "/") + if idx <= 0 || idx == len(rawPath)-1 { + problemJSON(c, http.StatusBadRequest, "path must contain {datasetId}/{filePath}") + + return + } + + datasetID := rawPath[:idx] + filePath := rawPath[idx+1:] + + authCtx, ok := middleware.GetAuthContext(c) + if !ok { + problemJSON(c, http.StatusUnauthorized, "authentication required") + + return + } + + if !hasDatasetAccess(authCtx.Datasets, datasetID) { + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return + } + + file, err := h.db.GetFileByPath(c.Request.Context(), datasetID, filePath) + if err != nil { + log.Errorf("failed to get file by path: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to retrieve file") + + return + } + + if file == nil { + problemJSON(c, http.StatusForbidden, "access denied") + h.auditDenied(c) + + return + } + + host := c.Request.Host + + checksums := []DrsChecksum{} + if file.DecryptedChecksum != "" { + checksums = append(checksums, DrsChecksum{ + Checksum: file.DecryptedChecksum, + Type: drsChecksumType(file.DecryptedChecksumType), + }) + } + + obj := DrsObject{ + ID: file.ID, + SelfURI: fmt.Sprintf("drs://%s/%s", host, file.ID), + Size: file.DecryptedSize, + CreatedTime: file.CreatedAt.UTC().Format("2006-01-02T15:04:05Z07:00"), + Checksums: checksums, + AccessMethods: []DrsAccessMethod{ + { + Type: "https", + AccessURL: DrsAccessURL{ + URL: fmt.Sprintf("https://%s/files/%s/content", host, file.ID), + }, + }, + }, + } + + c.Header("Cache-Control", "private, max-age=60, must-revalidate") + c.JSON(http.StatusOK, obj) +} diff --git a/sda/cmd/download/handlers/drs_test.go b/sda/cmd/download/handlers/drs_test.go new file mode 100644 index 0000000000..2ecd4f8af0 --- /dev/null +++ b/sda/cmd/download/handlers/drs_test.go @@ -0,0 +1,243 @@ +package handlers + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/neicnordic/sensitive-data-archive/cmd/download/database" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type drsObjectResponse struct { + ID string `json:"id"` + SelfURI string `json:"self_uri"` + Size int64 `json:"size"` + CreatedTime string `json:"created_time"` + Checksums []drsChecksumJSON `json:"checksums"` + AccessMethods []drsAccessMethodJSON `json:"access_methods"` +} + +type drsChecksumJSON struct { + Checksum string `json:"checksum"` + Type string `json:"type"` +} + +type drsAccessMethodJSON struct { + Type string `json:"type"` + AccessURL drsAccessURLJSON `json:"access_url"` +} + +type drsAccessURLJSON struct { + URL string `json:"url"` +} + +func TestGetDrsObject_Success(t *testing.T) { + router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) + mockDB := &mockDatabase{ + fileByPath: &database.File{ + ID: "urn:neic:001-002-003", + DatasetID: "EGAD00001000001", + SubmittedPath: "samples/controls/sample1.bam.c4gh", + DecryptedSize: 1048576, + DecryptedChecksum: "a1b2c3d4", + DecryptedChecksumType: "SHA256", + CreatedAt: time.Date(2026, 1, 15, 10, 30, 0, 0, time.UTC), + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, "/objects/EGAD00001000001/samples/controls/sample1.bam.c4gh", nil) + req.Host = "download.example.org" + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "private, max-age=60, must-revalidate", w.Header().Get("Cache-Control")) + + var resp drsObjectResponse + err = json.Unmarshal(w.Body.Bytes(), &resp) + require.NoError(t, err) + + assert.Equal(t, "urn:neic:001-002-003", resp.ID) + assert.Equal(t, "drs://download.example.org/urn:neic:001-002-003", resp.SelfURI) + assert.Equal(t, int64(1048576), resp.Size) + assert.Equal(t, "2026-01-15T10:30:00Z", resp.CreatedTime) + + require.Len(t, resp.Checksums, 1) + assert.Equal(t, "sha-256", resp.Checksums[0].Type) + assert.Equal(t, "a1b2c3d4", resp.Checksums[0].Checksum) + + require.Len(t, resp.AccessMethods, 1) + assert.Equal(t, "https", resp.AccessMethods[0].Type) + assert.Equal(t, "https://download.example.org/files/urn:neic:001-002-003/content", resp.AccessMethods[0].AccessURL.URL) +} + +func TestGetDrsObject_FileNotFound_Returns403(t *testing.T) { + router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) + mockDB := &mockDatabase{ + fileByPath: nil, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, "/objects/EGAD00001000001/no-such-file.bam", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) + + var resp ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &resp) + require.NoError(t, err) + assert.Equal(t, "access denied", resp.Detail) +} + +func TestGetDrsObject_NoDatasetAccess_Returns403(t *testing.T) { + router := setupTestRouterWithAuth([]string{"OTHER-DATASET"}) + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, "/objects/EGAD00001000001/file.bam", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusForbidden, w.Code) +} + +func TestGetDrsObject_MalformedPath_Returns400(t *testing.T) { + testCases := []struct { + name string + path string + }{ + {"no slash", "/objects/just-dataset-id"}, + {"trailing slash", "/objects/dataset/"}, + {"empty dataset", "/objects//file.bam"}, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + router := setupTestRouterWithAuth([]string{"dataset"}) + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, tc.path, nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + + var resp ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &resp) + require.NoError(t, err) + assert.Contains(t, resp.Detail, "path must contain") + }) + } +} + +func TestGetDrsObject_Unauthenticated_Returns401(t *testing.T) { + router := setupTestRouter() + mockDB := &mockDatabase{} + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, "/objects/EGAD00001000001/file.bam", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusUnauthorized, w.Code) +} + +func TestGetDrsObject_DBError_Returns500(t *testing.T) { + router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) + mockDB := &mockDatabase{ + err: errors.New("db error"), + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, "/objects/EGAD00001000001/file.bam", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusInternalServerError, w.Code) +} + +func TestGetDrsObject_NoChecksum_EmptyArray(t *testing.T) { + router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) + mockDB := &mockDatabase{ + fileByPath: &database.File{ + ID: "urn:neic:001-002-003", + DatasetID: "EGAD00001000001", + SubmittedPath: "file.bam", + DecryptedSize: 512, + DecryptedChecksum: "", + CreatedAt: time.Date(2026, 1, 15, 10, 30, 0, 0, time.UTC), + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, "/objects/EGAD00001000001/file.bam", nil) + req.Host = "download.example.org" + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + // Verify checksums is [] not null in raw JSON + var raw map[string]json.RawMessage + err = json.Unmarshal(w.Body.Bytes(), &raw) + require.NoError(t, err) + assert.Equal(t, "[]", string(raw["checksums"])) +} + +func TestDrsChecksumType(t *testing.T) { + testCases := []struct { + input string + expected string + }{ + {"SHA256", "sha-256"}, + {"sha256", "sha-256"}, + {"SHA-256", "sha-256"}, + {"SHA512", "sha-512"}, + {"MD5", "md5"}, + {"crc32c", "crc32c"}, + {"UNKNOWN", "unknown"}, + } + + for _, tc := range testCases { + t.Run(tc.input, func(t *testing.T) { + result := drsChecksumType(tc.input) + assert.Equal(t, tc.expected, result) + }) + } +} diff --git a/sda/cmd/download/handlers/handlers.go b/sda/cmd/download/handlers/handlers.go index 6e076db906..6c62e6889c 100644 --- a/sda/cmd/download/handlers/handlers.go +++ b/sda/cmd/download/handlers/handlers.go @@ -109,4 +109,11 @@ func (h *Handlers) RegisterRoutes(r *gin.Engine) { files.HEAD("/:fileId/content", h.HeadFileContent) files.GET("/:fileId/content", h.GetFileContent) } + + // DRS objects (auth required) + objects := r.Group("/objects") + objects.Use(middleware.TokenMiddleware(h.db, h.visaValidator, h.auditLogger)) + { + objects.GET("/*path", h.GetDrsObject) + } } diff --git a/sda/cmd/download/handlers/handlers_test.go b/sda/cmd/download/handlers/handlers_test.go index 4381713cce..dfca799cc6 100644 --- a/sda/cmd/download/handlers/handlers_test.go +++ b/sda/cmd/download/handlers/handlers_test.go @@ -156,6 +156,7 @@ func TestRoutes_AuthRequired(t *testing.T) { {http.MethodHead, "/files/test-file-id/header"}, {http.MethodGet, "/files/test-file-id/content"}, {http.MethodHead, "/files/test-file-id/content"}, + {http.MethodGet, "/objects/test-dataset/test-file.c4gh"}, } for _, p := range paths { From 935e69b7c8186a7e8f9af86a49c1fc43bf0a9a7d Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 4 Mar 2026 11:29:38 +0100 Subject: [PATCH 405/530] fix(download): use ArchiveSize and ARCHIVED checksums per DRS 1.5 spec DRS 1.5 requires size and checksums to describe the blob bytes served by access_url. Changed from DecryptedSize/UNENCRYPTED to ArchiveSize/ARCHIVED. Added GetFileChecksums DB method to return all checksums for a given source, supporting multiple algorithms. --- sda/cmd/download/database/cache.go | 5 ++ sda/cmd/download/database/cache_test.go | 9 +++ sda/cmd/download/database/database.go | 41 +++++++++++ sda/cmd/download/database/database_test.go | 4 ++ sda/cmd/download/handlers/drs.go | 25 +++++-- sda/cmd/download/handlers/drs_test.go | 69 +++++++++++++++---- .../download/handlers/mock_database_test.go | 9 +++ sda/cmd/download/swagger_v2.yml | 5 +- 8 files changed, 145 insertions(+), 22 deletions(-) diff --git a/sda/cmd/download/database/cache.go b/sda/cmd/download/database/cache.go index 22e5110e1c..8ef4dfa7cc 100644 --- a/sda/cmd/download/database/cache.go +++ b/sda/cmd/download/database/cache.go @@ -306,6 +306,11 @@ func (c *CachedDB) CheckDatasetExists(ctx context.Context, datasetID string) (bo return exists, nil } +// GetFileChecksums delegates to the underlying database without caching. +func (c *CachedDB) GetFileChecksums(ctx context.Context, fileID string, source string) ([]Checksum, error) { + return c.db.GetFileChecksums(ctx, fileID, source) +} + // GetDatasetFilesPaginated delegates to the underlying database without caching. // Paginated queries use ephemeral cursors, making caching impractical. func (c *CachedDB) GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) { diff --git a/sda/cmd/download/database/cache_test.go b/sda/cmd/download/database/cache_test.go index 32fbd295db..62a56ede09 100644 --- a/sda/cmd/download/database/cache_test.go +++ b/sda/cmd/download/database/cache_test.go @@ -103,6 +103,15 @@ func (m *MockDatabase) CheckDatasetExists(ctx context.Context, datasetID string) return args.Bool(0), args.Error(1) } +func (m *MockDatabase) GetFileChecksums(ctx context.Context, fileID string, source string) ([]Checksum, error) { + args := m.Called(ctx, fileID, source) + if args.Get(0) == nil { + return nil, args.Error(1) + } + + return args.Get(0).([]Checksum), args.Error(1) +} + func (m *MockDatabase) GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) { args := m.Called(ctx, datasetID, opts) if args.Get(0) == nil { diff --git a/sda/cmd/download/database/database.go b/sda/cmd/download/database/database.go index d46dc050f0..a625aa2f2d 100644 --- a/sda/cmd/download/database/database.go +++ b/sda/cmd/download/database/database.go @@ -30,6 +30,7 @@ const ( getDatasetFilesPageQuery = "getDatasetFilesPage" getDatasetFilesPageByPathQuery = "getDatasetFilesPageByPath" getDatasetFilesPageByPrefixQuery = "getDatasetFilesPageByPrefix" + getFileChecksumsQuery = "getFileChecksums" ) // paginatedFileBase is the shared SELECT+JOIN+LATERAL block for keyset-paginated @@ -163,6 +164,13 @@ var queries = map[string]string{ WHERE d.stable_id = $1 )`, + // getFileChecksums returns checksums for a file filtered by source (e.g., "ARCHIVED", "UNENCRYPTED"). + getFileChecksumsQuery: ` + SELECT c.checksum, c.type + FROM sda.checksums c + INNER JOIN sda.files f ON c.file_id = f.id + WHERE f.stable_id = $1 AND c.source = $2`, + // Keyset-paginated file queries compose from paginatedFileBase (defined below). // getDatasetFilesPage returns paginated files in a dataset (no path filter). @@ -237,6 +245,9 @@ type Database interface { // CheckDatasetExists checks if a dataset with the given stable_id exists. CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) + // GetFileChecksums returns checksums for a file filtered by source (e.g., "ARCHIVED", "UNENCRYPTED"). + GetFileChecksums(ctx context.Context, fileID string, source string) ([]Checksum, error) + // GetDatasetFilesPaginated returns files in a dataset with keyset cursor pagination. // Files are returned with aggregated checksums. Use FileListOptions to filter and paginate. GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) @@ -667,6 +678,31 @@ func (p *PostgresDB) CheckDatasetExists(ctx context.Context, datasetID string) ( return exists, nil } +// GetFileChecksums returns checksums for a file filtered by source (e.g., "ARCHIVED", "UNENCRYPTED"). +func (p *PostgresDB) GetFileChecksums(ctx context.Context, fileID string, source string) ([]Checksum, error) { + stmt := p.preparedStatements[getFileChecksumsQuery] + rows, err := stmt.QueryContext(ctx, fileID, source) + if err != nil { + return nil, fmt.Errorf("failed to query file checksums: %w", err) + } + defer rows.Close() + + var checksums []Checksum + for rows.Next() { + var c Checksum + if err := rows.Scan(&c.Checksum, &c.Type); err != nil { + return nil, fmt.Errorf("failed to scan checksum row: %w", err) + } + checksums = append(checksums, c) + } + + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating checksum rows: %w", err) + } + + return checksums, nil +} + // GetDatasetFilesPaginated returns files with keyset cursor pagination and aggregated checksums. func (p *PostgresDB) GetDatasetFilesPaginated(ctx context.Context, datasetID string, opts FileListOptions) ([]File, error) { var rows *sql.Rows @@ -767,3 +803,8 @@ func CheckFilePermission(ctx context.Context, fileID string, visas []string) (bo func CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { return db.CheckDatasetExists(ctx, datasetID) } + +// GetFileChecksums returns checksums for a file filtered by source (e.g., "ARCHIVED", "UNENCRYPTED"). +func GetFileChecksums(ctx context.Context, fileID string, source string) ([]Checksum, error) { + return db.GetFileChecksums(ctx, fileID, source) +} diff --git a/sda/cmd/download/database/database_test.go b/sda/cmd/download/database/database_test.go index 805898cdb2..ae5343fe4b 100644 --- a/sda/cmd/download/database/database_test.go +++ b/sda/cmd/download/database/database_test.go @@ -673,6 +673,10 @@ func (m *mockTestDatabase) CheckDatasetExists(_ context.Context, _ string) (bool return false, nil } +func (m *mockTestDatabase) GetFileChecksums(_ context.Context, _ string, _ string) ([]Checksum, error) { + return nil, nil +} + func (m *mockTestDatabase) GetDatasetFilesPaginated(_ context.Context, _ string, _ FileListOptions) ([]File, error) { return nil, nil } diff --git a/sda/cmd/download/handlers/drs.go b/sda/cmd/download/handlers/drs.go index 10b4e28666..893ed6a986 100644 --- a/sda/cmd/download/handlers/drs.go +++ b/sda/cmd/download/handlers/drs.go @@ -42,6 +42,8 @@ func drsChecksumType(sdaType string) string { switch strings.ToLower(sdaType) { case "sha256", "sha-256": return "sha-256" + case "sha384", "sha-384": + return "sha-384" case "sha512", "sha-512": return "sha-512" default: @@ -95,18 +97,27 @@ func (h *Handlers) GetDrsObject(c *gin.Context) { host := c.Request.Host - checksums := []DrsChecksum{} - if file.DecryptedChecksum != "" { - checksums = append(checksums, DrsChecksum{ - Checksum: file.DecryptedChecksum, - Type: drsChecksumType(file.DecryptedChecksumType), - }) + // Fetch ARCHIVED checksums (over the encrypted blob, per DRS 1.5 spec) + archivedChecksums, err := h.db.GetFileChecksums(c.Request.Context(), file.ID, "ARCHIVED") + if err != nil { + log.Errorf("failed to get file checksums: %v", err) + problemJSON(c, http.StatusInternalServerError, "failed to retrieve checksums") + + return + } + + checksums := make([]DrsChecksum, len(archivedChecksums)) + for i, ac := range archivedChecksums { + checksums[i] = DrsChecksum{ + Checksum: ac.Checksum, + Type: drsChecksumType(ac.Type), + } } obj := DrsObject{ ID: file.ID, SelfURI: fmt.Sprintf("drs://%s/%s", host, file.ID), - Size: file.DecryptedSize, + Size: file.ArchiveSize, CreatedTime: file.CreatedAt.UTC().Format("2006-01-02T15:04:05Z07:00"), Checksums: checksums, AccessMethods: []DrsAccessMethod{ diff --git a/sda/cmd/download/handlers/drs_test.go b/sda/cmd/download/handlers/drs_test.go index 2ecd4f8af0..4b2d572c07 100644 --- a/sda/cmd/download/handlers/drs_test.go +++ b/sda/cmd/download/handlers/drs_test.go @@ -40,14 +40,14 @@ func TestGetDrsObject_Success(t *testing.T) { router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) mockDB := &mockDatabase{ fileByPath: &database.File{ - ID: "urn:neic:001-002-003", - DatasetID: "EGAD00001000001", - SubmittedPath: "samples/controls/sample1.bam.c4gh", - DecryptedSize: 1048576, - DecryptedChecksum: "a1b2c3d4", - DecryptedChecksumType: "SHA256", - CreatedAt: time.Date(2026, 1, 15, 10, 30, 0, 0, time.UTC), + ID: "urn:neic:001-002-003", + DatasetID: "EGAD00001000001", + SubmittedPath: "samples/controls/sample1.bam.c4gh", + ArchiveSize: 2097152, + DecryptedSize: 1048576, + CreatedAt: time.Date(2026, 1, 15, 10, 30, 0, 0, time.UTC), }, + fileChecksums: []database.Checksum{{Type: "SHA256", Checksum: "a1b2c3d4"}}, } h, err := New(WithDatabase(mockDB)) require.NoError(t, err) @@ -69,7 +69,7 @@ func TestGetDrsObject_Success(t *testing.T) { assert.Equal(t, "urn:neic:001-002-003", resp.ID) assert.Equal(t, "drs://download.example.org/urn:neic:001-002-003", resp.SelfURI) - assert.Equal(t, int64(1048576), resp.Size) + assert.Equal(t, int64(2097152), resp.Size) assert.Equal(t, "2026-01-15T10:30:00Z", resp.CreatedTime) require.Len(t, resp.Checksums, 1) @@ -192,13 +192,13 @@ func TestGetDrsObject_NoChecksum_EmptyArray(t *testing.T) { router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) mockDB := &mockDatabase{ fileByPath: &database.File{ - ID: "urn:neic:001-002-003", - DatasetID: "EGAD00001000001", - SubmittedPath: "file.bam", - DecryptedSize: 512, - DecryptedChecksum: "", - CreatedAt: time.Date(2026, 1, 15, 10, 30, 0, 0, time.UTC), + ID: "urn:neic:001-002-003", + DatasetID: "EGAD00001000001", + SubmittedPath: "file.bam", + DecryptedSize: 512, + CreatedAt: time.Date(2026, 1, 15, 10, 30, 0, 0, time.UTC), }, + fileChecksums: []database.Checksum{}, } h, err := New(WithDatabase(mockDB)) require.NoError(t, err) @@ -220,6 +220,46 @@ func TestGetDrsObject_NoChecksum_EmptyArray(t *testing.T) { assert.Equal(t, "[]", string(raw["checksums"])) } +func TestGetDrsObject_MultipleChecksums(t *testing.T) { + router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) + mockDB := &mockDatabase{ + fileByPath: &database.File{ + ID: "urn:neic:001-002-003", + DatasetID: "EGAD00001000001", + SubmittedPath: "file.bam", + ArchiveSize: 4096, + DecryptedSize: 2048, + CreatedAt: time.Date(2026, 1, 15, 10, 30, 0, 0, time.UTC), + }, + fileChecksums: []database.Checksum{ + {Type: "SHA256", Checksum: "abc"}, + {Type: "MD5", Checksum: "def"}, + }, + } + h, err := New(WithDatabase(mockDB)) + require.NoError(t, err) + + router.GET("/objects/*path", h.GetDrsObject) + + req, _ := http.NewRequest(http.MethodGet, "/objects/EGAD00001000001/file.bam", nil) + req.Host = "download.example.org" + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var resp drsObjectResponse + err = json.Unmarshal(w.Body.Bytes(), &resp) + require.NoError(t, err) + + require.Len(t, resp.Checksums, 2) + assert.Equal(t, "sha-256", resp.Checksums[0].Type) + assert.Equal(t, "abc", resp.Checksums[0].Checksum) + assert.Equal(t, "md5", resp.Checksums[1].Type) + assert.Equal(t, "def", resp.Checksums[1].Checksum) +} + func TestDrsChecksumType(t *testing.T) { testCases := []struct { input string @@ -228,6 +268,7 @@ func TestDrsChecksumType(t *testing.T) { {"SHA256", "sha-256"}, {"sha256", "sha-256"}, {"SHA-256", "sha-256"}, + {"SHA384", "sha-384"}, {"SHA512", "sha-512"}, {"MD5", "md5"}, {"crc32c", "crc32c"}, diff --git a/sda/cmd/download/handlers/mock_database_test.go b/sda/cmd/download/handlers/mock_database_test.go index c936a8d05c..33ad4ca995 100644 --- a/sda/cmd/download/handlers/mock_database_test.go +++ b/sda/cmd/download/handlers/mock_database_test.go @@ -31,6 +31,7 @@ type mockDatabase struct { fileByPath *database.File hasPermission bool datasetNotFound bool + fileChecksums []database.Checksum err error pingErr error } @@ -115,6 +116,14 @@ func (m *mockDatabase) CheckDatasetExists(_ context.Context, _ string) (bool, er return !m.datasetNotFound, nil } +func (m *mockDatabase) GetFileChecksums(_ context.Context, _ string, _ string) ([]database.Checksum, error) { + if m.err != nil { + return nil, m.err + } + + return m.fileChecksums, nil +} + func (m *mockDatabase) GetDatasetFilesPaginated(_ context.Context, _ string, _ database.FileListOptions) ([]database.File, error) { if m.err != nil { return nil, m.err diff --git a/sda/cmd/download/swagger_v2.yml b/sda/cmd/download/swagger_v2.yml index 4abdeac852..d12c66690b 100644 --- a/sda/cmd/download/swagger_v2.yml +++ b/sda/cmd/download/swagger_v2.yml @@ -965,7 +965,7 @@ components: size: type: integer format: int64 - description: Decrypted file size in bytes. + description: Encrypted blob size in bytes (size of the data served by access_url). created_time: type: string format: date-time @@ -973,10 +973,13 @@ components: example: "2026-01-15T10:30:00Z" checksums: type: array + minItems: 1 + description: Checksums computed over the encrypted blob bytes. items: $ref: '#/components/schemas/DrsChecksum' access_methods: type: array + minItems: 1 items: $ref: '#/components/schemas/DrsAccessMethod' DrsChecksum: From 2cf425c171ca0508f83caed4e3742e99afe05f8b Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 6 Mar 2026 10:12:40 +0100 Subject: [PATCH 406/530] docs(download): add DRS object endpoint to download.md and TESTING.md --- sda/cmd/download/TESTING.md | 25 ++++++++++++++++++ sda/cmd/download/download.md | 50 ++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/sda/cmd/download/TESTING.md b/sda/cmd/download/TESTING.md index 4797ffd9b7..15904b76e8 100644 --- a/sda/cmd/download/TESTING.md +++ b/sda/cmd/download/TESTING.md @@ -30,6 +30,7 @@ This runs a minimal environment with: - File download with re-encryption - Range requests - Access control +- DRS object resolution ### Option 2: Full Pipeline Tests @@ -188,6 +189,30 @@ curl -H "Authorization: Bearer $TOKEN" \ http://localhost:8080/files/EGAF74900000001 ``` +#### Resolve a File via DRS + +```bash +# Resolve a dataset + file path to a DRS object with a download URL +curl -H "Authorization: Bearer $TOKEN" \ + http://localhost:8080/objects/EGAD74900000101/dummy_data.c4gh +``` + +The response is a GA4GH DRS 1.5 object with a pre-resolved `access_url`: + +```json +{ + "id": "EGAF74900000001", + "self_uri": "drs://localhost:8080/EGAF74900000001", + "size": 1048576, + "created_time": "2026-01-15T10:30:00Z", + "checksums": [{"checksum": "abc...", "type": "sha-256"}], + "access_methods": [{ + "type": "https", + "access_url": {"url": "https://localhost:8080/files/EGAF74900000001/content"} + }] +} +``` + ## Database Queries for Debugging Connect to the database: diff --git a/sda/cmd/download/download.md b/sda/cmd/download/download.md index 10a2155070..824b939861 100644 --- a/sda/cmd/download/download.md +++ b/sda/cmd/download/download.md @@ -304,6 +304,56 @@ curl -H "Authorization: Bearer $token" \ https://HOSTNAME/files/EGAF00000000001/content ``` +### DRS Object Endpoint + +#### `GET /objects/{datasetId}/{filePath}` + +Returns a minimal [GA4GH DRS 1.5](https://ga4gh.github.io/data-repository-service-schemas/preview/release/drs-1.5.0/docs/) +`DrsObject` with a pre-resolved `access_url` pointing to the file content endpoint. +This enables DRS-aware clients (e.g. htsget-rs) to resolve a dataset + file path +to a download URL without knowing the internal file ID. + +The path is composite: everything before the first `/` is the dataset ID, everything +after is the file path within the dataset. + +- Error codes + - `200` DRS object returned + - `400` Malformed path (missing dataset or file component) + - `401` Invalid or missing token + - `403` Access denied or file does not exist + +Example: + +```bash +curl -H "Authorization: Bearer $token" \ + https://HOSTNAME/objects/EGAD00000000001/samples/sample1.bam.c4gh +``` + +Response: + +```json +{ + "id": "EGAF00000000001", + "self_uri": "drs://HOSTNAME/EGAF00000000001", + "size": 1048576, + "created_time": "2026-01-15T10:30:00Z", + "checksums": [ + {"checksum": "a1b2c3d4...", "type": "sha-256"} + ], + "access_methods": [ + { + "type": "https", + "access_url": { + "url": "https://HOSTNAME/files/EGAF00000000001/content" + } + } + ] +} +``` + +The `size` and `checksums` describe the encrypted blob served by `access_url`, +per the DRS 1.5 specification. + ### Error Format All error responses use [RFC 9457 Problem Details](https://www.rfc-editor.org/rfc/rfc9457): From 28d6fc7c720298a47f23dc05a600af806583b0d8 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 18 Mar 2026 08:34:16 +0100 Subject: [PATCH 407/530] fix(download): use request scheme in DRS access_url and reject empty checksums Derive http/https from TLS state instead of hardcoding https in the DRS access_url, so local and integration test environments get correct URLs. Return 500 when a file has no ARCHIVED checksums, enforcing the DRS 1.5 minItems: 1 constraint on the checksums array. --- sda/cmd/download/handlers/drs.go | 13 ++++++++++++- sda/cmd/download/handlers/drs_test.go | 13 ++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/sda/cmd/download/handlers/drs.go b/sda/cmd/download/handlers/drs.go index 893ed6a986..a69f90755e 100644 --- a/sda/cmd/download/handlers/drs.go +++ b/sda/cmd/download/handlers/drs.go @@ -96,6 +96,10 @@ func (h *Handlers) GetDrsObject(c *gin.Context) { } host := c.Request.Host + scheme := "https" + if c.Request.TLS == nil { + scheme = "http" + } // Fetch ARCHIVED checksums (over the encrypted blob, per DRS 1.5 spec) archivedChecksums, err := h.db.GetFileChecksums(c.Request.Context(), file.ID, "ARCHIVED") @@ -106,6 +110,13 @@ func (h *Handlers) GetDrsObject(c *gin.Context) { return } + if len(archivedChecksums) == 0 { + log.Errorf("file %s has no ARCHIVED checksums", file.ID) + problemJSON(c, http.StatusInternalServerError, "file has no checksums") + + return + } + checksums := make([]DrsChecksum, len(archivedChecksums)) for i, ac := range archivedChecksums { checksums[i] = DrsChecksum{ @@ -124,7 +135,7 @@ func (h *Handlers) GetDrsObject(c *gin.Context) { { Type: "https", AccessURL: DrsAccessURL{ - URL: fmt.Sprintf("https://%s/files/%s/content", host, file.ID), + URL: fmt.Sprintf("%s://%s/files/%s/content", scheme, host, file.ID), }, }, }, diff --git a/sda/cmd/download/handlers/drs_test.go b/sda/cmd/download/handlers/drs_test.go index 4b2d572c07..1b85852f0f 100644 --- a/sda/cmd/download/handlers/drs_test.go +++ b/sda/cmd/download/handlers/drs_test.go @@ -78,7 +78,7 @@ func TestGetDrsObject_Success(t *testing.T) { require.Len(t, resp.AccessMethods, 1) assert.Equal(t, "https", resp.AccessMethods[0].Type) - assert.Equal(t, "https://download.example.org/files/urn:neic:001-002-003/content", resp.AccessMethods[0].AccessURL.URL) + assert.Equal(t, "http://download.example.org/files/urn:neic:001-002-003/content", resp.AccessMethods[0].AccessURL.URL) } func TestGetDrsObject_FileNotFound_Returns403(t *testing.T) { @@ -188,7 +188,7 @@ func TestGetDrsObject_DBError_Returns500(t *testing.T) { assert.Equal(t, http.StatusInternalServerError, w.Code) } -func TestGetDrsObject_NoChecksum_EmptyArray(t *testing.T) { +func TestGetDrsObject_NoChecksum_Returns500(t *testing.T) { router := setupTestRouterWithAuth([]string{"EGAD00001000001"}) mockDB := &mockDatabase{ fileByPath: &database.File{ @@ -211,13 +211,12 @@ func TestGetDrsObject_NoChecksum_EmptyArray(t *testing.T) { router.ServeHTTP(w, req) - assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, http.StatusInternalServerError, w.Code) - // Verify checksums is [] not null in raw JSON - var raw map[string]json.RawMessage - err = json.Unmarshal(w.Body.Bytes(), &raw) + var resp ProblemDetails + err = json.Unmarshal(w.Body.Bytes(), &resp) require.NoError(t, err) - assert.Equal(t, "[]", string(raw["checksums"])) + assert.Contains(t, resp.Detail, "no checksums") } func TestGetDrsObject_MultipleChecksums(t *testing.T) { From a17e3058aaf20dd04318ee3bbf3ea5b575fe5b41 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 20 Mar 2026 09:15:41 +0100 Subject: [PATCH 408/530] fix(download): match DRS access method type to request scheme Use the detected scheme for the access_methods type field instead of hardcoding "https", so the type accurately reflects the protocol. --- sda/cmd/download/handlers/drs.go | 2 +- sda/cmd/download/handlers/drs_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/download/handlers/drs.go b/sda/cmd/download/handlers/drs.go index a69f90755e..4d6725a8ba 100644 --- a/sda/cmd/download/handlers/drs.go +++ b/sda/cmd/download/handlers/drs.go @@ -133,7 +133,7 @@ func (h *Handlers) GetDrsObject(c *gin.Context) { Checksums: checksums, AccessMethods: []DrsAccessMethod{ { - Type: "https", + Type: scheme, AccessURL: DrsAccessURL{ URL: fmt.Sprintf("%s://%s/files/%s/content", scheme, host, file.ID), }, diff --git a/sda/cmd/download/handlers/drs_test.go b/sda/cmd/download/handlers/drs_test.go index 1b85852f0f..af47f259c5 100644 --- a/sda/cmd/download/handlers/drs_test.go +++ b/sda/cmd/download/handlers/drs_test.go @@ -77,7 +77,7 @@ func TestGetDrsObject_Success(t *testing.T) { assert.Equal(t, "a1b2c3d4", resp.Checksums[0].Checksum) require.Len(t, resp.AccessMethods, 1) - assert.Equal(t, "https", resp.AccessMethods[0].Type) + assert.Equal(t, "http", resp.AccessMethods[0].Type) assert.Equal(t, "http://download.example.org/files/urn:neic:001-002-003/content", resp.AccessMethods[0].AccessURL.URL) } From f405b1d4093a8f27c91ef3d997d3887c5f846d73 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 24 Mar 2026 14:16:02 +0100 Subject: [PATCH 409/530] refactor(download): rename cmd-download to download-v2 throughout Address review feedback: drop "cmd" naming, use "download-v2" for compose files, Makefile targets, workflow jobs, and docs. Remove duplicate cmd-download config files (already existed as download-config). --- .github/integration/sda-benchmark.yml | 2 +- ...on.yml => sda-download-v2-integration.yml} | 2 +- .github/integration/sda-s3-integration.yml | 2 +- .../sda/cmd-download-config-benchmark.yaml | 95 ------------------- .../integration/sda/cmd-download-config.yaml | 86 ----------------- .../workflows/pr_build_and_test_images.yml | 8 +- Makefile | 20 ++-- sda/cmd/download/TESTING.md | 2 +- sda/cmd/download/download.md | 6 +- 9 files changed, 21 insertions(+), 202 deletions(-) rename .github/integration/{sda-cmd-download-integration.yml => sda-download-v2-integration.yml} (99%) delete mode 100644 .github/integration/sda/cmd-download-config-benchmark.yaml delete mode 100644 .github/integration/sda/cmd-download-config.yaml diff --git a/.github/integration/sda-benchmark.yml b/.github/integration/sda-benchmark.yml index b5b9d3aaa0..351b543922 100644 --- a/.github/integration/sda-benchmark.yml +++ b/.github/integration/sda-benchmark.yml @@ -29,7 +29,7 @@ services: minio: condition: service_healthy volumes: - - ./sda/cmd-download-config-benchmark.yaml:/config.yaml + - ./sda/download-config-benchmark.yaml:/config.yaml - shared:/shared # Add network alias for minio so storage URLs work correctly diff --git a/.github/integration/sda-cmd-download-integration.yml b/.github/integration/sda-download-v2-integration.yml similarity index 99% rename from .github/integration/sda-cmd-download-integration.yml rename to .github/integration/sda-download-v2-integration.yml index 7089977b0a..718571a5f3 100644 --- a/.github/integration/sda-cmd-download-integration.yml +++ b/.github/integration/sda-download-v2-integration.yml @@ -229,7 +229,7 @@ services: - "8086:8081" restart: always volumes: - - ./sda/cmd-download-config.yaml:/config.yaml + - ./sda/download-config.yaml:/config.yaml - shared:/shared # Go-based integration test runner diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index add74d0fe8..c03ed298ee 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -280,7 +280,7 @@ services: - "8086:8081" restart: always volumes: - - ./sda/cmd-download-config.yaml:/config.yaml + - ./sda/download-config.yaml:/config.yaml - shared:/shared rotatekey: diff --git a/.github/integration/sda/cmd-download-config-benchmark.yaml b/.github/integration/sda/cmd-download-config-benchmark.yaml deleted file mode 100644 index f2312489e2..0000000000 --- a/.github/integration/sda/cmd-download-config-benchmark.yaml +++ /dev/null @@ -1,95 +0,0 @@ -# Configuration for sda/cmd/download service in BENCHMARK mode -# This benchmark compares public endpoints, not just internal implementation details. -# The endpoint uses http:// prefix for storage/v2 URL parsing - -# API Server -api: - host: "0.0.0.0" - port: 8080 - -# Logging -log: - level: "info" - -# gRPC Health check -health: - port: 8081 - -# Database -db: - host: "postgres" - port: 5432 - user: "postgres" - password: "rootpasswd" - database: "sda" - sslmode: "disable" - -# gRPC reencrypt service -grpc: - host: "reencrypt" - port: 50051 - timeout: 30 - -# JWT configuration - uses keys created by credentials script -jwt: - pubkey-path: "/shared/keys/pub/" - allow-all-data: false - -# Permission model: combined (ownership + visa) for benchmark -# This keeps the benchmark environment resilient if visa processing has issues. -permission: - model: "combined" - -# Auth: allow opaque tokens (old sda-download supported this) -auth: - allow-opaque: true - -# GA4GH visa configuration - matches old sda-download's OIDC/visa flow -visa: - enabled: true - source: "userinfo" - userinfo-url: "http://mockauth:8000/userinfo" - trusted-issuers-path: "/shared/trusted-issuers.json" - dataset-id-mode: "raw" - identity: - mode: "broker-bound" - validate-asserted: true - allow-insecure-jku: true # mockauth uses HTTP (testing only) - limits: - max-visas: 200 - max-jwks-per-request: 10 - max-visa-size: 16384 - cache: - token-ttl: 3600 - jwk-ttl: 300 - validation-ttl: 120 - userinfo-ttl: 60 - -# Session -session: - expiration: 3600 - secure: false - http-only: true - name: "sda_session" - -# Database query cache -# Disabled here so the benchmark does not compare a cached new endpoint -# against an uncached old endpoint. -cache: - enabled: false - file-ttl: 300 # 5 minutes - permission-ttl: 120 # 2 minutes - dataset-ttl: 300 # 5 minutes - -# Storage configuration (S3 backend) -# NOTE: endpoint includes http:// for proper URL parsing in storage/v2 -storage: - backend: "archive" - archive: - s3: - - endpoint: "http://s3:9000" - access_key: "access" - secret_key: "secretKey" - region: "us-east-1" - bucket_prefix: "archive" - disable_https: true diff --git a/.github/integration/sda/cmd-download-config.yaml b/.github/integration/sda/cmd-download-config.yaml deleted file mode 100644 index 0ca6bd1d3d..0000000000 --- a/.github/integration/sda/cmd-download-config.yaml +++ /dev/null @@ -1,86 +0,0 @@ -# Configuration for sda/cmd/download service in integration tests - -# API Server -api: - host: "0.0.0.0" - port: 8080 - -# gRPC Health check -health: - port: 8081 - -# Database -db: - host: "postgres" - port: 5432 - user: "postgres" - password: "rootpasswd" - database: "sda" - sslmode: "disable" - -# gRPC reencrypt service -grpc: - host: "reencrypt" - port: 50051 - timeout: 30 - -# JWT configuration - uses keys created by credentials script -jwt: - pubkey-path: "/shared/keys/pub/" - allow-all-data: true # For testing: allow all authenticated users access to all datasets - -# Permission model: combined (ownership + visa) -permission: - model: "combined" - -# Auth: allow opaque tokens (userinfo-based authentication) -auth: - allow-opaque: true - -# GA4GH visa configuration -visa: - enabled: true - source: "userinfo" - userinfo-url: "http://mockauth:8000/userinfo" - trusted-issuers-path: "/shared/trusted-issuers.json" - dataset-id-mode: "raw" - identity: - mode: "broker-bound" - validate-asserted: true - allow-insecure-jku: true # mockauth uses HTTP (testing only) - limits: - max-visas: 200 - max-jwks-per-request: 10 - max-visa-size: 16384 - cache: - token-ttl: 3600 - jwk-ttl: 300 - validation-ttl: 120 - userinfo-ttl: 60 - -# Session -session: - expiration: 3600 - secure: false - http-only: true - name: "sda_session" - -# Database query cache -cache: - enabled: true - file-ttl: 300 # 5 minutes - permission-ttl: 120 # 2 minutes - dataset-ttl: 300 # 5 minutes - -# Storage configuration (S3 backend) -storage: - backend: "archive" - archive: - s3: - # NOTE: include scheme to keep storage/v2 location parsing consistent (archive_location values are URL-parsed). - - endpoint: "http://s3:9000" - access_key: "access" - secret_key: "secretKey" - region: "us-east-1" - bucket_prefix: "archive" - disable_https: true diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index c565c775cf..8006d388e9 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -240,15 +240,15 @@ jobs: bash -x "$runscript"; done - sda-cmd-download_integration_test: + sda-download-v2_integration_test: needs: - build_images - name: sda/cmd/download integration tests + name: download v2 integration tests runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 - - name: Test sda/cmd/download - run: docker compose -f .github/integration/sda-cmd-download-integration.yml run --rm integration_test + - name: Test download v2 + run: docker compose -f .github/integration/sda-download-v2-integration.yml run --rm integration_test sda-doa_integration_test: name: sda-doa ${{ matrix.storage }} integration tests diff --git a/Makefile b/Makefile index 07dcc3445d..30e17a980c 100644 --- a/Makefile +++ b/Makefile @@ -159,20 +159,20 @@ integrationtest-sda-validator-orchestrator-down: # Download v2 integration tests (sda/cmd/download, Go-based) # Prerequisite: run `make build-all` first if container images are not already built. -integrationtest-sda-cmd-download: - @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml up -d +integrationtest-sda-download-v2: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-download-v2-integration.yml up -d @sleep 10 - @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml run --rm integration_test - @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml down -v --remove-orphans + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-download-v2-integration.yml run --rm integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-download-v2-integration.yml down -v --remove-orphans -integrationtest-sda-cmd-download-run: - @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml run --rm integration_test +integrationtest-sda-download-v2-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-download-v2-integration.yml run --rm integration_test -integrationtest-sda-cmd-download-up: build-all - @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml up -d +integrationtest-sda-download-v2-up: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-download-v2-integration.yml up -d -integrationtest-sda-cmd-download-down: - @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-cmd-download-integration.yml down -v --remove-orphans +integrationtest-sda-download-v2-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-download-v2-integration.yml down -v --remove-orphans # Download benchmark (compares old vs new public endpoints) # Uses sda-benchmark.yml which extends sda-s3-integration.yml with benchmark services diff --git a/sda/cmd/download/TESTING.md b/sda/cmd/download/TESTING.md index 15904b76e8..d6daf7abd6 100644 --- a/sda/cmd/download/TESTING.md +++ b/sda/cmd/download/TESTING.md @@ -12,7 +12,7 @@ The download service has its own Docker Compose-based Go integration tests: ```bash # From repository root -docker compose -f .github/integration/sda-cmd-download-integration.yml run integration_test +docker compose -f .github/integration/sda-download-v2-integration.yml run integration_test ``` This runs a minimal environment with: diff --git a/sda/cmd/download/download.md b/sda/cmd/download/download.md index 824b939861..eabc56b46a 100644 --- a/sda/cmd/download/download.md +++ b/sda/cmd/download/download.md @@ -545,9 +545,9 @@ cd sda && go test -tags visas -count=1 ./cmd/download/... ### Integration Tests ```bash -make integrationtest-sda-cmd-download-up # Start test environment -make integrationtest-sda-cmd-download-run # Run tests -make integrationtest-sda-cmd-download-down # Tear down +make integrationtest-sda-download-v2-up # Start test environment +make integrationtest-sda-download-v2-run # Run tests +make integrationtest-sda-download-v2-down # Tear down ``` ### Local Development From bb40bf53682369748e813bb19ead093aca3b0dd6 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 24 Mar 2026 16:45:06 +0100 Subject: [PATCH 410/530] =?UTF-8?q?fix(download):=20address=20review=20fee?= =?UTF-8?q?dback=20=E2=80=94=20audit=20path,=20timestamp,=20GetToken,=20pe?= =?UTF-8?q?rmission=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes from PR #2187 review by @KarlG-nbis: 1. Rename audit Event.Endpoint to Event.Path (JSON key: "path") since the field is populated from c.Request.URL.Path. 2. Always set audit timestamp at log time — the logger previously preserved caller-supplied timestamps, which is a footgun for audit integrity. 3. Convert GetToken from func variable to regular function — no reassignment or injection happens, so a plain function is simpler. 4. Validate permission.model at startup: reject unknown values and reject permission.model=visa when visa.enabled=false (would silently deny all requests). --- sda/cmd/download/audit/audit.go | 2 +- sda/cmd/download/audit/audit_test.go | 38 ++++++++++++---------- sda/cmd/download/audit/stdout_logger.go | 8 ++--- sda/cmd/download/handlers/datasets_test.go | 2 +- sda/cmd/download/handlers/files.go | 6 ++-- sda/cmd/download/handlers/handlers.go | 4 +-- sda/cmd/download/main.go | 24 ++++++++++++++ sda/cmd/download/main_test.go | 19 +++++++++++ sda/cmd/download/middleware/auth.go | 4 +-- sda/cmd/download/middleware/auth_test.go | 2 +- 10 files changed, 76 insertions(+), 33 deletions(-) diff --git a/sda/cmd/download/audit/audit.go b/sda/cmd/download/audit/audit.go index f5736a4fac..dd57f65384 100644 --- a/sda/cmd/download/audit/audit.go +++ b/sda/cmd/download/audit/audit.go @@ -17,7 +17,7 @@ type Event struct { FileID string `json:"fileId,omitempty"` DatasetID string `json:"datasetId,omitempty"` CorrelationID string `json:"correlationId"` - Endpoint string `json:"endpoint"` + Path string `json:"path"` HTTPStatus int `json:"httpStatus"` BytesTransferred int64 `json:"bytesTransferred,omitempty"` AuthType string `json:"authType,omitempty"` diff --git a/sda/cmd/download/audit/audit_test.go b/sda/cmd/download/audit/audit_test.go index 8c4e9c078f..ca4fd86f6f 100644 --- a/sda/cmd/download/audit/audit_test.go +++ b/sda/cmd/download/audit/audit_test.go @@ -17,7 +17,7 @@ func TestNoopLogger_LogDoesNotPanic(t *testing.T) { logger.Log(context.Background(), Event{ Event: "download.completed", CorrelationID: "abc-123", - Endpoint: "/files/test", + Path: "/files/test", HTTPStatus: 200, }) }) @@ -33,7 +33,7 @@ func TestStdoutLogger_OutputsValidJSON(t *testing.T) { FileID: "file-001", DatasetID: "EGAD00000000001", CorrelationID: "corr-123", - Endpoint: "/files/file-001", + Path: "/files/file-001", HTTPStatus: 200, }) @@ -45,7 +45,7 @@ func TestStdoutLogger_OutputsValidJSON(t *testing.T) { assert.Equal(t, "file-001", decoded.FileID) assert.Equal(t, "EGAD00000000001", decoded.DatasetID) assert.Equal(t, "corr-123", decoded.CorrelationID) - assert.Equal(t, "/files/file-001", decoded.Endpoint) + assert.Equal(t, "/files/file-001", decoded.Path) assert.Equal(t, 200, decoded.HTTPStatus) } @@ -57,7 +57,7 @@ func TestStdoutLogger_AlwaysSetsTypeToAudit(t *testing.T) { Type: "something-else", Event: "download.denied", CorrelationID: "corr-456", - Endpoint: "/files/file-002", + Path: "/files/file-002", HTTPStatus: 403, }) @@ -67,27 +67,31 @@ func TestStdoutLogger_AlwaysSetsTypeToAudit(t *testing.T) { assert.Equal(t, "audit", decoded.Type, "Type must always be 'audit' regardless of caller input") } -func TestStdoutLogger_TimestampIsUTC(t *testing.T) { +func TestStdoutLogger_TimestampAlwaysOverwritten(t *testing.T) { var buf bytes.Buffer logger := newStdoutLoggerWithWriter(&buf) - // Provide a non-UTC timestamp + // Provide a caller-supplied timestamp — logger must ignore it eastern := time.FixedZone("EST", -5*60*60) - ts := time.Date(2025, 6, 15, 10, 30, 0, 0, eastern) + callerTS := time.Date(2025, 6, 15, 10, 30, 0, 0, eastern) + before := time.Now().UTC() logger.Log(context.Background(), Event{ Event: "download.completed", - Timestamp: ts, + Timestamp: callerTS, CorrelationID: "corr-789", - Endpoint: "/files/file-003", + Path: "/files/file-003", HTTPStatus: 200, }) + after := time.Now().UTC() var decoded Event err := json.Unmarshal(buf.Bytes(), &decoded) require.NoError(t, err) assert.Equal(t, time.UTC, decoded.Timestamp.Location(), "timestamp must be in UTC") - assert.Equal(t, ts.UTC(), decoded.Timestamp) + assert.NotEqual(t, callerTS.UTC(), decoded.Timestamp, "caller-supplied timestamp must be overwritten") + assert.True(t, !decoded.Timestamp.Before(before) && !decoded.Timestamp.After(after), + "timestamp must be set at log time, not caller time") } func TestStdoutLogger_ZeroTimestampFilled(t *testing.T) { @@ -98,7 +102,7 @@ func TestStdoutLogger_ZeroTimestampFilled(t *testing.T) { logger.Log(context.Background(), Event{ Event: "download.failed", CorrelationID: "corr-000", - Endpoint: "/files/file-004", + Path: "/files/file-004", HTTPStatus: 500, }) after := time.Now().UTC() @@ -118,7 +122,7 @@ func TestStdoutLogger_OmitsEmptyOptionalFields(t *testing.T) { logger.Log(context.Background(), Event{ Event: "download.denied", CorrelationID: "corr-omit", - Endpoint: "/files/file-005", + Path: "/files/file-005", HTTPStatus: 401, }) @@ -138,7 +142,7 @@ func TestStdoutLogger_OmitsEmptyOptionalFields(t *testing.T) { assert.Contains(t, raw, "event") assert.Contains(t, raw, "timestamp") assert.Contains(t, raw, "correlationId") - assert.Contains(t, raw, "endpoint") + assert.Contains(t, raw, "path") assert.Contains(t, raw, "httpStatus") } @@ -149,13 +153,13 @@ func TestStdoutLogger_MultipleEvents(t *testing.T) { logger.Log(context.Background(), Event{ Event: "download.completed", CorrelationID: "corr-a", - Endpoint: "/files/a", + Path: "/files/a", HTTPStatus: 200, }) logger.Log(context.Background(), Event{ Event: "download.denied", CorrelationID: "corr-b", - Endpoint: "/files/b", + Path: "/files/b", HTTPStatus: 403, }) @@ -178,7 +182,7 @@ func TestAuditContract_RequiredFieldsPopulated(t *testing.T) { Event: "download.completed", UserID: "user@example.org", CorrelationID: "corr-contract", - Endpoint: "/files/file-006", + Path: "/files/file-006", HTTPStatus: 200, }) @@ -190,6 +194,6 @@ func TestAuditContract_RequiredFieldsPopulated(t *testing.T) { assert.NotEmpty(t, decoded.Event, "Event must be populated") assert.False(t, decoded.Timestamp.IsZero(), "Timestamp must be populated") assert.NotEmpty(t, decoded.CorrelationID, "CorrelationID must be populated") - assert.NotEmpty(t, decoded.Endpoint, "Endpoint must be populated") + assert.NotEmpty(t, decoded.Path, "Endpoint must be populated") assert.NotZero(t, decoded.HTTPStatus, "HTTPStatus must be populated") } diff --git a/sda/cmd/download/audit/stdout_logger.go b/sda/cmd/download/audit/stdout_logger.go index 3decff4f5b..5553b97a85 100644 --- a/sda/cmd/download/audit/stdout_logger.go +++ b/sda/cmd/download/audit/stdout_logger.go @@ -31,12 +31,8 @@ func newStdoutLoggerWithWriter(w io.Writer) *StdoutLogger { func (l *StdoutLogger) Log(_ context.Context, event Event) { // Enforce Type is always "audit" event.Type = "audit" - // Ensure timestamp is UTC - if event.Timestamp.IsZero() { - event.Timestamp = time.Now().UTC() - } else { - event.Timestamp = event.Timestamp.UTC() - } + // Always set timestamp at log time to prevent caller manipulation + event.Timestamp = time.Now().UTC() l.mu.Lock() defer l.mu.Unlock() diff --git a/sda/cmd/download/handlers/datasets_test.go b/sda/cmd/download/handlers/datasets_test.go index e83695894b..92b71e093c 100644 --- a/sda/cmd/download/handlers/datasets_test.go +++ b/sda/cmd/download/handlers/datasets_test.go @@ -267,7 +267,7 @@ func TestGetDataset_NoAccess_EmitsAuditDenied(t *testing.T) { evt := logger.events[0] assert.Equal(t, "download.denied", evt.Event) assert.Equal(t, http.StatusForbidden, evt.HTTPStatus) - assert.Equal(t, "/datasets/secret-dataset", evt.Endpoint) + assert.Equal(t, "/datasets/secret-dataset", evt.Path) assert.Equal(t, "test-user", evt.UserID) assert.NotEmpty(t, evt.CorrelationID) } diff --git a/sda/cmd/download/handlers/files.go b/sda/cmd/download/handlers/files.go index 8132b2d118..8c208925a9 100644 --- a/sda/cmd/download/handlers/files.go +++ b/sda/cmd/download/handlers/files.go @@ -270,7 +270,7 @@ func (h *Handlers) DownloadFile(c *gin.Context) { FileID: file.ID, DatasetID: file.DatasetID, CorrelationID: c.GetString("correlationId"), - Endpoint: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } @@ -352,7 +352,7 @@ func (h *Handlers) GetFileHeader(c *gin.Context) { FileID: file.ID, DatasetID: file.DatasetID, CorrelationID: c.GetString("correlationId"), - Endpoint: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } @@ -453,7 +453,7 @@ func (h *Handlers) GetFileContent(c *gin.Context) { FileID: file.ID, DatasetID: file.DatasetID, CorrelationID: c.GetString("correlationId"), - Endpoint: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } diff --git a/sda/cmd/download/handlers/handlers.go b/sda/cmd/download/handlers/handlers.go index 6c62e6889c..7b1e8a47dc 100644 --- a/sda/cmd/download/handlers/handlers.go +++ b/sda/cmd/download/handlers/handlers.go @@ -49,7 +49,7 @@ func (h *Handlers) auditDenied(c *gin.Context) { Event: "download.denied", UserID: authCtx.Subject, CorrelationID: c.GetString("correlationId"), - Endpoint: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } @@ -68,7 +68,7 @@ func (h *Handlers) auditFailed(c *gin.Context, authCtx middleware.AuthContext, f FileID: fileID, DatasetID: datasetID, CorrelationID: c.GetString("correlationId"), - Endpoint: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), ErrorReason: reason, }) diff --git a/sda/cmd/download/main.go b/sda/cmd/download/main.go index b14f1ab584..4c4dfe1c68 100644 --- a/sda/cmd/download/main.go +++ b/sda/cmd/download/main.go @@ -53,6 +53,11 @@ func run() error { return fmt.Errorf("failed to load config: %w", err) } + // Validate permission model + if err := validatePermissionModel(config.PermissionModel(), config.VisaEnabled()); err != nil { + return err + } + // Start gRPC health server go func() { if err := health.Start(config.HealthPort()); err != nil { @@ -323,6 +328,25 @@ type productionConfig struct { GRPCClientKey string } +// validatePermissionModel checks that the permission model is valid and +// that its dependencies are satisfied. +func validatePermissionModel(model string, visaEnabled bool) error { + switch model { + case "ownership": + return nil + case "visa": + if !visaEnabled { + return fmt.Errorf("permission.model is %q but visa.enabled is false — this combination would deny all requests", model) + } + + return nil + case "combined": + return nil + default: + return fmt.Errorf("invalid permission.model %q: must be ownership, visa, or combined", model) + } +} + // validateProductionConfig checks that dangerous testing flags are disabled // and required security configuration is present for production deployments. func validateProductionConfig(cfg productionConfig) error { diff --git a/sda/cmd/download/main_test.go b/sda/cmd/download/main_test.go index 445ee9f764..2c2d1580e4 100644 --- a/sda/cmd/download/main_test.go +++ b/sda/cmd/download/main_test.go @@ -7,6 +7,25 @@ import ( "github.com/stretchr/testify/require" ) +func TestValidatePermissionModel_Valid(t *testing.T) { + assert.NoError(t, validatePermissionModel("ownership", false)) + assert.NoError(t, validatePermissionModel("visa", true)) + assert.NoError(t, validatePermissionModel("combined", true)) + assert.NoError(t, validatePermissionModel("combined", false)) +} + +func TestValidatePermissionModel_Invalid(t *testing.T) { + err := validatePermissionModel("bogus", false) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid permission.model") +} + +func TestValidatePermissionModel_VisaWithoutEnabled(t *testing.T) { + err := validatePermissionModel("visa", false) + require.Error(t, err) + assert.Contains(t, err.Error(), "visa.enabled is false") +} + func TestValidateProductionConfig_AllValid(t *testing.T) { err := validateProductionConfig(productionConfig{ AllowAllData: false, diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index 51aeac0131..6377d25500 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -456,7 +456,7 @@ func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogge auditLogger.Log(c.Request.Context(), audit.Event{ Event: "download.denied", CorrelationID: c.GetString("correlationId"), - Endpoint: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: status, }) } @@ -635,7 +635,7 @@ func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogge // GetToken extracts the access token from request headers. // Supports both Authorization: Bearer and X-Amz-Security-Token headers. -var GetToken = func(headers http.Header) (string, int, error) { +func GetToken(headers http.Header) (string, int, error) { // Check X-Amz-Security-Token header first (for S3 compatibility) if token := headers.Get("X-Amz-Security-Token"); token != "" { return token, 0, nil diff --git a/sda/cmd/download/middleware/auth_test.go b/sda/cmd/download/middleware/auth_test.go index 02256f0471..01fdaab5fb 100644 --- a/sda/cmd/download/middleware/auth_test.go +++ b/sda/cmd/download/middleware/auth_test.go @@ -220,7 +220,7 @@ func TestTokenMiddleware_NoToken_EmitsAuditDenied(t *testing.T) { evt := logger.events[0] assert.Equal(t, "download.denied", evt.Event) assert.Equal(t, http.StatusUnauthorized, evt.HTTPStatus) - assert.Equal(t, "/datasets", evt.Endpoint) + assert.Equal(t, "/datasets", evt.Path) assert.Equal(t, "test-corr-123", evt.CorrelationID) assert.Empty(t, evt.UserID, "no user identity on 401") } From 76b4eac44ee5c07836ae5109a60bff804a6c43c7 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 25 Mar 2026 15:32:58 +0100 Subject: [PATCH 411/530] refactor(download): remove InitAuthForTesting, add typed audit event constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two changes from PR #2187 review discussion with @KarlG-nbis: 1. Remove InitAuthForTesting — tests now call InitAuth() after setting auth.allow-opaque=true via config, exercising the real init path. Added ensureTestConfig helpers for both middleware and handler tests. 2. Add typed string constants for audit event names (EventName type). Provides discoverability via IDE autocomplete while keeping the constant value as the JSON wire value — no String() mapping needed. --- sda/cmd/download/audit/audit.go | 13 +++++++- sda/cmd/download/audit/audit_test.go | 20 +++++------ sda/cmd/download/handlers/datasets_test.go | 3 +- sda/cmd/download/handlers/files.go | 6 ++-- sda/cmd/download/handlers/handlers.go | 4 +-- sda/cmd/download/handlers/handlers_test.go | 29 ++++++++++++++-- sda/cmd/download/middleware/auth.go | 32 +----------------- sda/cmd/download/middleware/auth_test.go | 11 ++++--- .../config_testhelper_novisas_test.go | 33 +++++++++++++++++++ 9 files changed, 97 insertions(+), 54 deletions(-) create mode 100644 sda/cmd/download/middleware/config_testhelper_novisas_test.go diff --git a/sda/cmd/download/audit/audit.go b/sda/cmd/download/audit/audit.go index dd57f65384..60e57a18b8 100644 --- a/sda/cmd/download/audit/audit.go +++ b/sda/cmd/download/audit/audit.go @@ -8,10 +8,21 @@ import ( "time" ) +// EventName is a typed string for audit event names. +type EventName string + +const ( + EventCompleted EventName = "download.completed" + EventDenied EventName = "download.denied" + EventFailed EventName = "download.failed" + EventContent EventName = "download.content" + EventHeader EventName = "download.header" +) + // Event represents an audit event for download operations. type Event struct { Type string `json:"type"` // routing tag: always "audit" - Event string `json:"event"` // "download.completed", "download.denied", "download.failed" + Event EventName `json:"event"` Timestamp time.Time `json:"timestamp"` UserID string `json:"userId"` FileID string `json:"fileId,omitempty"` diff --git a/sda/cmd/download/audit/audit_test.go b/sda/cmd/download/audit/audit_test.go index ca4fd86f6f..5c5848c825 100644 --- a/sda/cmd/download/audit/audit_test.go +++ b/sda/cmd/download/audit/audit_test.go @@ -15,7 +15,7 @@ func TestNoopLogger_LogDoesNotPanic(t *testing.T) { var logger Logger = NoopLogger{} assert.NotPanics(t, func() { logger.Log(context.Background(), Event{ - Event: "download.completed", + Event: EventCompleted, CorrelationID: "abc-123", Path: "/files/test", HTTPStatus: 200, @@ -28,7 +28,7 @@ func TestStdoutLogger_OutputsValidJSON(t *testing.T) { logger := newStdoutLoggerWithWriter(&buf) logger.Log(context.Background(), Event{ - Event: "download.completed", + Event: EventCompleted, UserID: "user@example.org", FileID: "file-001", DatasetID: "EGAD00000000001", @@ -40,7 +40,7 @@ func TestStdoutLogger_OutputsValidJSON(t *testing.T) { var decoded Event err := json.Unmarshal(buf.Bytes(), &decoded) require.NoError(t, err, "output must be valid JSON") - assert.Equal(t, "download.completed", decoded.Event) + assert.Equal(t, EventCompleted, decoded.Event) assert.Equal(t, "user@example.org", decoded.UserID) assert.Equal(t, "file-001", decoded.FileID) assert.Equal(t, "EGAD00000000001", decoded.DatasetID) @@ -55,7 +55,7 @@ func TestStdoutLogger_AlwaysSetsTypeToAudit(t *testing.T) { logger.Log(context.Background(), Event{ Type: "something-else", - Event: "download.denied", + Event: EventDenied, CorrelationID: "corr-456", Path: "/files/file-002", HTTPStatus: 403, @@ -77,7 +77,7 @@ func TestStdoutLogger_TimestampAlwaysOverwritten(t *testing.T) { before := time.Now().UTC() logger.Log(context.Background(), Event{ - Event: "download.completed", + Event: EventCompleted, Timestamp: callerTS, CorrelationID: "corr-789", Path: "/files/file-003", @@ -100,7 +100,7 @@ func TestStdoutLogger_ZeroTimestampFilled(t *testing.T) { before := time.Now().UTC() logger.Log(context.Background(), Event{ - Event: "download.failed", + Event: EventFailed, CorrelationID: "corr-000", Path: "/files/file-004", HTTPStatus: 500, @@ -120,7 +120,7 @@ func TestStdoutLogger_OmitsEmptyOptionalFields(t *testing.T) { logger := newStdoutLoggerWithWriter(&buf) logger.Log(context.Background(), Event{ - Event: "download.denied", + Event: EventDenied, CorrelationID: "corr-omit", Path: "/files/file-005", HTTPStatus: 401, @@ -151,13 +151,13 @@ func TestStdoutLogger_MultipleEvents(t *testing.T) { logger := newStdoutLoggerWithWriter(&buf) logger.Log(context.Background(), Event{ - Event: "download.completed", + Event: EventCompleted, CorrelationID: "corr-a", Path: "/files/a", HTTPStatus: 200, }) logger.Log(context.Background(), Event{ - Event: "download.denied", + Event: EventDenied, CorrelationID: "corr-b", Path: "/files/b", HTTPStatus: 403, @@ -179,7 +179,7 @@ func TestAuditContract_RequiredFieldsPopulated(t *testing.T) { logger := newStdoutLoggerWithWriter(&buf) logger.Log(context.Background(), Event{ - Event: "download.completed", + Event: EventCompleted, UserID: "user@example.org", CorrelationID: "corr-contract", Path: "/files/file-006", diff --git a/sda/cmd/download/handlers/datasets_test.go b/sda/cmd/download/handlers/datasets_test.go index 92b71e093c..9ce2d6a538 100644 --- a/sda/cmd/download/handlers/datasets_test.go +++ b/sda/cmd/download/handlers/datasets_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" "github.com/neicnordic/sensitive-data-archive/cmd/download/database" "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" "github.com/stretchr/testify/assert" @@ -265,7 +266,7 @@ func TestGetDataset_NoAccess_EmitsAuditDenied(t *testing.T) { assert.Equal(t, http.StatusForbidden, w.Code) require.Len(t, logger.events, 1) evt := logger.events[0] - assert.Equal(t, "download.denied", evt.Event) + assert.Equal(t, audit.EventDenied, evt.Event) assert.Equal(t, http.StatusForbidden, evt.HTTPStatus) assert.Equal(t, "/datasets/secret-dataset", evt.Path) assert.Equal(t, "test-user", evt.UserID) diff --git a/sda/cmd/download/handlers/files.go b/sda/cmd/download/handlers/files.go index 8c208925a9..b697030630 100644 --- a/sda/cmd/download/handlers/files.go +++ b/sda/cmd/download/handlers/files.go @@ -265,7 +265,7 @@ func (h *Handlers) DownloadFile(c *gin.Context) { // Audit event on completion h.auditLogger.Log(c.Request.Context(), audit.Event{ - Event: "download.completed", + Event: audit.EventCompleted, UserID: resolved.authCtx.Subject, FileID: file.ID, DatasetID: file.DatasetID, @@ -347,7 +347,7 @@ func (h *Handlers) GetFileHeader(c *gin.Context) { c.Data(http.StatusOK, "application/octet-stream", resolved.newHeader) h.auditLogger.Log(c.Request.Context(), audit.Event{ - Event: "download.header", + Event: audit.EventHeader, UserID: resolved.authCtx.Subject, FileID: file.ID, DatasetID: file.DatasetID, @@ -448,7 +448,7 @@ func (h *Handlers) GetFileContent(c *gin.Context) { // Audit event on completion h.auditLogger.Log(c.Request.Context(), audit.Event{ - Event: "download.content", + Event: audit.EventContent, UserID: resolved.authCtx.Subject, FileID: file.ID, DatasetID: file.DatasetID, diff --git a/sda/cmd/download/handlers/handlers.go b/sda/cmd/download/handlers/handlers.go index 7b1e8a47dc..e3191ac2c7 100644 --- a/sda/cmd/download/handlers/handlers.go +++ b/sda/cmd/download/handlers/handlers.go @@ -46,7 +46,7 @@ func New(options ...func(*Handlers)) (*Handlers, error) { func (h *Handlers) auditDenied(c *gin.Context) { authCtx, _ := middleware.GetAuthContext(c) h.auditLogger.Log(c.Request.Context(), audit.Event{ - Event: "download.denied", + Event: audit.EventDenied, UserID: authCtx.Subject, CorrelationID: c.GetString("correlationId"), Path: c.Request.URL.Path, @@ -63,7 +63,7 @@ func (h *Handlers) auditFailed(c *gin.Context, authCtx middleware.AuthContext, f } h.auditLogger.Log(c.Request.Context(), audit.Event{ - Event: "download.failed", + Event: audit.EventFailed, UserID: authCtx.Subject, FileID: fileID, DatasetID: datasetID, diff --git a/sda/cmd/download/handlers/handlers_test.go b/sda/cmd/download/handlers/handlers_test.go index dfca799cc6..a388f146af 100644 --- a/sda/cmd/download/handlers/handlers_test.go +++ b/sda/cmd/download/handlers/handlers_test.go @@ -4,14 +4,37 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "os" + "sync" "testing" "github.com/gin-gonic/gin" "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + configv2 "github.com/neicnordic/sensitive-data-archive/internal/config/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +var loadConfigOnce sync.Once + +func ensureTestConfig(t *testing.T) { + t.Helper() + + loadConfigOnce.Do(func() { + t.Setenv("DB_HOST", "localhost") + t.Setenv("DB_USER", "test") + t.Setenv("DB_PASSWORD", "test") + t.Setenv("GRPC_HOST", "localhost") + t.Setenv("AUTH_ALLOW_OPAQUE", "true") + + oldArgs := os.Args + os.Args = []string{oldArgs[0]} + defer func() { os.Args = oldArgs }() + + require.NoError(t, configv2.Load()) + }) +} + func init() { gin.SetMode(gin.TestMode) } @@ -135,7 +158,8 @@ func TestHasDatasetAccess(t *testing.T) { // v2 route wiring tests — verify RegisterRoutes applies auth middleware correctly func TestRoutes_AuthRequired(t *testing.T) { - err := middleware.InitAuthForTesting() + ensureTestConfig(t) + err := middleware.InitAuth() require.NoError(t, err) router := gin.New() @@ -184,7 +208,8 @@ func TestRoutes_ServiceInfoNoAuth(t *testing.T) { } func TestRoutes_OldV1PathsReturn404(t *testing.T) { - err := middleware.InitAuthForTesting() + ensureTestConfig(t) + err := middleware.InitAuth() require.NoError(t, err) router := gin.New() diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index 6377d25500..a10fe057f9 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -84,36 +84,6 @@ var ( legacyCookieName = "sda_session_key" ) -// InitAuthForTesting initializes the auth middleware with an empty keyset for testing. -// This allows middleware to run without valid keys, causing authentication to fail. -func InitAuthForTesting() error { - cache, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e6, - MaxCost: 100000, - BufferItems: 64, - }) - if err != nil { - return err - } - sessionCache = &SessionCache{cache: cache} - - tokenCacheInst, err := ristretto.NewCache(&ristretto.Config{ - NumCounters: 1e6, - MaxCost: 100000, - BufferItems: 64, - }) - if err != nil { - return err - } - tokenCache = &SessionCache{cache: tokenCacheInst} - - auth = &Authenticator{ - Keyset: jwk.NewSet(), - } - - return nil -} - // InitAuth initializes the authentication middleware. // This should be called during application startup. // It loads JWT public keys from either a local path or remote JWKS URL. @@ -454,7 +424,7 @@ func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogge auditDenied := func(c *gin.Context, status int) { auditLogger.Log(c.Request.Context(), audit.Event{ - Event: "download.denied", + Event: audit.EventDenied, CorrelationID: c.GetString("correlationId"), Path: c.Request.URL.Path, HTTPStatus: status, diff --git a/sda/cmd/download/middleware/auth_test.go b/sda/cmd/download/middleware/auth_test.go index 01fdaab5fb..db8463c5aa 100644 --- a/sda/cmd/download/middleware/auth_test.go +++ b/sda/cmd/download/middleware/auth_test.go @@ -176,12 +176,13 @@ func TestSessionCache_GetNotExists(t *testing.T) { } func TestTokenMiddleware_NoToken(t *testing.T) { + ensureTestConfig(t) + w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest("GET", "/test", nil) - // Initialize auth middleware for testing - err := InitAuthForTesting() + err := InitAuth() assert.NoError(t, err) handler := TokenMiddleware(nil, nil, audit.NoopLogger{}) // nil database, nil visa - not testing dataset lookup @@ -203,12 +204,14 @@ func (l *capturingLogger) Log(_ context.Context, event audit.Event) { } func TestTokenMiddleware_NoToken_EmitsAuditDenied(t *testing.T) { + ensureTestConfig(t) + w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest("GET", "/datasets", nil) c.Set("correlationId", "test-corr-123") - err := InitAuthForTesting() + err := InitAuth() assert.NoError(t, err) logger := &capturingLogger{} @@ -218,7 +221,7 @@ func TestTokenMiddleware_NoToken_EmitsAuditDenied(t *testing.T) { assert.Equal(t, http.StatusUnauthorized, w.Code) require.Len(t, logger.events, 1) evt := logger.events[0] - assert.Equal(t, "download.denied", evt.Event) + assert.Equal(t, audit.EventDenied, evt.Event) assert.Equal(t, http.StatusUnauthorized, evt.HTTPStatus) assert.Equal(t, "/datasets", evt.Path) assert.Equal(t, "test-corr-123", evt.CorrelationID) diff --git a/sda/cmd/download/middleware/config_testhelper_novisas_test.go b/sda/cmd/download/middleware/config_testhelper_novisas_test.go new file mode 100644 index 0000000000..9dd0bb5668 --- /dev/null +++ b/sda/cmd/download/middleware/config_testhelper_novisas_test.go @@ -0,0 +1,33 @@ +//go:build !visas +// +build !visas + +package middleware + +import ( + "os" + "sync" + "testing" + + configv2 "github.com/neicnordic/sensitive-data-archive/internal/config/v2" + "github.com/stretchr/testify/require" +) + +var loadConfigOnceNoVisas sync.Once + +func ensureTestConfig(t *testing.T) { + t.Helper() + + loadConfigOnceNoVisas.Do(func() { + t.Setenv("DB_HOST", "localhost") + t.Setenv("DB_USER", "test") + t.Setenv("DB_PASSWORD", "test") + t.Setenv("GRPC_HOST", "localhost") + t.Setenv("AUTH_ALLOW_OPAQUE", "true") + + oldArgs := os.Args + os.Args = []string{oldArgs[0]} + defer func() { os.Args = oldArgs }() + + require.NoError(t, configv2.Load()) + }) +} From 144ebb0d51a0bd3fa94ab0cbe409ee43d213ab24 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 25 Mar 2026 15:43:39 +0100 Subject: [PATCH 412/530] style(download): fix gofmt alignment and merge identical switch branches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix gofmt issues from Endpoint→Path rename (field alignment in struct literals) and merge identical ownership/combined switch branches to satisfy revive identical-switch-branches lint. --- sda/cmd/download/audit/audit.go | 2 +- sda/cmd/download/audit/audit_test.go | 16 ++++++++-------- sda/cmd/download/handlers/files.go | 6 +++--- sda/cmd/download/handlers/handlers.go | 4 ++-- sda/cmd/download/main.go | 4 +--- sda/cmd/download/middleware/auth.go | 2 +- 6 files changed, 16 insertions(+), 18 deletions(-) diff --git a/sda/cmd/download/audit/audit.go b/sda/cmd/download/audit/audit.go index 60e57a18b8..570b665be0 100644 --- a/sda/cmd/download/audit/audit.go +++ b/sda/cmd/download/audit/audit.go @@ -21,7 +21,7 @@ const ( // Event represents an audit event for download operations. type Event struct { - Type string `json:"type"` // routing tag: always "audit" + Type string `json:"type"` // routing tag: always "audit" Event EventName `json:"event"` Timestamp time.Time `json:"timestamp"` UserID string `json:"userId"` diff --git a/sda/cmd/download/audit/audit_test.go b/sda/cmd/download/audit/audit_test.go index 5c5848c825..781c3cd67f 100644 --- a/sda/cmd/download/audit/audit_test.go +++ b/sda/cmd/download/audit/audit_test.go @@ -17,7 +17,7 @@ func TestNoopLogger_LogDoesNotPanic(t *testing.T) { logger.Log(context.Background(), Event{ Event: EventCompleted, CorrelationID: "abc-123", - Path: "/files/test", + Path: "/files/test", HTTPStatus: 200, }) }) @@ -33,7 +33,7 @@ func TestStdoutLogger_OutputsValidJSON(t *testing.T) { FileID: "file-001", DatasetID: "EGAD00000000001", CorrelationID: "corr-123", - Path: "/files/file-001", + Path: "/files/file-001", HTTPStatus: 200, }) @@ -57,7 +57,7 @@ func TestStdoutLogger_AlwaysSetsTypeToAudit(t *testing.T) { Type: "something-else", Event: EventDenied, CorrelationID: "corr-456", - Path: "/files/file-002", + Path: "/files/file-002", HTTPStatus: 403, }) @@ -102,7 +102,7 @@ func TestStdoutLogger_ZeroTimestampFilled(t *testing.T) { logger.Log(context.Background(), Event{ Event: EventFailed, CorrelationID: "corr-000", - Path: "/files/file-004", + Path: "/files/file-004", HTTPStatus: 500, }) after := time.Now().UTC() @@ -122,7 +122,7 @@ func TestStdoutLogger_OmitsEmptyOptionalFields(t *testing.T) { logger.Log(context.Background(), Event{ Event: EventDenied, CorrelationID: "corr-omit", - Path: "/files/file-005", + Path: "/files/file-005", HTTPStatus: 401, }) @@ -153,13 +153,13 @@ func TestStdoutLogger_MultipleEvents(t *testing.T) { logger.Log(context.Background(), Event{ Event: EventCompleted, CorrelationID: "corr-a", - Path: "/files/a", + Path: "/files/a", HTTPStatus: 200, }) logger.Log(context.Background(), Event{ Event: EventDenied, CorrelationID: "corr-b", - Path: "/files/b", + Path: "/files/b", HTTPStatus: 403, }) @@ -182,7 +182,7 @@ func TestAuditContract_RequiredFieldsPopulated(t *testing.T) { Event: EventCompleted, UserID: "user@example.org", CorrelationID: "corr-contract", - Path: "/files/file-006", + Path: "/files/file-006", HTTPStatus: 200, }) diff --git a/sda/cmd/download/handlers/files.go b/sda/cmd/download/handlers/files.go index b697030630..e4c8fc0ff2 100644 --- a/sda/cmd/download/handlers/files.go +++ b/sda/cmd/download/handlers/files.go @@ -270,7 +270,7 @@ func (h *Handlers) DownloadFile(c *gin.Context) { FileID: file.ID, DatasetID: file.DatasetID, CorrelationID: c.GetString("correlationId"), - Path: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } @@ -352,7 +352,7 @@ func (h *Handlers) GetFileHeader(c *gin.Context) { FileID: file.ID, DatasetID: file.DatasetID, CorrelationID: c.GetString("correlationId"), - Path: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } @@ -453,7 +453,7 @@ func (h *Handlers) GetFileContent(c *gin.Context) { FileID: file.ID, DatasetID: file.DatasetID, CorrelationID: c.GetString("correlationId"), - Path: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } diff --git a/sda/cmd/download/handlers/handlers.go b/sda/cmd/download/handlers/handlers.go index e3191ac2c7..e285f7a72a 100644 --- a/sda/cmd/download/handlers/handlers.go +++ b/sda/cmd/download/handlers/handlers.go @@ -49,7 +49,7 @@ func (h *Handlers) auditDenied(c *gin.Context) { Event: audit.EventDenied, UserID: authCtx.Subject, CorrelationID: c.GetString("correlationId"), - Path: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), }) } @@ -68,7 +68,7 @@ func (h *Handlers) auditFailed(c *gin.Context, authCtx middleware.AuthContext, f FileID: fileID, DatasetID: datasetID, CorrelationID: c.GetString("correlationId"), - Path: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: c.Writer.Status(), ErrorReason: reason, }) diff --git a/sda/cmd/download/main.go b/sda/cmd/download/main.go index 4c4dfe1c68..97c2868055 100644 --- a/sda/cmd/download/main.go +++ b/sda/cmd/download/main.go @@ -332,15 +332,13 @@ type productionConfig struct { // that its dependencies are satisfied. func validatePermissionModel(model string, visaEnabled bool) error { switch model { - case "ownership": + case "ownership", "combined": return nil case "visa": if !visaEnabled { return fmt.Errorf("permission.model is %q but visa.enabled is false — this combination would deny all requests", model) } - return nil - case "combined": return nil default: return fmt.Errorf("invalid permission.model %q: must be ownership, visa, or combined", model) diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index a10fe057f9..a61b268989 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -426,7 +426,7 @@ func TokenMiddleware(db DatasetLookup, visaValidator *visa.Validator, auditLogge auditLogger.Log(c.Request.Context(), audit.Event{ Event: audit.EventDenied, CorrelationID: c.GetString("correlationId"), - Path: c.Request.URL.Path, + Path: c.Request.URL.Path, HTTPStatus: status, }) } From d61c6ceb5a7e52ae966783452c44e1c7362ae5d9 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 25 Mar 2026 16:26:55 +0100 Subject: [PATCH 413/530] fix(download): fix S3/sync integration test failures for download v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two issues caused 55_download_test.sh to fail in the S3 and sync stacks: 1. S3 stack: download-config.yaml referenced mockauth:8000 (visa endpoints) which doesn't exist in the S3 stack — it uses mock-aai (LS AAI mock) instead. Created download-config-s3.yaml with ownership-only auth and visas disabled. Full visa testing happens in the dedicated download-v2 integration tests. 2. Sync stack: no download service exists, but the test script ran via the shared tests/sda/ directory. Added a connectivity check that skips gracefully when the download service is unavailable. --- .github/integration/sda-s3-integration.yml | 2 +- .../integration/sda/download-config-s3.yaml | 71 +++++++++++++++++++ .../integration/tests/sda/55_download_test.sh | 7 ++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 .github/integration/sda/download-config-s3.yaml diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index c03ed298ee..5424e14203 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -280,7 +280,7 @@ services: - "8086:8081" restart: always volumes: - - ./sda/download-config.yaml:/config.yaml + - ./sda/download-config-s3.yaml:/config.yaml - shared:/shared rotatekey: diff --git a/.github/integration/sda/download-config-s3.yaml b/.github/integration/sda/download-config-s3.yaml new file mode 100644 index 0000000000..72969860d9 --- /dev/null +++ b/.github/integration/sda/download-config-s3.yaml @@ -0,0 +1,71 @@ +# Configuration for sda/cmd/download service in the S3 integration stack. +# Uses ownership-only auth with allow-all-data — visa testing is handled +# by the dedicated download-v2 integration tests, not here. + +# API Server +api: + host: "0.0.0.0" + port: 8080 + +# gRPC Health check +health: + port: 8081 + +# Database +db: + host: "postgres" + port: 5432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +# gRPC reencrypt service +grpc: + host: "reencrypt" + port: 50051 + timeout: 30 + +# JWT configuration +jwt: + pubkey-path: "/shared/keys/pub/" + allow-all-data: true + +# Permission model: ownership only (no visa in this stack) +permission: + model: "ownership" + +# Auth: allow opaque tokens +auth: + allow-opaque: true + +# Visa disabled — the S3 stack uses mock-aai (LS AAI mock), +# not the Python mockauth that serves visa endpoints. +visa: + enabled: false + +# Session +session: + expiration: 3600 + secure: false + http-only: true + name: "sda_session" + +# Database query cache +cache: + enabled: true + file-ttl: 300 + permission-ttl: 120 + dataset-ttl: 300 + +# Storage configuration (S3 backend) +storage: + backend: "archive" + archive: + s3: + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true diff --git a/.github/integration/tests/sda/55_download_test.sh b/.github/integration/tests/sda/55_download_test.sh index 20223aa2b4..54dcc92894 100755 --- a/.github/integration/tests/sda/55_download_test.sh +++ b/.github/integration/tests/sda/55_download_test.sh @@ -3,11 +3,18 @@ set -e # Test the new sda/cmd/download service (v2 API) # This test runs AFTER files have been mapped to a dataset (40_mapper_test.sh) +# Skips gracefully if the download service is not available (e.g. in sync stack) cd shared || true echo "=== Testing Download Service (v2 API) ===" +# Check if the download service is reachable before running tests +if ! curl -s --connect-timeout 3 -o /dev/null "http://download:8080/health/live" 2>/dev/null; then + echo "Download service not available, skipping download v2 tests" + exit 0 +fi + token="$(cat /shared/token)" # Test 1: Health endpoints From 0994afa49f6acd4786cd772a2ef3ed2233a28a98 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 25 Mar 2026 16:41:25 +0100 Subject: [PATCH 414/530] style(download): remove redundant +build tag from test helper --- sda/cmd/download/middleware/config_testhelper_novisas_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sda/cmd/download/middleware/config_testhelper_novisas_test.go b/sda/cmd/download/middleware/config_testhelper_novisas_test.go index 9dd0bb5668..9806af7f39 100644 --- a/sda/cmd/download/middleware/config_testhelper_novisas_test.go +++ b/sda/cmd/download/middleware/config_testhelper_novisas_test.go @@ -1,5 +1,4 @@ //go:build !visas -// +build !visas package middleware From 0605ae37606be24c0fc61e881378156cd8f85479 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 25 Mar 2026 16:42:53 +0100 Subject: [PATCH 415/530] fix(download): simplify mergeDatasets to avoid CodeQL overflow warning --- sda/cmd/download/middleware/auth.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/sda/cmd/download/middleware/auth.go b/sda/cmd/download/middleware/auth.go index a61b268989..fe3f88cb53 100644 --- a/sda/cmd/download/middleware/auth.go +++ b/sda/cmd/download/middleware/auth.go @@ -9,7 +9,6 @@ import ( "encoding/json" "errors" "fmt" - "math" "net/http" "net/url" "os" @@ -666,12 +665,7 @@ func (sc *SessionCache) Set(key string, value AuthContext, ttl time.Duration) { // mergeDatasets returns a deduplicated union of two string slices. func mergeDatasets(a, b []string) []string { - hint := 0 - if len(a) <= math.MaxInt-len(b) { - hint = len(a) + len(b) - } - - seen := make(map[string]bool, hint) + seen := make(map[string]bool, len(a)) var result []string for _, s := range a { From 465e1f48f4aede081ed3b902d2c361e0d63e8122 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 26 Mar 2026 11:52:15 +0100 Subject: [PATCH 416/530] refactor(download): remove dead code, use errors.Is for sql.ErrNoRows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback from @KarlG-nbis: 1. Replace err == sql.ErrNoRows with errors.Is(err, sql.ErrNoRows) at three locations (database.go:489, 565, 624). 2. Remove unused GetDatasetFiles — handlers use GetDatasetFilesPaginated. Removed from Database interface, PostgresDB, CachedDB, and all test mocks. 3. Remove unused package-level delegate functions (GetAllDatasets, GetUserDatasets, GetDatasetInfo, GetFileByID, GetFileByPath, CheckFilePermission, CheckDatasetExists, GetFileChecksums) — handlers use the injected Database interface directly. --- sda/cmd/download/database/cache.go | 23 ---- sda/cmd/download/database/cache_test.go | 41 ------ sda/cmd/download/database/database.go | 118 +----------------- sda/cmd/download/database/database_test.go | 64 ---------- .../download/handlers/mock_database_test.go | 8 -- 5 files changed, 3 insertions(+), 251 deletions(-) diff --git a/sda/cmd/download/database/cache.go b/sda/cmd/download/database/cache.go index 8ef4dfa7cc..caf5010a3f 100644 --- a/sda/cmd/download/database/cache.go +++ b/sda/cmd/download/database/cache.go @@ -179,29 +179,6 @@ func (c *CachedDB) GetDatasetInfo(ctx context.Context, datasetID string) (*Datas return info, nil } -// GetDatasetFiles returns files in a dataset. -// Results are cached with DatasetTTL. -func (c *CachedDB) GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { - key := "dataset:files:" + datasetID - - if val, found := c.cache.Get(key); found { - if rval, ok := val.([]File); ok { - log.Debugf("cache hit: GetDatasetFiles(%s)", datasetID) - - return rval, nil - } - } - - log.Debugf("cache miss: GetDatasetFiles(%s)", datasetID) - files, err := c.db.GetDatasetFiles(ctx, datasetID) - if err != nil { - return nil, err - } - - c.cache.SetWithTTL(key, files, 1, c.config.DatasetTTL) - - return files, nil -} // GetFileByID returns file information by stable ID. // Results are cached with FileTTL. This is particularly important diff --git a/sda/cmd/download/database/cache_test.go b/sda/cmd/download/database/cache_test.go index 62a56ede09..e70465563b 100644 --- a/sda/cmd/download/database/cache_test.go +++ b/sda/cmd/download/database/cache_test.go @@ -64,14 +64,6 @@ func (m *MockDatabase) GetDatasetInfo(ctx context.Context, datasetID string) (*D return args.Get(0).(*DatasetInfo), args.Error(1) } -func (m *MockDatabase) GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { - args := m.Called(ctx, datasetID) - if args.Get(0) == nil { - return nil, args.Error(1) - } - - return args.Get(0).([]File), args.Error(1) -} func (m *MockDatabase) GetFileByID(ctx context.Context, fileID string) (*File, error) { args := m.Called(ctx, fileID) @@ -381,39 +373,6 @@ func TestCachedDB_GetDatasetInfo_CacheHit(t *testing.T) { mockDB.AssertExpectations(t) } -func TestCachedDB_GetDatasetFiles_CacheHit(t *testing.T) { - mockDB := new(MockDatabase) - cfg := CacheConfig{ - FileTTL: 1 * time.Minute, - PermissionTTL: 1 * time.Minute, - DatasetTTL: 1 * time.Minute, - } - - cachedDB, err := NewCachedDB(mockDB, cfg) - require.NoError(t, err) - - ctx := context.Background() - expectedFiles := []File{ - {ID: "file1", DatasetID: "dataset1", SubmittedPath: "/file1.txt"}, - {ID: "file2", DatasetID: "dataset1", SubmittedPath: "/file2.txt"}, - } - - mockDB.On("GetDatasetFiles", ctx, "dataset1").Return(expectedFiles, nil).Once() - - files1, err := cachedDB.GetDatasetFiles(ctx, "dataset1") - require.NoError(t, err) - assert.Equal(t, expectedFiles, files1) - - // Wait for ristretto to process the set - time.Sleep(10 * time.Millisecond) - - // Second call should hit cache - files2, err := cachedDB.GetDatasetFiles(ctx, "dataset1") - require.NoError(t, err) - assert.Equal(t, expectedFiles, files2) - - mockDB.AssertExpectations(t) -} func TestCachedDB_GetAllDatasets_CacheHit(t *testing.T) { mockDB := new(MockDatabase) diff --git a/sda/cmd/download/database/database.go b/sda/cmd/download/database/database.go index a625aa2f2d..9e95eb1c6a 100644 --- a/sda/cmd/download/database/database.go +++ b/sda/cmd/download/database/database.go @@ -22,7 +22,6 @@ const ( getDatasetIDsByUserQuery = "getDatasetIDsByUser" getUserDatasetsQuery = "getUserDatasets" getDatasetInfoQuery = "getDatasetInfo" - getDatasetFilesQuery = "getDatasetFiles" getFileByIDQuery = "getFileByID" getFileByPathQuery = "getFileByPath" checkFilePermissionQuery = "checkFilePermission" @@ -87,26 +86,6 @@ var queries = map[string]string{ WHERE d.stable_id = $1 GROUP BY d.id, d.stable_id, d.title, d.description, d.created_at`, - // getDatasetFiles returns files in a dataset - // Checksums are stored in sda.checksums table, not in files table - getDatasetFilesQuery: ` - SELECT - f.stable_id, - d.stable_id as dataset_id, - f.submission_file_path, - f.archive_file_path, - f.archive_location, - f.archive_file_size, - f.decrypted_file_size, - c.checksum as decrypted_checksum, - c.type as decrypted_checksum_type - FROM sda.files f - INNER JOIN sda.file_dataset fd ON f.id = fd.file_id - INNER JOIN sda.datasets d ON fd.dataset_id = d.id - LEFT JOIN sda.checksums c ON f.id = c.file_id AND c.source = 'UNENCRYPTED' - WHERE d.stable_id = $1 AND f.stable_id IS NOT NULL - ORDER BY f.submission_file_path`, - getFileByIDQuery: ` SELECT f.stable_id, @@ -230,9 +209,6 @@ type Database interface { // GetDatasetInfo returns metadata for a specific dataset. GetDatasetInfo(ctx context.Context, datasetID string) (*DatasetInfo, error) - // GetDatasetFiles returns files in a dataset. - GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) - // GetFileByID returns file information by stable ID. GetFileByID(ctx context.Context, fileID string) (*File, error) @@ -486,7 +462,7 @@ func (p *PostgresDB) GetDatasetInfo(ctx context.Context, datasetID string) (*Dat &info.FileCount, &info.TotalSize, ) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } if err != nil { @@ -499,48 +475,6 @@ func (p *PostgresDB) GetDatasetInfo(ctx context.Context, datasetID string) (*Dat return &info, nil } -// GetDatasetFiles returns files in a dataset. -func (p *PostgresDB) GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { - stmt := p.preparedStatements[getDatasetFilesQuery] - rows, err := stmt.QueryContext(ctx, datasetID) - if err != nil { - return nil, fmt.Errorf("failed to query dataset files: %w", err) - } - defer rows.Close() - - var files []File - for rows.Next() { - var f File - var archivePath, archiveLocation, decryptedChecksum, decryptedChecksumType sql.NullString - var archiveSize, decryptedSize sql.NullInt64 - if err := rows.Scan( - &f.ID, - &f.DatasetID, - &f.SubmittedPath, - &archivePath, - &archiveLocation, - &archiveSize, - &decryptedSize, - &decryptedChecksum, - &decryptedChecksumType, - ); err != nil { - return nil, fmt.Errorf("failed to scan file row: %w", err) - } - f.ArchivePath = archivePath.String - f.ArchiveLocation = archiveLocation.String - f.ArchiveSize = archiveSize.Int64 - f.DecryptedSize = decryptedSize.Int64 - f.DecryptedChecksum = decryptedChecksum.String - f.DecryptedChecksumType = decryptedChecksumType.String - files = append(files, f) - } - - if err := rows.Err(); err != nil { - return nil, fmt.Errorf("error iterating file rows: %w", err) - } - - return files, nil -} // GetFileByID returns file information by stable ID. func (p *PostgresDB) GetFileByID(ctx context.Context, fileID string) (*File, error) { @@ -562,7 +496,7 @@ func (p *PostgresDB) GetFileByID(ctx context.Context, fileID string) (*File, err &headerHex, &f.CreatedAt, ) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } if err != nil { @@ -621,7 +555,7 @@ func (p *PostgresDB) GetFileByPath(ctx context.Context, datasetID, filePath stri &headerHex, &f.CreatedAt, ) - if err == sql.ErrNoRows { + if errors.Is(err, sql.ErrNoRows) { return nil, nil } if err != nil { @@ -762,49 +696,3 @@ func escapeLikePrefix(prefix string) string { return r.Replace(prefix) + "%" } -// Package-level functions that delegate to the registered database - -// GetAllDatasets returns all datasets (for allow-all-data mode). -func GetAllDatasets(ctx context.Context) ([]Dataset, error) { - return db.GetAllDatasets(ctx) -} - -// GetUserDatasets returns datasets the user has access to based on their allowed dataset IDs. -func GetUserDatasets(ctx context.Context, datasetIDs []string) ([]Dataset, error) { - return db.GetUserDatasets(ctx, datasetIDs) -} - -// GetDatasetInfo returns metadata for a specific dataset. -func GetDatasetInfo(ctx context.Context, datasetID string) (*DatasetInfo, error) { - return db.GetDatasetInfo(ctx, datasetID) -} - -// GetDatasetFiles returns files in a dataset. -func GetDatasetFiles(ctx context.Context, datasetID string) ([]File, error) { - return db.GetDatasetFiles(ctx, datasetID) -} - -// GetFileByID returns file information by stable ID. -func GetFileByID(ctx context.Context, fileID string) (*File, error) { - return db.GetFileByID(ctx, fileID) -} - -// GetFileByPath returns file information by dataset and submitted path. -func GetFileByPath(ctx context.Context, datasetID, filePath string) (*File, error) { - return db.GetFileByPath(ctx, datasetID, filePath) -} - -// CheckFilePermission verifies the user has access to download a file. -func CheckFilePermission(ctx context.Context, fileID string, visas []string) (bool, error) { - return db.CheckFilePermission(ctx, fileID, visas) -} - -// CheckDatasetExists checks if a dataset with the given stable_id exists. -func CheckDatasetExists(ctx context.Context, datasetID string) (bool, error) { - return db.CheckDatasetExists(ctx, datasetID) -} - -// GetFileChecksums returns checksums for a file filtered by source (e.g., "ARCHIVED", "UNENCRYPTED"). -func GetFileChecksums(ctx context.Context, fileID string, source string) ([]Checksum, error) { - return db.GetFileChecksums(ctx, fileID, source) -} diff --git a/sda/cmd/download/database/database_test.go b/sda/cmd/download/database/database_test.go index ae5343fe4b..b025e1d6b9 100644 --- a/sda/cmd/download/database/database_test.go +++ b/sda/cmd/download/database/database_test.go @@ -203,34 +203,6 @@ func TestGetDatasetInfo_NotFound(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } -func TestGetDatasetFiles(t *testing.T) { - db, mock, cleanup := setupMockDB(t) - defer cleanup() - - rows := sqlmock.NewRows([]string{ - "stable_id", "dataset_id", "submission_file_path", "archive_file_path", - "archive_location", "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", "decrypted_file_checksum_type", - }). - AddRow("file-1", "dataset-1", "/path/to/file.txt", "/archive/file.c4gh", "s3:9000/archive", int64(1024), int64(900), "abc123", "SHA256"). - AddRow("file-2", "dataset-1", "/path/to/file2.txt", nil, nil, nil, nil, nil, nil) - - mock.ExpectQuery(queries[getDatasetFilesQuery]). - WithArgs("dataset-1"). - WillReturnRows(rows) - - files, err := db.GetDatasetFiles(context.Background(), "dataset-1") - - assert.NoError(t, err) - assert.Len(t, files, 2) - assert.Equal(t, "file-1", files[0].ID) - assert.Equal(t, "/path/to/file.txt", files[0].SubmittedPath) - assert.Equal(t, "s3:9000/archive", files[0].ArchiveLocation) - assert.Equal(t, int64(1024), files[0].ArchiveSize) - assert.Equal(t, "file-2", files[1].ID) - assert.Empty(t, files[1].ArchivePath) - assert.Empty(t, files[1].ArchiveLocation) - assert.NoError(t, mock.ExpectationsWereMet()) -} func TestGetFileByID(t *testing.T) { db, mock, cleanup := setupMockDB(t) @@ -487,40 +459,7 @@ func TestGetDatasetInfo_QueryError(t *testing.T) { assert.Contains(t, err.Error(), "failed to query dataset info") } -func TestGetDatasetFiles_QueryError(t *testing.T) { - db, mock, cleanup := setupMockDB(t) - defer cleanup() - - mock.ExpectQuery(queries[getDatasetFilesQuery]). - WithArgs("dataset-1"). - WillReturnError(sql.ErrConnDone) - - files, err := db.GetDatasetFiles(context.Background(), "dataset-1") - - assert.Error(t, err) - assert.Nil(t, files) - assert.Contains(t, err.Error(), "failed to query dataset files") -} - -func TestGetDatasetFiles_Empty(t *testing.T) { - db, mock, cleanup := setupMockDB(t) - defer cleanup() - rows := sqlmock.NewRows([]string{ - "stable_id", "dataset_id", "submission_file_path", "archive_file_path", - "archive_file_size", "decrypted_file_size", "decrypted_file_checksum", "decrypted_file_checksum_type", - }) - - mock.ExpectQuery(queries[getDatasetFilesQuery]). - WithArgs("empty-dataset"). - WillReturnRows(rows) - - files, err := db.GetDatasetFiles(context.Background(), "empty-dataset") - - assert.NoError(t, err) - assert.Empty(t, files) - assert.NoError(t, mock.ExpectationsWereMet()) -} func TestGetFileByID_QueryError(t *testing.T) { db, mock, cleanup := setupMockDB(t) @@ -653,9 +592,6 @@ func (m *mockTestDatabase) GetDatasetInfo(_ context.Context, _ string) (*Dataset return nil, nil } -func (m *mockTestDatabase) GetDatasetFiles(_ context.Context, _ string) ([]File, error) { - return nil, nil -} func (m *mockTestDatabase) GetFileByID(_ context.Context, _ string) (*File, error) { return nil, nil diff --git a/sda/cmd/download/handlers/mock_database_test.go b/sda/cmd/download/handlers/mock_database_test.go index 33ad4ca995..2d3cd23eb5 100644 --- a/sda/cmd/download/handlers/mock_database_test.go +++ b/sda/cmd/download/handlers/mock_database_test.go @@ -25,7 +25,6 @@ type mockDatabase struct { datasets []database.Dataset datasetIDs []string datasetInfo *database.DatasetInfo - datasetFiles []database.File datasetFilesPaged []database.File fileByID *database.File fileByPath *database.File @@ -76,13 +75,6 @@ func (m *mockDatabase) GetDatasetInfo(_ context.Context, _ string) (*database.Da return m.datasetInfo, nil } -func (m *mockDatabase) GetDatasetFiles(_ context.Context, _ string) ([]database.File, error) { - if m.err != nil { - return nil, m.err - } - - return m.datasetFiles, nil -} func (m *mockDatabase) GetFileByID(_ context.Context, _ string) (*database.File, error) { if m.err != nil { From 3e54fabba8892f4080e5a55ca00a8f251a385b20 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 26 Mar 2026 13:20:10 +0100 Subject: [PATCH 417/530] refactor(download): remove gRPC health server, storage.backend flag, clarify HealthCheck MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback from @KarlG-nbis: 1. Remove gRPC health server — redundant with HTTP health endpoints. The download chart uses HTTP probes, not gRPC. Removes health package, health.port config flag, and port 8081 mappings from compose files. 2. Remove storage.backend config flag — hardcode "archive" like the old sda-download service. The storage.archive config is still needed. 3. Clarify reencrypt HealthCheck — document that grpc.NewClient does no I/O, so HealthCheck only validates client construction. Added TODO for a real RPC health check. --- .../sda-download-v2-integration.yml | 1 - .github/integration/sda-s3-integration.yml | 1 - .../sda/download-config-benchmark.yaml | 5 -- .../integration/sda/download-config-s3.yaml | 5 -- .github/integration/sda/download-config.yaml | 5 -- sda/cmd/download/config/config.go | 35 ------------- sda/cmd/download/database/cache.go | 1 - sda/cmd/download/dev_config.yaml | 8 +-- sda/cmd/download/download.md | 5 +- sda/cmd/download/health/health.go | 51 ------------------- sda/cmd/download/health/health_test.go | 31 ----------- sda/cmd/download/main.go | 21 ++------ sda/cmd/download/reencrypt/reencrypt.go | 5 +- 13 files changed, 9 insertions(+), 165 deletions(-) delete mode 100644 sda/cmd/download/health/health.go delete mode 100644 sda/cmd/download/health/health_test.go diff --git a/.github/integration/sda-download-v2-integration.yml b/.github/integration/sda-download-v2-integration.yml index 718571a5f3..767ce01179 100644 --- a/.github/integration/sda-download-v2-integration.yml +++ b/.github/integration/sda-download-v2-integration.yml @@ -226,7 +226,6 @@ services: - "localhost:host-gateway" ports: - "8085:8080" - - "8086:8081" restart: always volumes: - ./sda/download-config.yaml:/config.yaml diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 5424e14203..60e008da43 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -277,7 +277,6 @@ services: - "localhost:host-gateway" ports: - "8085:8080" - - "8086:8081" restart: always volumes: - ./sda/download-config-s3.yaml:/config.yaml diff --git a/.github/integration/sda/download-config-benchmark.yaml b/.github/integration/sda/download-config-benchmark.yaml index cea66446c0..b4c43cc92c 100644 --- a/.github/integration/sda/download-config-benchmark.yaml +++ b/.github/integration/sda/download-config-benchmark.yaml @@ -11,10 +11,6 @@ api: log: level: "debug" -# gRPC Health check -health: - port: 8081 - # Database db: host: "postgres" @@ -82,7 +78,6 @@ cache: # Storage configuration (S3 backend) # NOTE: endpoint includes http:// for proper URL parsing in storage/v2 storage: - backend: "archive" archive: s3: - endpoint: "http://s3:9000" diff --git a/.github/integration/sda/download-config-s3.yaml b/.github/integration/sda/download-config-s3.yaml index 72969860d9..2282369027 100644 --- a/.github/integration/sda/download-config-s3.yaml +++ b/.github/integration/sda/download-config-s3.yaml @@ -7,10 +7,6 @@ api: host: "0.0.0.0" port: 8080 -# gRPC Health check -health: - port: 8081 - # Database db: host: "postgres" @@ -60,7 +56,6 @@ cache: # Storage configuration (S3 backend) storage: - backend: "archive" archive: s3: - endpoint: "http://s3:9000" diff --git a/.github/integration/sda/download-config.yaml b/.github/integration/sda/download-config.yaml index 0ca6bd1d3d..9896e12b30 100644 --- a/.github/integration/sda/download-config.yaml +++ b/.github/integration/sda/download-config.yaml @@ -5,10 +5,6 @@ api: host: "0.0.0.0" port: 8080 -# gRPC Health check -health: - port: 8081 - # Database db: host: "postgres" @@ -74,7 +70,6 @@ cache: # Storage configuration (S3 backend) storage: - backend: "archive" archive: s3: # NOTE: include scheme to keep storage/v2 location parsing consistent (archive_location values are URL-parsed). diff --git a/sda/cmd/download/config/config.go b/sda/cmd/download/config/config.go index bc04838387..b5ead9b1d3 100644 --- a/sda/cmd/download/config/config.go +++ b/sda/cmd/download/config/config.go @@ -13,7 +13,6 @@ var ( apiPort int apiServerCert string apiServerKey string - healthPort int // Database configuration dbHost string @@ -25,7 +24,6 @@ var ( dbCACert string // Storage configuration - storageBackend string // gRPC reencrypt service configuration grpcHost string @@ -133,17 +131,6 @@ func init() { apiServerKey = viper.GetString(flagName) }, }, - &config.Flag{ - Name: "health.port", - RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { - flagSet.Int(flagName, 8081, "Port for gRPC health check server") - }, - Required: false, - AssignFunc: func(flagName string) { - healthPort = viper.GetInt(flagName) - }, - }, - // Database flags &config.Flag{ Name: "db.host", @@ -216,18 +203,6 @@ func init() { }, }, - // Storage flags - &config.Flag{ - Name: "storage.backend", - RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { - flagSet.String(flagName, "archive", "Storage backend name (used for storage/v2 configuration)") - }, - Required: false, - AssignFunc: func(flagName string) { - storageBackend = viper.GetString(flagName) - }, - }, - // gRPC reencrypt service flags &config.Flag{ Name: "grpc.host", @@ -672,11 +647,6 @@ func APIServerKey() string { return apiServerKey } -// HealthPort returns the port for the gRPC health check server. -func HealthPort() int { - return healthPort -} - // DBHost returns the database host. func DBHost() string { return dbHost @@ -712,11 +682,6 @@ func DBCACert() string { return dbCACert } -// StorageBackend returns the storage backend name. -func StorageBackend() string { - return storageBackend -} - // GRPCHost returns the gRPC service host. func GRPCHost() string { return grpcHost diff --git a/sda/cmd/download/database/cache.go b/sda/cmd/download/database/cache.go index caf5010a3f..2cf1755619 100644 --- a/sda/cmd/download/database/cache.go +++ b/sda/cmd/download/database/cache.go @@ -179,7 +179,6 @@ func (c *CachedDB) GetDatasetInfo(ctx context.Context, datasetID string) (*Datas return info, nil } - // GetFileByID returns file information by stable ID. // Results are cached with FileTTL. This is particularly important // for streaming use cases where the same file may be requested diff --git a/sda/cmd/download/dev_config.yaml b/sda/cmd/download/dev_config.yaml index 51505c31ab..db2eb1ba18 100644 --- a/sda/cmd/download/dev_config.yaml +++ b/sda/cmd/download/dev_config.yaml @@ -6,10 +6,6 @@ api: host: "0.0.0.0" port: 8080 -# gRPC Health check server -health: - port: 8081 - # Database configuration (connects to integration test PostgreSQL) db: host: "localhost" @@ -45,10 +41,8 @@ session: http-only: true name: "sda_session" -# Storage configuration for storage/v2 -# The backend name is "archive" by default (see config.StorageBackend()) +# Storage configuration (archive backend via storage/v2) storage: - backend: "archive" archive: s3: - endpoint: "localhost:19000" diff --git a/sda/cmd/download/download.md b/sda/cmd/download/download.md index eabc56b46a..835d3aef16 100644 --- a/sda/cmd/download/download.md +++ b/sda/cmd/download/download.md @@ -395,7 +395,6 @@ db: | `API_PORT` | `api.port` | Port to listen on | `8080` | | `API_SERVER_CERT`| `api.server-cert`| Path to TLS certificate | | | `API_SERVER_KEY` | `api.server-key` | Path to TLS private key | | -| `HEALTH_PORT` | `health.port` | gRPC health check port | `8081` | ### Database @@ -466,14 +465,12 @@ At least one of `jwt.pubkey-path` or `jwt.pubkey-url` must be configured. ### Storage -Storage is configured using the `storage/v2` package. The `storage.backend` flag -(default: `archive`) selects the named storage backend. +Storage is configured using the `storage/v2` package with the `archive` backend. Example S3 configuration: ```yaml storage: - backend: "archive" archive: s3: - endpoint: "s3.example.com" diff --git a/sda/cmd/download/health/health.go b/sda/cmd/download/health/health.go deleted file mode 100644 index d06cb8fd89..0000000000 --- a/sda/cmd/download/health/health.go +++ /dev/null @@ -1,51 +0,0 @@ -// Package health provides gRPC health check server for Kubernetes probes. -package health - -import ( - "fmt" - "net" - - "google.golang.org/grpc" - "google.golang.org/grpc/health" - healthpb "google.golang.org/grpc/health/grpc_health_v1" -) - -var healthServer *health.Server -var grpcServer *grpc.Server - -func init() { - healthServer = health.NewServer() - healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING) -} - -// Start starts the gRPC health check server on the specified port. -func Start(port int) error { - grpcServer = grpc.NewServer() - - healthpb.RegisterHealthServer(grpcServer, healthServer) - - listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port)) - if err != nil { - return fmt.Errorf("failed to listen on port %d: %w", port, err) - } - - return grpcServer.Serve(listener) -} - -// SetServingStatus sets the serving status for the health server. -func SetServingStatus(status healthpb.HealthCheckResponse_ServingStatus) { - healthServer.SetServingStatus("", status) -} - -// Stop gracefully stops the gRPC health server. -func Stop() { - if grpcServer != nil { - grpcServer.GracefulStop() - } -} - -// ServingStatus constants for convenience. -const ( - Serving = healthpb.HealthCheckResponse_SERVING - NotServing = healthpb.HealthCheckResponse_NOT_SERVING -) diff --git a/sda/cmd/download/health/health_test.go b/sda/cmd/download/health/health_test.go deleted file mode 100644 index aef064c218..0000000000 --- a/sda/cmd/download/health/health_test.go +++ /dev/null @@ -1,31 +0,0 @@ -package health - -import ( - "testing" - - "github.com/stretchr/testify/assert" - healthpb "google.golang.org/grpc/health/grpc_health_v1" -) - -func TestServingStatusConstants(t *testing.T) { - assert.Equal(t, healthpb.HealthCheckResponse_SERVING, Serving) - assert.Equal(t, healthpb.HealthCheckResponse_NOT_SERVING, NotServing) -} - -func TestSetServingStatus(t *testing.T) { - // Test that SetServingStatus doesn't panic - assert.NotPanics(t, func() { - SetServingStatus(Serving) - }) - - assert.NotPanics(t, func() { - SetServingStatus(NotServing) - }) -} - -func TestStop_WhenNotStarted(t *testing.T) { - // Test that Stop doesn't panic when server hasn't been started - assert.NotPanics(t, func() { - Stop() - }) -} diff --git a/sda/cmd/download/main.go b/sda/cmd/download/main.go index 97c2868055..cd5f898a99 100644 --- a/sda/cmd/download/main.go +++ b/sda/cmd/download/main.go @@ -20,7 +20,6 @@ import ( "github.com/neicnordic/sensitive-data-archive/cmd/download/config" "github.com/neicnordic/sensitive-data-archive/cmd/download/database" "github.com/neicnordic/sensitive-data-archive/cmd/download/handlers" - "github.com/neicnordic/sensitive-data-archive/cmd/download/health" "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" "github.com/neicnordic/sensitive-data-archive/cmd/download/reencrypt" "github.com/neicnordic/sensitive-data-archive/cmd/download/visa" @@ -58,15 +57,6 @@ func run() error { return err } - // Start gRPC health server - go func() { - if err := health.Start(config.HealthPort()); err != nil { - log.Errorf("health server error: %v", err) - cancel() - } - }() - defer health.Stop() - // Initialize database if err := database.Init(); err != nil { return fmt.Errorf("failed to initialize database: %w", err) @@ -145,11 +135,11 @@ func run() error { } // Initialize storage/v2 reader - storageReader, err := storage.NewReader(ctx, config.StorageBackend()) + storageReader, err := storage.NewReader(ctx, "archive") if err != nil { return fmt.Errorf("failed to initialize storage reader: %w", err) } - log.Infof("storage reader initialized for backend: %s", config.StorageBackend()) + log.Info("storage reader initialized") // Initialize gRPC reencrypt client reencryptOpts := []reencrypt.ClientOption{ @@ -241,9 +231,7 @@ func run() error { } }() - // Mark service as ready - health.SetServingStatus(health.Serving) - log.Infof("health server listening at: :%d", config.HealthPort()) + log.Info("service ready") // Wait for shutdown signal select { @@ -253,9 +241,6 @@ func run() error { log.Info("context cancelled") } - // Mark service as not serving - health.SetServingStatus(health.NotServing) - // Graceful shutdown log.Info("shutting down server...") diff --git a/sda/cmd/download/reencrypt/reencrypt.go b/sda/cmd/download/reencrypt/reencrypt.go index 1791dd3da1..9215da3e98 100644 --- a/sda/cmd/download/reencrypt/reencrypt.go +++ b/sda/cmd/download/reencrypt/reencrypt.go @@ -193,7 +193,10 @@ func (c *Client) ReencryptHeaderWithEditList(ctx context.Context, oldHeader []by return res.GetHeader(), nil } -// HealthCheck verifies the gRPC connection can be established. +// HealthCheck validates that the gRPC client can be constructed with the +// configured options. Note: grpc.NewClient does no I/O, so this does NOT +// verify that the remote reencrypt service is reachable. +// TODO: perform a real RPC (e.g. grpc health check) to verify reachability. func (c *Client) HealthCheck() error { return c.connect() } From a3600dc6bb18f4cf8bdb27284a37fd73ae805675 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 26 Mar 2026 14:05:09 +0100 Subject: [PATCH 418/530] style(download): resolve all golangci-lint warnings Fix all remaining lint issues across cmd/download/: - Remove redundant // +build tags (Go 1.17+, only //go:build needed) - Replace interface{} with any (Go 1.18+) - Add blank lines before returns (nlreturn) - Fix gofmt alignment in database files - Add nolint:gosec for intentional unsafe.Pointer in go:linkname test --- sda/cmd/download/database/cache_test.go | 2 -- sda/cmd/download/database/database.go | 2 -- sda/cmd/download/database/database_test.go | 4 ---- sda/cmd/download/handlers/mock_database_test.go | 1 - sda/cmd/download/middleware/auth_visa_test.go | 7 +++---- sda/cmd/download/middleware/cache_ttl_test.go | 1 - sda/cmd/download/middleware/config_linkname_test.go | 3 +-- sda/cmd/download/middleware/config_testhelper_test.go | 1 - sda/cmd/download/middleware/visa_middleware_test.go | 11 ++++++----- sda/cmd/download/visa/extract_test.go | 1 - sda/cmd/download/visa/testhelpers_test.go | 5 ++--- sda/cmd/download/visa/trust_test.go | 1 - sda/cmd/download/visa/userinfo_test.go | 1 - sda/cmd/download/visa/validator_extra_test.go | 5 ++--- sda/cmd/download/visa/validator_test.go | 7 +++---- 15 files changed, 17 insertions(+), 35 deletions(-) diff --git a/sda/cmd/download/database/cache_test.go b/sda/cmd/download/database/cache_test.go index e70465563b..941e632f41 100644 --- a/sda/cmd/download/database/cache_test.go +++ b/sda/cmd/download/database/cache_test.go @@ -64,7 +64,6 @@ func (m *MockDatabase) GetDatasetInfo(ctx context.Context, datasetID string) (*D return args.Get(0).(*DatasetInfo), args.Error(1) } - func (m *MockDatabase) GetFileByID(ctx context.Context, fileID string) (*File, error) { args := m.Called(ctx, fileID) if args.Get(0) == nil { @@ -373,7 +372,6 @@ func TestCachedDB_GetDatasetInfo_CacheHit(t *testing.T) { mockDB.AssertExpectations(t) } - func TestCachedDB_GetAllDatasets_CacheHit(t *testing.T) { mockDB := new(MockDatabase) cfg := CacheConfig{ diff --git a/sda/cmd/download/database/database.go b/sda/cmd/download/database/database.go index 9e95eb1c6a..c548e49d89 100644 --- a/sda/cmd/download/database/database.go +++ b/sda/cmd/download/database/database.go @@ -475,7 +475,6 @@ func (p *PostgresDB) GetDatasetInfo(ctx context.Context, datasetID string) (*Dat return &info, nil } - // GetFileByID returns file information by stable ID. func (p *PostgresDB) GetFileByID(ctx context.Context, fileID string) (*File, error) { stmt := p.preparedStatements[getFileByIDQuery] @@ -695,4 +694,3 @@ func escapeLikePrefix(prefix string) string { return r.Replace(prefix) + "%" } - diff --git a/sda/cmd/download/database/database_test.go b/sda/cmd/download/database/database_test.go index b025e1d6b9..c3ec19bfac 100644 --- a/sda/cmd/download/database/database_test.go +++ b/sda/cmd/download/database/database_test.go @@ -203,7 +203,6 @@ func TestGetDatasetInfo_NotFound(t *testing.T) { assert.NoError(t, mock.ExpectationsWereMet()) } - func TestGetFileByID(t *testing.T) { db, mock, cleanup := setupMockDB(t) defer cleanup() @@ -459,8 +458,6 @@ func TestGetDatasetInfo_QueryError(t *testing.T) { assert.Contains(t, err.Error(), "failed to query dataset info") } - - func TestGetFileByID_QueryError(t *testing.T) { db, mock, cleanup := setupMockDB(t) defer cleanup() @@ -592,7 +589,6 @@ func (m *mockTestDatabase) GetDatasetInfo(_ context.Context, _ string) (*Dataset return nil, nil } - func (m *mockTestDatabase) GetFileByID(_ context.Context, _ string) (*File, error) { return nil, nil } diff --git a/sda/cmd/download/handlers/mock_database_test.go b/sda/cmd/download/handlers/mock_database_test.go index 2d3cd23eb5..55421a0b41 100644 --- a/sda/cmd/download/handlers/mock_database_test.go +++ b/sda/cmd/download/handlers/mock_database_test.go @@ -75,7 +75,6 @@ func (m *mockDatabase) GetDatasetInfo(_ context.Context, _ string) (*database.Da return m.datasetInfo, nil } - func (m *mockDatabase) GetFileByID(_ context.Context, _ string) (*database.File, error) { if m.err != nil { return nil, m.err diff --git a/sda/cmd/download/middleware/auth_visa_test.go b/sda/cmd/download/middleware/auth_visa_test.go index 9b95f4f4ba..6f758ae3ce 100644 --- a/sda/cmd/download/middleware/auth_visa_test.go +++ b/sda/cmd/download/middleware/auth_visa_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package middleware @@ -24,8 +23,8 @@ import ( ) func TestLooksLikeJWT(t *testing.T) { - header := map[string]interface{}{"alg": "RS256", "typ": "JWT"} - payload := map[string]interface{}{"sub": "user", "exp": time.Now().Add(1 * time.Hour).Unix()} + header := map[string]any{"alg": "RS256", "typ": "JWT"} + payload := map[string]any{"sub": "user", "exp": time.Now().Add(1 * time.Hour).Unix()} token := b64JSON(t, header) + "." + b64JSON(t, payload) + ".signature" assert.True(t, looksLikeJWT(token)) @@ -109,7 +108,7 @@ func TestAuthenticate_OpaqueToken_Disallowed(t *testing.T) { assert.Error(t, err) } -func b64JSON(t *testing.T, value interface{}) string { +func b64JSON(t *testing.T, value any) string { t.Helper() data, err := json.Marshal(value) diff --git a/sda/cmd/download/middleware/cache_ttl_test.go b/sda/cmd/download/middleware/cache_ttl_test.go index 25af944993..e3b06755a7 100644 --- a/sda/cmd/download/middleware/cache_ttl_test.go +++ b/sda/cmd/download/middleware/cache_ttl_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package middleware diff --git a/sda/cmd/download/middleware/config_linkname_test.go b/sda/cmd/download/middleware/config_linkname_test.go index d615f38953..0c910bad11 100644 --- a/sda/cmd/download/middleware/config_linkname_test.go +++ b/sda/cmd/download/middleware/config_linkname_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package middleware @@ -18,7 +17,7 @@ var authAllowOpaque bool var oidcIssuer string // Use unsafe to ensure go:linkname is permitted. -var _ = unsafe.Pointer(nil) +var _ = unsafe.Pointer(nil) //nolint:gosec // required for go:linkname func setPermissionModel(t *testing.T, value string) { prev := permissionModel diff --git a/sda/cmd/download/middleware/config_testhelper_test.go b/sda/cmd/download/middleware/config_testhelper_test.go index 2cc5abf2dd..ae7b275d41 100644 --- a/sda/cmd/download/middleware/config_testhelper_test.go +++ b/sda/cmd/download/middleware/config_testhelper_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package middleware diff --git a/sda/cmd/download/middleware/visa_middleware_test.go b/sda/cmd/download/middleware/visa_middleware_test.go index 2ba1feb257..7256e6a77d 100644 --- a/sda/cmd/download/middleware/visa_middleware_test.go +++ b/sda/cmd/download/middleware/visa_middleware_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package middleware @@ -33,6 +32,7 @@ type fakeDatasetLookup struct { func (f *fakeDatasetLookup) GetDatasetIDsByUser(ctx context.Context, user string) ([]string, error) { atomic.AddInt32(&f.calls, 1) + return f.datasets, nil } @@ -285,6 +285,7 @@ func newVisaValidator(t *testing.T, datasetID, issuer string) *visa.Validator { t.Helper() validator, _ := newVisaValidatorWithUserinfoCounter(t, datasetID, issuer) + return validator } @@ -365,7 +366,7 @@ func newUserinfoServer(t *testing.T, passports []string, status int, calls *int3 if status != http.StatusOK { return } - payload, _ := json.Marshal(map[string]interface{}{ + payload, _ := json.Marshal(map[string]any{ "sub": "user-123", "ga4gh_passport_v1": passports, }) @@ -388,8 +389,8 @@ func newJWKSServer(t *testing.T, key jwk.Key) *httptest.Server { })) } -func visaClaim(dataset string) map[string]interface{} { - return map[string]interface{}{ +func visaClaim(dataset string) map[string]any { + return map[string]any{ "type": "ControlledAccessGrants", "by": "https://example.org/issuer", "value": dataset, @@ -399,7 +400,7 @@ func visaClaim(dataset string) map[string]interface{} { } } -func signVisaJWT(t *testing.T, priv *rsa.PrivateKey, jku, kid, iss, sub string, visaClaim map[string]interface{}, exp time.Time) string { +func signVisaJWT(t *testing.T, priv *rsa.PrivateKey, jku, kid, iss, sub string, visaClaim map[string]any, exp time.Time) string { t.Helper() token := jwt.New() diff --git a/sda/cmd/download/visa/extract_test.go b/sda/cmd/download/visa/extract_test.go index b69d37b452..16c7f1968e 100644 --- a/sda/cmd/download/visa/extract_test.go +++ b/sda/cmd/download/visa/extract_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package visa diff --git a/sda/cmd/download/visa/testhelpers_test.go b/sda/cmd/download/visa/testhelpers_test.go index 063d5f2c96..dcccb6d66b 100644 --- a/sda/cmd/download/visa/testhelpers_test.go +++ b/sda/cmd/download/visa/testhelpers_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package visa @@ -69,7 +68,7 @@ func newFailingJWKSServer(t *testing.T) *httptest.Server { func newUserinfoServer(t *testing.T, passports []string) *httptest.Server { t.Helper() - payload, err := json.Marshal(map[string]interface{}{ + payload, err := json.Marshal(map[string]any{ "sub": "user-123", "ga4gh_passport_v1": passports, }) @@ -81,7 +80,7 @@ func newUserinfoServer(t *testing.T, passports []string) *httptest.Server { })) } -func signVisaJWT(t *testing.T, priv *rsa.PrivateKey, jku, kid, iss, sub string, visa map[string]interface{}, exp time.Time) string { +func signVisaJWT(t *testing.T, priv *rsa.PrivateKey, jku, kid, iss, sub string, visa map[string]any, exp time.Time) string { t.Helper() token := jwt.New() diff --git a/sda/cmd/download/visa/trust_test.go b/sda/cmd/download/visa/trust_test.go index fac4ccf359..4355e37f69 100644 --- a/sda/cmd/download/visa/trust_test.go +++ b/sda/cmd/download/visa/trust_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package visa diff --git a/sda/cmd/download/visa/userinfo_test.go b/sda/cmd/download/visa/userinfo_test.go index 1d02994bde..d3eb740b29 100644 --- a/sda/cmd/download/visa/userinfo_test.go +++ b/sda/cmd/download/visa/userinfo_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package visa diff --git a/sda/cmd/download/visa/validator_extra_test.go b/sda/cmd/download/visa/validator_extra_test.go index 068183cd47..2e98c10fbc 100644 --- a/sda/cmd/download/visa/validator_extra_test.go +++ b/sda/cmd/download/visa/validator_extra_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package visa @@ -291,7 +290,7 @@ func setupValidatorWithPassports(t *testing.T, passports []string, trusted []Tru return validator } -func setupValidatorWithUntrustedIssuer(t *testing.T, visaClaim map[string]interface{}) (*Validator, Identity, string) { +func setupValidatorWithUntrustedIssuer(t *testing.T, visaClaim map[string]any) (*Validator, Identity, string) { t.Helper() priv, pub, kid := newRSAKeyPair(t) @@ -324,7 +323,7 @@ func setupValidatorWithUntrustedIssuer(t *testing.T, visaClaim map[string]interf return validator, identity, "opaque-token" } -func setupValidatorWithIdentityMode(t *testing.T, visaClaim map[string]interface{}, mode, subject, issuer string) (*Validator, Identity, string) { +func setupValidatorWithIdentityMode(t *testing.T, visaClaim map[string]any, mode, subject, issuer string) (*Validator, Identity, string) { t.Helper() priv, pub, kid := newRSAKeyPair(t) diff --git a/sda/cmd/download/visa/validator_test.go b/sda/cmd/download/visa/validator_test.go index 449953b1d9..7b0b5ae4ef 100644 --- a/sda/cmd/download/visa/validator_test.go +++ b/sda/cmd/download/visa/validator_test.go @@ -1,5 +1,4 @@ //go:build visas -// +build visas package visa @@ -96,8 +95,8 @@ func TestGetVisaDatasets_SuffixExtraction(t *testing.T) { assert.Equal(t, []string{"EGAD005"}, result.Datasets) } -func baseVisaClaim(value string) map[string]interface{} { - return map[string]interface{}{ +func baseVisaClaim(value string) map[string]any { + return map[string]any{ "type": "ControlledAccessGrants", "by": "https://example.org/issuer", "value": value, @@ -107,7 +106,7 @@ func baseVisaClaim(value string) map[string]interface{} { } } -func setupValidatorWithVisa(t *testing.T, visaClaim map[string]interface{}, datasetIDMode string) (*Validator, Identity, string) { +func setupValidatorWithVisa(t *testing.T, visaClaim map[string]any, datasetIDMode string) (*Validator, Identity, string) { t.Helper() priv, pub, kid := newRSAKeyPair(t) From 08a12bfc4365b81c34327a7681a1f319518ebb92 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 27 Mar 2026 10:13:15 +0100 Subject: [PATCH 419/530] feat(download): add database client certificate support for mTLS Add db.clientcert and db.clientkey config flags to support PostgreSQL mTLS authentication (sslcert/sslkey in DSN). Validates both must be set together when using sslmode=verify-full. Matches the old sda-download service's database TLS support. Review feedback from @jbygdell (database.go:294). --- sda/cmd/download/config/config.go | 47 +++++++++++++++++++++++---- sda/cmd/download/database/database.go | 14 ++++++++ sda/cmd/download/download.md | 2 ++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/sda/cmd/download/config/config.go b/sda/cmd/download/config/config.go index b5ead9b1d3..6923ed5c67 100644 --- a/sda/cmd/download/config/config.go +++ b/sda/cmd/download/config/config.go @@ -15,13 +15,15 @@ var ( apiServerKey string // Database configuration - dbHost string - dbPort int - dbUser string - dbPassword string - dbDatabase string - dbSSLMode string - dbCACert string + dbHost string + dbPort int + dbUser string + dbPassword string + dbDatabase string + dbSSLMode string + dbCACert string + dbClientCert string + dbClientKey string // Storage configuration @@ -203,6 +205,27 @@ func init() { }, }, + &config.Flag{ + Name: "db.clientcert", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to client certificate for database mTLS") + }, + Required: false, + AssignFunc: func(flagName string) { + dbClientCert = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "db.clientkey", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Path to client key for database mTLS") + }, + Required: false, + AssignFunc: func(flagName string) { + dbClientKey = viper.GetString(flagName) + }, + }, + // gRPC reencrypt service flags &config.Flag{ Name: "grpc.host", @@ -682,6 +705,16 @@ func DBCACert() string { return dbCACert } +// DBClientCert returns the path to the client certificate for database mTLS. +func DBClientCert() string { + return dbClientCert +} + +// DBClientKey returns the path to the client key for database mTLS. +func DBClientKey() string { + return dbClientKey +} + // GRPCHost returns the gRPC service host. func GRPCHost() string { return grpcHost diff --git a/sda/cmd/download/database/database.go b/sda/cmd/download/database/database.go index c548e49d89..e90b156f5a 100644 --- a/sda/cmd/download/database/database.go +++ b/sda/cmd/download/database/database.go @@ -284,6 +284,12 @@ func GetDB() Database { // Init initializes the database connection using configuration values. // All queries are prepared at startup to verify correctness and improve runtime performance. func Init() error { + if config.DBSSLMode() == "verify-full" { + if config.DBClientCert() == "" || config.DBClientKey() == "" { + return errors.New("both db.clientcert and db.clientkey are required when db.sslmode=verify-full") + } + } + connStr := fmt.Sprintf( "host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", config.DBHost(), @@ -298,6 +304,14 @@ func Init() error { connStr += fmt.Sprintf(" sslrootcert=%s", config.DBCACert()) } + if config.DBClientCert() != "" { + connStr += fmt.Sprintf(" sslcert=%s", config.DBClientCert()) + } + + if config.DBClientKey() != "" { + connStr += fmt.Sprintf(" sslkey=%s", config.DBClientKey()) + } + sqlDB, err := sql.Open("postgres", connStr) if err != nil { return fmt.Errorf("failed to open database connection: %w", err) diff --git a/sda/cmd/download/download.md b/sda/cmd/download/download.md index 835d3aef16..b63fe77ad7 100644 --- a/sda/cmd/download/download.md +++ b/sda/cmd/download/download.md @@ -407,6 +407,8 @@ db: | `DB_DATABASE` | `db.database` | Database name | `sda` | | `DB_SSLMODE` | `db.sslmode` | SSL mode (disable, allow, prefer, require, verify-ca, verify-full) | `prefer` | | `DB_CACERT` | `db.cacert` | Path to CA certificate for database TLS | | +| `DB_CLIENTCERT`| `db.clientcert`| Path to client certificate for database mTLS | | +| `DB_CLIENTKEY` | `db.clientkey` | Path to client key for database mTLS | | ### JWT / Authentication From 2283e71b962039e257efb5d4ba5c8cbdd1c8eeb1 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 27 Mar 2026 10:27:14 +0100 Subject: [PATCH 420/530] fix(download): make service-info ID and organization configurable Replace hardcoded NBIS-specific values in /service-info with configurable service.id, service.org-name, and service.org-url flags. Default ID is "sda.download". Review feedback from @jbygdell (serviceinfo.go:34, :44). --- .../sda/download-config-benchmark.yaml | 6 +++ .../integration/sda/download-config-s3.yaml | 6 +++ .github/integration/sda/download-config.yaml | 6 +++ sda/cmd/download/config/config.go | 52 +++++++++++++++++++ sda/cmd/download/database/database.go | 8 +-- sda/cmd/download/dev_config.yaml | 6 +++ sda/cmd/download/download.md | 8 +++ sda/cmd/download/handlers/handlers.go | 3 ++ sda/cmd/download/handlers/handlers_test.go | 2 + sda/cmd/download/handlers/options.go | 9 ++++ sda/cmd/download/handlers/serviceinfo.go | 6 +-- sda/cmd/download/handlers/serviceinfo_test.go | 14 +++-- sda/cmd/download/main.go | 1 + .../config_testhelper_novisas_test.go | 2 + .../middleware/config_testhelper_test.go | 2 + 15 files changed, 117 insertions(+), 14 deletions(-) diff --git a/.github/integration/sda/download-config-benchmark.yaml b/.github/integration/sda/download-config-benchmark.yaml index b4c43cc92c..d1d15a6fba 100644 --- a/.github/integration/sda/download-config-benchmark.yaml +++ b/.github/integration/sda/download-config-benchmark.yaml @@ -7,6 +7,12 @@ api: host: "0.0.0.0" port: 8080 +# Service info +service: + id: "neicnordic.sda.download" + org-name: "NBIS" + org-url: "https://nbis.se" + # Logging log: level: "debug" diff --git a/.github/integration/sda/download-config-s3.yaml b/.github/integration/sda/download-config-s3.yaml index 2282369027..4a25966d66 100644 --- a/.github/integration/sda/download-config-s3.yaml +++ b/.github/integration/sda/download-config-s3.yaml @@ -7,6 +7,12 @@ api: host: "0.0.0.0" port: 8080 +# Service info +service: + id: "neicnordic.sda.download" + org-name: "NBIS" + org-url: "https://nbis.se" + # Database db: host: "postgres" diff --git a/.github/integration/sda/download-config.yaml b/.github/integration/sda/download-config.yaml index 9896e12b30..8ad7a37ee1 100644 --- a/.github/integration/sda/download-config.yaml +++ b/.github/integration/sda/download-config.yaml @@ -5,6 +5,12 @@ api: host: "0.0.0.0" port: 8080 +# Service info +service: + id: "neicnordic.sda.download" + org-name: "NBIS" + org-url: "https://nbis.se" + # Database db: host: "postgres" diff --git a/sda/cmd/download/config/config.go b/sda/cmd/download/config/config.go index 6923ed5c67..814d2ff4c4 100644 --- a/sda/cmd/download/config/config.go +++ b/sda/cmd/download/config/config.go @@ -14,6 +14,11 @@ var ( apiServerCert string apiServerKey string + // Service info + serviceID string + serviceOrgName string + serviceOrgURL string + // Database configuration dbHost string dbPort int @@ -133,6 +138,38 @@ func init() { apiServerKey = viper.GetString(flagName) }, }, + // Service info flags + &config.Flag{ + Name: "service.id", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "neicnordic.sda.download", "GA4GH service-info ID (reverse domain notation)") + }, + Required: false, + AssignFunc: func(flagName string) { + serviceID = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "service.org-name", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Organization name for service-info") + }, + Required: true, + AssignFunc: func(flagName string) { + serviceOrgName = viper.GetString(flagName) + }, + }, + &config.Flag{ + Name: "service.org-url", + RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) { + flagSet.String(flagName, "", "Organization URL for service-info") + }, + Required: true, + AssignFunc: func(flagName string) { + serviceOrgURL = viper.GetString(flagName) + }, + }, + // Database flags &config.Flag{ Name: "db.host", @@ -670,6 +707,21 @@ func APIServerKey() string { return apiServerKey } +// ServiceID returns the GA4GH service-info ID. +func ServiceID() string { + return serviceID +} + +// ServiceOrgName returns the organization name for service-info. +func ServiceOrgName() string { + return serviceOrgName +} + +// ServiceOrgURL returns the organization URL for service-info. +func ServiceOrgURL() string { + return serviceOrgURL +} + // DBHost returns the database host. func DBHost() string { return dbHost diff --git a/sda/cmd/download/database/database.go b/sda/cmd/download/database/database.go index e90b156f5a..d32f8e938f 100644 --- a/sda/cmd/download/database/database.go +++ b/sda/cmd/download/database/database.go @@ -304,12 +304,8 @@ func Init() error { connStr += fmt.Sprintf(" sslrootcert=%s", config.DBCACert()) } - if config.DBClientCert() != "" { - connStr += fmt.Sprintf(" sslcert=%s", config.DBClientCert()) - } - - if config.DBClientKey() != "" { - connStr += fmt.Sprintf(" sslkey=%s", config.DBClientKey()) + if config.DBClientCert() != "" && config.DBClientKey() != "" { + connStr += fmt.Sprintf(" sslcert=%s sslkey=%s", config.DBClientCert(), config.DBClientKey()) } sqlDB, err := sql.Open("postgres", connStr) diff --git a/sda/cmd/download/dev_config.yaml b/sda/cmd/download/dev_config.yaml index db2eb1ba18..01b957f761 100644 --- a/sda/cmd/download/dev_config.yaml +++ b/sda/cmd/download/dev_config.yaml @@ -6,6 +6,12 @@ api: host: "0.0.0.0" port: 8080 +# Service info (GA4GH service-info endpoint) +service: + id: "neicnordic.sda.download" + org-name: "NBIS" + org-url: "https://nbis.se" + # Database configuration (connects to integration test PostgreSQL) db: host: "localhost" diff --git a/sda/cmd/download/download.md b/sda/cmd/download/download.md index b63fe77ad7..390109d61a 100644 --- a/sda/cmd/download/download.md +++ b/sda/cmd/download/download.md @@ -396,6 +396,14 @@ db: | `API_SERVER_CERT`| `api.server-cert`| Path to TLS certificate | | | `API_SERVER_KEY` | `api.server-key` | Path to TLS private key | | +### Service Info + +| Variable | Config Key | Description | Default | +|---------------------|-------------------|--------------------------------------------------|----------------| +| `SERVICE_ID` | `service.id` | GA4GH service-info ID (reverse domain notation) | `neicnordic.sda.download` | +| `SERVICE_ORG_NAME` | `service.org-name`| Organization name for service-info | *required* | +| `SERVICE_ORG_URL` | `service.org-url` | Organization URL for service-info | *required* | + ### Database | Variable | Config Key | Description | Default | diff --git a/sda/cmd/download/handlers/handlers.go b/sda/cmd/download/handlers/handlers.go index e285f7a72a..198ea4103b 100644 --- a/sda/cmd/download/handlers/handlers.go +++ b/sda/cmd/download/handlers/handlers.go @@ -22,6 +22,9 @@ type Handlers struct { auditLogger audit.Logger grpcHost string grpcPort int + serviceID string + serviceOrgName string + serviceOrgURL string } // New creates a new Handlers instance with the given options. diff --git a/sda/cmd/download/handlers/handlers_test.go b/sda/cmd/download/handlers/handlers_test.go index a388f146af..77a1ae191c 100644 --- a/sda/cmd/download/handlers/handlers_test.go +++ b/sda/cmd/download/handlers/handlers_test.go @@ -26,6 +26,8 @@ func ensureTestConfig(t *testing.T) { t.Setenv("DB_PASSWORD", "test") t.Setenv("GRPC_HOST", "localhost") t.Setenv("AUTH_ALLOW_OPAQUE", "true") + t.Setenv("SERVICE_ORG_NAME", "Test") + t.Setenv("SERVICE_ORG_URL", "https://test.example.com") oldArgs := os.Args os.Args = []string{oldArgs[0]} diff --git a/sda/cmd/download/handlers/options.go b/sda/cmd/download/handlers/options.go index fec83e4064..7d6ab7787d 100644 --- a/sda/cmd/download/handlers/options.go +++ b/sda/cmd/download/handlers/options.go @@ -43,6 +43,15 @@ func WithAuditLogger(l audit.Logger) func(*Handlers) { } } +// WithServiceInfo sets the GA4GH service-info fields. +func WithServiceInfo(id, orgName, orgURL string) func(*Handlers) { + return func(h *Handlers) { + h.serviceID = id + h.serviceOrgName = orgName + h.serviceOrgURL = orgURL + } +} + // WithGRPCReencryptHost sets the gRPC reencrypt service host (deprecated, use WithReencryptClient). func WithGRPCReencryptHost(host string) func(*Handlers) { return func(h *Handlers) { diff --git a/sda/cmd/download/handlers/serviceinfo.go b/sda/cmd/download/handlers/serviceinfo.go index 8487c3bffd..4af4420b36 100644 --- a/sda/cmd/download/handlers/serviceinfo.go +++ b/sda/cmd/download/handlers/serviceinfo.go @@ -31,7 +31,7 @@ type serviceInfoOrg struct { func (h *Handlers) ServiceInfo(c *gin.Context) { c.Header("Cache-Control", "public, max-age=300") c.JSON(http.StatusOK, serviceInfoResponse{ - ID: "se.nbis.sda.download", + ID: h.serviceID, Name: "SDA Download", Type: serviceInfoType{ Group: "org.ga4gh", @@ -39,8 +39,8 @@ func (h *Handlers) ServiceInfo(c *gin.Context) { Version: "1.0.0", }, Organization: serviceInfoOrg{ - Name: "NBIS", - URL: "https://nbis.se", + Name: h.serviceOrgName, + URL: h.serviceOrgURL, }, Version: "2.0.0", }) diff --git a/sda/cmd/download/handlers/serviceinfo_test.go b/sda/cmd/download/handlers/serviceinfo_test.go index f68b09deb9..f269944000 100644 --- a/sda/cmd/download/handlers/serviceinfo_test.go +++ b/sda/cmd/download/handlers/serviceinfo_test.go @@ -16,7 +16,11 @@ func TestServiceInfo_Response(t *testing.T) { c, _ := gin.CreateTestContext(w) c.Request, _ = http.NewRequest(http.MethodGet, "/service-info", nil) - h := newTestHandlers(t) + h, err := New( + WithDatabase(&mockDatabase{}), + WithServiceInfo("fi.csc.sda.download", "CSC", "https://csc.fi"), + ) + require.NoError(t, err) h.ServiceInfo(c) assert.Equal(t, http.StatusOK, w.Code) @@ -24,15 +28,15 @@ func TestServiceInfo_Response(t *testing.T) { assert.Equal(t, "public, max-age=300", w.Header().Get("Cache-Control")) var response serviceInfoResponse - err := json.Unmarshal(w.Body.Bytes(), &response) + err = json.Unmarshal(w.Body.Bytes(), &response) require.NoError(t, err) - assert.Equal(t, "se.nbis.sda.download", response.ID) + assert.Equal(t, "fi.csc.sda.download", response.ID) assert.Equal(t, "SDA Download", response.Name) assert.Equal(t, "org.ga4gh", response.Type.Group) assert.Equal(t, "drs", response.Type.Artifact) assert.Equal(t, "1.0.0", response.Type.Version) - assert.Equal(t, "NBIS", response.Organization.Name) - assert.Equal(t, "https://nbis.se", response.Organization.URL) + assert.Equal(t, "CSC", response.Organization.Name) + assert.Equal(t, "https://csc.fi", response.Organization.URL) assert.Equal(t, "2.0.0", response.Version) } diff --git a/sda/cmd/download/main.go b/sda/cmd/download/main.go index cd5f898a99..7322ac2292 100644 --- a/sda/cmd/download/main.go +++ b/sda/cmd/download/main.go @@ -190,6 +190,7 @@ func run() error { handlers.WithStorageReader(storageReader), handlers.WithReencryptClient(reencryptClient), handlers.WithAuditLogger(auditLogger), + handlers.WithServiceInfo(config.ServiceID(), config.ServiceOrgName(), config.ServiceOrgURL()), } if visaValidator != nil { handlerOpts = append(handlerOpts, handlers.WithVisaValidator(visaValidator)) diff --git a/sda/cmd/download/middleware/config_testhelper_novisas_test.go b/sda/cmd/download/middleware/config_testhelper_novisas_test.go index 9806af7f39..5774e898ec 100644 --- a/sda/cmd/download/middleware/config_testhelper_novisas_test.go +++ b/sda/cmd/download/middleware/config_testhelper_novisas_test.go @@ -22,6 +22,8 @@ func ensureTestConfig(t *testing.T) { t.Setenv("DB_PASSWORD", "test") t.Setenv("GRPC_HOST", "localhost") t.Setenv("AUTH_ALLOW_OPAQUE", "true") + t.Setenv("SERVICE_ORG_NAME", "Test") + t.Setenv("SERVICE_ORG_URL", "https://test.example.com") oldArgs := os.Args os.Args = []string{oldArgs[0]} diff --git a/sda/cmd/download/middleware/config_testhelper_test.go b/sda/cmd/download/middleware/config_testhelper_test.go index ae7b275d41..2f2b4d5ad1 100644 --- a/sda/cmd/download/middleware/config_testhelper_test.go +++ b/sda/cmd/download/middleware/config_testhelper_test.go @@ -24,6 +24,8 @@ func ensureTestConfig(t *testing.T) { t.Setenv("AUTH_ALLOW_OPAQUE", "true") t.Setenv("OIDC_ISSUER", "https://issuer.example") t.Setenv("VISA_CACHE_TOKEN_TTL", "7200") + t.Setenv("SERVICE_ORG_NAME", "Test") + t.Setenv("SERVICE_ORG_URL", "https://test.example.com") oldArgs := os.Args os.Args = []string{oldArgs[0]} From 67f10892abc9029fe5518e1a0f90ca894b9fe55f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 8 Apr 2026 03:06:01 +0000 Subject: [PATCH 421/530] build(deps): bump github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream Updates `github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream` from 1.7.3 to 1.7.8 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/m2/v1.7.3...service/m2/v1.7.8) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream dependency-version: 1.7.8 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- .../tests/validator-orchestrator/go.mod | 22 +++++----- .../tests/validator-orchestrator/go.sum | 44 +++++++++---------- 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/integration/tests/validator-orchestrator/go.mod b/.github/integration/tests/validator-orchestrator/go.mod index e684f09d91..ae38379604 100644 --- a/.github/integration/tests/validator-orchestrator/go.mod +++ b/.github/integration/tests/validator-orchestrator/go.mod @@ -3,11 +3,11 @@ module validator-orchestrator-tester go 1.25.0 require ( - github.com/aws/aws-sdk-go-v2 v1.39.6 + github.com/aws/aws-sdk-go-v2 v1.41.5 github.com/aws/aws-sdk-go-v2/config v1.31.20 github.com/aws/aws-sdk-go-v2/credentials v1.18.24 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7 - github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 github.com/golang-jwt/jwt/v5 v5.3.0 github.com/google/uuid v1.6.0 github.com/neicnordic/crypt4gh v1.14.0 @@ -16,20 +16,20 @@ require ( require ( filippo.io/edwards25519 v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 // indirect - github.com/aws/smithy-go v1.23.2 // indirect + github.com/aws/smithy-go v1.24.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/.github/integration/tests/validator-orchestrator/go.sum b/.github/integration/tests/validator-orchestrator/go.sum index d8fe3dc9fe..d9c34bbc69 100644 --- a/.github/integration/tests/validator-orchestrator/go.sum +++ b/.github/integration/tests/validator-orchestrator/go.sum @@ -1,9 +1,9 @@ filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw= filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= -github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk= -github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc= github.com/aws/aws-sdk-go-v2/config v1.31.20/go.mod h1:95Hh1Tc5VYKL9NJ7tAkDcqeKt+MCXQB1hQZaRdJIZE0= github.com/aws/aws-sdk-go-v2/credentials v1.18.24 h1:iJ2FmPT35EaIB0+kMa6TnQ+PwG5A1prEdAw+PsMzfHg= @@ -12,32 +12,32 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBK github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7 h1:u8danF+A2Zv//pFZvj5V23v/6XG4AxuSVup5s6nxSnI= github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7/go.mod h1:uvLIvU8iJPEU5so7b6lLDNArWpOX6sRBfL5wBABmlfc= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 h1:DhdbtDl4FdNlj31+xiRXANxEE+eC7n8JQz+/ilwQ8Uc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 h1:NjShtS1t8r5LUfFVtFeI8xLAHQNTa7UI0VawXlrBMFQ= github.com/aws/aws-sdk-go-v2/service/sso v1.30.3/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 h1:gTsnx0xXNQ6SBbymoDvcoRHL+q4l/dAFsQuKfDWSaGc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo= github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 h1:HK5ON3KmQV2HcAunnx4sKLB9aPf3gKGwVAf7xnx0QT0= github.com/aws/aws-sdk-go-v2/service/sts v1.40.2/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk= -github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM= -github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= +github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= From 9a6d4119ad7591e51d10a2de564d6d9d5a1e7104 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Apr 2026 03:42:53 +0000 Subject: [PATCH 422/530] build(deps): bump github.com/go-jose/go-jose/v3 in /sda Bumps [github.com/go-jose/go-jose/v3](https://github.com/go-jose/go-jose) from 3.0.4 to 3.0.5. - [Release notes](https://github.com/go-jose/go-jose/releases) - [Commits](https://github.com/go-jose/go-jose/compare/v3.0.4...v3.0.5) --- updated-dependencies: - dependency-name: github.com/go-jose/go-jose/v3 dependency-version: 3.0.5 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index de52b21fe7..98508d67d0 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -95,7 +95,7 @@ require ( github.com/fsnotify/fsnotify v1.9.0 // indirect github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.4 // indirect + github.com/go-jose/go-jose/v3 v3.0.5 // indirect github.com/go-jose/go-jose/v4 v4.1.3 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index 941e24199d..dc19c92afc 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -147,8 +147,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= -github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= -github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ= +github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= From ad9dc44f4b1be255ed137230042b97a77408ae63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 7 Apr 2026 10:05:11 +0200 Subject: [PATCH 423/530] feat(sda-svc chart): expose location broker cache ttl configuration for ingest, finalize, and sync services --- charts/sda-svc/README.md | 345 +++++++++--------- .../sda-svc/templates/finalize-secrets.yaml | 8 +- charts/sda-svc/templates/ingest-secrets.yaml | 4 + charts/sda-svc/templates/sync-secrets.yaml | 4 + charts/sda-svc/values.yaml | 3 + 5 files changed, 191 insertions(+), 173 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 1e68e675d0..87578818c3 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -36,177 +36,180 @@ containing the credentials to perform the upgrade needs to be created. The following table lists the configurable parameters of the `sda-svc` chart and their default values. -| Parameter | Description | Default | -|----------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------| -| `image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` | -| `image.tag` | Tag version to deploy | `chart appVersion` | -| `image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` | -| `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` | -| `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` | -| `global.tlsPath` | This path will be a subpath to the secretsPath | `tls` | -| `global.jwtPath` | This path will be a subpath to the secretsPath | `jwt` | -| `global.confFile` | Name of config file, used when secrets are handled by hasicorp vault | `config.yaml` | -| `global.confFilePath` | This path will be a subpath to the secretsPath | `""` | -| `global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all` | -| `global.schemaType` | Standalone version requires a custom JOSN schema, available options are `federated` or `isolated`, unset defaults to federated. | `""` | -| `global.ingress.deploy` | | `false` | -| `global.ingress.hostName.auth` | | `""` | -| `global.ingress.hostName.doa` | | `""` | -| `global.ingress.hostName.s3Inbox` | | `""` | -| `global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | -| `global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | -| `global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | -| `global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | -| `global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | -| `global.ingress.annotations` | extra annotations shared by all ingress definitions | `""` | -| `global.ingress.ingressClassName` | Class of ingress controller to use | `""` | -| `global.log.format` | Log format for all services, JSON or TEXT. | `json` | -| `global.log.level` | Log level for all services. | `info` | -| `global.networkPolicy.create` | Use network isolation. | `false` | -| `global.networkPolicy.brokerNamespace` | Namespace where the broker is deployed. | `""` | -| `global.networkPolicy.databaseNamespace` | Namespace where the database is deployed. | `""` | -| `global.networkPolicy.externalNamespace` | Namespace where the external components are deployed. | `""` | -| `global.networkPolicy.internalNamespace` | Namespace where the internal components are deployed. | `""` | -| `global.networkPolicy.ingressNamespace` | Namespace where the ingress is deployed. | `""` | -| `global.networkPolicy.ingressLabelMatch` | Match to use to allow connections from ingress controller pod. | `""` | -| `global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` | -| `global.podAnnotations` | Annotations applied to pods of all services. | `{}` | -| `global.pkiService` | If an external PKI infrastructure is used set this to true. | `false` | -| `global.rbacEnabled` | Use role based access control. | `true` | -| `global.vaultSecrets` | If Hashicorp Vault is used for secrets management | `false` | -| `global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. | `s3` | -| `global.archive.s3Url` | URL to S3 archive instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.archive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"archive"` | -| `global.archive.s3Region` | S3 archive region. | `us-east-1` | -| `global.archive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `50mb` | -| `global.archive.s3AccessKey` | Access key to S3 archive . | `null` | -| `global.archive.s3SecretKey` | Secret key to S3 archive. | `null` | -| `global.archive.s3Port` | Port that the S3 S3 archive is available on. | `` | -| `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | -| `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | -| `global.archive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | -| `global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive` | -| `global.archive.nfsServer` | URL or IP address to a NFS server. | `""` | -| `global.archive.nfsPath` | Path on the NFS server for the archive. | `""` | -| `global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. | `null` | -| `global.backupArchive.s3Url` | URL to S3 backup archive instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.backupArchive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"backup"` | -| `global.backupArchive.s3Region` | S3 backup archive region. | `us-east-1` | -| `global.backupArchive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `15` | -| `global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` | -| `global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` | -| `global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `` | -| `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | -| `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | -| `global.backupArchive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | -| `global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup` | -| `global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""` | -| `global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. | `""` | -| `global.api.jwtPubKeyName` | Public key used to verify the JWT. | `` | -| `global.api.jwtSecret` | The name of the secret holding the JWT public key | `` | -| `global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) | `` | -| `global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type | `"ES256"` | -| `global.auth.jwtKey` | Private key used to sign the JWT. | `""` | -| `global.auth.jwtPub` | Public key ues to verify the JWT. | `""` | -| `global.auth.jwtTTL` | TTL of the resigned token (hours). | `168` | -| `global.auth.resignJWT` | Resign the LS-AAI JWTs. | `true` | -| `global.auth.useTLS` | Run a TLS secured server. | `true` | -| `global.auth.corsOrigins` | Domain name allowed for cross-domain requests. | `""` | -| `global.auth.corsMethods` | Allowed cross-domain request methods. | `""` | -| `global.auth.corsCreds` | Include credentials in cross-domain requests. | `false` | -| `global.broker.host` | Domain name or IP address to the message broker. | `""` | -| `global.broker.exchange` | Exchange to publish messages to. | `""` | -| `global.broker.port` | Port for the message broker. | `5671` | -| `global.broker.verifyPeer` | Use Client/Server verification. | `true` | -| `global.broker.vhost` | Virtual host to connect to. | `/` | -| `global.broker.password` | Shared password to the message broker. | `/` | -| `global.broker.username` | Shared user to the message broker. | `/` | -| `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` | -| `global.cega.host` | Full URI to the EGA user authentication service. | `""` | -| `global.cega.user` | Username for the EGA user authentication service. | `""` | -| `global.cega.password` | Password for the EGA user authentication service. | `""` | -| `global.c4gh.privateKeys` | List of Private C4GH keys. | `` | -| `global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. | `""` | -| `global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. | `""` | -| `global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""` | -| `global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""` | -| `global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""` | -| `global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` | -| `global.db.admin.passKey` | Key in the secret that holds the password. | `""` | -| `global.db.admin.userkey` | Key in the secret that holds the username. | `""` | -| `global.db.admin.username` | If secretName is not set will default to this to populate PGUSER used in migration jobs | `""` | -| `global.db.admin.password` | If secretName is not set will default to this to populate PGPASSWORD used in migration jobs | `""` | -| `global.db.host` | Hostname for the database. | `""` | -| `global.db.name` | Database to connect to. | `lega` | -| `global.db.passIngest` | Password used for `data in` services. | `""` | -| `global.db.passOutgest` | Password used for `data out` services. | `""` | -| `global.db.port` | Port that the database is listening on. | `5432` | -| `global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. | `verify-full` | -| `global.doa.enabled` | Deploy the DOA service | `false` | -| `global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env` | -| `global.doa.serviceport` | Port that the DOA service is accessible on | `443` | -| `global.doa.outbox.enabled` | Enable Outbox functionality of Data Out API | `false` | -| `global.doa.outbox.queue` | MQ queue name for files/datasets export requests | `""` | -| `global.doa.outbox.type` | Outbox type can be either S3 or POSIX | `""` | -| `global.doa.outbox.path` | Posix outbox location with placeholder for the username | `""` | -| `global.doa.outbox.s3Url` | Outbox S3 URL | `""` | -| `global.doa.outbox.s3Port` | Outbox S3 port | `443` | -| `global.doa.outbox.s3Region` | Outbox S3 region | `""` | -| `global.doa.outbox.s3Bucket` | Outbox S3 bucket | `""` | -| `global.doa.outbox.s3CaFile` | Outbox S3 CA certificate to use | `null` | -| `global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null` | -| `global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null` | -| `global.download.enabled` | Deploy the download service | `true` | -| `global.download.serveDecrypted.c4ghKeyFile` | Transient private C4GH key | `""` | -| `global.download.serveDecrypted.secretName` | Secret holding the transient private C4GH key and its passphrase | `""` | -| `global.download.sessionExpiration` | Session key expiration time in seconds | `28800` | -| `global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` | -| `global.download.trusted.configFile` | Name of ISS config file | `iss.json` | -| `global.download.trusted.iss` | Array of trusted OIDC endpoints | `` | -| `global.download.trusted.iss[iss]` | URI to the OIDC service | `https://proxy.aai.lifescience-ri.eu` | -| `global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://proxy.aai.lifescience-ri.eu/OIDC/jwks` | -| `global.oidc.provider` | URL to the OIDc service. | `"https://proxy.aai.lifescience-ri.eu"` | -| `global.oidc.jwkPath` | Public key path on the OIDC host. | `/OIDC/jwks` | -| `global.oidc.id` | User ID to the OIDC host. | `` | -| `global.oidc.secret` | User credentials to the OIDC host. | `` | -| `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` | -| `global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. | `posix` | -| `global.inbox.volumePath` | Path to the mounted `posix` volume. | `/inbox` | -| `global.inbox.nfsServer` | URL or IP address to a NFS server. | `""` | -| `global.inbox.nfsPath` | Path on the NFS server for the inbox. | `""` | -| `global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""` | -| `global.inbox.s3Url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.inbox.s3Port` | Port that the S3 inbox is available on. | `` | -| `global.inbox.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | -| `global.inbox.s3Region` | S3 inbox region. | `"us-east-1"` | -| `global.inbox.s3AccessKey` | Access key to S3 inbox . | `null` | -| `global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null` | -| `global.s3Inbox.url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.s3Inbox.port` | Port that the S3 inbox is available on. | `` | -| `global.s3Inbox.bucket` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | -| `global.s3Inbox.accessKey` | Access key to S3 inbox . | `null` | -| `global.s3Inbox.secretKey` | Secret key to S3 inbox. | `null` | -| `global.s3Inbox.region` | S3 inbox region. | `"us-east-1"` | -| `global.s3Inbox.readyPath` | Path to the s3 ready/healthy endpoint. | `null` | -| `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` | -| `global.sync.api.user` | User for authenticating to the syncAPI server | `null` | -| `global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` | -| `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` | -| `global.sync.destination.s3Url` | URL to S3 sync destination instance. | `null` | -| `global.sync.destination.s3Port` | Port that the S3 sync destination instance is available on | `` | -| `global.sync.destination.s3Accesskey` | Access key to S3 sync destination | `null` | -| `global.sync.destination.s3Secretkey` | Secret key to S3 sync destination | `null` | -| `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | -| `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | -| `global.sync.remote.host` | URL to the remote syncAPI host | `null` | -| `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | -| `global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null` | -| `global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null` | -| `global.tls.enabled` | Use TLS for all connections. | `true` | -| `global.tls.issuer` | Issuer for TLS certificate creation. | `""` | -| `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` | -| `global.reencrypt.host` | gRPC host for reencryption | `""` | -| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` | +| Parameter | Description | Default | +|--------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------| +| `image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` | +| `image.tag` | Tag version to deploy | `chart appVersion` | +| `image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` | +| `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` | +| `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` | +| `global.tlsPath` | This path will be a subpath to the secretsPath | `tls` | +| `global.jwtPath` | This path will be a subpath to the secretsPath | `jwt` | +| `global.confFile` | Name of config file, used when secrets are handled by hasicorp vault | `config.yaml` | +| `global.confFilePath` | This path will be a subpath to the secretsPath | `""` | +| `global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all` | +| `global.schemaType` | Standalone version requires a custom JOSN schema, available options are `federated` or `isolated`, unset defaults to federated. | `""` | +| `global.ingress.deploy` | | `false` | +| `global.ingress.hostName.auth` | | `""` | +| `global.ingress.hostName.doa` | | `""` | +| `global.ingress.hostName.s3Inbox` | | `""` | +| `global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | +| `global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | +| `global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | +| `global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | +| `global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | +| `global.ingress.annotations` | extra annotations shared by all ingress definitions | `""` | +| `global.ingress.ingressClassName` | Class of ingress controller to use | `""` | +| `global.log.format` | Log format for all services, JSON or TEXT. | `json` | +| `global.log.level` | Log level for all services. | `info` | +| `global.networkPolicy.create` | Use network isolation. | `false` | +| `global.networkPolicy.brokerNamespace` | Namespace where the broker is deployed. | `""` | +| `global.networkPolicy.databaseNamespace` | Namespace where the database is deployed. | `""` | +| `global.networkPolicy.externalNamespace` | Namespace where the external components are deployed. | `""` | +| `global.networkPolicy.internalNamespace` | Namespace where the internal components are deployed. | `""` | +| `global.networkPolicy.ingressNamespace` | Namespace where the ingress is deployed. | `""` | +| `global.networkPolicy.ingressLabelMatch` | Match to use to allow connections from ingress controller pod. | `""` | +| `global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` | +| `global.podAnnotations` | Annotations applied to pods of all services. | `{}` | +| `global.pkiService` | If an external PKI infrastructure is used set this to true. | `false` | +| `global.rbacEnabled` | Use role based access control. | `true` | +| `global.vaultSecrets` | If Hashicorp Vault is used for secrets management | `false` | +| `global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. | `s3` | +| `global.archive.s3Url` | URL to S3 archive instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.archive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"archive"` | +| `global.archive.s3Region` | S3 archive region. | `us-east-1` | +| `global.archive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `50mb` | +| `global.archive.s3AccessKey` | Access key to S3 archive . | `null` | +| `global.archive.s3SecretKey` | Secret key to S3 archive. | `null` | +| `global.archive.s3Port` | Port that the S3 S3 archive is available on. | `` | +| `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | +| `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | +| `global.archive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | +| `global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive` | +| `global.archive.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.archive.nfsPath` | Path on the NFS server for the archive. | `""` | +| `global.archive.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the archive storage | `""` | +| `global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. | `null` | +| `global.backupArchive.s3Url` | URL to S3 backup archive instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.backupArchive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"backup"` | +| `global.backupArchive.s3Region` | S3 backup archive region. | `us-east-1` | +| `global.backupArchive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `15` | +| `global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` | +| `global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` | +| `global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `` | +| `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | +| `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | +| `global.backupArchive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | +| `global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup` | +| `global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. | `""` | +| `global.backupArchive.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the backup storage | `""` | +| `global.api.jwtPubKeyName` | Public key used to verify the JWT. | `` | +| `global.api.jwtSecret` | The name of the secret holding the JWT public key | `` | +| `global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) | `` | +| `global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type | `"ES256"` | +| `global.auth.jwtKey` | Private key used to sign the JWT. | `""` | +| `global.auth.jwtPub` | Public key ues to verify the JWT. | `""` | +| `global.auth.jwtTTL` | TTL of the resigned token (hours). | `168` | +| `global.auth.resignJWT` | Resign the LS-AAI JWTs. | `true` | +| `global.auth.useTLS` | Run a TLS secured server. | `true` | +| `global.auth.corsOrigins` | Domain name allowed for cross-domain requests. | `""` | +| `global.auth.corsMethods` | Allowed cross-domain request methods. | `""` | +| `global.auth.corsCreds` | Include credentials in cross-domain requests. | `false` | +| `global.broker.host` | Domain name or IP address to the message broker. | `""` | +| `global.broker.exchange` | Exchange to publish messages to. | `""` | +| `global.broker.port` | Port for the message broker. | `5671` | +| `global.broker.verifyPeer` | Use Client/Server verification. | `true` | +| `global.broker.vhost` | Virtual host to connect to. | `/` | +| `global.broker.password` | Shared password to the message broker. | `/` | +| `global.broker.username` | Shared user to the message broker. | `/` | +| `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` | +| `global.cega.host` | Full URI to the EGA user authentication service. | `""` | +| `global.cega.user` | Username for the EGA user authentication service. | `""` | +| `global.cega.password` | Password for the EGA user authentication service. | `""` | +| `global.c4gh.privateKeys` | List of Private C4GH keys. | `` | +| `global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. | `""` | +| `global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. | `""` | +| `global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""` | +| `global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""` | +| `global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""` | +| `global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` | +| `global.db.admin.passKey` | Key in the secret that holds the password. | `""` | +| `global.db.admin.userkey` | Key in the secret that holds the username. | `""` | +| `global.db.admin.username` | If secretName is not set will default to this to populate PGUSER used in migration jobs | `""` | +| `global.db.admin.password` | If secretName is not set will default to this to populate PGPASSWORD used in migration jobs | `""` | +| `global.db.host` | Hostname for the database. | `""` | +| `global.db.name` | Database to connect to. | `lega` | +| `global.db.passIngest` | Password used for `data in` services. | `""` | +| `global.db.passOutgest` | Password used for `data out` services. | `""` | +| `global.db.port` | Port that the database is listening on. | `5432` | +| `global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. | `verify-full` | +| `global.doa.enabled` | Deploy the DOA service | `false` | +| `global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env` | +| `global.doa.serviceport` | Port that the DOA service is accessible on | `443` | +| `global.doa.outbox.enabled` | Enable Outbox functionality of Data Out API | `false` | +| `global.doa.outbox.queue` | MQ queue name for files/datasets export requests | `""` | +| `global.doa.outbox.type` | Outbox type can be either S3 or POSIX | `""` | +| `global.doa.outbox.path` | Posix outbox location with placeholder for the username | `""` | +| `global.doa.outbox.s3Url` | Outbox S3 URL | `""` | +| `global.doa.outbox.s3Port` | Outbox S3 port | `443` | +| `global.doa.outbox.s3Region` | Outbox S3 region | `""` | +| `global.doa.outbox.s3Bucket` | Outbox S3 bucket | `""` | +| `global.doa.outbox.s3CaFile` | Outbox S3 CA certificate to use | `null` | +| `global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null` | +| `global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null` | +| `global.download.enabled` | Deploy the download service | `true` | +| `global.download.serveDecrypted.c4ghKeyFile` | Transient private C4GH key | `""` | +| `global.download.serveDecrypted.secretName` | Secret holding the transient private C4GH key and its passphrase | `""` | +| `global.download.sessionExpiration` | Session key expiration time in seconds | `28800` | +| `global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` | +| `global.download.trusted.configFile` | Name of ISS config file | `iss.json` | +| `global.download.trusted.iss` | Array of trusted OIDC endpoints | `` | +| `global.download.trusted.iss[iss]` | URI to the OIDC service | `https://proxy.aai.lifescience-ri.eu` | +| `global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://proxy.aai.lifescience-ri.eu/OIDC/jwks` | +| `global.oidc.provider` | URL to the OIDc service. | `"https://proxy.aai.lifescience-ri.eu"` | +| `global.oidc.jwkPath` | Public key path on the OIDC host. | `/OIDC/jwks` | +| `global.oidc.id` | User ID to the OIDC host. | `` | +| `global.oidc.secret` | User credentials to the OIDC host. | `` | +| `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` | +| `global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. | `posix` | +| `global.inbox.volumePath` | Path to the mounted `posix` volume. | `/inbox` | +| `global.inbox.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.inbox.nfsPath` | Path on the NFS server for the inbox. | `""` | +| `global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""` | +| `global.inbox.s3Url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.inbox.s3Port` | Port that the S3 inbox is available on. | `` | +| `global.inbox.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | +| `global.inbox.s3Region` | S3 inbox region. | `"us-east-1"` | +| `global.inbox.s3AccessKey` | Access key to S3 inbox . | `null` | +| `global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null` | +| `global.s3Inbox.url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.s3Inbox.port` | Port that the S3 inbox is available on. | `` | +| `global.s3Inbox.bucket` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | +| `global.s3Inbox.accessKey` | Access key to S3 inbox . | `null` | +| `global.s3Inbox.secretKey` | Secret key to S3 inbox. | `null` | +| `global.s3Inbox.region` | S3 inbox region. | `"us-east-1"` | +| `global.s3Inbox.readyPath` | Path to the s3 ready/healthy endpoint. | `null` | +| `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` | +| `global.sync.api.user` | User for authenticating to the syncAPI server | `null` | +| `global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` | +| `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` | +| `global.sync.destination.s3Url` | URL to S3 sync destination instance. | `null` | +| `global.sync.destination.s3Port` | Port that the S3 sync destination instance is available on | `` | +| `global.sync.destination.s3Accesskey` | Access key to S3 sync destination | `null` | +| `global.sync.destination.s3Secretkey` | Secret key to S3 sync destination | `null` | +| `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | +| `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | +| `global.sync.destination.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the sync storage | `""` | +| `global.sync.remote.host` | URL to the remote syncAPI host | `null` | +| `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | +| `global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null` | +| `global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null` | +| `global.tls.enabled` | Use TLS for all connections. | `true` | +| `global.tls.issuer` | Issuer for TLS certificate creation. | `""` | +| `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` | +| `global.reencrypt.host` | gRPC host for reencryption | `""` | +| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` | ### Credentials diff --git a/charts/sda-svc/templates/finalize-secrets.yaml b/charts/sda-svc/templates/finalize-secrets.yaml index 0a362d5132..77a0138a6f 100644 --- a/charts/sda-svc/templates/finalize-secrets.yaml +++ b/charts/sda-svc/templates/finalize-secrets.yaml @@ -8,7 +8,7 @@ metadata: type: Opaque stringData: config.yaml: |- - {{- if .Values.global.backupArchive.storageType }} + {{- if .Values.global.backupArchive.storageType }} storage: archive: {{- if eq "s3" .Values.global.archive.storageType }} @@ -57,7 +57,11 @@ stringData: {{- if .Values.global.backupArchive.maxSize }} max_size: {{ .Values.global.backupArchive.maxSize }} {{- end }} - {{- end }} + {{- end }} + {{- if .Values.global.backupArchive.locationBrokerCacheTTL }} + location_broker: + cache_ttl: {{ .Values.global.backupArchive.locationBrokerCacheTTL }} + {{- end }} broker: {{- if .Values.global.tls.enabled }} caCert: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/templates/ingest-secrets.yaml b/charts/sda-svc/templates/ingest-secrets.yaml index 381c974d84..d70b67c0f1 100644 --- a/charts/sda-svc/templates/ingest-secrets.yaml +++ b/charts/sda-svc/templates/ingest-secrets.yaml @@ -50,6 +50,10 @@ stringData: posix: - path: {{ .Values.global.inbox.volumePath }} {{- end }} + {{- if .Values.global.archive.locationBrokerCacheTTL }} + location_broker: + cache_ttl: {{ .Values.global.archive.locationBrokerCacheTTL }} + {{- end }} broker: {{- if .Values.global.tls.enabled }} caCert: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index e72f037215..35ac44b8cb 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -41,6 +41,10 @@ stringData: {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} + {{- if .Values.global.sync.destination.locationBrokerCacheTTL }} + location_broker: + cache_ttl: {{ .Values.global.sync.destination.locationBrokerCacheTTL }} + {{- end }} broker: {{- if .Values.global.tls.enabled }} caCert: {{ template "tlsPath" . }}/ca.crt diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 93409540ea..c4415d0433 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -121,6 +121,7 @@ global: s3SecretKey: null s3Port: 443 maxBuckets: 1 + locationBrokerCacheTTL: "" # The lines below is only used with POSIX backend volumePath: "/archive" nfsServer: "" @@ -139,6 +140,7 @@ global: s3SecretKey: null s3Port: 443 maxBuckets: 1 + locationBrokerCacheTTL: "" # The lines below is only used with POSIX backend volumePath: "/backup" nfsServer: "" @@ -300,6 +302,7 @@ global: s3SecretKey: "" s3BucketPrefix: "sync" s3Region: "us-east-1" + locationBrokerCacheTTL: "" remote: host: "" port: "" From de089b5f10e13627ede035de046d8ffdc97bab32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 7 Apr 2026 10:35:58 +0200 Subject: [PATCH 424/530] feat(sda-svc chart): bump chart version -> v3.2.2 --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 1f4cdd765c..561e9b8edd 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.2.1 +version: 3.2.2 appVersion: v3.1.37 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From 0df750d4a0ccfcd5d4dc68c23cb7b1ee352e0d9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 7 Apr 2026 10:53:01 +0200 Subject: [PATCH 425/530] feat(sda-svc chart): expose maxBuckets, maxObjects, and maxSize configuration for sync storge writer --- charts/sda-svc/README.md | 15 +++++++++------ charts/sda-svc/templates/sync-secrets.yaml | 9 +++++++++ charts/sda-svc/values.yaml | 3 +++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 87578818c3..3c6e004ece 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -82,8 +82,8 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.archive.s3AccessKey` | Access key to S3 archive . | `null` | | `global.archive.s3SecretKey` | Secret key to S3 archive. | `null` | | `global.archive.s3Port` | Port that the S3 S3 archive is available on. | `` | -| `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | -| `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | +| `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `""` | +| `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `""` | | `global.archive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | | `global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive` | | `global.archive.nfsServer` | URL or IP address to a NFS server. | `""` | @@ -97,8 +97,8 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` | | `global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` | | `global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `` | -| `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `0` | -| `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `0` | +| `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `""` | +| `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `""` | | `global.backupArchive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | | `global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup` | | `global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""` | @@ -196,10 +196,13 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` | | `global.sync.destination.s3Url` | URL to S3 sync destination instance. | `null` | | `global.sync.destination.s3Port` | Port that the S3 sync destination instance is available on | `` | -| `global.sync.destination.s3Accesskey` | Access key to S3 sync destination | `null` | -| `global.sync.destination.s3Secretkey` | Secret key to S3 sync destination | `null` | +| `global.sync.destination.s3AccessKey` | Access key to S3 sync destination | `null` | +| `global.sync.destination.s3SecretKey` | Secret key to S3 sync destination | `null` | | `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | | `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | +| `global.sync.destination.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `"1"` | +| `global.sync.destination.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `""` | +| `global.sync.destination.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `""` | | `global.sync.destination.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the sync storage | `""` | | `global.sync.remote.host` | URL to the remote syncAPI host | `null` | | `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index 35ac44b8cb..026bc3cdff 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -41,6 +41,15 @@ stringData: {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} + {{- if .Values.global.sync.destination.maxBuckets }} + max_buckets: {{ .Values.global.sync.destination.maxBuckets }} + {{- end }} + {{- if .Values.global.sync.destination.maxObjects }} + max_objects: {{ .Values.global.sync.destination.maxObjects }} + {{- end }} + {{- if .Values.global.sync.destination.maxSize }} + max_size: {{ .Values.global.sync.destination.maxSize }} + {{- end }} {{- if .Values.global.sync.destination.locationBrokerCacheTTL }} location_broker: cache_ttl: {{ .Values.global.sync.destination.locationBrokerCacheTTL }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index c4415d0433..f1b847bae9 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -303,6 +303,9 @@ global: s3BucketPrefix: "sync" s3Region: "us-east-1" locationBrokerCacheTTL: "" + maxBuckets: 1 + maxObjects: "" + maxSize: "" remote: host: "" port: "" From 465481356470938b8718b525ead6ef2bab7d2c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 7 Apr 2026 11:39:00 +0200 Subject: [PATCH 426/530] feat(sda-svc chart): update README.md expand locationBrokerCacheTTL description, and fix some typos --- charts/sda-svc/README.md | 354 +++++++++++++++++++-------------------- 1 file changed, 177 insertions(+), 177 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 3c6e004ece..0082d29b9f 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -36,183 +36,183 @@ containing the credentials to perform the upgrade needs to be created. The following table lists the configurable parameters of the `sda-svc` chart and their default values. -| Parameter | Description | Default | -|--------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------| -| `image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` | -| `image.tag` | Tag version to deploy | `chart appVersion` | -| `image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` | -| `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` | -| `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` | -| `global.tlsPath` | This path will be a subpath to the secretsPath | `tls` | -| `global.jwtPath` | This path will be a subpath to the secretsPath | `jwt` | -| `global.confFile` | Name of config file, used when secrets are handled by hasicorp vault | `config.yaml` | -| `global.confFilePath` | This path will be a subpath to the secretsPath | `""` | -| `global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all` | -| `global.schemaType` | Standalone version requires a custom JOSN schema, available options are `federated` or `isolated`, unset defaults to federated. | `""` | -| `global.ingress.deploy` | | `false` | -| `global.ingress.hostName.auth` | | `""` | -| `global.ingress.hostName.doa` | | `""` | -| `global.ingress.hostName.s3Inbox` | | `""` | -| `global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | -| `global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | -| `global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""` | -| `global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | -| `global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | -| `global.ingress.annotations` | extra annotations shared by all ingress definitions | `""` | -| `global.ingress.ingressClassName` | Class of ingress controller to use | `""` | -| `global.log.format` | Log format for all services, JSON or TEXT. | `json` | -| `global.log.level` | Log level for all services. | `info` | -| `global.networkPolicy.create` | Use network isolation. | `false` | -| `global.networkPolicy.brokerNamespace` | Namespace where the broker is deployed. | `""` | -| `global.networkPolicy.databaseNamespace` | Namespace where the database is deployed. | `""` | -| `global.networkPolicy.externalNamespace` | Namespace where the external components are deployed. | `""` | -| `global.networkPolicy.internalNamespace` | Namespace where the internal components are deployed. | `""` | -| `global.networkPolicy.ingressNamespace` | Namespace where the ingress is deployed. | `""` | -| `global.networkPolicy.ingressLabelMatch` | Match to use to allow connections from ingress controller pod. | `""` | -| `global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` | -| `global.podAnnotations` | Annotations applied to pods of all services. | `{}` | -| `global.pkiService` | If an external PKI infrastructure is used set this to true. | `false` | -| `global.rbacEnabled` | Use role based access control. | `true` | -| `global.vaultSecrets` | If Hashicorp Vault is used for secrets management | `false` | -| `global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. | `s3` | -| `global.archive.s3Url` | URL to S3 archive instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.archive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"archive"` | -| `global.archive.s3Region` | S3 archive region. | `us-east-1` | -| `global.archive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `50mb` | -| `global.archive.s3AccessKey` | Access key to S3 archive . | `null` | -| `global.archive.s3SecretKey` | Secret key to S3 archive. | `null` | -| `global.archive.s3Port` | Port that the S3 S3 archive is available on. | `` | -| `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `""` | -| `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `""` | -| `global.archive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | -| `global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive` | -| `global.archive.nfsServer` | URL or IP address to a NFS server. | `""` | -| `global.archive.nfsPath` | Path on the NFS server for the archive. | `""` | -| `global.archive.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the archive storage | `""` | -| `global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. | `null` | -| `global.backupArchive.s3Url` | URL to S3 backup archive instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.backupArchive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"backup"` | -| `global.backupArchive.s3Region` | S3 backup archive region. | `us-east-1` | -| `global.backupArchive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: 500mb, etc), needs to between 5mb and 1gb. | `15` | -| `global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` | -| `global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` | -| `global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `` | -| `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `""` | -| `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `""` | -| `global.backupArchive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | -| `global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup` | -| `global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""` | -| `global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. | `""` | -| `global.backupArchive.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the backup storage | `""` | -| `global.api.jwtPubKeyName` | Public key used to verify the JWT. | `` | -| `global.api.jwtSecret` | The name of the secret holding the JWT public key | `` | -| `global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) | `` | -| `global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type | `"ES256"` | -| `global.auth.jwtKey` | Private key used to sign the JWT. | `""` | -| `global.auth.jwtPub` | Public key ues to verify the JWT. | `""` | -| `global.auth.jwtTTL` | TTL of the resigned token (hours). | `168` | -| `global.auth.resignJWT` | Resign the LS-AAI JWTs. | `true` | -| `global.auth.useTLS` | Run a TLS secured server. | `true` | -| `global.auth.corsOrigins` | Domain name allowed for cross-domain requests. | `""` | -| `global.auth.corsMethods` | Allowed cross-domain request methods. | `""` | -| `global.auth.corsCreds` | Include credentials in cross-domain requests. | `false` | -| `global.broker.host` | Domain name or IP address to the message broker. | `""` | -| `global.broker.exchange` | Exchange to publish messages to. | `""` | -| `global.broker.port` | Port for the message broker. | `5671` | -| `global.broker.verifyPeer` | Use Client/Server verification. | `true` | -| `global.broker.vhost` | Virtual host to connect to. | `/` | -| `global.broker.password` | Shared password to the message broker. | `/` | -| `global.broker.username` | Shared user to the message broker. | `/` | -| `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` | -| `global.cega.host` | Full URI to the EGA user authentication service. | `""` | -| `global.cega.user` | Username for the EGA user authentication service. | `""` | -| `global.cega.password` | Password for the EGA user authentication service. | `""` | -| `global.c4gh.privateKeys` | List of Private C4GH keys. | `` | -| `global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. | `""` | -| `global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. | `""` | -| `global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""` | -| `global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""` | -| `global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""` | -| `global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` | -| `global.db.admin.passKey` | Key in the secret that holds the password. | `""` | -| `global.db.admin.userkey` | Key in the secret that holds the username. | `""` | -| `global.db.admin.username` | If secretName is not set will default to this to populate PGUSER used in migration jobs | `""` | -| `global.db.admin.password` | If secretName is not set will default to this to populate PGPASSWORD used in migration jobs | `""` | -| `global.db.host` | Hostname for the database. | `""` | -| `global.db.name` | Database to connect to. | `lega` | -| `global.db.passIngest` | Password used for `data in` services. | `""` | -| `global.db.passOutgest` | Password used for `data out` services. | `""` | -| `global.db.port` | Port that the database is listening on. | `5432` | -| `global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. | `verify-full` | -| `global.doa.enabled` | Deploy the DOA service | `false` | -| `global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env` | -| `global.doa.serviceport` | Port that the DOA service is accessible on | `443` | -| `global.doa.outbox.enabled` | Enable Outbox functionality of Data Out API | `false` | -| `global.doa.outbox.queue` | MQ queue name for files/datasets export requests | `""` | -| `global.doa.outbox.type` | Outbox type can be either S3 or POSIX | `""` | -| `global.doa.outbox.path` | Posix outbox location with placeholder for the username | `""` | -| `global.doa.outbox.s3Url` | Outbox S3 URL | `""` | -| `global.doa.outbox.s3Port` | Outbox S3 port | `443` | -| `global.doa.outbox.s3Region` | Outbox S3 region | `""` | -| `global.doa.outbox.s3Bucket` | Outbox S3 bucket | `""` | -| `global.doa.outbox.s3CaFile` | Outbox S3 CA certificate to use | `null` | -| `global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null` | -| `global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null` | -| `global.download.enabled` | Deploy the download service | `true` | -| `global.download.serveDecrypted.c4ghKeyFile` | Transient private C4GH key | `""` | -| `global.download.serveDecrypted.secretName` | Secret holding the transient private C4GH key and its passphrase | `""` | -| `global.download.sessionExpiration` | Session key expiration time in seconds | `28800` | -| `global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` | -| `global.download.trusted.configFile` | Name of ISS config file | `iss.json` | -| `global.download.trusted.iss` | Array of trusted OIDC endpoints | `` | -| `global.download.trusted.iss[iss]` | URI to the OIDC service | `https://proxy.aai.lifescience-ri.eu` | -| `global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://proxy.aai.lifescience-ri.eu/OIDC/jwks` | -| `global.oidc.provider` | URL to the OIDc service. | `"https://proxy.aai.lifescience-ri.eu"` | -| `global.oidc.jwkPath` | Public key path on the OIDC host. | `/OIDC/jwks` | -| `global.oidc.id` | User ID to the OIDC host. | `` | -| `global.oidc.secret` | User credentials to the OIDC host. | `` | -| `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` | -| `global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. | `posix` | -| `global.inbox.volumePath` | Path to the mounted `posix` volume. | `/inbox` | -| `global.inbox.nfsServer` | URL or IP address to a NFS server. | `""` | -| `global.inbox.nfsPath` | Path on the NFS server for the inbox. | `""` | -| `global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""` | -| `global.inbox.s3Url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.inbox.s3Port` | Port that the S3 inbox is available on. | `` | -| `global.inbox.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | -| `global.inbox.s3Region` | S3 inbox region. | `"us-east-1"` | -| `global.inbox.s3AccessKey` | Access key to S3 inbox . | `null` | -| `global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null` | -| `global.s3Inbox.url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | -| `global.s3Inbox.port` | Port that the S3 inbox is available on. | `` | -| `global.s3Inbox.bucket` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | -| `global.s3Inbox.accessKey` | Access key to S3 inbox . | `null` | -| `global.s3Inbox.secretKey` | Secret key to S3 inbox. | `null` | -| `global.s3Inbox.region` | S3 inbox region. | `"us-east-1"` | -| `global.s3Inbox.readyPath` | Path to the s3 ready/healthy endpoint. | `null` | -| `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` | -| `global.sync.api.user` | User for authenticating to the syncAPI server | `null` | -| `global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` | -| `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` | -| `global.sync.destination.s3Url` | URL to S3 sync destination instance. | `null` | -| `global.sync.destination.s3Port` | Port that the S3 sync destination instance is available on | `` | -| `global.sync.destination.s3AccessKey` | Access key to S3 sync destination | `null` | -| `global.sync.destination.s3SecretKey` | Secret key to S3 sync destination | `null` | -| `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | -| `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | -| `global.sync.destination.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `"1"` | -| `global.sync.destination.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `""` | -| `global.sync.destination.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc) | `""` | -| `global.sync.destination.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the sync storage | `""` | -| `global.sync.remote.host` | URL to the remote syncAPI host | `null` | -| `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | -| `global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null` | -| `global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null` | -| `global.tls.enabled` | Use TLS for all connections. | `true` | -| `global.tls.issuer` | Issuer for TLS certificate creation. | `""` | -| `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` | -| `global.reencrypt.host` | gRPC host for reencryption | `""` | -| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` | +| Parameter | Description | Default | +|--------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------| +| `image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` | +| `image.tag` | Tag version to deploy | `chart appVersion` | +| `image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` | +| `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` | +| `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` | +| `global.tlsPath` | This path will be a subpath to the secretsPath | `tls` | +| `global.jwtPath` | This path will be a subpath to the secretsPath | `jwt` | +| `global.confFile` | Name of config file, used when secrets are handled by hasicorp vault | `config.yaml` | +| `global.confFilePath` | This path will be a subpath to the secretsPath | `""` | +| `global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all` | +| `global.schemaType` | Standalone version requires a custom JSON schema, available options are `federated` or `isolated`, unset defaults to federated. | `""` | +| `global.ingress.deploy` | | `false` | +| `global.ingress.hostName.auth` | | `""` | +| `global.ingress.hostName.doa` | | `""` | +| `global.ingress.hostName.s3Inbox` | | `""` | +| `global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingress enpoint. | `""` | +| `global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingress enpoint. | `""` | +| `global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingress enpoint. | `""` | +| `global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | +| `global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""` | +| `global.ingress.annotations` | extra annotations shared by all ingress definitions | `""` | +| `global.ingress.ingressClassName` | Class of ingress controller to use | `""` | +| `global.log.format` | Log format for all services, JSON or TEXT. | `json` | +| `global.log.level` | Log level for all services. | `info` | +| `global.networkPolicy.create` | Use network isolation. | `false` | +| `global.networkPolicy.brokerNamespace` | Namespace where the broker is deployed. | `""` | +| `global.networkPolicy.databaseNamespace` | Namespace where the database is deployed. | `""` | +| `global.networkPolicy.externalNamespace` | Namespace where the external components are deployed. | `""` | +| `global.networkPolicy.internalNamespace` | Namespace where the internal components are deployed. | `""` | +| `global.networkPolicy.ingressNamespace` | Namespace where the ingress is deployed. | `""` | +| `global.networkPolicy.ingressLabelMatch` | Match to use to allow connections from ingress controller pod. | `""` | +| `global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3` | +| `global.podAnnotations` | Annotations applied to pods of all services. | `{}` | +| `global.pkiService` | If an external PKI infrastructure is used set this to true. | `false` | +| `global.rbacEnabled` | Use role based access control. | `true` | +| `global.vaultSecrets` | If Hashicorp Vault is used for secrets management | `false` | +| `global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. | `s3` | +| `global.archive.s3Url` | URL to S3 archive instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.archive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"archive"` | +| `global.archive.s3Region` | S3 archive region. | `us-east-1` | +| `global.archive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: `15mb`, `500mb`), needs to be between `5mb` and `1gb`. If unset, the application default is used. | `""` | +| `global.archive.s3AccessKey` | Access key to S3 archive . | `null` | +| `global.archive.s3SecretKey` | Secret key to S3 archive. | `null` | +| `global.archive.s3Port` | Port that the S3 S3 archive is available on. | `` | +| `global.archive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `null` | +| `global.archive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc). If unset, no size checks is done before writing. | `""` | +| `global.archive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | +| `global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive` | +| `global.archive.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.archive.nfsPath` | Path on the NFS server for the archive. | `""` | +| `global.archive.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the archive storage. Expects a duration string (eg: `60s`, `5m`). If unset, the application default is used. | `""` | +| `global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. | `null` | +| `global.backupArchive.s3Url` | URL to S3 backup archive instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.backupArchive.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"backup"` | +| `global.backupArchive.s3Region` | S3 backup archive region. | `us-east-1` | +| `global.backupArchive.s3ChunkSize` | S3 chunk size in human readable byte size (eg: `15mb`, `500mb`), needs to be between `5mb` and `1gb`. If unset, the application default is used. | `""` | +| `global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null` | +| `global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null` | +| `global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `` | +| `global.backupArchive.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `null` | +| `global.backupArchive.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc). If unset, no size checks is done before writing. | `""` | +| `global.backupArchive.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | +| `global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup` | +| `global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. | `""` | +| `global.backupArchive.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the backup storage. Expects a duration string (eg: `60s`, `5m`). If unset, the application default is used. | `""` | +| `global.api.jwtPubKeyName` | Public key used to verify the JWT. | `` | +| `global.api.jwtSecret` | The name of the secret holding the JWT public key | `` | +| `global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) | `` | +| `global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type | `"ES256"` | +| `global.auth.jwtKey` | Private key used to sign the JWT. | `""` | +| `global.auth.jwtPub` | Public key ues to verify the JWT. | `""` | +| `global.auth.jwtTTL` | TTL of the resigned token (hours). | `168` | +| `global.auth.resignJWT` | Resign the LS-AAI JWTs. | `true` | +| `global.auth.useTLS` | Run a TLS secured server. | `true` | +| `global.auth.corsOrigins` | Domain name allowed for cross-domain requests. | `""` | +| `global.auth.corsMethods` | Allowed cross-domain request methods. | `""` | +| `global.auth.corsCreds` | Include credentials in cross-domain requests. | `false` | +| `global.broker.host` | Domain name or IP address to the message broker. | `""` | +| `global.broker.exchange` | Exchange to publish messages to. | `""` | +| `global.broker.port` | Port for the message broker. | `5671` | +| `global.broker.verifyPeer` | Use Client/Server verification. | `true` | +| `global.broker.vhost` | Virtual host to connect to. | `/` | +| `global.broker.password` | Shared password to the message broker. | `/` | +| `global.broker.username` | Shared user to the message broker. | `/` | +| `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` | +| `global.cega.host` | Full URI to the EGA user authentication service. | `""` | +| `global.cega.user` | Username for the EGA user authentication service. | `""` | +| `global.cega.password` | Password for the EGA user authentication service. | `""` | +| `global.c4gh.privateKeys` | List of Private C4GH keys. | `` | +| `global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. | `""` | +| `global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. | `""` | +| `global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""` | +| `global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""` | +| `global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""` | +| `global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` | +| `global.db.admin.passKey` | Key in the secret that holds the password. | `""` | +| `global.db.admin.userkey` | Key in the secret that holds the username. | `""` | +| `global.db.admin.username` | If secretName is not set will default to this to populate PGUSER used in migration jobs | `""` | +| `global.db.admin.password` | If secretName is not set will default to this to populate PGPASSWORD used in migration jobs | `""` | +| `global.db.host` | Hostname for the database. | `""` | +| `global.db.name` | Database to connect to. | `lega` | +| `global.db.passIngest` | Password used for `data in` services. | `""` | +| `global.db.passOutgest` | Password used for `data out` services. | `""` | +| `global.db.port` | Port that the database is listening on. | `5432` | +| `global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. | `verify-full` | +| `global.doa.enabled` | Deploy the DOA service | `false` | +| `global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env` | +| `global.doa.serviceport` | Port that the DOA service is accessible on | `443` | +| `global.doa.outbox.enabled` | Enable Outbox functionality of Data Out API | `false` | +| `global.doa.outbox.queue` | MQ queue name for files/datasets export requests | `""` | +| `global.doa.outbox.type` | Outbox type can be either S3 or POSIX | `""` | +| `global.doa.outbox.path` | Posix outbox location with placeholder for the username | `""` | +| `global.doa.outbox.s3Url` | Outbox S3 URL | `""` | +| `global.doa.outbox.s3Port` | Outbox S3 port | `443` | +| `global.doa.outbox.s3Region` | Outbox S3 region | `""` | +| `global.doa.outbox.s3Bucket` | Outbox S3 bucket | `""` | +| `global.doa.outbox.s3CaFile` | Outbox S3 CA certificate to use | `null` | +| `global.doa.outbox.s3AccessKey` | Outbox S3 Access Key | `null` | +| `global.doa.outbox.s3SecretKey` | Outbox S3 Secret key | `null` | +| `global.download.enabled` | Deploy the download service | `true` | +| `global.download.serveDecrypted.c4ghKeyFile` | Transient private C4GH key | `""` | +| `global.download.serveDecrypted.secretName` | Secret holding the transient private C4GH key and its passphrase | `""` | +| `global.download.sessionExpiration` | Session key expiration time in seconds | `28800` | +| `global.download.trusted.configPath` | Path to the ISS config file | `$secrets/iss` | +| `global.download.trusted.configFile` | Name of ISS config file | `iss.json` | +| `global.download.trusted.iss` | Array of trusted OIDC endpoints | `` | +| `global.download.trusted.iss[iss]` | URI to the OIDC service | `https://proxy.aai.lifescience-ri.eu` | +| `global.download.trusted.iss[jku]` | The URI to the OIDCs jwk endpoint | `https://proxy.aai.lifescience-ri.eu/OIDC/jwks` | +| `global.oidc.provider` | URL to the OIDc service. | `"https://proxy.aai.lifescience-ri.eu"` | +| `global.oidc.jwkPath` | Public key path on the OIDC host. | `/OIDC/jwks` | +| `global.oidc.id` | User ID to the OIDC host. | `` | +| `global.oidc.secret` | User credentials to the OIDC host. | `` | +| `global.inbox.servicePort` | The port that the inbox is accessible via. | `2222` | +| `global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. | `posix` | +| `global.inbox.volumePath` | Path to the mounted `posix` volume. | `/inbox` | +| `global.inbox.nfsServer` | URL or IP address to a NFS server. | `""` | +| `global.inbox.nfsPath` | Path on the NFS server for the inbox. | `""` | +| `global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""` | +| `global.inbox.s3Url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.inbox.s3Port` | Port that the S3 inbox is available on. | `` | +| `global.inbox.s3BucketPrefix` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | +| `global.inbox.s3Region` | S3 inbox region. | `"us-east-1"` | +| `global.inbox.s3AccessKey` | Access key to S3 inbox . | `null` | +| `global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null` | +| `global.s3Inbox.url` | URL to S3 inbox instance (if port is included in url s3Port should be set as 0). | `""` | +| `global.s3Inbox.port` | Port that the S3 inbox is available on. | `` | +| `global.s3Inbox.bucket` | S3 archive bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"inbox"` | +| `global.s3Inbox.accessKey` | Access key to S3 inbox . | `null` | +| `global.s3Inbox.secretKey` | Secret key to S3 inbox. | `null` | +| `global.s3Inbox.region` | S3 inbox region. | `"us-east-1"` | +| `global.s3Inbox.readyPath` | Path to the s3 ready/healthy endpoint. | `null` | +| `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` | +| `global.sync.api.user` | User for authenticating to the syncAPI server | `null` | +| `global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` | +| `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` | +| `global.sync.destination.s3Url` | URL to S3 sync destination instance. | `null` | +| `global.sync.destination.s3Port` | Port that the S3 sync destination instance is available on | `` | +| `global.sync.destination.s3AccessKey` | Access key to S3 sync destination | `null` | +| `global.sync.destination.s3SecretKey` | Secret key to S3 sync destination | `null` | +| `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | +| `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | +| `global.sync.destination.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `"1"` | +| `global.sync.destination.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `null` | +| `global.sync.destination.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc). If unset, no size checks is done before writing. | `""` | +| `global.sync.destination.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the sync storage. Expects a duration string (eg: `60s`, `5m`). If unset, the application default is used. | `null` | +| `global.sync.remote.host` | URL to the remote syncAPI host | `null` | +| `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | +| `global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null` | +| `global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null` | +| `global.tls.enabled` | Use TLS for all connections. | `true` | +| `global.tls.issuer` | Issuer for TLS certificate creation. | `""` | +| `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` | +| `global.reencrypt.host` | gRPC host for reencryption | `""` | +| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` | ### Credentials From 2643a94f1d33dd70ee9857eba517015024e21e0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 7 Apr 2026 11:41:11 +0200 Subject: [PATCH 427/530] feat(sda-svc chart): fix some values.yaml comments --- charts/sda-svc/values.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index f1b847bae9..5da0787ce4 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -112,7 +112,8 @@ global: storageType: "" # s3 or posix maxObjects: null maxSize: "" - # The six lines below is only used with S3 backend + locationBrokerCacheTTL: "" + # The lines below are only used when storageType is "S3" s3Url: "" s3BucketPrefix: "archive" s3Region: "us-east-1" @@ -121,8 +122,7 @@ global: s3SecretKey: null s3Port: 443 maxBuckets: 1 - locationBrokerCacheTTL: "" - # The lines below is only used with POSIX backend + # The lines below are only used when storageType is "posix" volumePath: "/archive" nfsServer: "" nfsPath: "" @@ -131,7 +131,8 @@ global: storageType: "" # s3 or posix, leave empty for no backup maxObjects: null maxSize: "" - # The six lines below is only used with S3 backend + locationBrokerCacheTTL: "" + # The lines below are only used when storageType is "S3" s3Url: "" s3BucketPrefix: "backup" s3Region: "us-east-1" @@ -140,8 +141,7 @@ global: s3SecretKey: null s3Port: 443 maxBuckets: 1 - locationBrokerCacheTTL: "" - # The lines below is only used with POSIX backend + # The lines below are only used when storageType is "posix" volumePath: "/backup" nfsServer: "" nfsPath: "" From 665d181bc45c0668c66c4c9fce5ebf7d87c22307 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 7 Apr 2026 14:54:45 +0200 Subject: [PATCH 428/530] feat(sda-svc chart): expose chunkSize for sync destination --- charts/sda-svc/README.md | 5 +++-- charts/sda-svc/templates/sync-secrets.yaml | 3 +++ charts/sda-svc/values.yaml | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 0082d29b9f..9fda2e30c1 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -200,10 +200,11 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.sync.destination.s3SecretKey` | Secret key to S3 sync destination | `null` | | `global.sync.destination.s3BucketPrefix` | S3 bucket prefix, buckets will be created with the following naming convention ${BUCKET_PREFIX}${1-MAX_BUCKETS} | `"sync"` | | `global.sync.destination.s3Region` | Sync S3 region | `"us-east-1"` | -| `global.sync.destination.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `"1"` | +| `global.sync.destination.s3ChunkSize` | S3 chunk size in human readable byte size (eg: `15mb`, `500mb`), needs to be between `5mb` and `1gb`. If unset, the application default is used. | `""` | +| `global.sync.destination.maxBuckets` | How many buckets will automatically be created once existing buckets has reached either their maxObjects or maxSize quota. | `1` | | `global.sync.destination.maxObjects` | How many objects will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on object count will be done. | `null` | | `global.sync.destination.maxSize` | How many bytes will be written to a location(s3 bucket or posix volume) before switching to a new one, if set to 0 this will not be evaluated and no checking on byte size will done. Expects human readable byte size (eg: 5tb, 500mb, etc). If unset, no size checks is done before writing. | `""` | -| `global.sync.destination.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the sync storage. Expects a duration string (eg: `60s`, `5m`). If unset, the application default is used. | `null` | +| `global.sync.destination.locationBrokerCacheTTL` | Sets the TTL for the object count and size cache used for the sync storage. Expects a duration string (eg: `60s`, `5m`). If unset, the application default is used. | `""` | | `global.sync.remote.host` | URL to the remote syncAPI host | `null` | | `global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null` | | `global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null` | diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index 026bc3cdff..b53e7658e9 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -41,6 +41,9 @@ stringData: {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt {{- end }} + {{- if .Values.global.sync.destination.s3ChunkSize }} + chunk_size: {{ .Values.global.sync.destination.s3ChunkSize }} + {{- end }} {{- if .Values.global.sync.destination.maxBuckets }} max_buckets: {{ .Values.global.sync.destination.maxBuckets }} {{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 5da0787ce4..4304497c73 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -302,9 +302,10 @@ global: s3SecretKey: "" s3BucketPrefix: "sync" s3Region: "us-east-1" + s3ChunkSize: "" locationBrokerCacheTTL: "" maxBuckets: 1 - maxObjects: "" + maxObjects: null maxSize: "" remote: host: "" From 839814885474534747f9c6dae9ae96657710e654 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Apr 2026 19:05:09 +0000 Subject: [PATCH 429/530] build(deps): bump oras-project/setup-oras in the github-actions group Bumps the github-actions group with 1 update: [oras-project/setup-oras](https://github.com/oras-project/setup-oras). Updates `oras-project/setup-oras` from 1 to 2 - [Release notes](https://github.com/oras-project/setup-oras/releases) - [Commits](https://github.com/oras-project/setup-oras/compare/v1...v2) --- updated-dependencies: - dependency-name: oras-project/setup-oras dependency-version: '2' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/update_trivy_cache.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_trivy_cache.yml b/.github/workflows/update_trivy_cache.yml index 1330f0090b..2a5b158a6e 100644 --- a/.github/workflows/update_trivy_cache.yml +++ b/.github/workflows/update_trivy_cache.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup oras - uses: oras-project/setup-oras@v1 + uses: oras-project/setup-oras@v2 - name: Get current date id: date From 1950b2b4c7dd27186ea1870460b8641bc24527a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Apr 2026 06:24:31 +0000 Subject: [PATCH 430/530] build(deps): bump the go group across 3 directories with 10 updates Bumps the go group with 8 updates in the /sda directory: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.13` | `1.32.14` | | [github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager](https://github.com/aws/aws-sdk-go-v2) | `0.1.12` | `0.1.15` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.24.2` | `1.24.3` | | [github.com/coreos/go-oidc/v3](https://github.com/coreos/go-oidc) | `3.17.0` | `3.18.0` | | [github.com/dgraph-io/ristretto](https://github.com/dgraph-io/ristretto) | `0.0.3-0.20200630154024-f66de99634de` | `0.2.0` | | [github.com/lib/pq](https://github.com/lib/pq) | `1.12.0` | `1.12.3` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.49.0` | `0.50.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.79.3` | `1.80.0` | Bumps the go group with 6 updates in the /sda-download directory: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.13` | `1.32.14` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.97.3` | `1.99.0` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.24.2` | `1.24.3` | | [github.com/lib/pq](https://github.com/lib/pq) | `1.12.0` | `1.12.3` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.49.0` | `0.50.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.79.3` | `1.80.0` | Bumps the go group with 2 updates in the /sda-validator/orchestrator directory: [github.com/lib/pq](https://github.com/lib/pq) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.13 to 1.32.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.13...config/v1.32.14) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.13 to 1.19.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.13...credentials/v1.19.14) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager` from 0.1.12 to 0.1.15 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/transfermanager/v0.1.12...feature/s3/transfermanager/v0.1.15) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.3 to 1.99.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.3...service/s3/v1.99.0) Updates `github.com/aws/smithy-go` from 1.24.2 to 1.24.3 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.24.2...v1.24.3) Updates `github.com/coreos/go-oidc/v3` from 3.17.0 to 3.18.0 - [Release notes](https://github.com/coreos/go-oidc/releases) - [Commits](https://github.com/coreos/go-oidc/compare/v3.17.0...v3.18.0) Updates `github.com/dgraph-io/ristretto` from 0.0.3-0.20200630154024-f66de99634de to 0.2.0 - [Release notes](https://github.com/dgraph-io/ristretto/releases) - [Changelog](https://github.com/dgraph-io/ristretto/blob/main/CHANGELOG.md) - [Commits](https://github.com/dgraph-io/ristretto/commits/v0.2.0) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `golang.org/x/crypto` from 0.49.0 to 0.50.0 - [Commits](https://github.com/golang/crypto/compare/v0.49.0...v0.50.0) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.13 to 1.32.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.13...config/v1.32.14) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.13 to 1.19.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.13...credentials/v1.19.14) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.3 to 1.99.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.3...service/s3/v1.99.0) Updates `github.com/aws/smithy-go` from 1.24.2 to 1.24.3 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.24.2...v1.24.3) Updates `github.com/dgraph-io/ristretto` from 0.0.3-0.20200630154024-f66de99634de to 0.2.0 - [Release notes](https://github.com/dgraph-io/ristretto/releases) - [Changelog](https://github.com/dgraph-io/ristretto/blob/main/CHANGELOG.md) - [Commits](https://github.com/dgraph-io/ristretto/commits/v0.2.0) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `golang.org/x/crypto` from 0.49.0 to 0.50.0 - [Commits](https://github.com/golang/crypto/compare/v0.49.0...v0.50.0) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.13 to 1.32.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.13...config/v1.32.14) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.13 to 1.19.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.13...credentials/v1.19.14) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.3 to 1.99.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.3...service/s3/v1.99.0) Updates `github.com/aws/smithy-go` from 1.24.2 to 1.24.3 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.24.2...v1.24.3) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `golang.org/x/crypto` from 0.49.0 to 0.50.0 - [Commits](https://github.com/golang/crypto/compare/v0.49.0...v0.50.0) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.13 to 1.32.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.13...config/v1.32.14) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.13 to 1.19.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.13...credentials/v1.19.14) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.97.3 to 1.99.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.97.3...service/s3/v1.99.0) Updates `github.com/aws/smithy-go` from 1.24.2 to 1.24.3 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.24.2...v1.24.3) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `golang.org/x/crypto` from 0.49.0 to 0.50.0 - [Commits](https://github.com/golang/crypto/compare/v0.49.0...v0.50.0) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) Updates `github.com/lib/pq` from 1.12.0 to 1.12.3 - [Release notes](https://github.com/lib/pq/releases) - [Changelog](https://github.com/lib/pq/blob/master/CHANGELOG.md) - [Commits](https://github.com/lib/pq/compare/v1.12.0...v1.12.3) Updates `google.golang.org/grpc` from 1.79.3 to 1.80.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.79.3...v1.80.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager dependency-version: 0.1.15 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.99.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/smithy-go dependency-version: 1.24.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/coreos/go-oidc/v3 dependency-version: 3.18.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/dgraph-io/ristretto dependency-version: 0.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: golang.org/x/crypto dependency-version: 0.50.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.99.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/smithy-go dependency-version: 1.24.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/dgraph-io/ristretto dependency-version: 0.2.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: golang.org/x/crypto dependency-version: 0.50.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.99.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/smithy-go dependency-version: 1.24.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: golang.org/x/crypto dependency-version: 0.50.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.99.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/smithy-go dependency-version: 1.24.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: golang.org/x/crypto dependency-version: 0.50.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/lib/pq dependency-version: 1.12.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 24 ++++----- sda-download/go.sum | 56 ++++++++++---------- sda-validator/orchestrator/go.mod | 12 ++--- sda-validator/orchestrator/go.sum | 32 ++++++------ sda/go.mod | 34 ++++++------ sda/go.sum | 86 ++++++++++++++----------------- 6 files changed, 119 insertions(+), 125 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 79721cc529..6936d4c9df 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -6,24 +6,24 @@ require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/Masterminds/semver/v3 v3.4.0 github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.32.13 - github.com/aws/aws-sdk-go-v2/credentials v1.19.13 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 - github.com/aws/smithy-go v1.24.2 + github.com/aws/aws-sdk-go-v2/config v1.32.14 + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 + github.com/aws/smithy-go v1.24.3 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.12.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/uuid v1.6.0 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.12.0 + github.com/lib/pq v1.12.3 github.com/neicnordic/crypt4gh v1.15.0 github.com/sirupsen/logrus v1.9.4 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.49.0 + golang.org/x/crypto v0.50.0 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa - google.golang.org/grpc v1.79.3 + google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 ) @@ -40,8 +40,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect @@ -87,9 +87,9 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect - golang.org/x/net v0.51.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 8414bcb834..47fef7121b 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -8,10 +8,10 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg= -github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY= -github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk= +github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= +github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= @@ -30,18 +30,18 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3x github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= +github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -125,8 +125,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.12.0 h1:mC1zeiNamwKBecjHarAr26c/+d8V5w/u4J0I/yASbJo= -github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -200,25 +200,25 @@ go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index f889788428..c3fa655b96 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -8,7 +8,7 @@ require ( github.com/gin-gonic/gin v1.12.0 github.com/google/uuid v1.6.0 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.12.0 + github.com/lib/pq v1.12.3 github.com/neicnordic/crypt4gh v1.15.0 github.com/rabbitmq/amqp091-go v1.10.0 github.com/sirupsen/logrus v1.9.4 @@ -16,7 +16,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - google.golang.org/grpc v1.79.3 + google.golang.org/grpc v1.80.0 ) require ( @@ -66,10 +66,10 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect - golang.org/x/crypto v0.49.0 // indirect - golang.org/x/net v0.51.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/crypto v0.50.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect google.golang.org/protobuf v1.36.11 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index 04d4013057..3b5ada3182 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -90,8 +90,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.12.0 h1:mC1zeiNamwKBecjHarAr26c/+d8V5w/u4J0I/yASbJo= -github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -174,28 +174,28 @@ go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.24.0 h1:qlJ3M9upxvFfwRM51tTg3Yl+8CP9vCC1E7vlFpgv99Y= golang.org/x/arch v0.24.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sda/go.mod b/sda/go.mod index 98508d67d0..1af2657c40 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -5,15 +5,15 @@ go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.32.13 - github.com/aws/aws-sdk-go-v2/credentials v1.19.13 - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.12 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 - github.com/aws/smithy-go v1.24.2 + github.com/aws/aws-sdk-go-v2/config v1.32.14 + github.com/aws/aws-sdk-go-v2/credentials v1.19.14 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.15 + github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 + github.com/aws/smithy-go v1.24.3 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 - github.com/coreos/go-oidc/v3 v3.17.0 - github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de + github.com/coreos/go-oidc/v3 v3.18.0 + github.com/dgraph-io/ristretto v0.2.0 github.com/gin-gonic/gin v1.12.0 github.com/go-viper/mapstructure/v2 v2.5.0 github.com/google/uuid v1.6.0 @@ -21,7 +21,7 @@ require ( github.com/iris-contrib/middleware/cors v0.0.0-20251225090426-92c6f28facda github.com/kataras/iris/v12 v12.2.11 github.com/lestrrat-go/jwx/v2 v2.1.6 - github.com/lib/pq v1.12.0 + github.com/lib/pq v1.12.3 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 github.com/neicnordic/crypt4gh v1.15.0 @@ -36,9 +36,9 @@ require ( github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.49.0 + golang.org/x/crypto v0.50.0 golang.org/x/oauth2 v0.36.0 - google.golang.org/grpc v1.79.3 + google.golang.org/grpc v1.80.0 google.golang.org/protobuf v1.36.11 ) @@ -65,8 +65,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect @@ -76,7 +76,6 @@ require ( github.com/casbin/govaluate v1.10.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/containerd/continuity v0.4.5 // indirect @@ -89,6 +88,7 @@ require ( github.com/docker/cli v29.2.0+incompatible // indirect github.com/docker/go-connections v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect @@ -96,7 +96,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.13 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.5 // indirect - github.com/go-jose/go-jose/v4 v4.1.3 // indirect + github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-playground/locales v0.14.1 // indirect @@ -174,9 +174,9 @@ require ( go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect - golang.org/x/net v0.51.0 // indirect - golang.org/x/sys v0.42.0 // indirect - golang.org/x/text v0.35.0 // indirect + golang.org/x/net v0.52.0 // indirect + golang.org/x/sys v0.43.0 // indirect + golang.org/x/text v0.36.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 // indirect gopkg.in/ini.v1 v1.67.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index dc19c92afc..a3c5d829b5 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -21,8 +21,6 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= -github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 h1:dG7/gLroJGht/jSQtHiLvT48Hxn+crbmvyItZC8cWOs= github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= @@ -33,14 +31,14 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.13 h1:5KgbxMaS2coSWRrx9TX/QtWbqzgQkOdEa3sZPhBhCSg= -github.com/aws/aws-sdk-go-v2/config v1.32.13/go.mod h1:8zz7wedqtCbw5e9Mi2doEwDyEgHcEE9YOJp6a8jdSMY= -github.com/aws/aws-sdk-go-v2/credentials v1.19.13 h1:mA59E3fokBvyEGHKFdnpNNrvaR351cqiHgRg+JzOSRI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.13/go.mod h1:yoTXOQKea18nrM69wGF9jBdG4WocSZA1h38A+t/MAsk= +github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= +github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.12 h1:JNrcxEplOFBfHDrbqIxf2pMzmuszlvXp8QTCz8Vl5Po= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.12/go.mod h1:sp1ofAB9uYALR+Q7s3BddGbb6Pne0Kucm9lNh45e/Uo= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.15 h1:92MfpwB6KjsPIEq9g3DniRPxOe92ew5hUz1h8W8cX7E= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.15/go.mod h1:7O129SmOn4acM++3oVfTLAeHmNOsj0y7AA7zmbgnGOk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= @@ -57,18 +55,18 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3x github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.14 h1:GcLE9ba5ehAQma6wlopUesYg/hbcOhFNWTjELkiWkh4= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.14/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18 h1:mP49nTpfKtpXLt5SLn8Uv8z6W+03jYVoOSAl/c02nog= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.18/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= +github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= @@ -91,8 +89,6 @@ github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEe github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= -github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= @@ -103,8 +99,8 @@ github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc= -github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8= +github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A= +github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= @@ -116,10 +112,10 @@ github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFn github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA= -github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgraph-io/ristretto v0.2.0 h1:XAfl+7cmoUDWW/2Lx8TGZQjjxIQ2Ley9DSf52dru4WE= +github.com/dgraph-io/ristretto v0.2.0/go.mod h1:8uBHCU/PBV4Ag0CJrP47b9Ofby5dqWNh4FicAdoqFNU= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= github.com/docker/cli v29.2.0+incompatible h1:9oBd9+YM7rxjZLfyMGxjraKBKE4/nVyvVfN4qNl9XRM= @@ -129,6 +125,8 @@ github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vma github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= @@ -149,8 +147,8 @@ github.com/gin-gonic/gin v1.12.0 h1:b3YAbrZtnf8N//yjKeU2+MQsh2mY5htkZidOM7O0wG8= github.com/gin-gonic/gin v1.12.0/go.mod h1:VxccKfsSllpKshkBWgVgRniFFAzFb9csfngsqANjnLc= github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ= github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs= -github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= +github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= +github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -258,8 +256,8 @@ github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVf github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= -github.com/lib/pq v1.12.0 h1:mC1zeiNamwKBecjHarAr26c/+d8V5w/u4J0I/yASbJo= -github.com/lib/pq v1.12.0/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.9.1 h1:LbtsOm5WAswyWbvTEOqhypdPeZzHavpZx96/n553mR8= @@ -347,8 +345,6 @@ github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= -github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= @@ -367,7 +363,6 @@ github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -442,8 +437,8 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4= -golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA= +golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI= +golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa h1:Zt3DZoOFFYkKhDT3v7Lm9FDMEV06GpzjG2jrqW+QTE0= golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa/go.mod h1:K79w1Vqn7PoiZn+TkNpx3BUWUQksGO3JcVX6qIjytmA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= @@ -459,8 +454,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= +golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= +golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw= golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs= golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -482,15 +477,15 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= -golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= +golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU= -golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A= +golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY= +golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -498,8 +493,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= -golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= +golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= +golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -512,12 +507,12 @@ golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= +google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= +google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -527,7 +522,6 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EV gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.1 h1:tVBILHy0R6e4wkYOn3XmiITt/hEVH4TFMYvAX2Ytz6k= gopkg.in/ini.v1 v1.67.1/go.mod h1:x/cyOwCgZqOkJoDIJ3c1KNHMo10+nLGAhh+kn3Zizss= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From 5cf743955cc63b8d59b3abd7f27dc6a214c8f7b9 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 16 Mar 2026 15:02:34 +0100 Subject: [PATCH 431/530] docs(adr): propose ADR-0003 replace s3inbox in-memory file ID cache The s3inbox in-memory map[uniqueFileID]string caused a fatal crash in production due to concurrent map writes (#2295). The existing DB fallback for cross-pod reconnection (#1358) also has a bug: it sends a broker message with an empty file ID before looking up the ID. Proposes removing the in-memory cache entirely and using PostgreSQL as the source of truth. The DB lookup moves before the broker send, fixing both the concurrency crash and the empty-fileID bug. Evaluates 4 options: DB-only (recommended), sync.RWMutex, ristretto, and Redis (deferred). Includes Mermaid sequence diagrams showing the current bug vs proposed fix across Pod A/Pod B. --- ...-state-strategy-for-s3inbox-and-caching.md | 234 ++++++++++++++++++ docs/decisions/README.md | 1 + 2 files changed, 235 insertions(+) create mode 100644 docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md diff --git a/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md b/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md new file mode 100644 index 0000000000..a6d5577734 --- /dev/null +++ b/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md @@ -0,0 +1,234 @@ +--- +status: proposed +date: "2026-03-13" +decision-makers: + - "@neicnordic/sensitive-data-development-collaboration" +consulted: [] +informed: [] +--- + +# Replace s3inbox In-Memory File ID Cache with Database Lookups + +## Context and Problem Statement + +The s3inbox service maintains an in-memory `map[uniqueFileID]string` +([proxy.go#L43][proxy-go-L43]) to track file IDs between the initial +`RegisterFile` call (on PUT) and the upload completion handler. This map is +accessed concurrently by HTTP handler goroutines with no synchronization, which +caused a **fatal crash in production** due to concurrent map writes +([#2295][issue-2295]): + +```text +fatal error: concurrent map writes + +goroutine 13331 [running]: +main.(*Proxy).allowedResponse(0xc0000a8180, ...) + /go/cmd/s3inbox/proxy.go:254 +0x10f6 +main.(*Proxy).ServeHTTP(0xc0000a8180, ...) + /go/cmd/s3inbox/proxy.go:109 +0x17b +``` + +A hotfix ([822354f][hotfix-commit]) added `sync.RWMutex` to the v3.1.0 +production branch. However, the underlying design has several remaining +problems: + +1. **Per-pod state** — when a client reconnects to a different s3inbox pod + (e.g., after a network interruption or load-balancer rotation), the file ID + is not available on the new pod. The code has a DB fallback for this case + ([proxy.go#L237][proxy-go-L237], added for [#1358][issue-1358]), but the + fallback runs *after* `checkAndSendMessage` has already been called with an + empty file ID ([proxy.go#L228][proxy-go-L228]), meaning a message with a + missing correlation ID is sent to the broker before the DB lookup occurs. +2. **TOCTOU races** — multiple map reads within the same handler can return + different results if another goroutine deletes or modifies the entry between + calls. +3. **Memory leak on incomplete uploads** — if an upload starts (registering in + the map) but never completes (no delete), the entry stays in memory forever. +4. **Duplicate overwrite messages** — `sendMessageOnOverwrite` + ([proxy.go#L578][proxy-go-L578]) fires based on an S3 `HeadObject` check, + not the in-memory map. Multiple pods receiving concurrent PUT requests for + the same file can each independently send an overwrite message. Note: this + problem is not caused by the in-memory cache and will not be solved by + replacing it — it requires separate coordination (out of scope for this ADR). + +[issue-2295]: https://github.com/neicnordic/sensitive-data-archive/issues/2295 +[issue-1358]: https://github.com/neicnordic/sensitive-data-archive/issues/1358 +[hotfix-commit]: https://github.com/neicnordic/sensitive-data-archive/commit/822354f668c08f3a3531cfa9dfc0ad8d3292805f +[proxy-go-L43]: https://github.com/neicnordic/sensitive-data-archive/blob/1001e975/sda/cmd/s3inbox/proxy.go#L43 +[proxy-go-L228]: https://github.com/neicnordic/sensitive-data-archive/blob/1001e975/sda/cmd/s3inbox/proxy.go#L228 +[proxy-go-L237]: https://github.com/neicnordic/sensitive-data-archive/blob/1001e975/sda/cmd/s3inbox/proxy.go#L237 +[proxy-go-L578]: https://github.com/neicnordic/sensitive-data-archive/blob/1001e975/sda/cmd/s3inbox/proxy.go#L578 + +## Decision Drivers + +* **Production stability** — the concurrent map crash already hit production; + the fix must be robust, not a band-aid. +* **Multi-pod correctness** — s3inbox runs behind a load balancer; any request + in an upload session may land on any pod. +* **Simplicity** — the team is small; the fix should reduce complexity, not add + it. + +## Considered Options + +1. **Remove the in-memory cache — use PostgreSQL as source of truth** +2. **sync.RWMutex with read-once pattern** (port the hotfix to main, improve it) +3. **Ristretto (in-process cache with TTL)** +4. **Redis as shared state layer** + +## Decision Outcome + +Proposed: **Option 1** — remove the in-memory cache entirely and retrieve the +file ID from PostgreSQL when needed. To be confirmed at the NeIC SDA-Devs +bi-weekly meet-up. + +### Consequences + +* Good, because all four in-memory cache problems (crash, per-pod state, TOCTOU, + memory leak) are eliminated. +* Good, because the multi-pod reconnection case becomes the normal path rather + than a buggy fallback. +* Neutral, because one additional DB query is added per completed upload, but + this is negligible next to the S3, broker, and DB operations already in the + same code path. +* Neutral, because the duplicate overwrite message problem (item 4) remains — + it is a separate concern requiring its own solution. + +### Confirmation + +The fix is confirmed when: + +* The `fileIDs` map and its associated `sync.RWMutex` (from the hotfix) are + removed from the `Proxy` struct. +* The existing integration tests in `proxy_test.go` pass without modification + (they exercise both single-part and multipart upload flows). +* A concurrent upload test reproducing the crash from #2295 can be added and + passes. + +## Pros and Cons of the Options + +### Option 1: Remove the in-memory cache — use PostgreSQL + +Eliminate the `fileIDs` map entirely. On upload completion — whether a +single-part PUT or a multipart POST (both handled by `uploadFinishedSuccessfully` +at [proxy.go#L326][proxy-go-L326]) — call `GetFileIDByUserPathAndStatus` to +retrieve the file ID from the database *before* sending the broker message or +updating the event log: + +[proxy-go-L326]: https://github.com/neicnordic/sensitive-data-archive/blob/1001e975/sda/cmd/s3inbox/proxy.go#L326 + +**Current flow** (simplified, showing the bug): + +```mermaid +sequenceDiagram + participant Client + participant PodA as s3inbox (Pod A) + participant PodB as s3inbox (Pod B) + participant DB as PostgreSQL + participant MQ as RabbitMQ + + Client->>PodA: PUT (upload start) + PodA->>DB: RegisterFile() + DB-->>PodA: fileID + Note over PodA: fileID stored in local map + + Note over Client,PodB: Client reconnects — routed to Pod B (no cached fileID) + + Client->>PodB: POST (multipart complete) + PodB->>MQ: checkAndSendMessage(fileID="") ❌ empty! + PodB->>DB: GetFileIDByUserPathAndStatus() (too late) + DB-->>PodB: fileID + PodB->>DB: storeObjectSizeInDB(fileID) + PodB->>DB: UpdateFileEventLog(fileID) +``` + +**Proposed flow** (Option 1): + +```mermaid +sequenceDiagram + participant Client + participant s3inbox + participant DB as PostgreSQL + participant MQ as RabbitMQ + + Client->>s3inbox: PUT (upload start) + s3inbox->>DB: RegisterFile() + DB-->>s3inbox: fileID + + Note over Client,s3inbox: Any pod can handle completion + + Client->>s3inbox: POST (multipart complete) + s3inbox->>DB: GetFileIDByUserPathAndStatus() + DB-->>s3inbox: fileID + s3inbox->>MQ: checkAndSendMessage(fileID) ✓ + s3inbox->>DB: storeObjectSizeInDB(fileID) + s3inbox->>DB: UpdateFileEventLog(fileID) +``` + +* Good, because it eliminates the entire class of concurrency bugs (no shared + mutable state). +* Good, because it is the simplest change — removes code rather than adding it. +* Good, because it works correctly across multiple pods with zero coordination. +* Good, because it eliminates the memory leak on incomplete uploads. +* Good, because it fixes the existing bug where the broker message is sent with + an empty file ID in the reconnection case. +* Neutral, because it adds one indexed DB query per completed upload. The same + code path already performs `RegisterFile`, `UpdateFileEventLog`, + `SetSubmissionFileSize`, S3 `HeadObject`, and broker `SendMessage`. + +### Option 2: sync.RWMutex with read-once pattern + +Port the production hotfix to `main` and improve it by reading the cached value +once per handler invocation, avoiding repeated lock acquisitions and TOCTOU +races. + +```go +p.mu.RLock() +cachedFileID := p.fileIDs[fileIdentifier] +p.mu.RUnlock() +// Use cachedFileID throughout — no further map reads +``` + +* Good, because it fixes the crash. +* Good, because it is the minimal change from the production hotfix. +* Good, because it addresses the TOCTOU race (Nanjiang's feedback in #2295). +* Bad, because per-pod state remains — the DB fallback is still needed for + cross-pod reconnection, and the existing fallback has the empty-fileID bug + described above. +* Bad, because the memory leak on incomplete uploads remains. +* Bad, because it adds synchronization complexity that must be maintained. + +### Option 3: Ristretto (in-process cache with TTL) + +Replace the `map` with a [ristretto][ristretto] cache (already used in the +download service). Entries auto-evict after a configurable TTL. + +[ristretto]: https://github.com/hypermodeinc/ristretto + +* Good, because it fixes the crash (ristretto is thread-safe). +* Good, because TTL-based eviction fixes the memory leak. +* Neutral, because it still has per-pod state — does not fix the multi-pod + problem or the empty-fileID bug. +* Bad, because TTL introduces a new failure mode: if the TTL is shorter than the + upload duration, the entry evicts mid-upload. + +### Option 4: Redis as shared state layer + +Add Redis to the SDA infrastructure for the s3inbox file ID mapping. + +* Good, because it solves per-pod state (shared across pods). +* Good, because TTL prevents memory leaks. +* Bad, because it adds a new infrastructure dependency (Redis). +* Bad, because it introduces a new failure mode (Redis unavailability). +* Bad, because the problem can be solved without new infrastructure (Option 1). + +Redis may still be worth evaluating for other cross-service caching needs +(e.g., shared session caching in the download service) if a measured need +arises. + +## More Information + +### Related issues + +* [#2295][issue-2295] — concurrent map read/write crash in s3inbox (production) +* [#1358][issue-1358] — client reconnecting to different s3inbox pod loses + file ID state diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f96474007d..f53eb67ac0 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -94,6 +94,7 @@ superseded-by: "0005-use-new-approach.md" | --- | --- | --- | | [0000](0000-use-markdown-architectural-decision-records.md) | Use Markdown Architectural Decision Records | accepted | | [0002](0002-merge-dependabot-package-managers.md) | Merge Dependabot package managers | accepted | +| [0003](0003-shared-state-strategy-for-s3inbox-and-caching.md) | Replace s3inbox In-Memory File ID Cache with Database Lookups | proposed | ## More information From bf0a5d237c71d2c8ffb55a4634a79ab6f4e33568 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 16 Mar 2026 16:59:34 +0100 Subject: [PATCH 432/530] docs(adr): clarify that round-robin triggers the per-pod state bug MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The empty-fileID bug is not limited to client reconnections — normal multipart uploads with parallel chunks are round-robin distributed across pods by default, so the completion request routinely lands on a different pod than the one that registered the file. Addresses review feedback from @jbygdell on PR #2319. --- ...0003-shared-state-strategy-for-s3inbox-and-caching.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md b/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md index a6d5577734..0f8107c957 100644 --- a/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md +++ b/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md @@ -32,9 +32,10 @@ A hotfix ([822354f][hotfix-commit]) added `sync.RWMutex` to the v3.1.0 production branch. However, the underlying design has several remaining problems: -1. **Per-pod state** — when a client reconnects to a different s3inbox pod - (e.g., after a network interruption or load-balancer rotation), the file ID - is not available on the new pod. The code has a DB fallback for this case +1. **Per-pod state** — when a request lands on a different s3inbox pod + (e.g., due to round-robin distribution of multipart upload chunks, or + client reconnection after a network interruption), the file ID is not + available on the new pod. The code has a DB fallback for this case ([proxy.go#L237][proxy-go-L237], added for [#1358][issue-1358]), but the fallback runs *after* `checkAndSendMessage` has already been called with an empty file ID ([proxy.go#L228][proxy-go-L228]), meaning a message with a @@ -131,7 +132,7 @@ sequenceDiagram DB-->>PodA: fileID Note over PodA: fileID stored in local map - Note over Client,PodB: Client reconnects — routed to Pod B (no cached fileID) + Note over Client,PodB: Multipart chunks round-robin across pods —
completion may land on Pod B (no cached fileID) Client->>PodB: POST (multipart complete) PodB->>MQ: checkAndSendMessage(fileID="") ❌ empty! From 6aae4c90d60c203b08a3e740309d2662b1c1b4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 13 Apr 2026 09:45:55 +0200 Subject: [PATCH 433/530] docs(adr-0003): update status to accepted --- .../0003-shared-state-strategy-for-s3inbox-and-caching.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md b/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md index 0f8107c957..6e388bc168 100644 --- a/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md +++ b/docs/decisions/0003-shared-state-strategy-for-s3inbox-and-caching.md @@ -1,5 +1,5 @@ --- -status: proposed +status: accepted date: "2026-03-13" decision-makers: - "@neicnordic/sensitive-data-development-collaboration" From 3abc301d3fe3dc7f35d1006ec75c9dc0e1e1c6c1 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 1 Apr 2026 15:04:37 +0200 Subject: [PATCH 434/530] feat(download): add lightweight dev compose for v2 API Adds a minimal Docker Compose stack for webapp development against the download v2 API, replacing the need for the full sda-s3-up pipeline. Services: postgres, minio, mockoidc (with /tokens endpoint), reencrypt, and the download service. Database is seeded with a real Crypt4GH encrypted test file with matching header, checksums, and sizes. Features: - GET /tokens returns a ready-to-use JWT (like v1's compose-no-tls) - Real c4gh file: all three download tiers work (combined, header, content) - Idempotent seed: safe to re-run against persistent volumes - Self-contained: all keys generated at startup, no host files needed - No container name collisions with other SDA compose stacks Usage: make dev-download-v2-up Refs: #2353 --- Makefile | 11 + dev-tools/download-v2-dev/README.md | 117 +++++++++ dev-tools/download-v2-dev/compose.yml | 228 ++++++++++++++++++ dev-tools/download-v2-dev/config.yaml | 76 ++++++ dev-tools/download-v2-dev/mockoidc.py | 198 +++++++++++++++ .../download-v2-dev/reencrypt-config.yaml | 10 + dev-tools/download-v2-dev/seed.sh | 102 ++++++++ 7 files changed, 742 insertions(+) create mode 100644 dev-tools/download-v2-dev/README.md create mode 100644 dev-tools/download-v2-dev/compose.yml create mode 100644 dev-tools/download-v2-dev/config.yaml create mode 100644 dev-tools/download-v2-dev/mockoidc.py create mode 100644 dev-tools/download-v2-dev/reencrypt-config.yaml create mode 100755 dev-tools/download-v2-dev/seed.sh diff --git a/Makefile b/Makefile index 30e17a980c..f7aaef8d81 100644 --- a/Makefile +++ b/Makefile @@ -174,6 +174,17 @@ integrationtest-sda-download-v2-up: build-all integrationtest-sda-download-v2-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-download-v2-integration.yml down -v --remove-orphans +# Lightweight dev stack for download API v2 (webapp development) +dev-download-v2-up: build-all + @PR_NUMBER=$$(date +%F) docker compose -f dev-tools/download-v2-dev/compose.yml up -d + @echo "" + @echo "Download API v2 ready at http://localhost:8085" + @echo "Get a token: TOKEN=\$$(curl -s http://localhost:8000/tokens | jq -r '.[0]')" + @echo "Try it: curl -H \"Authorization: Bearer \$$TOKEN\" http://localhost:8085/datasets" + +dev-download-v2-down: + @PR_NUMBER=$$(date +%F) docker compose -f dev-tools/download-v2-dev/compose.yml down -v --remove-orphans + # Download benchmark (compares old vs new public endpoints) # Uses sda-benchmark.yml which extends sda-s3-integration.yml with benchmark services # The benchmark runs in a container with auto-configuration from the environment diff --git a/dev-tools/download-v2-dev/README.md b/dev-tools/download-v2-dev/README.md new file mode 100644 index 0000000000..f9e84b8d37 --- /dev/null +++ b/dev-tools/download-v2-dev/README.md @@ -0,0 +1,117 @@ +# Download API v2 - Local Development + +Lightweight dev stack for building applications against the sda-download v2 API. + +## Quick Start + +```bash +# Build images first (only needed once, or after code changes) +make build-all + +# Start all services +make dev-download-v2-up + +# Get a token +TOKEN=$(curl -s http://localhost:8000/tokens | jq -r '.[0]') + +# List datasets +curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/datasets + +# List files in dataset +curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/datasets/EGAD00000000001/files +``` + +## Services + +| Service | Port | Description | +|----------|-------|--------------------------------------| +| download | 8085 | Download API v2 | +| mockauth | 8000 | Mock OIDC (JWKS, userinfo, /tokens) | +| postgres | 15432 | PostgreSQL with SDA schema | +| minio | 19000 | S3 storage (console at 19001) | +| reencrypt| 50051 | gRPC re-encryption for file downloads| + +## Getting Tokens + +```bash +# Get dev token (JSON array with one entry) +curl http://localhost:8000/tokens + +# Token for integration_test@example.org (has access to all test datasets) +TOKEN=$(curl -s http://localhost:8000/tokens | jq -r '.[0]') +``` + +## API Endpoints + +Full documentation: [sda/cmd/download/download.md](../../sda/cmd/download/download.md) + +```bash +# Health check +curl http://localhost:8085/health/ready + +# List datasets +curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/datasets + +# Dataset details +curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/datasets/EGAD00000000001 + +# List files in dataset +curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/datasets/EGAD00000000001/files + +# File metadata +curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/files/EGAF00000000001 + +# DRS object resolution +curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/objects/EGAD00000000001/test-file.c4gh + +# Get the c4gh public key from the running stack +docker cp $(docker ps -qf label=com.docker.compose.project=download-v2-dev -f label=com.docker.compose.service=reencrypt | head -1):/shared/c4gh.pub.pem /tmp/dev-c4gh.pub.pem +C4GH_KEY=$(base64 -w0 /tmp/dev-c4gh.pub.pem) + +# Download a file (requires c4gh public key) +curl -H "Authorization: Bearer $TOKEN" \ + -H "X-C4GH-Public-Key: $C4GH_KEY" \ + http://localhost:8085/files/EGAF00000000001 -o file.c4gh + +# Download content only (stable ETag, supports Range requests) +curl -H "Authorization: Bearer $TOKEN" \ + -H "X-C4GH-Public-Key: $C4GH_KEY" \ + http://localhost:8085/files/EGAF00000000001/content -o content.c4gh +``` + +## Test Data + +The stack is pre-seeded with: + +- **Dataset**: `EGAD00000000001` ("Test Dataset") +- **File**: `EGAF00000000001` (`test-file.c4gh`, 1000 bytes archived, 500 bytes decrypted) +- **User**: `integration_test@example.org` (file owner) + +## Using with sda-auth + +A JWT from `sda-auth` works with this download service as long as the public key +matches. Configure the download service with `JWT_PUBKEY_PATH` or `JWT_PUBKEY_URL` +pointing to sda-auth's public key. + +## Interactive Demo + +Run the sprint review demo script to see all endpoints in action: + +```bash +./dev-tools/download-v2-dev/demo.sh # all 7 steps +./dev-tools/download-v2-dev/demo.sh 6 # single step (e.g. file download) +``` + +## Limitations + +- **Authorization is bypassed.** The config sets `jwt.allow-all-data: true` so any valid + JWT can access all datasets. This simplifies getting started but means permission + denied paths (ownership model, visa grants) are not exercised locally. When building + auth flows against sda-auth, be aware that access control will behave differently in + real deployments. + +## Cleanup + +```bash +make dev-download-v2-down +``` diff --git a/dev-tools/download-v2-dev/compose.yml b/dev-tools/download-v2-dev/compose.yml new file mode 100644 index 0000000000..b858920cc2 --- /dev/null +++ b/dev-tools/download-v2-dev/compose.yml @@ -0,0 +1,228 @@ +# Lightweight dev stack for the sda-download v2 API. +# +# Starts the minimum services needed for local webapp development: +# postgres, minio, mockoidc, reencrypt, and the download service. +# Database is pre-seeded with a test dataset and file. +# +# Usage: +# docker compose up -d +# TOKEN=$(curl -s http://localhost:8000/tokens | jq -r '.[0]') +# curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/datasets + +services: + postgres: + build: + context: ../../postgresql + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER:-dev}-postgres + environment: + - POSTGRES_PASSWORD=rootpasswd + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres"] + interval: 10s + timeout: 2s + retries: 6 + ports: + - "15432:5432" + restart: always + volumes: + - postgres_data:/var/lib/postgresql/data + + minio: + image: minio/minio + command: server /data --console-address ":9001" + networks: + default: + aliases: + - s3 + environment: + - MINIO_ROOT_USER=access + - MINIO_ROOT_PASSWORD=secretKey + - MINIO_SERVER_URL=http://127.0.0.1:9000 + healthcheck: + test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"] + interval: 10s + timeout: 2s + retries: 6 + ports: + - "19000:9000" + - "19001:9001" + volumes: + - minio_data:/data + + # Generates JWT keys, c4gh keys, and a dev token + credentials: + command: + - "/bin/sh" + - "-c" + - | + set -e + apt-get -o DPkg::Lock::Timeout=60 update > /dev/null + apt-get -o DPkg::Lock::Timeout=60 install -y postgresql-client > /dev/null + pip install --upgrade pip > /dev/null + pip install joserfc crypt4gh > /dev/null + + # Create database role for download service + psql -U postgres -h postgres -d sda -c "ALTER ROLE download LOGIN PASSWORD 'download';" || true + psql -U postgres -h postgres -d sda -c "GRANT base TO download;" || true + + # Create RSA key pair for JWT signing + # Uses the same hardcoded key as mockoidc.py so tokens are interchangeable + mkdir -p /shared/keys/pub + if [ ! -f "/shared/keys/jwt.key" ]; then + cat > /shared/keys/jwt.key << 'PRIVKEY' + -----BEGIN PRIVATE KEY----- + MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDhuZjxPmOGUIW1 + LhxzKfxkN+1aTbvI5w+AptqT33X+bWuzfjvhEodiNz0bBfQgJJpQ3TZ8J1IZpM2F + Tnzox+FGxKPe5T9Mgngzd4N6eByWVPXoNMk7IdmBXMdPZBFSyjMW4ba1MELCpiKV + 05de4J5opRDwmHmyMqYJxBk78e3iiYYixVk+j1Ku+yFl4d2R29y2+O9PlZegJloe + 8FGnKIGZApS/8t9iyCkXg8WbjSPzgYCTQKxn/E4lcGdTrAt/McKrWmAuppcr+rpP + +BInm3l5Zu/QiRSZcMb5O460ojP9eKnaUlDpGZv9CY5j4x4lq8vjU2kK77YXBO8I + 2oxse5a5AgMBAAECggEABbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKP + RushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF+eqipSQXKghdKZCV + AsY8io0MmpXB1omDSygp7h3j52yHdayE2muav+VTAPOYn5QwG0/gGgVqYrR9x7CM + iTuyOIuGNO4Wlly4/5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI + 3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq/77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsn + mccF9plVTzem7N/vo+T+hFRPUO9TZUao91mMV8iV5QKBgQD1nZbQW3NHdol0fXA8 + nw5JRkTLZx1zcZ5l36WVPkwCjJOyXQ2vWHm4lz7F81Rr8dQnMKLWMDKjrBT9Dbfs + xYK2bYxENS1W/n+0jOIaX/792DY9tfX7vvHU9yGSdoJE5os6DGCHYInOD0xnRmnl + 3vS7gKv8miDwDzFsbjtDg6WfSwKBgQDrRLkmmfZCMcmLA02YSrErAlUseuyad7lY + HEJApXKfn262iHELlQa2zOBZpJGXIcHsNf1XGpMeU5pH+ILKE4Y5qbclq+AzFCcZ + nBFUfDeawmWdV5FJqNDd1L8Mb8aE+6q0Y5rNb3RL7A2ypH2ZeYKSGpHz3C7Rn5KW + voWAXRWriwKBgQCH4bxK3x0ivxiCgtcyIojDzwVGRnDLqmMIVzeDHqjsjBs2BTcJ + 9/e3QK1w1BKzeWF2oPilaJrLY+tkqE9FxWtwQ6DjJ0xDIZ9DIuH/13X5t8EiWOWS + devSdzpyje+58JW78pcArk7u2hXZ2OHDU5qvlRsRL6/jP3SHWWCeFFnviwKBgGov + M02r0YygwfEfBYeFtp7Nx7lypZU2Eg4levWIdsp6f9KclEEA+u3IXD25XAiVMNw2 + pegJU3stioWPMSCZXUxrQAEdqOwE3XzehqfWBJaxxIEWQ7m2Gsb0PWIUlMnyeGJA + Tl8IPboCiVAmk5WQVREyMsuYhf0Qg23MAZ8k5CHvAoGBAJm55NQZVKAEDGd4a21q + TDcRddtPwwL2oP3qa0gbGk4YFRUCrX99hIejOTvQW1xf6vGxTd7E1QizvFse4yRz + ZRKyXIc7DCcdzOnpMrSd1+aXwZtRHLSw0EDS6PWeJZdjJYHxl2YpAmMdURdcGTrH + b6b/6vhU90+xL14CX7Awofp/ + -----END PRIVATE KEY----- + PRIVKEY + cat > /shared/keys/pub/jwt.pub << 'PUBKEY' + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4bmY8T5jhlCFtS4ccyn8 + ZDftWk27yOcPgKbak991/m1rs3474RKHYjc9GwX0ICSaUN02fCdSGaTNhU586Mfh + RsSj3uU/TIJ4M3eDengcllT16DTJOyHZgVzHT2QRUsozFuG2tTBCwqYildOXXuCe + aKUQ8Jh5sjKmCcQZO/Ht4omGIsVZPo9SrvshZeHdkdvctvjvT5WXoCZaHvBRpyiB + mQKUv/LfYsgpF4PFm40j84GAk0CsZ/xOJXBnU6wLfzHCq1pgLqaXK/q6T/gSJ5t5 + eWbv0IkUmXDG+TuOtKIz/Xip2lJQ6Rmb/QmOY+MeJavL41NpCu+2FwTvCNqMbHuW + uQIDAQAB + -----END PUBLIC KEY----- + PUBKEY + chmod 644 /shared/keys/pub/jwt.pub /shared/keys/jwt.key + fi + + # Create trusted issuers for visa validation + cat > "/shared/trusted-issuers.json" <<'ISSUERS' + [ + { + "iss": "https://demo.example", + "jku": "http://mockauth:8000/jwks" + } + ] + ISSUERS + + # Generate c4gh key pair (used by reencrypt and seed) + if [ ! -f "/shared/c4gh.pub.pem" ]; then + printf 'c4ghpass\nc4ghpass\n' | crypt4gh-keygen --sk /shared/c4gh.sec.pem --pk /shared/c4gh.pub.pem + chmod 644 /shared/c4gh.sec.pem /shared/c4gh.pub.pem + fi + + echo "Credentials ready" + depends_on: + postgres: + condition: service_healthy + environment: + - PGPASSWORD=rootpasswd + image: python:3.11-slim-bookworm + volumes: + - shared:/shared + + # Create a real Crypt4GH test file, upload to MinIO, seed database + seed: + command: + - /bin/sh + - -c + - | + apt-get -o DPkg::Lock::Timeout=60 update > /dev/null + apt-get -o DPkg::Lock::Timeout=60 install -y curl postgresql-client > /dev/null + pip install --upgrade pip > /dev/null + pip install crypt4gh > /dev/null + curl -sL https://dl.min.io/client/mc/release/linux-amd64/mc -o /usr/local/bin/mc + chmod +x /usr/local/bin/mc + sh /seed.sh + depends_on: + credentials: + condition: service_completed_successfully + minio: + condition: service_healthy + postgres: + condition: service_healthy + environment: + - PGPASSWORD=rootpasswd + image: python:3.11-slim-bookworm + volumes: + - ./seed.sh:/seed.sh:ro + - shared:/shared + + # Mock OIDC server with /tokens endpoint for easy token retrieval + mockauth: + command: + - /bin/sh + - -c + - | + pip install --upgrade pip > /dev/null + pip install aiohttp Authlib joserfc > /dev/null + python -u /mockoidc.py + image: python:3.10-slim + volumes: + - ./mockoidc.py:/mockoidc.py + ports: + - "8000:8000" + restart: always + + # gRPC re-encryption service (needed for file downloads) + reencrypt: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER:-dev} + command: [sda-reencrypt] + depends_on: + credentials: + condition: service_completed_successfully + ports: + - "50051:50051" + restart: always + volumes: + - ./reencrypt-config.yaml:/config.yaml + - shared:/shared + + # Download service v2 + download: + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER:-dev} + command: + - sda-download + - "--config-file=/config.yaml" + depends_on: + credentials: + condition: service_completed_successfully + seed: + condition: service_completed_successfully + postgres: + condition: service_healthy + reencrypt: + condition: service_started + mockauth: + condition: service_started + environment: + - CONFIG_FILE=/config.yaml + ports: + - "8085:8080" + restart: always + volumes: + - ./config.yaml:/config.yaml + - shared:/shared + +volumes: + minio_data: + postgres_data: + shared: diff --git a/dev-tools/download-v2-dev/config.yaml b/dev-tools/download-v2-dev/config.yaml new file mode 100644 index 0000000000..9267b36ad9 --- /dev/null +++ b/dev-tools/download-v2-dev/config.yaml @@ -0,0 +1,76 @@ +# Download service v2 - local development configuration + +api: + host: "0.0.0.0" + port: 8080 + +service: + id: "neicnordic.sda.download" + org-name: "NBIS" + org-url: "https://nbis.se" + +db: + host: "postgres" + port: 5432 + user: "postgres" + password: "rootpasswd" + database: "sda" + sslmode: "disable" + +grpc: + host: "reencrypt" + port: 50051 + timeout: 30 + +jwt: + pubkey-path: "/shared/keys/pub/" + allow-all-data: true + +permission: + model: "combined" + +auth: + allow-opaque: false + +visa: + enabled: true + source: "userinfo" + userinfo-url: "http://mockauth:8000/userinfo" + trusted-issuers-path: "/shared/trusted-issuers.json" + dataset-id-mode: "raw" + identity: + mode: "broker-bound" + validate-asserted: true + allow-insecure-jku: true + limits: + max-visas: 200 + max-jwks-per-request: 10 + max-visa-size: 16384 + cache: + token-ttl: 3600 + jwk-ttl: 300 + validation-ttl: 120 + userinfo-ttl: 60 + +session: + expiration: 3600 + secure: false + http-only: true + name: "sda_session" + +cache: + enabled: true + file-ttl: 300 + permission-ttl: 120 + dataset-ttl: 300 + +storage: + backend: "archive" + archive: + s3: + - endpoint: "http://s3:9000" + access_key: "access" + secret_key: "secretKey" + region: "us-east-1" + bucket_prefix: "archive" + disable_https: true diff --git a/dev-tools/download-v2-dev/mockoidc.py b/dev-tools/download-v2-dev/mockoidc.py new file mode 100644 index 0000000000..99734649b6 --- /dev/null +++ b/dev-tools/download-v2-dev/mockoidc.py @@ -0,0 +1,198 @@ +"""Mock OIDC server for download service local development. + +Provides JWT/JWKS validation, userinfo with GA4GH visas, and a /tokens +endpoint for easy token retrieval (matching the v1 developer experience). + +Uses the same RSA key as make_download_credentials.sh so tokens are +accepted by the download service. +""" + +from aiohttp import web +from authlib.jose import jwt, RSAKey +import json +import base64 +import time + +# RSA private key - same as in make_download_credentials.sh +PRIVATE_KEY_PEM = """-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDhuZjxPmOGUIW1 +LhxzKfxkN+1aTbvI5w+AptqT33X+bWuzfjvhEodiNz0bBfQgJJpQ3TZ8J1IZpM2F +Tnzox+FGxKPe5T9Mgngzd4N6eByWVPXoNMk7IdmBXMdPZBFSyjMW4ba1MELCpiKV +05de4J5opRDwmHmyMqYJxBk78e3iiYYixVk+j1Ku+yFl4d2R29y2+O9PlZegJloe +8FGnKIGZApS/8t9iyCkXg8WbjSPzgYCTQKxn/E4lcGdTrAt/McKrWmAuppcr+rpP ++BInm3l5Zu/QiRSZcMb5O460ojP9eKnaUlDpGZv9CY5j4x4lq8vjU2kK77YXBO8I +2oxse5a5AgMBAAECggEABbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKP +RushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF+eqipSQXKghdKZCV +AsY8io0MmpXB1omDSygp7h3j52yHdayE2muav+VTAPOYn5QwG0/gGgVqYrR9x7CM +iTuyOIuGNO4Wlly4/5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI +3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq/77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsn +mccF9plVTzem7N/vo+T+hFRPUO9TZUao91mMV8iV5QKBgQD1nZbQW3NHdol0fXA8 +nw5JRkTLZx1zcZ5l36WVPkwCjJOyXQ2vWHm4lz7F81Rr8dQnMKLWMDKjrBT9Dbfs +xYK2bYxENS1W/n+0jOIaX/792DY9tfX7vvHU9yGSdoJE5os6DGCHYInOD0xnRmnl +3vS7gKv8miDwDzFsbjtDg6WfSwKBgQDrRLkmmfZCMcmLA02YSrErAlUseuyad7lY +HEJApXKfn262iHELlQa2zOBZpJGXIcHsNf1XGpMeU5pH+ILKE4Y5qbclq+AzFCcZ +nBFUfDeawmWdV5FJqNDd1L8Mb8aE+6q0Y5rNb3RL7A2ypH2ZeYKSGpHz3C7Rn5KW +voWAXRWriwKBgQCH4bxK3x0ivxiCgtcyIojDzwVGRnDLqmMIVzeDHqjsjBs2BTcJ +9/e3QK1w1BKzeWF2oPilaJrLY+tkqE9FxWtwQ6DjJ0xDIZ9DIuH/13X5t8EiWOWS +devSdzpyje+58JW78pcArk7u2hXZ2OHDU5qvlRsRL6/jP3SHWWCeFFnviwKBgGov +M02r0YygwfEfBYeFtp7Nx7lypZU2Eg4levWIdsp6f9KclEEA+u3IXD25XAiVMNw2 +pegJU3stioWPMSCZXUxrQAEdqOwE3XzehqfWBJaxxIEWQ7m2Gsb0PWIUlMnyeGJA +Tl8IPboCiVAmk5WQVREyMsuYhf0Qg23MAZ8k5CHvAoGBAJm55NQZVKAEDGd4a21q +TDcRddtPwwL2oP3qa0gbGk4YFRUCrX99hIejOTvQW1xf6vGxTd7E1QizvFse4yRz +ZRKyXIc7DCcdzOnpMrSd1+aXwZtRHLSw0EDS6PWeJZdjJYHxl2YpAmMdURdcGTrH +b6b/6vhU90+xL14CX7Awofp/ +-----END PRIVATE KEY-----""" + +# Load RSA key +KEY = RSAKey.import_key(PRIVATE_KEY_PEM) +PUBLIC_JWK = KEY.as_dict(is_private=False) +PUBLIC_JWK["kid"] = "rsa1" +PUBLIC_JWK["use"] = "sig" +PUBLIC_JWK["alg"] = "RS256" +PRIVATE_JWK = dict(KEY) + + +def generate_token(sub: str) -> str: + """Generate a signed JWT for a user.""" + header = { + "alg": "RS256", + "typ": "JWT", + "kid": "rsa1", + } + now = int(time.time()) + payload = { + "iss": "http://mockauth:8000", + "sub": sub, + "aud": "XC56EL11xx", + "iat": now, + "exp": now + 86400, # 24 hours + "jti": f"dev-token-{sub}", + } + return jwt.encode(header, payload, PRIVATE_JWK).decode("utf-8") + + +def generate_visa(dataset: str, sub: str) -> str: + """Generate a signed visa JWT for a dataset.""" + header = { + "alg": "RS256", + "typ": "JWT", + "kid": "rsa1", + "jku": "http://mockauth:8000/jwks", + } + payload = { + "iss": "https://demo.example", + "sub": sub, + "ga4gh_visa_v1": { + "type": "ControlledAccessGrants", + "value": dataset, + "source": "https://doi.example/no_org", + "by": "self", + "asserted": 1568699331, + }, + "iat": 1571144438, + "exp": 9999999999, + "jti": f"visa-{dataset}", + } + return jwt.encode(header, payload, PRIVATE_JWK).decode("utf-8") + + +# Pre-generate visas for test datasets +VISAS = {} + + +def get_visas_for_user(sub: str) -> list: + """Get visas for a user, generating them if needed.""" + if sub not in VISAS: + VISAS[sub] = [ + generate_visa("EGAD00000000001", sub), + ] + return VISAS[sub] + + +async def well_known(request: web.Request) -> web.Response: + """Return OIDC discovery document.""" + print("[mockoidc] GET /.well-known/openid-configuration") + config = { + "issuer": "http://mockauth:8000", + "authorization_endpoint": "http://mockauth:8000/authorize", + "token_endpoint": "http://mockauth:8000/token", + "userinfo_endpoint": "http://mockauth:8000/userinfo", + "jwks_uri": "http://mockauth:8000/jwks", + "response_types_supported": ["code", "token"], + "subject_types_supported": ["public"], + "id_token_signing_alg_values_supported": ["RS256"], + "scopes_supported": ["openid", "ga4gh_passport_v1"], + "claims_supported": ["sub", "iss", "aud", "exp", "iat", "ga4gh_passport_v1"], + } + return web.json_response(config) + + +async def jwks(request: web.Request) -> web.Response: + """Return JSON Web Key Set.""" + print("[mockoidc] GET /jwks") + return web.json_response({"keys": [PUBLIC_JWK]}) + + +async def tokens(request: web.Request) -> web.Response: + """Serve pre-generated tokens for development. + + Returns a JSON array with one token for integration_test@example.org + which has access to all seeded test datasets. + """ + print("[mockoidc] GET /tokens — generating dev token") + data = [ + generate_token("integration_test@example.org"), + ] + return web.json_response(data) + + +async def userinfo(request: web.Request) -> web.Response: + """Return user info with GA4GH visas.""" + auth_header = request.headers.get("Authorization", "") + if not auth_header.startswith("Bearer "): + return web.json_response( + {"error": "invalid_token", "error_description": "Missing bearer token"}, + status=401, + ) + + print("[mockoidc] GET /userinfo") + token = auth_header.split(" ")[1] + try: + parts = token.split(".") + if len(parts) != 3: + raise ValueError("Not a JWT") + + payload_b64 = parts[1].replace("-", "+").replace("_", "/") + padding = 4 - len(payload_b64) % 4 + if padding != 4: + payload_b64 += "=" * padding + + payload = json.loads(base64.b64decode(payload_b64)) + sub = payload.get("sub", "unknown@example.org") + except Exception: + sub = token + + visas = get_visas_for_user(sub) + response = { + "sub": sub, + "ga4gh_passport_v1": visas, + } + return web.json_response(response) + + +def init() -> web.Application: + """Initialize the web application.""" + app = web.Application() + app.router.add_get("/.well-known/openid-configuration", well_known) + app.router.add_get("/jwks", jwks) + app.router.add_get("/tokens", tokens) + app.router.add_get("/userinfo", userinfo) + return app + + +if __name__ == "__main__": + print("[mockoidc] Starting mock OIDC server on port 8000") + print("[mockoidc] GET /tokens - fetch dev tokens") + print("[mockoidc] GET /jwks - JSON Web Key Set") + print("[mockoidc] GET /userinfo - user info with visas") + web.run_app(init(), port=8000) diff --git a/dev-tools/download-v2-dev/reencrypt-config.yaml b/dev-tools/download-v2-dev/reencrypt-config.yaml new file mode 100644 index 0000000000..73d415cc3e --- /dev/null +++ b/dev-tools/download-v2-dev/reencrypt-config.yaml @@ -0,0 +1,10 @@ +log: + format: "json" + level: "debug" + +c4gh: + filePath: /shared/c4gh.sec.pem + passphrase: "c4ghpass" + privateKeys: + - filePath: /shared/c4gh.sec.pem + passphrase: "c4ghpass" diff --git a/dev-tools/download-v2-dev/seed.sh b/dev-tools/download-v2-dev/seed.sh new file mode 100755 index 0000000000..a9c77217cb --- /dev/null +++ b/dev-tools/download-v2-dev/seed.sh @@ -0,0 +1,102 @@ +#!/bin/sh +# Seed script for download v2 dev compose. +# Creates a real Crypt4GH encrypted test file, uploads data segments to MinIO, +# and inserts matching metadata (header, checksums, sizes) into the database. +# Idempotent: safe to re-run against existing volumes. +set -e + +echo "=== Seeding test data ===" + +# Create deterministic plaintext test data (1 KB) +python3 -c "import hashlib; open('/tmp/plaintext.bin','wb').write(hashlib.sha256(b'sda-download-v2-dev-test-data').digest() * 32)" + +# Encrypt with crypt4gh using the server's public key +crypt4gh encrypt --recipient_pk /shared/c4gh.pub.pem < /tmp/plaintext.bin > /tmp/encrypted.c4gh + +# Split: extract header and data segments, compute checksums +python3 << 'PYEOF' +import struct, hashlib + +with open("/tmp/encrypted.c4gh", "rb") as f: + data = f.read() + +# Parse c4gh header +magic = data[:8] +assert magic == b"crypt4gh", f"Bad magic: {magic}" +packet_count = struct.unpack(" Date: Wed, 1 Apr 2026 15:04:48 +0200 Subject: [PATCH 435/530] feat(download): add interactive API demo script Sprint review demo showcasing all v2 endpoints: health, service-info, auth, datasets, files, DRS resolution, three-tier downloads with Range requests, and error handling with RFC 9457 Problem Details. Supports --tmux flag for split-screen with live service logs. Auto-starts the dev stack if not already running. --- dev-tools/download-v2-dev/demo.sh | 233 ++++++++++++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100755 dev-tools/download-v2-dev/demo.sh diff --git a/dev-tools/download-v2-dev/demo.sh b/dev-tools/download-v2-dev/demo.sh new file mode 100755 index 0000000000..eeada03a4a --- /dev/null +++ b/dev-tools/download-v2-dev/demo.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# Sprint Review Demo Script — SDA Download API v2 +# +# Starts the dev stack automatically if not already running. +# Run from the repository root. +# +# Usage: +# ./dev-tools/download-v2-dev/demo.sh # run all steps interactively +# ./dev-tools/download-v2-dev/demo.sh # run a single step (1-7) +# ./dev-tools/download-v2-dev/demo.sh --tmux # tmux split with service logs +# ./dev-tools/download-v2-dev/demo.sh --tmux # tmux + single step + +set -euo pipefail + +BASE="http://localhost:8085" +AUTH="http://localhost:8000" +BOLD='\033[1m' +DIM='\033[2m' +CYAN='\033[1;36m' +GREEN='\033[1;32m' +YELLOW='\033[1;33m' +RED='\033[1;31m' +RESET='\033[0m' +TOKEN="" + +# ── helpers ────────────────────────────────────────────────────────── + +banner() { + echo "" + echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" + echo -e "${BOLD} $1${RESET}" + echo -e "${CYAN}━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━${RESET}" +} + +narrate() { + echo -e "\n${DIM}# $1${RESET}" +} + +run() { + local display_cmd="$1" + # Show the command with $TOKEN truncated for readability + if [[ -n "$TOKEN" ]]; then + local short_token="${TOKEN:0:20}..." + echo -e "${YELLOW}\$ ${display_cmd//$TOKEN/$short_token}${RESET}" + else + echo -e "${YELLOW}\$ $display_cmd${RESET}" + fi + eval "$1" + echo "" +} + +pause() { + echo -e "${DIM} [press Enter to continue]${RESET}" + read -r +} + +# ── token setup ────────────────────────────────────────────────────── + +get_token() { + TOKEN=$(curl -sf "$AUTH/tokens" | jq -r '.[0]') + if [[ -z "$TOKEN" || "$TOKEN" == "null" ]]; then + echo -e "${RED}ERROR: Could not get token from mockauth. Is docker compose running?${RESET}" + exit 1 + fi + echo -e "${GREEN}Token acquired from mock OIDC.${RESET}" +} + +# ── demo steps ─────────────────────────────────────────────────────── + +step1() { + banner "1/7 Health Check" + narrate "Liveness — is the process up?" + run "curl -s $BASE/health/live | jq ." + + narrate "Readiness — are all dependencies healthy? (DB, storage, gRPC)" + run "curl -s $BASE/health/ready | jq ." + pause +} + +step2() { + banner "2/7 GA4GH Service Info" + narrate "Public endpoint, no auth required." + narrate "Follows the GA4GH service-info spec — ID and organization are configurable." + run "curl -s $BASE/service-info | jq ." + pause +} + +step3() { + banner "3/7 Authentication & Dataset Listing" + narrate "Get a JWT from the mock OIDC provider, then list accessible datasets." + get_token + echo "" + run "curl -s -H \"Authorization: Bearer $TOKEN\" $BASE/datasets | jq ." + pause +} + +step4() { + banner "4/7 Dataset Details & Files" + narrate "Drill into a dataset and list its files." + run "curl -s -H \"Authorization: Bearer $TOKEN\" $BASE/datasets/EGAD00000000001 | jq ." + + narrate "Files in the dataset:" + run "curl -s -H \"Authorization: Bearer $TOKEN\" $BASE/datasets/EGAD00000000001/files | jq ." + pause +} + +step5() { + banner "5/7 DRS Object Resolution (GA4GH DRS 1.5)" + narrate "Resolve dataset + file path → DRS object with download URL." + narrate "This is how htsget-rs and other GA4GH clients discover files." + run "curl -s -H \"Authorization: Bearer $TOKEN\" $BASE/objects/EGAD00000000001/test-file.c4gh | jq ." + pause +} + +step6() { + banner "6/7 File Download (Crypt4GH)" + narrate "Three-tier download: combined, header-only, content-only." + narrate "Downloads require a Crypt4GH public key in the X-C4GH-Public-Key header." + + # Get c4gh public key from the running stack + local c4gh_key="" + local container_id + container_id=$(docker ps -qf 'label=com.docker.compose.project=download-v2-dev' -f 'label=com.docker.compose.service=reencrypt' | head -1) + if [[ -n "$container_id" ]]; then + docker cp "$container_id:/shared/c4gh.pub.pem" /tmp/dev-c4gh.pub.pem 2>/dev/null + c4gh_key=$(base64 -w0 /tmp/dev-c4gh.pub.pem 2>/dev/null) + fi + if [[ -z "$c4gh_key" ]]; then + narrate "WARNING: Could not get c4gh key from container — download will fail with 400." + c4gh_key="missing" + fi + + narrate "HEAD request — file size and ETag without downloading:" + run "curl -sI -H \"Authorization: Bearer $TOKEN\" -H \"X-C4GH-Public-Key: $c4gh_key\" $BASE/files/EGAF00000000001" + + narrate "Combined download (re-encrypted header + data segments):" + run "curl -s -o /dev/null -w 'HTTP %{http_code} Size: %{size_download} bytes\n' -H \"Authorization: Bearer $TOKEN\" -H \"X-C4GH-Public-Key: $c4gh_key\" $BASE/files/EGAF00000000001" + + narrate "Content-only with Range request (first 512 bytes):" + run "curl -s -o /dev/null -w 'HTTP %{http_code} Size: %{size_download} bytes\n' -H \"Authorization: Bearer $TOKEN\" -H \"X-C4GH-Public-Key: $c4gh_key\" -H 'Range: bytes=0-511' $BASE/files/EGAF00000000001/content" + pause +} + +step7() { + banner "7/7 Error Handling (RFC 9457 Problem Details)" + narrate "No token → 401:" + run "curl -s $BASE/datasets | jq ." + + narrate "Non-existent resource → 403 (prevents existence leakage):" + run "curl -s -H \"Authorization: Bearer $TOKEN\" $BASE/datasets/DOES_NOT_EXIST | jq ." + pause +} + +# ── auto-start ────────────────────────────────────────────────────── + +ensure_running() { + if curl -sf "$BASE/health/live" > /dev/null 2>&1; then + return + fi + REPO_ROOT=$(git rev-parse --show-toplevel 2>/dev/null || echo ".") + echo -e "${YELLOW}Dev stack not running — starting with make dev-download-v2-up ...${RESET}" + make -C "$REPO_ROOT" dev-download-v2-up + echo "Waiting for download service ..." + for _ in $(seq 1 30); do + if curl -sf "$BASE/health/live" > /dev/null 2>&1; then + echo -e "${GREEN}Ready.${RESET}" + return + fi + sleep 2 + done + echo "ERROR: download service did not become ready in time." + exit 1 +} + +# ── tmux mode ─────────────────────────────────────────────────────── + +launch_tmux() { + local step="${1:-all}" + local script + script=$(readlink -f "$0") + local session="sda-demo" + + if tmux has-session -t "$session" 2>/dev/null; then + tmux kill-session -t "$session" + fi + + local repo_root + repo_root=$(git rev-parse --show-toplevel 2>/dev/null || echo ".") + + # Start logs pane first so it catches all requests + tmux new-session -d -s "$session" + tmux send-keys -t "$session" "PR_NUMBER=\$(date +%F) docker compose -f $repo_root/dev-tools/download-v2-dev/compose.yml logs -f --tail 0 download mockauth" Enter + + # Demo pane on the left + tmux split-window -h -b -t "$session" + tmux send-keys -t "$session" "sleep 2 && $script $step" Enter + + tmux attach -t "$session" +} + +# ── main ───────────────────────────────────────────────────────────── + +main() { + ensure_running + banner "SDA Download API v2 — Sprint Review Demo" + echo "" + echo -e " Download API: ${BOLD}$BASE${RESET}" + echo -e " Mock OIDC: ${BOLD}$AUTH${RESET}" + echo -e " Test dataset: ${BOLD}EGAD00000000001${RESET}" + echo -e " Test file: ${BOLD}EGAF00000000001${RESET}" + echo "" + + if [[ ${1:-all} == "all" ]]; then + get_token + pause + step1; step2; step3; step4; step5; step6; step7 + banner "Demo Complete!" + else + if [[ $1 -ge 3 ]]; then + get_token + fi + "step$1" + fi +} + +# ── entrypoint ────────────────────────────────────────────────────── + +if [[ "${1:-}" == "--tmux" ]]; then + shift + launch_tmux "${1:-all}" +else + main "${1:-all}" +fi From d7153078a31ec0a623bf221bc9fa93714a3ba636 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 2 Apr 2026 08:41:00 +0200 Subject: [PATCH 436/530] fix(download): update Python images to 3.14-slim-trixie Bookworm is EOL June 2026 and Python 3.11/3.10 are in security maintenance mode. --- dev-tools/download-v2-dev/compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-tools/download-v2-dev/compose.yml b/dev-tools/download-v2-dev/compose.yml index b858920cc2..6cf66706bd 100644 --- a/dev-tools/download-v2-dev/compose.yml +++ b/dev-tools/download-v2-dev/compose.yml @@ -135,7 +135,7 @@ services: condition: service_healthy environment: - PGPASSWORD=rootpasswd - image: python:3.11-slim-bookworm + image: python:3.14-slim-trixie volumes: - shared:/shared @@ -161,7 +161,7 @@ services: condition: service_healthy environment: - PGPASSWORD=rootpasswd - image: python:3.11-slim-bookworm + image: python:3.14-slim-trixie volumes: - ./seed.sh:/seed.sh:ro - shared:/shared @@ -175,7 +175,7 @@ services: pip install --upgrade pip > /dev/null pip install aiohttp Authlib joserfc > /dev/null python -u /mockoidc.py - image: python:3.10-slim + image: python:3.14-slim-trixie volumes: - ./mockoidc.py:/mockoidc.py ports: From 5da3b99d8f9ed8b7b1c0b5782cedc2feabbcbcf5 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 2 Apr 2026 08:43:02 +0200 Subject: [PATCH 437/530] fix(download): suppress shellcheck SC1091 for runtime-generated env file --- dev-tools/download-v2-dev/seed.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/dev-tools/download-v2-dev/seed.sh b/dev-tools/download-v2-dev/seed.sh index a9c77217cb..1146a8f01b 100755 --- a/dev-tools/download-v2-dev/seed.sh +++ b/dev-tools/download-v2-dev/seed.sh @@ -57,6 +57,7 @@ mc pipe myminio/archive/test-file.c4gh < /tmp/body.bin echo "Uploaded to MinIO: archive/test-file.c4gh" # Load computed metadata +# shellcheck source=/dev/null . /tmp/seed_metadata.env # Seed database (upserts so reruns are safe with persistent volumes) From 608878fa453770fcb3962bf0239366905c125982 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 15 Apr 2026 09:49:17 +0200 Subject: [PATCH 438/530] fix(download): address Copilot review comments on dev compose MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Pin compose project name with `name: download-v2-dev` so label-based container lookups (in demo.sh and README) work regardless of how the stack is invoked (e.g. from a different working directory or with a custom COMPOSE_PROJECT_NAME). - Update mockoidc.py docstring to clarify that /userinfo does not verify JWT signatures — it only decodes the payload to extract sub. - Remove stale byte counts from README test data section; the seed script now produces different sizes than the old fake-data era docs. --- dev-tools/download-v2-dev/README.md | 2 +- dev-tools/download-v2-dev/compose.yml | 2 ++ dev-tools/download-v2-dev/mockoidc.py | 7 +++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/dev-tools/download-v2-dev/README.md b/dev-tools/download-v2-dev/README.md index f9e84b8d37..fb424e29c4 100644 --- a/dev-tools/download-v2-dev/README.md +++ b/dev-tools/download-v2-dev/README.md @@ -84,7 +84,7 @@ curl -H "Authorization: Bearer $TOKEN" \ The stack is pre-seeded with: - **Dataset**: `EGAD00000000001` ("Test Dataset") -- **File**: `EGAF00000000001` (`test-file.c4gh`, 1000 bytes archived, 500 bytes decrypted) +- **File**: `EGAF00000000001` (`test-file.c4gh`) - **User**: `integration_test@example.org` (file owner) ## Using with sda-auth diff --git a/dev-tools/download-v2-dev/compose.yml b/dev-tools/download-v2-dev/compose.yml index 6cf66706bd..7817566d9e 100644 --- a/dev-tools/download-v2-dev/compose.yml +++ b/dev-tools/download-v2-dev/compose.yml @@ -9,6 +9,8 @@ # TOKEN=$(curl -s http://localhost:8000/tokens | jq -r '.[0]') # curl -H "Authorization: Bearer $TOKEN" http://localhost:8085/datasets +name: download-v2-dev + services: postgres: build: diff --git a/dev-tools/download-v2-dev/mockoidc.py b/dev-tools/download-v2-dev/mockoidc.py index 99734649b6..8fff09ff2b 100644 --- a/dev-tools/download-v2-dev/mockoidc.py +++ b/dev-tools/download-v2-dev/mockoidc.py @@ -1,7 +1,10 @@ """Mock OIDC server for download service local development. -Provides JWT/JWKS validation, userinfo with GA4GH visas, and a /tokens -endpoint for easy token retrieval (matching the v1 developer experience). +Provides OIDC discovery, JWKS metadata, userinfo with GA4GH visas, and a +/tokens endpoint for easy token retrieval (matching the v1 developer +experience). Tokens are generated for local dev; the /userinfo handler +does not verify JWT signatures or claims — it only decodes the payload +to extract the subject. Uses the same RSA key as make_download_credentials.sh so tokens are accepted by the download service. From 70a206b9cbee7dc86eb6462d2864c812232f9c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 11:36:55 +0200 Subject: [PATCH 439/530] feat(s3inbox): use postgres instead of in memory cache for fileIds - Add new DB func GetFileIDInInbox which returns the file id of a file if the latest file event deems it should exist in the inbox - Also update dedection and mapping of s3 actions - Only explicitly allow GetBucketLocation, ListObjects, ListObjectsV2, PutObject, UploadPart, CreateMultiPartUpload, CompleteMultiPartUpload, AbortMultiPartUpload, ListMultiPartUpload - Remove unecessary logging which do not provide any useful information - Update handling of s3 action to hopefully be more readable - Reduce about of information exposed on errors, ie do not return errors, etc instead log them and return generic error message - Update AlwaysAllow authenticator to set "dummy" as the subject on the token --- .../integration/tests/sda/10_upload_test.sh | 4 +- .../tests/sda/15_s3cmd-errorcodes_test.sh | 4 +- sda/cmd/s3inbox/proxy.go | 556 ++++++++---------- sda/cmd/s3inbox/proxy_test.go | 69 +-- sda/internal/database/db_functions.go | 31 + sda/internal/database/db_functions_test.go | 13 + sda/internal/helper/helper.go | 6 +- 7 files changed, 327 insertions(+), 356 deletions(-) diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh index e21f474bea..11b7dd3abe 100644 --- a/.github/integration/tests/sda/10_upload_test.sh +++ b/.github/integration/tests/sda/10_upload_test.sh @@ -67,8 +67,8 @@ if [ "$num_rows" -ne 5 ]; then fi num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") -if [ "$num_log_rows" -ne 12 ]; then - echo "database queries for file_event_logs failed, expected 12 got $num_log_rows" +if [ "$num_log_rows" -ne 11 ]; then + echo "database queries for file_event_logs failed, expected 11 got $num_log_rows" exit 1 fi diff --git a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh index 1936f90d62..f6ec0374e2 100644 --- a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh +++ b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh @@ -8,7 +8,7 @@ cd shared || true # test that listing all buckets is not allowed list_all_buckets=$(s3cmd -c s3cfg -q ls 2>&1) -if ! [[ "$list_all_buckets" =~ "not allowed response" ]]; then +if ! [[ "$list_all_buckets" =~ "Forbidden" ]]; then echo "list buckets should fail" exit 1 fi @@ -25,7 +25,7 @@ cp s3cfg bads3cfg sed -i "s/access_token=.*/access_token=invalid/" bads3cfg unathorized=$(s3cmd -c bads3cfg -q ls s3://test_dummy.org/ 2>&1) -if ! [[ "$unathorized" =~ "not authorized" ]]; then +if ! [[ "$unathorized" =~ "Unauthorized" ]]; then echo "testing unathorized call failed" exit 1 fi diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 1ba4f8a4ee..2fa4d00519 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -28,10 +28,6 @@ import ( log "github.com/sirupsen/logrus" ) -type uniqueFileID struct { - username, filePath string -} - // Proxy represents the toplevel object in this application type Proxy struct { s3Conf config.S3InboxConf @@ -40,7 +36,6 @@ type Proxy struct { messenger *broker.AMQPBroker database *database.SDAdb client *http.Client - fileIDs map[uniqueFileID]string } // The Event struct @@ -70,15 +65,16 @@ type ErrorResponse struct { // The different types of requests const ( - MakeBucket S3RequestType = iota - RemoveBucket - List - Put - Get - Delete - AbortMultipart - Policy - Other + Unsupported S3RequestType = iota + ListObjectsV2 + ListObjects + PutObject + UploadPart + CreateMultiPartUpload + CompleteMultiPartUpload + ListMultiPartUpload + AbortMultiPartUpload + GetBucketLocation ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. @@ -93,208 +89,205 @@ func NewProxy(s3conf config.S3InboxConf, s3Client *s3.Client, auth userauth.Auth messenger: messenger, database: db, client: client, - fileIDs: make(map[uniqueFileID]string), } } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { token, err := p.auth.Authenticate(r) if err != nil { - log.Debugln("Request not authenticated") - p.notAuthorized(w, r) + p.notAuthorized(w, err.Error()) return } - switch t := p.detectRequestType(r); t { - case MakeBucket, RemoveBucket, Delete, Policy, Get: - // Not allowed - log.Debug("not allowed known") - p.notAllowedResponse(w, r) - case Put, List, Other, AbortMultipart: - // Allowed - log.Debug("allowed known") - p.allowedResponse(w, r, token) + s3RequestType := detectS3RequestType(r) + switch s3RequestType { + // These actions we just forward to the s3 backend after ensuring that requests have been made user specific by + // prepareForwardPathAndQuery + case ListObjects, ListObjectsV2, GetBucketLocation, UploadPart, ListMultiPartUpload, AbortMultiPartUpload: + p.forwardRequest(s3RequestType, w, r, token) + case PutObject, CreateMultiPartUpload, CompleteMultiPartUpload: + p.handleUpload(s3RequestType, w, r, token) default: - log.Debugf("Unexpected request (%v) not allowed", r) - p.notAllowedResponse(w, r) + p.notAllowedResponse(w, fmt.Sprintf("user: %s, attempted to do not allowed request: method: %s, path: %s, query: %s", token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery)) } } // Report 500 to the user, log the original error -func (p *Proxy) internalServerError(w http.ResponseWriter, r *http.Request, err string) { +func (p *Proxy) internalServerError(w http.ResponseWriter, err string) { log.Error(err) - msg := fmt.Sprintf("Internal server error for request (%v)", r) - reportError(http.StatusInternalServerError, msg, w) + reportError(http.StatusInternalServerError, "Internal Error", w) } -func (p *Proxy) notAllowedResponse(w http.ResponseWriter, _ *http.Request) { - reportError(http.StatusForbidden, "not allowed response", w) +func (p *Proxy) notAllowedResponse(w http.ResponseWriter, err string) { + log.Warn(err) + reportError(http.StatusForbidden, "Forbidden", w) } -func (p *Proxy) notAuthorized(w http.ResponseWriter, _ *http.Request) { - reportError(http.StatusUnauthorized, "not authorized", w) +func (p *Proxy) notAuthorized(w http.ResponseWriter, err string) { + log.Warn(err) + reportError(http.StatusUnauthorized, "Unauthorized", w) } -func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jwt.Token) { - log.Debug("prepend") - // Check whether token username and filepath match - str, err := url.ParseRequestURI(r.URL.Path) - if err != nil || str.Path == "" { - reportError(http.StatusBadRequest, err.Error(), w) +// prepareForwardPathAndQuery prepares the new path and query to be used for the s3 request to be user specific when +// reaching the s3 backend +func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPath, originQuery, tokenSubject string) (string, string, error) { + str, err := url.ParseRequestURI(originPath) + if err != nil { + return "", "", err + } + + if str.Path == "" { + return "", "", fmt.Errorf("invalid path: %s", originPath) } path := strings.Split(str.Path, "/") - if strings.Contains(token.Subject(), "@") { - if strings.ReplaceAll(token.Subject(), "@", "_") != path[1] { - reportError(http.StatusBadRequest, fmt.Sprintf("token supplied username: %s, but URL had: %s", token.Subject(), path[1]), w) + if strings.Contains(tokenSubject, "@") { + tokenSubject = strings.ReplaceAll(tokenSubject, "@", "_") + } - return + userNameInPath := path[1] + if tokenSubject != userNameInPath { + return "", "", fmt.Errorf("token supplied username: %s, but URL had: %s", tokenSubject, path[1]) + } + + var newPath, newQuery string + switch s3RequestType { + case ListObjects, ListObjectsV2: + newPath = "/" + p.s3Conf.Bucket + if strings.Contains(originQuery, "prefix") { + params := strings.Split(originQuery, "prefix=") + if !strings.HasPrefix(params[1], tokenSubject) { + newQuery = params[0] + "prefix=" + tokenSubject + "%2F" + params[1] + } + } else { + newQuery = originQuery + "&prefix=" + tokenSubject + "%2F" } - } else if token.Subject() != path[1] { - reportError(http.StatusBadRequest, fmt.Sprintf("token supplied username: %s, but URL had: %s", token.Subject(), path[1]), w) + default: + newPath = "/" + p.s3Conf.Bucket + originPath + newQuery = originQuery + } + + return newPath, newQuery, nil +} + +// forwardRequest forwards the request to the s3 backend after making request user specific, then forwards response to client +func (p *Proxy) forwardRequest(s3RequestType S3RequestType, w http.ResponseWriter, r *http.Request, token jwt.Token) { + var err error + r.URL.Path, r.URL.RawQuery, err = p.prepareForwardPathAndQuery(s3RequestType, r.URL.Path, r.URL.RawQuery, token.Subject()) + if err != nil { + reportError(http.StatusBadRequest, err.Error(), w) return } - err = p.prependBucketToHostPath(r) + + s3response, err := p.forwardRequestToBackend(r) if err != nil { - reportError(http.StatusBadRequest, err.Error(), w) + p.internalServerError(w, fmt.Sprintf("forwarding error: %v", err)) + + return } + if err := p.forwardResponseToClient(s3response, w); err != nil { + p.internalServerError(w, fmt.Sprintf("failed to forward reponse to client: %v", err)) + } +} +func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, r *http.Request, token jwt.Token) { username := token.Subject() - rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3Conf.Bucket+"/", "", 1) - anonymizedFilepath := helper.AnonymizeFilepath(rawFilepath, username) - filePath, err := formatUploadFilePath(anonymizedFilepath) + var err error + r.URL.Path, r.URL.RawQuery, err = p.prepareForwardPathAndQuery(s3RequestType, r.URL.Path, r.URL.RawQuery, username) if err != nil { - reportError(http.StatusNotAcceptable, err.Error(), w) + reportError(http.StatusBadRequest, err.Error(), w) return } - fileIdentifier := uniqueFileID{ - username: username, - filePath: filePath, + s3FilePath := strings.Replace(r.URL.Path, "/"+p.s3Conf.Bucket+"/", "", 1) + filePath, err := formatUploadFilePath(helper.AnonymizeFilepath(s3FilePath, username)) + if err != nil { + reportError(http.StatusBadRequest, err.Error(), w) + + return + } + + fileID, err := p.database.GetFileIDInInbox(r.Context(), username, filePath) + if err != nil { + p.internalServerError(w, fmt.Sprintf("failed to check/get existing file id from database: %v", err)) + + return } - location := p.s3Conf.Endpoint + "/" + p.s3Conf.Bucket // if this is an upload request - if p.detectRequestType(r) == Put && p.fileIDs[fileIdentifier] == "" { - // register file in database - log.Debugf("registering file %v in the database with location: %s", r.URL.Path, location) - p.fileIDs[fileIdentifier], err = p.database.RegisterFile(nil, location, filePath, username) - log.Debugf("fileId: %v", p.fileIDs[fileIdentifier]) + if fileID == "" { + fileID, err = p.database.RegisterFile(nil, p.s3Conf.Endpoint+"/"+p.s3Conf.Bucket, filePath, username) if err != nil { - p.internalServerError(w, r, fmt.Sprintf("failed to register file in database: %v", err)) + p.internalServerError(w, fmt.Sprintf("failed to register file in database: %v", err)) return } + } - // check if the file already exists, in that case send an overwrite message, - // so that the FEGA portal is informed that a new version of the file exists. - err = p.sendMessageOnOverwrite(r, p.fileIDs[fileIdentifier], rawFilepath, token) - if err != nil { - p.internalServerError(w, r, err.Error()) + // check if the file already exists when an upload completes, in that case send an overwrite message, + // so that the FEGA portal is informed that a new version + if s3RequestType == PutObject || s3RequestType == CompleteMultiPartUpload { + if err := p.sendMessageOnOverwrite(r.Context(), username, fileID, s3FilePath, filePath); err != nil { + p.internalServerError(w, err.Error()) return } } - log.Debug("Forwarding to backend") - s3response, err := p.forwardToBackend(r) + s3Response, err := p.forwardRequestToBackend(r) if err != nil { - p.internalServerError(w, r, fmt.Sprintf("forwarding error: %v", err)) + p.internalServerError(w, fmt.Sprintf("forwarding error: %v", err)) return } // Send message to upstream and set file as uploaded in the database // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections - if p.uploadFinishedSuccessfully(r, s3response) { - log.Debug("create message") - message, err := p.CreateMessageFromRequest(r, token, anonymizedFilepath) + if s3Response.StatusCode == 200 && (s3RequestType == PutObject || s3RequestType == CompleteMultiPartUpload) { + message, err := p.CreateMessageFromRequest(r.Context(), token.Subject(), s3FilePath) if err != nil { - p.internalServerError(w, r, err.Error()) + p.internalServerError(w, err.Error()) return } jsonMessage, err := json.Marshal(message) if err != nil { - p.internalServerError(w, r, fmt.Sprintf("failed to marshal rabbitmq message to json: %v", err)) + p.internalServerError(w, fmt.Sprintf("failed to marshal rabbitmq message to json: %v", err)) return } - err = p.checkAndSendMessage(p.fileIDs[fileIdentifier], jsonMessage, r) - if err != nil { - p.internalServerError(w, r, fmt.Sprintf("broker error: %v", err)) + if err = p.checkAndSendMessage(fileID, jsonMessage); err != nil { + p.internalServerError(w, fmt.Sprintf("broker error: %v", err)) return } - // The following block is for treating the case when the client loses connection to the server and then it reconnects to a - // different instance of s3inbox. For more details see #1358. - if p.fileIDs[fileIdentifier] == "" { - p.fileIDs[fileIdentifier], err = p.database.GetFileIDByUserPathAndStatus(username, filePath, "registered") - if err != nil { - p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) - - return - } - - log.Debugf("resuming work on file with fileId: %v", p.fileIDs[fileIdentifier]) - } - - if err := p.storeObjectSizeInDB(r.Context(), rawFilepath, p.fileIDs[fileIdentifier]); err != nil { - log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) - p.internalServerError(w, r, "storeObjectSizeInDB failed") + if err := p.storeObjectSizeInDB(r.Context(), s3FilePath, fileID); err != nil { + p.internalServerError(w, fmt.Sprintf("storeObjectSizeInDB failed because: %v", err)) return } - log.Debugf("marking file %v as 'uploaded' in database", p.fileIDs[fileIdentifier]) - err = p.database.UpdateFileEventLog(p.fileIDs[fileIdentifier], "uploaded", "inbox", "{}", string(jsonMessage)) - if err != nil { - p.internalServerError(w, r, fmt.Sprintf("could not connect to db: %v", err)) + if err := p.database.UpdateFileEventLog(fileID, "uploaded", "inbox", "{}", string(jsonMessage)); err != nil { + p.internalServerError(w, fmt.Sprintf("could not connect to db: %v", err)) return } - - delete(p.fileIDs, fileIdentifier) - } - - // Writing non-200 to the response before the headers propagate the error - // to the s3cmd client. - // Writing 200 here breaks uploads though, and writing non-200 codes after - // the headers results in the error message always being - // "MD5 Sums don't match!". - if s3response.StatusCode < 200 || s3response.StatusCode > 299 { - w.WriteHeader(s3response.StatusCode) - } - - // Redirect answer - log.Debug("redirect answer") - for header, values := range s3response.Header { - for _, value := range values { - w.Header().Add(header, value) - } + log.Infof("user: %s, uploaded file: %s, with id: %s", username, filePath, fileID) } - _, err = io.Copy(w, s3response.Body) - if err != nil { - p.internalServerError(w, r, fmt.Sprintf("redirect error: %v", err)) - - return + if err := p.forwardResponseToClient(s3Response, w); err != nil { + p.internalServerError(w, fmt.Sprintf("failed to forward reponse to client: %v", err)) } - - // Read any remaining data in the connection and - // Close so connection can be reused. - _, _ = io.ReadAll(s3response.Body) - _ = s3response.Body.Close() } // Renew the connection to MQ if necessary, then send message -func (p *Proxy) checkAndSendMessage(fileID string, jsonMessage []byte, r *http.Request) error { +func (p *Proxy) checkAndSendMessage(fileID string, jsonMessage []byte) error { var err error if p.messenger == nil { return errors.New("messenger is down") @@ -315,7 +308,6 @@ func (p *Proxy) checkAndSendMessage(fileID string, jsonMessage []byte, r *http.R } } - log.Debugf("Sending message with id %s", fileID) if err := p.messenger.SendMessage(fileID, p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { return fmt.Errorf("error when sending message to broker: %v", err) } @@ -323,115 +315,51 @@ func (p *Proxy) checkAndSendMessage(fileID string, jsonMessage []byte, r *http.R return nil } -func (p *Proxy) uploadFinishedSuccessfully(req *http.Request, response *http.Response) bool { - if response.StatusCode != 200 { - return false - } - - switch req.Method { - case http.MethodPut: - if !strings.Contains(req.URL.String(), "partNumber") { - return true - } - - return false - case http.MethodPost: - if strings.Contains(req.URL.String(), "uploadId") { - return true - } - - return false - default: - - return false - } -} - -func (p *Proxy) forwardToBackend(r *http.Request) (*http.Response, error) { +func (p *Proxy) forwardRequestToBackend(r *http.Request) (*http.Response, error) { p.resignHeader(r) - // Redirect request - nr, err := http.NewRequest(r.Method, p.s3Conf.Endpoint+r.URL.String(), r.Body) // #nosec G704 -- endpoint and port controlled by configuration, TODO verify if r.URL needs to be sanitized + nr, err := http.NewRequest(r.Method, p.s3Conf.Endpoint+r.URL.String(), r.Body) // #nosec G704 -- endpoint and port controlled by configuration if err != nil { - log.Debug("error when redirecting the request") - log.Debug(err) - return nil, err } nr.Header = r.Header contentLength, _ := strconv.ParseInt(r.Header.Get("content-length"), 10, 64) nr.ContentLength = contentLength - return p.client.Do(nr) // #nosec G704 -- endpoint and port controlled by configuration, TODO verify if r.URL needs to be sanitized + return p.client.Do(nr) // #nosec G704 -- endpoint and port controlled by configuration } - -// Add bucket to host path -func (p *Proxy) prependBucketToHostPath(r *http.Request) error { - bucket := p.s3Conf.Bucket - - // Extract username for request's url path - str, err := url.ParseRequestURI(r.URL.Path) - if err != nil || str.Path == "" { - return fmt.Errorf("failed to get path from query (%v)", r.URL.Path) +func (p *Proxy) forwardResponseToClient(s3response *http.Response, w http.ResponseWriter) error { + // Writing non-200 to the response before the headers propagate the error + // to the s3cmd client. + // Writing 200 here breaks uploads though, and writing non-200 codes after + // the headers results in the error message always being + // "MD5 Sums don't match!". + if s3response.StatusCode < 200 || s3response.StatusCode > 299 { + w.WriteHeader(s3response.StatusCode) } - path := strings.Split(str.Path, "/") - username := path[1] - - log.Debugf("incoming path: %s", r.URL.Path) - log.Debugf("incoming raw: %s", r.URL.RawQuery) - - // Restructure request to query the users folder instead of the general bucket - switch r.Method { - case http.MethodGet: - switch { - case strings.Contains(r.URL.String(), "?uploadId"): - // resume multipart upload - r.URL.Path = "/" + bucket + r.URL.Path - case strings.Contains(r.URL.String(), "?uploads"): - // list multipart upload - r.URL.Path = "/" + bucket - r.URL.RawQuery = "uploads&prefix=" + username + "%2F" - case strings.Contains(r.URL.String(), "?delimiter"): - r.URL.Path = "/" + bucket + "/" - if strings.Contains(r.URL.RawQuery, "&prefix") { - params := strings.Split(r.URL.RawQuery, "&prefix=") - r.URL.RawQuery = params[0] + "&prefix=" + username + "%2F" + params[1] - } else { - r.URL.RawQuery = r.URL.RawQuery + "&prefix=" + username + "%2F" - } - log.Debug("new Raw Query: ", r.URL.RawQuery) - case strings.Contains(r.URL.String(), "?location") || strings.Contains(r.URL.String(), "&prefix"): - r.URL.Path = "/" + bucket + "/" - log.Debug("new Path: ", r.URL.Path) - default: - r.URL.Path = "/" + bucket + "/" - } - case http.MethodPost: - r.URL.Path = "/" + bucket + r.URL.Path - log.Debug("new Path: ", r.URL.Path) - case http.MethodPut: - r.URL.Path = "/" + bucket + r.URL.Path - log.Info("new PUT Path: ", r.URL.Path) - case http.MethodDelete: - if strings.Contains(r.URL.String(), "?uploadId") { - // abort multipart upload - r.URL.Path = "/" + bucket + r.URL.Path + + for header, values := range s3response.Header { + for _, value := range values { + w.Header().Add(header, value) } - default: - log.Errorf("unknown request: %s", r.Method) + } - return errors.New("unknown request method") + if _, err := io.Copy(w, s3response.Body); err != nil { + return err } - log.Infof("User: %v, Request type %v, Path: %v", username, r.Method, r.URL.Path) + + // Read any remaining data in the connection and + // Close so connection can be reused. + _, _ = io.ReadAll(s3response.Body) + _ = s3response.Body.Close() return nil } // Function for signing the headers of the s3 requests -// Used for for creating a signature for with the default +// Used for creating a signature for with the default // credentials of the s3 service and the user's signature (authentication) func (p *Proxy) resignHeader(r *http.Request) *http.Request { - log.Debugf("Generating resigning header for %s", p.s3Conf.Endpoint) r.Header.Del("X-Amz-Security-Token") r.Header.Del("X-Forwarded-Port") r.Header.Del("X-Forwarded-Proto") @@ -449,94 +377,109 @@ func (p *Proxy) resignHeader(r *http.Request) *http.Request { return signer.SignV4(*r, p.s3Conf.AccessKey, p.s3Conf.SecretKey, "", p.s3Conf.Region) } -// Not necessarily a function on the struct since it does not use any of the -// members. -func (p *Proxy) detectRequestType(r *http.Request) S3RequestType { - switch r.Method { - case http.MethodGet: - switch { - case strings.HasSuffix(r.URL.String(), "/"): - log.Debug("detect Get") - - return Get - case strings.Contains(r.URL.String(), "?acl"): - log.Debug("detect Policy") - - return Policy - default: - log.Debug("detect List") - - return List - } - case http.MethodDelete: - switch { - case strings.HasSuffix(r.URL.String(), "/"): - log.Debug("detect RemoveBucket") - - return RemoveBucket - case strings.Contains(r.URL.String(), "uploadId"): - log.Debug("detect AbortMultipart") - - return AbortMultipart - default: - // Do we allow deletion of files? - log.Debug("detect Delete") - - return Delete - } - case http.MethodPut: - switch { - case strings.HasSuffix(r.URL.String(), "/"): - log.Debug("detect MakeBucket") - - return MakeBucket - case strings.Contains(r.URL.String(), "?policy"): - log.Debug("detect Policy") - - return Policy - default: - // Should decide if we will handle copy here or through authentication - log.Debug("detect Put") - - return Put - } +// detectS3RequestType detects which s3 actions is being taken based upon the http method, path and query +// Allowed actions: +// +// * GetBucketLocation == GET /${bucket}?location +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html +// +// * ListObjectsV2 == GET /${bucket}?list-type=2 +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html +// For ListObjectsV2 we enforce that the prefix query argument starts with the token.Subject() such that a user can +// only see files within a directory named by the token subject +// +// * ListObjects == GET /${bucket} +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html +// Checked by makings sure there are nu query params that indicate other actions +// Checks that any of the following are not present in the query: +// "acl", "policy", "cors", "lifecycle", "versioning", "logging", "tagging", "encryption", "website", "notification", +// +// "replication", "analytics", "metrics", "inventory", "ownershipControls", "publicAccessBlock", "object-lock" +// +// * PutObject == PUT /${bucket}/${object} +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html +// partNumber and uploadId query arguments not present +// We ensure x-amz-copy-source is not present to now allow CopyObject +// +// * UploadPart == PUT /${bucket}/${object} +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html +// partNumber and uploadId query arguments present +// +// * CreateMultiPartUpload == POST /${bucket}/${object}?uploads +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html +// +// * CompleteMultiPartUpload == POST /${bucket}/${object}?uploads +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html +// +// * AbortMultiPartUpload == DELETE /${bucket}/${object}?uploads +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html +// +// * ListMultiPartUpload == Get /${bucket}/${object}?uploadId +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html +func detectS3RequestType(r *http.Request) S3RequestType { + query := r.URL.Query() + + pathParts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") + isBucketPath := len(pathParts) == 1 && pathParts[0] != "" + isObjectPath := len(pathParts) > 1 && pathParts[0] != "" && pathParts[1] != "" + + switch { + // ListObjectsV2 + case r.Method == http.MethodGet && isBucketPath && query.Get("list-type") == "2": + return ListObjectsV2 + case r.Method == http.MethodGet && isObjectPath && query.Has("uploadId"): + return ListMultiPartUpload + case r.Method == http.MethodGet && isBucketPath && query.Has("location"): + return GetBucketLocation + case r.Method == http.MethodGet && isBucketPath && !query.Has("acl") && !query.Has("policy") && + !query.Has("cors") && !query.Has("lifecycle") && !query.Has("versioning") && + !query.Has("location") && !query.Has("logging") && !query.Has("tagging") && + !query.Has("encryption") && !query.Has("website") && !query.Has("notification") && + !query.Has("replication") && !query.Has("analytics") && !query.Has("metrics") && + !query.Has("inventory") && !query.Has("ownershipControls") && !query.Has("publicAccessBlock") && + !query.Has("object-lock"): + return ListObjects + case r.Method == http.MethodPut && isObjectPath && !query.Has("partNumber") && !query.Has("uploadId") && r.Header.Get("x-amz-copy-source") == "": + return PutObject + case r.Method == http.MethodPut && isObjectPath && query.Has("partNumber") && query.Has("uploadId") && r.Header.Get("x-amz-copy-source") == "": + return UploadPart + case r.Method == http.MethodPost && isObjectPath && query.Has("uploads"): + return CreateMultiPartUpload + case r.Method == http.MethodPost && isObjectPath && query.Has("uploadId"): + return CompleteMultiPartUpload + case r.Method == http.MethodDelete && isObjectPath && query.Has("uploadId"): + return AbortMultiPartUpload default: - log.Debug("detect Other") - - return Other + return Unsupported } } // CreateMessageFromRequest is a function that can take a http request and // figure out the correct rabbitmq message to send from it. -func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anonymizedFilepath string) (Event, error) { +func (p *Proxy) CreateMessageFromRequest(ctx context.Context, username, s3FilePath string) (Event, error) { event := Event{} checksum := Checksum{} var err error - checksum.Value, event.Filesize, err = p.requestInfo(r.Context(), r.URL.Path) + checksum.Value, event.Filesize, err = p.requestInfo(ctx, s3FilePath) if err != nil { return event, fmt.Errorf("could not get checksum information: %s", err) } // Case for simple upload event.Operation = "upload" - event.Filepath = anonymizedFilepath + event.Filepath = s3FilePath - event.Username = claims.Subject() + event.Username = username checksum.Type = "md5" event.Checksum = []any{checksum} - privateClaims := claims.PrivateClaims() - log.Info("user ", event.Username, " with pilot ", privateClaims["pilot"], " uploaded file ", event.Filepath, " with checksum ", checksum.Value, " at ", time.Now()) return event, nil } // RequestInfo is a function that makes a request to the S3 and collects // the etag and size information for the uploaded document -func (p *Proxy) requestInfo(ctx context.Context, fullPath string) (string, int64, error) { - filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) - +func (p *Proxy) requestInfo(ctx context.Context, filePath string) (string, int64, error) { input := &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), Key: aws.String(filePath), @@ -544,8 +487,6 @@ func (p *Proxy) requestInfo(ctx context.Context, fullPath string) (string, int64 result, err := p.s3Client.HeadObject(ctx, input) if err != nil { - log.Debug(err.Error()) - return "", 0, err } @@ -558,48 +499,43 @@ func (p *Proxy) requestInfo(ctx context.Context, fullPath string) (string, int64 // checkFileExists makes a request to the S3 to check whether the file already // is uploaded. Returns a bool indicating whether the file was found. -func (p *Proxy) checkFileExists(ctx context.Context, fullPath string) (bool, error) { - filePath := strings.Replace(fullPath, "/"+p.s3Conf.Bucket+"/", "", 1) - +func (p *Proxy) checkFileExists(ctx context.Context, s3FilePath string) (bool, error) { result, err := p.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), - Key: aws.String(filePath), + Key: aws.String(s3FilePath), }) if err != nil && strings.Contains(err.Error(), "StatusCode: 404") { - log.Debugf("s3 could not find file %s: %s", fullPath, err.Error()) - return false, nil } return result != nil, err } -func (p *Proxy) sendMessageOnOverwrite(r *http.Request, fileID, rawFilepath string, token jwt.Token) error { - exist, err := p.checkFileExists(r.Context(), r.URL.Path) +func (p *Proxy) sendMessageOnOverwrite(ctx context.Context, username, fileID, s3FilePath, filepath string) error { + exist, err := p.checkFileExists(ctx, s3FilePath) if err != nil { return err } - if exist { - username := token.Subject() - msg := schema.InboxRemove{ - User: username, - FilePath: rawFilepath, - Operation: "remove", - } + if !exist { + return nil + } - privateClaims := token.PrivateClaims() - log.Info("user ", msg.User, " with pilot ", privateClaims["pilot"], " will overwrite file ", msg.FilePath, " at ", time.Now()) + log.Infof("user: %s, reuploaded file: %s, with id: %s", username, filepath, fileID) + msg := schema.InboxRemove{ + User: username, + FilePath: s3FilePath, + Operation: "remove", + } - jsonMessage, err := json.Marshal(msg) - if err != nil { - return err - } + jsonMessage, err := json.Marshal(msg) + if err != nil { + return err + } - err = p.checkAndSendMessage(fileID, jsonMessage, r) - if err != nil { - return err - } + err = p.checkAndSendMessage(fileID, jsonMessage) + if err != nil { + return err } return nil @@ -651,10 +587,10 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { } } -func (p *Proxy) storeObjectSizeInDB(ctx context.Context, path, fileID string) error { +func (p *Proxy) storeObjectSizeInDB(ctx context.Context, s3FilePath, fileID string) error { o, err := p.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ Bucket: aws.String(p.s3Conf.Bucket), - Key: aws.String(path), + Key: aws.String(s3FilePath), }) if err != nil { return err diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index eb4139d78d..ef9971be6a 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -134,6 +134,9 @@ func (s *ProxyTests) SetupTest() { if err != nil { s.T().FailNow() } + if err := s.token.Set("sub", "dummy"); err != nil { + s.T().FailNow() + } s3cfg, err := s3config.LoadDefaultConfig(context.TODO(), s3config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.s3Conf.AccessKey, s.s3Conf.SecretKey, ""))) if err != nil { @@ -268,14 +271,6 @@ func (s *ProxyTests) TestServeHTTP_disallowed() { assert.Equal(s.T(), 403, w.Result().StatusCode) assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) - // Normal get is dissallowed - w = httptest.NewRecorder() - r.Method = "GET" - r.URL, _ = url.Parse("/asdf/") - proxy.ServeHTTP(w, r) - assert.Equal(s.T(), 403, w.Result().StatusCode) - assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) - // Put policy is disallowed w = httptest.NewRecorder() r.Method = "PUT" @@ -317,8 +312,8 @@ func (s *ProxyTests) TestServeHTTPS3Unresponsive() { // Just try to list the files r.Method = "GET" - r.URL, _ = url.Parse("/dummy/asdf") - proxy.allowedResponse(w, r, s.token) + r.URL, _ = url.Parse("/dummy") + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 500, w.Result().StatusCode) // nolint:bodyclose } @@ -336,7 +331,7 @@ func (s *ProxyTests) TestServeHTTP_MQConnectionClosed() { w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose assert.False(s.T(), proxy.messenger.Connection.IsClosed()) } @@ -355,7 +350,7 @@ func (s *ProxyTests) TestServeHTTP_MQChannelClosed() { w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose assert.False(s.T(), proxy.messenger.Channel.IsClosed()) } @@ -375,7 +370,7 @@ func (s *ProxyTests) TestServeHTTP_MQ_Unavailable() { w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 500, w.Result().StatusCode) // nolint:bodyclose } @@ -387,10 +382,10 @@ func (s *ProxyTests) TestServeHTTP_allowed() { proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // List files works - r, err := http.NewRequest("GET", "/dummy/file", nil) + r, err := http.NewRequest("GET", "/dummy", nil) assert.NoError(s.T(), err) w := httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Testing the pinged interface @@ -401,15 +396,15 @@ func (s *ProxyTests) TestServeHTTP_allowed() { assert.NoError(s.T(), err) s.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\";", "Content-Length": "5"} - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Put with partnumber sends no message w = httptest.NewRecorder() r.Method = "PUT" - r.URL, _ = url.Parse("/dummy/file?partNumber=5") - proxy.allowedResponse(w, r, s.token) + r.URL, _ = url.Parse("/dummy/file?partNumber=5&uploadId=1") + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) @@ -417,15 +412,7 @@ func (s *ProxyTests) TestServeHTTP_allowed() { r.Method = "POST" r.URL, _ = url.Parse("/dummy/file?uploadId=5") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) - assert.Equal(s.T(), 200, w.Result().StatusCode) - assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) - - // Post without uploadId sends no message - r.Method = "POST" - r.URL, _ = url.Parse("/dummy/file") - w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) @@ -433,7 +420,7 @@ func (s *ProxyTests) TestServeHTTP_allowed() { r.Method = "DELETE" r.URL, _ = url.Parse("/dummy/asdf?uploadId=123") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) @@ -441,33 +428,33 @@ func (s *ProxyTests) TestServeHTTP_allowed() { // that trigger different code paths in the code. // Delimiter alone r.Method = "GET" - r.URL, _ = url.Parse("/dummy/file?delimiter=puppe") + r.URL, _ = url.Parse("/dummy?delimiter=puppe") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Show multiparts uploads r.Method = "GET" - r.URL, _ = url.Parse("/dummy/file?uploads") + r.URL, _ = url.Parse("/dummy/file?uploadId=1") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Delimiter alone together with prefix r.Method = "GET" - r.URL, _ = url.Parse("/dummy/file?delimiter=puppe&prefix=asdf") + r.URL, _ = url.Parse("/dummy?delimiter=puppe&prefix=asdf") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Location parameter r.Method = "GET" - r.URL, _ = url.Parse("/dummy/file?location=fnuffe") + r.URL, _ = url.Parse("/dummy?location=fnuffe") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) assert.Equal(s.T(), 200, w.Result().StatusCode) assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) @@ -476,8 +463,8 @@ func (s *ProxyTests) TestServeHTTP_allowed() { r.Method = "PUT" r.URL, _ = url.Parse("/dummy/fi|le") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, s.token) - assert.Equal(s.T(), 406, w.Result().StatusCode) + proxy.ServeHTTP(w, r) + assert.Equal(s.T(), 400, w.Result().StatusCode) assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) } @@ -496,7 +483,7 @@ func (s *ProxyTests) TestMessageFormatting() { proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "1234"} - msg, err := proxy.CreateMessageFromRequest(r, claims, "new_file.txt") + msg, err := proxy.CreateMessageFromRequest(r.Context(), claims.Subject(), "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) @@ -512,7 +499,7 @@ func (s *ProxyTests) TestMessageFormatting() { // Test single shot upload r.Method = "PUT" - msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), "new_file.txt") + msg, err = proxy.CreateMessageFromRequest(r.Context(), claims.Subject(), "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), "upload", msg.Operation) @@ -537,7 +524,7 @@ func (s *ProxyTests) TestDatabaseConnection() { w := httptest.NewRecorder() s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "5"} - proxy.allowedResponse(w, r, s.token) + proxy.ServeHTTP(w, r) res := w.Result() defer res.Body.Close() assert.Equal(s.T(), 200, res.StatusCode) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index b3edf8b85c..e2a6b4040f 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -118,6 +118,37 @@ WHERE id_and_event.event = $3;` return fileID, nil } +// GetFileIDInInbox gets the file id of a file which last known event is either 'registered', 'uploaded', or 'disabled' +// as that means that ingestion has not been triggered and users is allowed continue uploading or reupload the file to the inbox +// if no row is found does not return sql.ErrNoRows, just empty string in id return field +func (dbs *SDAdb) GetFileIDInInbox(ctx context.Context, submissionUser, filePath string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const getFileID = ` +SELECT id_and_event.id +FROM ( + SELECT DISTINCT ON (f.id) f.id, fel.event FROM sda.files AS f + LEFT JOIN sda.file_event_log AS fel ON fel.file_id = f.id + WHERE f.submission_user = $1 + AND f.submission_file_path = $2 + AND f.stable_id IS NULL + ORDER BY f.id, fel.started_at DESC LIMIT 1 + ) AS id_and_event +WHERE id_and_event.event IN ('registered', 'uploaded', 'disabled');` + + var fileID string + + if err := db.QueryRowContext(ctx, getFileID, submissionUser, filePath).Scan(&fileID); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return "", err + } + + return fileID, nil +} + // CheckStableIDOwnedByUser checks if the file a stableID links to belongs to the user // Returns true if a file is found by the stableID and user, false if not found func (dbs *SDAdb) CheckStableIDOwnedByUser(stableID, user string) (bool, error) { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 98d35513d7..2b90783b9f 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1904,3 +1904,16 @@ func (suite *DatabaseTests) TestSetBackedUp_FileID_Not_Exists() { notExistingFileID := uuid.NewString() assert.EqualError(suite.T(), db.SetBackedUp("/backup", notExistingFileID, notExistingFileID), sql.ErrNoRows.Error()) } + +func (suite *DatabaseTests) TestGetFileIDInInbox() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + defer db.Close() + + fileID, err := db.RegisterFile(nil, "/inbox", "TestGetFileIDInInbox.c4gh", "testuser") + assert.NoError(suite.T(), err, "failed to register file in database") + + fileIdFromDB, err := db.GetFileIDInInbox(context.TODO(), "testuser", "TestGetFileIDInInbox.c4gh") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), fileID, fileIdFromDB) +} diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index 08ad03e002..9969e3f4ee 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -90,7 +90,11 @@ func NewAlwaysAllow() *AlwaysAllow { // Authenticate authenticates everyone. func (u *AlwaysAllow) Authenticate(_ *http.Request) (jwt.Token, error) { - return jwt.New(), nil + token := jwt.New() + if err := token.Set("sub", "dummy"); err != nil { + return nil, err + } + return token, nil } // AlwaysAllow is an Authenticator that always authenticates From 19a236e2962fd758c63f25c4751f4b1350df1b8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 12:02:55 +0200 Subject: [PATCH 440/530] fix(s3inbox): fix some comments, update forwardResponseToClient to add headers before writing respone code, update prepareForwardPathAndQuery list actions prefix logic, fix some linter issues --- sda/cmd/s3inbox/proxy.go | 40 ++++++++++++++-------- sda/internal/database/db_functions.go | 3 +- sda/internal/database/db_functions_test.go | 4 +-- sda/internal/helper/helper.go | 1 + 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 2fa4d00519..78d567feb2 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -155,14 +155,24 @@ func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPa switch s3RequestType { case ListObjects, ListObjectsV2: newPath = "/" + p.s3Conf.Bucket - if strings.Contains(originQuery, "prefix") { - params := strings.Split(originQuery, "prefix=") - if !strings.HasPrefix(params[1], tokenSubject) { - newQuery = params[0] + "prefix=" + tokenSubject + "%2F" + params[1] - } - } else { - newQuery = originQuery + "&prefix=" + tokenSubject + "%2F" + + queryValues, err := url.ParseQuery(originQuery) + if err != nil { + return "", "", err } + requiredPrefix := tokenSubject + "/" + existingPrefix := queryValues.Get("prefix") + switch { + case existingPrefix == "": + queryValues.Set("prefix", requiredPrefix) + case existingPrefix == tokenSubject: + queryValues.Set("prefix", requiredPrefix) + case strings.HasPrefix(existingPrefix, requiredPrefix): + queryValues.Set("prefix", existingPrefix) + default: + queryValues.Set("prefix", requiredPrefix+existingPrefix) + } + newQuery = queryValues.Encode() default: newPath = "/" + p.s3Conf.Bucket + originPath newQuery = originQuery @@ -329,6 +339,12 @@ func (p *Proxy) forwardRequestToBackend(r *http.Request) (*http.Response, error) return p.client.Do(nr) // #nosec G704 -- endpoint and port controlled by configuration } func (p *Proxy) forwardResponseToClient(s3response *http.Response, w http.ResponseWriter) error { + for header, values := range s3response.Header { + for _, value := range values { + w.Header().Add(header, value) + } + } + // Writing non-200 to the response before the headers propagate the error // to the s3cmd client. // Writing 200 here breaks uploads though, and writing non-200 codes after @@ -338,12 +354,6 @@ func (p *Proxy) forwardResponseToClient(s3response *http.Response, w http.Respon w.WriteHeader(s3response.StatusCode) } - for header, values := range s3response.Header { - for _, value := range values { - w.Header().Add(header, value) - } - } - if _, err := io.Copy(w, s3response.Body); err != nil { return err } @@ -408,10 +418,10 @@ func (p *Proxy) resignHeader(r *http.Request) *http.Request { // * CreateMultiPartUpload == POST /${bucket}/${object}?uploads // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html // -// * CompleteMultiPartUpload == POST /${bucket}/${object}?uploads +// * CompleteMultiPartUpload == POST /${bucket}/${object}?uploadId // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html // -// * AbortMultiPartUpload == DELETE /${bucket}/${object}?uploads +// * AbortMultiPartUpload == DELETE /${bucket}/${object}?uploadId // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html // // * ListMultiPartUpload == Get /${bucket}/${object}?uploadId diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index e2a6b4040f..5c9e494f46 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -119,7 +119,7 @@ WHERE id_and_event.event = $3;` } // GetFileIDInInbox gets the file id of a file which last known event is either 'registered', 'uploaded', or 'disabled' -// as that means that ingestion has not been triggered and users is allowed continue uploading or reupload the file to the inbox +// as that means that ingestion has not been triggered and users are allowed continue uploading or reupload the file to the inbox // if no row is found does not return sql.ErrNoRows, just empty string in id return field func (dbs *SDAdb) GetFileIDInInbox(ctx context.Context, submissionUser, filePath string) (string, error) { dbs.checkAndReconnectIfNeeded() @@ -143,6 +143,7 @@ WHERE id_and_event.event IN ('registered', 'uploaded', 'disabled');` if errors.Is(err, sql.ErrNoRows) { return "", nil } + return "", err } diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 2b90783b9f..8f162e5f25 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1913,7 +1913,7 @@ func (suite *DatabaseTests) TestGetFileIDInInbox() { fileID, err := db.RegisterFile(nil, "/inbox", "TestGetFileIDInInbox.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileIdFromDB, err := db.GetFileIDInInbox(context.TODO(), "testuser", "TestGetFileIDInInbox.c4gh") + fileIDFromDB, err := db.GetFileIDInInbox(context.TODO(), "testuser", "TestGetFileIDInInbox.c4gh") assert.NoError(suite.T(), err) - assert.Equal(suite.T(), fileID, fileIdFromDB) + assert.Equal(suite.T(), fileID, fileIDFromDB) } diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index 9969e3f4ee..dec2370fe7 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -94,6 +94,7 @@ func (u *AlwaysAllow) Authenticate(_ *http.Request) (jwt.Token, error) { if err := token.Set("sub", "dummy"); err != nil { return nil, err } + return token, nil } From 58482b22d189f4dfd06a5bfaeba2a4aeb3ad4532 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 12:25:46 +0200 Subject: [PATCH 441/530] fix(s3inbox): fix typo, fix ListMultiPartUpload detection and ensure it has prefix query arg --- sda/cmd/s3inbox/proxy.go | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 78d567feb2..2f7f3f57f5 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -153,7 +153,7 @@ func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPa var newPath, newQuery string switch s3RequestType { - case ListObjects, ListObjectsV2: + case ListObjects, ListObjectsV2, ListMultiPartUpload: newPath = "/" + p.s3Conf.Bucket queryValues, err := url.ParseQuery(originQuery) @@ -199,7 +199,7 @@ func (p *Proxy) forwardRequest(s3RequestType S3RequestType, w http.ResponseWrite } if err := p.forwardResponseToClient(s3response, w); err != nil { - p.internalServerError(w, fmt.Sprintf("failed to forward reponse to client: %v", err)) + p.internalServerError(w, fmt.Sprintf("failed to forward response to client: %v", err)) } } func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, r *http.Request, token jwt.Token) { @@ -292,7 +292,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, } if err := p.forwardResponseToClient(s3Response, w); err != nil { - p.internalServerError(w, fmt.Sprintf("failed to forward reponse to client: %v", err)) + p.internalServerError(w, fmt.Sprintf("failed to forward response to client: %v", err)) } } @@ -354,15 +354,16 @@ func (p *Proxy) forwardResponseToClient(s3response *http.Response, w http.Respon w.WriteHeader(s3response.StatusCode) } + defer func() { + // Read any remaining data in the connection and + // Close so connection can be reused. + _, _ = io.ReadAll(s3response.Body) + _ = s3response.Body.Close() + }() if _, err := io.Copy(w, s3response.Body); err != nil { return err } - // Read any remaining data in the connection and - // Close so connection can be reused. - _, _ = io.ReadAll(s3response.Body) - _ = s3response.Body.Close() - return nil } @@ -424,7 +425,7 @@ func (p *Proxy) resignHeader(r *http.Request) *http.Request { // * AbortMultiPartUpload == DELETE /${bucket}/${object}?uploadId // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html // -// * ListMultiPartUpload == Get /${bucket}/${object}?uploadId +// * ListMultiPartUpload == Get /${bucket}/${object}?uploads // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html func detectS3RequestType(r *http.Request) S3RequestType { query := r.URL.Query() @@ -437,7 +438,7 @@ func detectS3RequestType(r *http.Request) S3RequestType { // ListObjectsV2 case r.Method == http.MethodGet && isBucketPath && query.Get("list-type") == "2": return ListObjectsV2 - case r.Method == http.MethodGet && isObjectPath && query.Has("uploadId"): + case r.Method == http.MethodGet && isObjectPath && query.Has("uploads"): return ListMultiPartUpload case r.Method == http.MethodGet && isBucketPath && query.Has("location"): return GetBucketLocation @@ -447,7 +448,7 @@ func detectS3RequestType(r *http.Request) S3RequestType { !query.Has("encryption") && !query.Has("website") && !query.Has("notification") && !query.Has("replication") && !query.Has("analytics") && !query.Has("metrics") && !query.Has("inventory") && !query.Has("ownershipControls") && !query.Has("publicAccessBlock") && - !query.Has("object-lock"): + !query.Has("object-lock") && !query.Has("uploads"): return ListObjects case r.Method == http.MethodPut && isObjectPath && !query.Has("partNumber") && !query.Has("uploadId") && r.Header.Get("x-amz-copy-source") == "": return PutObject From b6387ff4c9ff2c2cdc13f3e7e4922e7aad177b1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 12:26:42 +0200 Subject: [PATCH 442/530] feat(s3inbox): add TestGetFileIDInInbox_NotFound unit test to test behaviour when file not found --- sda/internal/database/db_functions_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 8f162e5f25..240b9ea58e 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1917,3 +1917,13 @@ func (suite *DatabaseTests) TestGetFileIDInInbox() { assert.NoError(suite.T(), err) assert.Equal(suite.T(), fileID, fileIDFromDB) } + +func (suite *DatabaseTests) TestGetFileIDInInbox_NotFound() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got %v when creating new connection", err) + defer db.Close() + + fileIDFromDB, err := db.GetFileIDInInbox(context.TODO(), "testuser", "TestGetFileIDInInbox.c4gh") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), "", fileIDFromDB) +} From 84742ee6757396ef3ae8a535f2f8842aa8e35986 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 12:39:12 +0200 Subject: [PATCH 443/530] feat(s3inbox): fix mix and match between ListMultiPartUploads and ListParts s3 actions --- sda/cmd/s3inbox/proxy.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 2f7f3f57f5..eddf813281 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -72,7 +72,8 @@ const ( UploadPart CreateMultiPartUpload CompleteMultiPartUpload - ListMultiPartUpload + ListMultiPartUploads + ListParts AbortMultiPartUpload GetBucketLocation ) @@ -104,7 +105,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch s3RequestType { // These actions we just forward to the s3 backend after ensuring that requests have been made user specific by // prepareForwardPathAndQuery - case ListObjects, ListObjectsV2, GetBucketLocation, UploadPart, ListMultiPartUpload, AbortMultiPartUpload: + case ListObjects, ListObjectsV2, GetBucketLocation, UploadPart, ListMultiPartUploads, AbortMultiPartUpload, ListParts: p.forwardRequest(s3RequestType, w, r, token) case PutObject, CreateMultiPartUpload, CompleteMultiPartUpload: p.handleUpload(s3RequestType, w, r, token) @@ -153,7 +154,7 @@ func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPa var newPath, newQuery string switch s3RequestType { - case ListObjects, ListObjectsV2, ListMultiPartUpload: + case ListObjects, ListObjectsV2, ListMultiPartUploads: newPath = "/" + p.s3Conf.Bucket queryValues, err := url.ParseQuery(originQuery) @@ -401,7 +402,7 @@ func (p *Proxy) resignHeader(r *http.Request) *http.Request { // // * ListObjects == GET /${bucket} // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html -// Checked by makings sure there are nu query params that indicate other actions +// Checked by making sure there are no query params that indicate other actions // Checks that any of the following are not present in the query: // "acl", "policy", "cors", "lifecycle", "versioning", "logging", "tagging", "encryption", "website", "notification", // @@ -410,7 +411,7 @@ func (p *Proxy) resignHeader(r *http.Request) *http.Request { // * PutObject == PUT /${bucket}/${object} // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html // partNumber and uploadId query arguments not present -// We ensure x-amz-copy-source is not present to now allow CopyObject +// We ensure x-amz-copy-source is not present to not allow CopyObject // // * UploadPart == PUT /${bucket}/${object} // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html @@ -425,8 +426,11 @@ func (p *Proxy) resignHeader(r *http.Request) *http.Request { // * AbortMultiPartUpload == DELETE /${bucket}/${object}?uploadId // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html // -// * ListMultiPartUpload == Get /${bucket}/${object}?uploads +// * ListParts == Get /${bucket}/${object}?uploadId // For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html +// +// * ListMultiPartUploads == Get /${bucket}?uploads +// For aws docs see: https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html func detectS3RequestType(r *http.Request) S3RequestType { query := r.URL.Query() @@ -438,8 +442,10 @@ func detectS3RequestType(r *http.Request) S3RequestType { // ListObjectsV2 case r.Method == http.MethodGet && isBucketPath && query.Get("list-type") == "2": return ListObjectsV2 - case r.Method == http.MethodGet && isObjectPath && query.Has("uploads"): - return ListMultiPartUpload + case r.Method == http.MethodGet && isBucketPath && query.Has("uploads"): + return ListMultiPartUploads + case r.Method == http.MethodGet && isObjectPath && query.Has("uploadId"): + return ListParts case r.Method == http.MethodGet && isBucketPath && query.Has("location"): return GetBucketLocation case r.Method == http.MethodGet && isBucketPath && !query.Has("acl") && !query.Has("policy") && @@ -448,7 +454,7 @@ func detectS3RequestType(r *http.Request) S3RequestType { !query.Has("encryption") && !query.Has("website") && !query.Has("notification") && !query.Has("replication") && !query.Has("analytics") && !query.Has("metrics") && !query.Has("inventory") && !query.Has("ownershipControls") && !query.Has("publicAccessBlock") && - !query.Has("object-lock") && !query.Has("uploads"): + !query.Has("object-lock") && !query.Has("uploads") && !query.Has("uploadId"): return ListObjects case r.Method == http.MethodPut && isObjectPath && !query.Has("partNumber") && !query.Has("uploadId") && r.Header.Get("x-amz-copy-source") == "": return PutObject From 94020266a7de6cf2174e066f5bfd7b81d8d03823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 12:54:22 +0200 Subject: [PATCH 444/530] fix(s3inbox): move closing of s3 reponse so that linter will be happy --- sda/cmd/s3inbox/proxy.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index eddf813281..dd4c790a8c 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -192,16 +192,18 @@ func (p *Proxy) forwardRequest(s3RequestType S3RequestType, w http.ResponseWrite return } - s3response, err := p.forwardRequestToBackend(r) + s3Response, err := p.forwardRequestToBackend(r) if err != nil { p.internalServerError(w, fmt.Sprintf("forwarding error: %v", err)) return } - if err := p.forwardResponseToClient(s3response, w); err != nil { + if err := p.forwardResponseToClient(s3Response, w); err != nil { p.internalServerError(w, fmt.Sprintf("failed to forward response to client: %v", err)) } + + _ = s3Response.Body.Close() } func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, r *http.Request, token jwt.Token) { username := token.Subject() @@ -295,6 +297,8 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, if err := p.forwardResponseToClient(s3Response, w); err != nil { p.internalServerError(w, fmt.Sprintf("failed to forward response to client: %v", err)) } + + _ = s3Response.Body.Close() } // Renew the connection to MQ if necessary, then send message @@ -339,8 +343,8 @@ func (p *Proxy) forwardRequestToBackend(r *http.Request) (*http.Response, error) return p.client.Do(nr) // #nosec G704 -- endpoint and port controlled by configuration } -func (p *Proxy) forwardResponseToClient(s3response *http.Response, w http.ResponseWriter) error { - for header, values := range s3response.Header { +func (p *Proxy) forwardResponseToClient(s3Response *http.Response, w http.ResponseWriter) error { + for header, values := range s3Response.Header { for _, value := range values { w.Header().Add(header, value) } @@ -351,20 +355,17 @@ func (p *Proxy) forwardResponseToClient(s3response *http.Response, w http.Respon // Writing 200 here breaks uploads though, and writing non-200 codes after // the headers results in the error message always being // "MD5 Sums don't match!". - if s3response.StatusCode < 200 || s3response.StatusCode > 299 { - w.WriteHeader(s3response.StatusCode) + if s3Response.StatusCode < 200 || s3Response.StatusCode > 299 { + w.WriteHeader(s3Response.StatusCode) } - defer func() { - // Read any remaining data in the connection and - // Close so connection can be reused. - _, _ = io.ReadAll(s3response.Body) - _ = s3response.Body.Close() - }() - if _, err := io.Copy(w, s3response.Body); err != nil { + if _, err := io.Copy(w, s3Response.Body); err != nil { return err } + // Read any remaining data in the connection + _, _ = io.ReadAll(s3Response.Body) + return nil } From 8b62fd36acd3787510b2a486ff1b5d492deadcf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 13:30:27 +0200 Subject: [PATCH 445/530] fix(s3inbox): protect against index out of bounds in prepareForwardPathAndQuery if path does not follow expected structure --- sda/cmd/s3inbox/proxy.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index dd4c790a8c..cf31b85bc1 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -142,11 +142,14 @@ func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPa return "", "", fmt.Errorf("invalid path: %s", originPath) } - path := strings.Split(str.Path, "/") if strings.Contains(tokenSubject, "@") { tokenSubject = strings.ReplaceAll(tokenSubject, "@", "_") } + path := strings.Split(str.Path, "/") + if len(path) < 2 || path[1] == "" { + return "", "", fmt.Errorf("invalid path: %s", originPath) + } userNameInPath := path[1] if tokenSubject != userNameInPath { return "", "", fmt.Errorf("token supplied username: %s, but URL had: %s", tokenSubject, path[1]) From 93637e873096cefd3f12be14b598aca00821afb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 13:59:08 +0200 Subject: [PATCH 446/530] feat(s3inbox): extend unit test to trigger ListObjectsV2 and ListMultiPartUpload handling --- sda/cmd/s3inbox/proxy_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index ef9971be6a..814413787b 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -466,6 +466,22 @@ func (s *ProxyTests) TestServeHTTP_allowed() { proxy.ServeHTTP(w, r) assert.Equal(s.T(), 400, w.Result().StatusCode) assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) + + // Get with list-type=2 + w = httptest.NewRecorder() + r.Method = "GET" + r.URL, _ = url.Parse("/dummy?list-type=2") + proxy.ServeHTTP(w, r) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) + + // Get with uploads + w = httptest.NewRecorder() + r.Method = "GET" + r.URL, _ = url.Parse("/dummy?uploads") + proxy.ServeHTTP(w, r) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) } func (s *ProxyTests) TestMessageFormatting() { From 9eff120c6810d95d86f0aa53eceb3aa773e60727 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 14:42:43 +0200 Subject: [PATCH 447/530] fix(s3inbox): update prepareForwardPathAndQuery fix GetBucketLocation newPath --- sda/cmd/s3inbox/proxy.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index cf31b85bc1..aaa3058255 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -157,6 +157,8 @@ func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPa var newPath, newQuery string switch s3RequestType { + case GetBucketLocation: + newPath = "/" + p.s3Conf.Bucket case ListObjects, ListObjectsV2, ListMultiPartUploads: newPath = "/" + p.s3Conf.Bucket From 673ba2d6163229227b163dd4d4b141699b5fc16e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 10 Apr 2026 14:44:12 +0200 Subject: [PATCH 448/530] fix(s3inbox): update storeObjectSizeInDB add nil check for o.ContentLength --- sda/cmd/s3inbox/proxy.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index aaa3058255..e23463fd86 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -619,5 +619,9 @@ func (p *Proxy) storeObjectSizeInDB(ctx context.Context, s3FilePath, fileID stri return err } + if o.ContentLength == nil { + return errors.New("s3 content length not available") + } + return p.database.SetSubmissionFileSize(fileID, *o.ContentLength) } From 640fbf3759a4c9cf712cfc9a7421f9e446c72b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 13 Apr 2026 09:05:13 +0200 Subject: [PATCH 449/530] fix(s3inbox): send file overwrite message after upload has completed (but check if it will be an overwrite before), and fix defer of s3Response to also ensure its closed incase error --- sda/cmd/s3inbox/proxy.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index e23463fd86..4a18e7d981 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -246,10 +246,12 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, } } - // check if the file already exists when an upload completes, in that case send an overwrite message, + isReupload := false + // check if the file already exists when an upload completes, in that case send an overwrite message when the s3 has responded with 200, // so that the FEGA portal is informed that a new version if s3RequestType == PutObject || s3RequestType == CompleteMultiPartUpload { - if err := p.sendMessageOnOverwrite(r.Context(), username, fileID, s3FilePath, filePath); err != nil { + isReupload, err = p.checkFileExists(r.Context(), s3FilePath) + if err != nil { p.internalServerError(w, err.Error()) return @@ -262,6 +264,9 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, return } + defer func() { + _ = s3Response.Body.Close() + }() // Send message to upstream and set file as uploaded in the database // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections @@ -296,14 +301,22 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, return } - log.Infof("user: %s, uploaded file: %s, with id: %s", username, filePath, fileID) + + if isReupload { + log.Infof("user: %s, reuploaded file: %s, with id: %s", username, filePath, fileID) + if err := p.sendMessageOnOverwrite(username, fileID, s3FilePath); err != nil { + p.internalServerError(w, err.Error()) + + return + } + } else { + log.Infof("user: %s, uploaded file: %s, with id: %s", username, filePath, fileID) + } } if err := p.forwardResponseToClient(s3Response, w); err != nil { p.internalServerError(w, fmt.Sprintf("failed to forward response to client: %v", err)) } - - _ = s3Response.Body.Close() } // Renew the connection to MQ if necessary, then send message @@ -535,16 +548,7 @@ func (p *Proxy) checkFileExists(ctx context.Context, s3FilePath string) (bool, e return result != nil, err } -func (p *Proxy) sendMessageOnOverwrite(ctx context.Context, username, fileID, s3FilePath, filepath string) error { - exist, err := p.checkFileExists(ctx, s3FilePath) - if err != nil { - return err - } - if !exist { - return nil - } - - log.Infof("user: %s, reuploaded file: %s, with id: %s", username, filepath, fileID) +func (p *Proxy) sendMessageOnOverwrite(username, fileID, s3FilePath string) error { msg := schema.InboxRemove{ User: username, FilePath: s3FilePath, From f1065ebe5649f747e453c4dc3a715b1e0cc50d7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 13 Apr 2026 10:57:58 +0200 Subject: [PATCH 450/530] fix(s3inbox): remove redundant error log message in reportError --- sda/cmd/s3inbox/proxy.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 4a18e7d981..4e640db2f5 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -593,7 +593,6 @@ func formatUploadFilePath(filePath string) (string, error) { // Write the error and its status code to the response func reportError(errorCode int, message string, w http.ResponseWriter) { - log.Error(message) errorResponse := ErrorResponse{ Code: http.StatusText(errorCode), Message: message, From 6a40b84a659c84175afda6c11208fd9e690ae9e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 13 Apr 2026 11:05:08 +0200 Subject: [PATCH 451/530] fix(s3inbox): fix detectS3RequestType to classify any request containing unsupported query params as unsupported no matter the http method --- sda/cmd/s3inbox/proxy.go | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 4e640db2f5..f771f8fb7b 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -458,6 +458,14 @@ func detectS3RequestType(r *http.Request) S3RequestType { isObjectPath := len(pathParts) > 1 && pathParts[0] != "" && pathParts[1] != "" switch { + // Unsupported actions + case query.Has("acl") || query.Has("policy") || + query.Has("cors") || query.Has("lifecycle") || query.Has("versioning") || + query.Has("logging") || query.Has("tagging") || query.Has("encryption") || + query.Has("website") || query.Has("notification") || query.Has("replication") || + query.Has("analytics") || query.Has("metrics") || query.Has("inventory") || + query.Has("ownershipControls") || query.Has("publicAccessBlock") || query.Has("object-lock"): + return Unsupported // ListObjectsV2 case r.Method == http.MethodGet && isBucketPath && query.Get("list-type") == "2": return ListObjectsV2 @@ -467,13 +475,7 @@ func detectS3RequestType(r *http.Request) S3RequestType { return ListParts case r.Method == http.MethodGet && isBucketPath && query.Has("location"): return GetBucketLocation - case r.Method == http.MethodGet && isBucketPath && !query.Has("acl") && !query.Has("policy") && - !query.Has("cors") && !query.Has("lifecycle") && !query.Has("versioning") && - !query.Has("location") && !query.Has("logging") && !query.Has("tagging") && - !query.Has("encryption") && !query.Has("website") && !query.Has("notification") && - !query.Has("replication") && !query.Has("analytics") && !query.Has("metrics") && - !query.Has("inventory") && !query.Has("ownershipControls") && !query.Has("publicAccessBlock") && - !query.Has("object-lock") && !query.Has("uploads") && !query.Has("uploadId"): + case r.Method == http.MethodGet && isBucketPath: return ListObjects case r.Method == http.MethodPut && isObjectPath && !query.Has("partNumber") && !query.Has("uploadId") && r.Header.Get("x-amz-copy-source") == "": return PutObject From e7ce0ce881b4acfaa68655d0a8bf1021d8019c55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 14 Apr 2026 12:51:09 +0200 Subject: [PATCH 452/530] feat(s3inbox): add check to integration test to ensure we dont expose other users files with s3 list actions --- .../tests/sda/01_install_dependencies.sh | 2 +- .../tests/sda/15_s3cmd-errorcodes_test.sh | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/01_install_dependencies.sh b/.github/integration/tests/sda/01_install_dependencies.sh index 174d96c6dd..9d089fb79d 100644 --- a/.github/integration/tests/sda/01_install_dependencies.sh +++ b/.github/integration/tests/sda/01_install_dependencies.sh @@ -2,7 +2,7 @@ set -e # install tools if missing -for t in curl expect jq openssh-client postgresql-client xxd; do +for t in curl expect jq openssh-client postgresql-client xxd awscli; do if [ ! "$(command -v $t)" ]; then if [ "$(id -u)" != 0 ]; then echo "$t is missing, unable to install it" diff --git a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh index f6ec0374e2..c468ed486e 100644 --- a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh +++ b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh @@ -30,5 +30,36 @@ if ! [[ "$unathorized" =~ "Unauthorized" ]]; then exit 1 fi +aws configure set aws_access_key_id dummy # Value dont matter as we authenticate with the aws_session_token +aws configure set aws_secret_access_key dummy # Value dont matter as we authenticate with the aws_session_token +aws configure set aws_session_token "$(grep 'access_token' s3cfg | cut -d "=" -f 2 | xargs)" + +listResponse=$(aws s3api list-objects --endpoint http://s3inbox:8000 --bucket test_dummy.org) +if (( $(echo "$listResponse" | grep -c "test_dummy.org") == 0 )); then + echo "found no files for users when listing towards s3inbox with ListObjects call" + exit 1 +fi + +listV2Response=$(aws s3api list-objects --endpoint http://s3inbox:8000 --bucket test_dummy.org) +if (( $(echo "$listV2Response" | grep -c "test_dummy.org") == 0 )); then + echo "found no files for users when listing towards s3inbox with ListObjectsV2 call" + exit 1 +fi + + +aws configure set aws_session_token "$(cat token)" + +listResponse=$(aws s3api list-objects --endpoint http://s3inbox:8000 --bucket testu_lifescience-ri.eu) +if ! (( $(echo "$listResponse" | grep -c "test_dummy.org") == 0 )); then + echo "found other users files when listing towards s3inbox with ListObjects call" + exit 1 +fi + +listV2Response=$(aws s3api list-objects-v2 --endpoint http://s3inbox:8000 --bucket testu_lifescience-ri.eu) +if ! (( $(echo "$listV2Response" | grep -c "test_dummy.org") == 0 )); then + echo "found other users files when listing towards s3inbox with ListObjectsV2 call" + exit 1 +fi + echo "s3cmd error messages tested successfully" From 0ec28839602f9cf6594b340fe16113c432bca790 Mon Sep 17 00:00:00 2001 From: KarlG-nbis Date: Wed, 15 Apr 2026 09:45:09 +0200 Subject: [PATCH 453/530] fix(integration test): update .github/integration/tests/sda/15_s3cmd-errorcodes_test.sh fix to use list-objects-v2 with user expecting files Co-authored-by: Jonas Hagberg --- .github/integration/tests/sda/15_s3cmd-errorcodes_test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh index c468ed486e..78e04d8d71 100644 --- a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh +++ b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh @@ -40,7 +40,7 @@ if (( $(echo "$listResponse" | grep -c "test_dummy.org") == 0 )); then exit 1 fi -listV2Response=$(aws s3api list-objects --endpoint http://s3inbox:8000 --bucket test_dummy.org) +listV2Response=$(aws s3api list-objects-v2 --endpoint http://s3inbox:8000 --bucket test_dummy.org) if (( $(echo "$listV2Response" | grep -c "test_dummy.org") == 0 )); then echo "found no files for users when listing towards s3inbox with ListObjectsV2 call" exit 1 From efd30e56f0ff771202cd2f0771cd171cc836ce80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 15 Apr 2026 09:53:34 +0200 Subject: [PATCH 454/530] fix(s3inbox): fix not propagating origin query when handling GetBucketLocation --- sda/cmd/s3inbox/proxy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index f771f8fb7b..c1bf291df1 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -159,6 +159,7 @@ func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPa switch s3RequestType { case GetBucketLocation: newPath = "/" + p.s3Conf.Bucket + newQuery = originQuery case ListObjects, ListObjectsV2, ListMultiPartUploads: newPath = "/" + p.s3Conf.Bucket From 98f453c12b22d60b1b493061b42cffffd0b85a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 15 Apr 2026 10:14:35 +0200 Subject: [PATCH 455/530] fix(s3inbox): fix log warnings on bad requests and return generic error to user, update integration test to expect generic error, and update aws s3api cmd to use --endpoint-url arg instead of --endpoint --- .../tests/sda/15_s3cmd-errorcodes_test.sh | 12 ++++++------ sda/cmd/s3inbox/proxy.go | 17 ++++++++++------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh index 78e04d8d71..700a8ee50c 100644 --- a/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh +++ b/.github/integration/tests/sda/15_s3cmd-errorcodes_test.sh @@ -14,8 +14,8 @@ if ! [[ "$list_all_buckets" =~ "Forbidden" ]]; then fi # test disallowed characters -disallowed_characters=$(s3cmd -c s3cfg -q put s3cfg s3://test_dummy.org/fai\| 2>&1) -if ! [[ "$disallowed_characters" =~ "filepath contains disallowed characters" ]];then +bad_request=$(s3cmd -c s3cfg -q put s3cfg s3://test_dummy.org/fai\| 2>&1) +if ! [[ "$bad_request" =~ "Bad Request" ]];then echo "test with disallowed characters failed" exit 1 fi @@ -34,13 +34,13 @@ aws configure set aws_access_key_id dummy # Value dont matter as we authenticate aws configure set aws_secret_access_key dummy # Value dont matter as we authenticate with the aws_session_token aws configure set aws_session_token "$(grep 'access_token' s3cfg | cut -d "=" -f 2 | xargs)" -listResponse=$(aws s3api list-objects --endpoint http://s3inbox:8000 --bucket test_dummy.org) +listResponse=$(aws s3api list-objects --endpoint-url http://s3inbox:8000 --bucket test_dummy.org) if (( $(echo "$listResponse" | grep -c "test_dummy.org") == 0 )); then echo "found no files for users when listing towards s3inbox with ListObjects call" exit 1 fi -listV2Response=$(aws s3api list-objects-v2 --endpoint http://s3inbox:8000 --bucket test_dummy.org) +listV2Response=$(aws s3api list-objects-v2 --endpoint-url http://s3inbox:8000 --bucket test_dummy.org) if (( $(echo "$listV2Response" | grep -c "test_dummy.org") == 0 )); then echo "found no files for users when listing towards s3inbox with ListObjectsV2 call" exit 1 @@ -49,13 +49,13 @@ fi aws configure set aws_session_token "$(cat token)" -listResponse=$(aws s3api list-objects --endpoint http://s3inbox:8000 --bucket testu_lifescience-ri.eu) +listResponse=$(aws s3api list-objects --endpoint-url http://s3inbox:8000 --bucket testu_lifescience-ri.eu) if ! (( $(echo "$listResponse" | grep -c "test_dummy.org") == 0 )); then echo "found other users files when listing towards s3inbox with ListObjects call" exit 1 fi -listV2Response=$(aws s3api list-objects-v2 --endpoint http://s3inbox:8000 --bucket testu_lifescience-ri.eu) +listV2Response=$(aws s3api list-objects-v2 --endpoint-url http://s3inbox:8000 --bucket testu_lifescience-ri.eu) if ! (( $(echo "$listV2Response" | grep -c "test_dummy.org") == 0 )); then echo "found other users files when listing towards s3inbox with ListObjectsV2 call" exit 1 diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index c1bf291df1..73e7f5638b 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -117,17 +117,17 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { // Report 500 to the user, log the original error func (p *Proxy) internalServerError(w http.ResponseWriter, err string) { log.Error(err) - reportError(http.StatusInternalServerError, "Internal Error", w) + reportErrorToClient(http.StatusInternalServerError, "Internal Error", w) } func (p *Proxy) notAllowedResponse(w http.ResponseWriter, err string) { log.Warn(err) - reportError(http.StatusForbidden, "Forbidden", w) + reportErrorToClient(http.StatusForbidden, "Forbidden", w) } func (p *Proxy) notAuthorized(w http.ResponseWriter, err string) { log.Warn(err) - reportError(http.StatusUnauthorized, "Unauthorized", w) + reportErrorToClient(http.StatusUnauthorized, "Unauthorized", w) } // prepareForwardPathAndQuery prepares the new path and query to be used for the s3 request to be user specific when @@ -193,7 +193,8 @@ func (p *Proxy) forwardRequest(s3RequestType S3RequestType, w http.ResponseWrite var err error r.URL.Path, r.URL.RawQuery, err = p.prepareForwardPathAndQuery(s3RequestType, r.URL.Path, r.URL.RawQuery, token.Subject()) if err != nil { - reportError(http.StatusBadRequest, err.Error(), w) + log.Warnf("bad request from user %s: %v", token.Subject(), err) + reportErrorToClient(http.StatusBadRequest, "Bad Request", w) return } @@ -217,7 +218,8 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, var err error r.URL.Path, r.URL.RawQuery, err = p.prepareForwardPathAndQuery(s3RequestType, r.URL.Path, r.URL.RawQuery, username) if err != nil { - reportError(http.StatusBadRequest, err.Error(), w) + log.Warnf("bad request from user %s: %v", token.Subject(), err) + reportErrorToClient(http.StatusBadRequest, "Bad Request", w) return } @@ -225,7 +227,8 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, s3FilePath := strings.Replace(r.URL.Path, "/"+p.s3Conf.Bucket+"/", "", 1) filePath, err := formatUploadFilePath(helper.AnonymizeFilepath(s3FilePath, username)) if err != nil { - reportError(http.StatusBadRequest, err.Error(), w) + log.Warnf("bad request from user %s: %v", token.Subject(), err) + reportErrorToClient(http.StatusBadRequest, "Bad Request", w) return } @@ -595,7 +598,7 @@ func formatUploadFilePath(filePath string) (string, error) { } // Write the error and its status code to the response -func reportError(errorCode int, message string, w http.ResponseWriter) { +func reportErrorToClient(errorCode int, message string, w http.ResponseWriter) { errorResponse := ErrorResponse{ Code: http.StatusText(errorCode), Message: message, From c2d51bca0e01b79a46cad98d6715eea1a49f6854 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 15 Apr 2026 10:43:03 +0200 Subject: [PATCH 456/530] feat(s3inbox): add user, http method, path, and query when logging errors/warning of failed requests --- sda/cmd/s3inbox/proxy.go | 46 +++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 27 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 73e7f5638b..8edf57f9f2 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -96,7 +96,8 @@ func NewProxy(s3conf config.S3InboxConf, s3Client *s3.Client, auth userauth.Auth func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { token, err := p.auth.Authenticate(r) if err != nil { - p.notAuthorized(w, err.Error()) + log.Warnf("unauthorized user attempted: method: %s, path: %s, query: %s", r.Method, r.URL.Path, r.URL.RawQuery) + reportErrorToClient(http.StatusUnauthorized, "Unauthorized", w) return } @@ -110,26 +111,17 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { case PutObject, CreateMultiPartUpload, CompleteMultiPartUpload: p.handleUpload(s3RequestType, w, r, token) default: - p.notAllowedResponse(w, fmt.Sprintf("user: %s, attempted to do not allowed request: method: %s, path: %s, query: %s", token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery)) + log.Warnf("user: %s, attempted to do not allowed request: method: %s, path: %s, query: %s", token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery) + reportErrorToClient(http.StatusForbidden, "Forbidden", w) } } // Report 500 to the user, log the original error -func (p *Proxy) internalServerError(w http.ResponseWriter, err string) { - log.Error(err) +func (p *Proxy) internalServerError(w http.ResponseWriter, tokenSubject, httpMethod, path, query, err string) { + log.Errorf("user: %s, method: %s, path: %s, query: %s, encountered internal error: %s", tokenSubject, httpMethod, path, query, err) reportErrorToClient(http.StatusInternalServerError, "Internal Error", w) } -func (p *Proxy) notAllowedResponse(w http.ResponseWriter, err string) { - log.Warn(err) - reportErrorToClient(http.StatusForbidden, "Forbidden", w) -} - -func (p *Proxy) notAuthorized(w http.ResponseWriter, err string) { - log.Warn(err) - reportErrorToClient(http.StatusUnauthorized, "Unauthorized", w) -} - // prepareForwardPathAndQuery prepares the new path and query to be used for the s3 request to be user specific when // reaching the s3 backend func (p *Proxy) prepareForwardPathAndQuery(s3RequestType S3RequestType, originPath, originQuery, tokenSubject string) (string, string, error) { @@ -201,13 +193,13 @@ func (p *Proxy) forwardRequest(s3RequestType S3RequestType, w http.ResponseWrite s3Response, err := p.forwardRequestToBackend(r) if err != nil { - p.internalServerError(w, fmt.Sprintf("forwarding error: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("forwarding error: %v", err)) return } if err := p.forwardResponseToClient(s3Response, w); err != nil { - p.internalServerError(w, fmt.Sprintf("failed to forward response to client: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("failed to forward response to client: %v", err)) } _ = s3Response.Body.Close() @@ -235,7 +227,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, fileID, err := p.database.GetFileIDInInbox(r.Context(), username, filePath) if err != nil { - p.internalServerError(w, fmt.Sprintf("failed to check/get existing file id from database: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("failed to check/get existing file id from database: %v", err)) return } @@ -244,7 +236,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, if fileID == "" { fileID, err = p.database.RegisterFile(nil, p.s3Conf.Endpoint+"/"+p.s3Conf.Bucket, filePath, username) if err != nil { - p.internalServerError(w, fmt.Sprintf("failed to register file in database: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("failed to register file in database: %v", err)) return } @@ -256,7 +248,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, if s3RequestType == PutObject || s3RequestType == CompleteMultiPartUpload { isReupload, err = p.checkFileExists(r.Context(), s3FilePath) if err != nil { - p.internalServerError(w, err.Error()) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, err.Error()) return } @@ -264,7 +256,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, s3Response, err := p.forwardRequestToBackend(r) if err != nil { - p.internalServerError(w, fmt.Sprintf("forwarding error: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("forwarding error: %v", err)) return } @@ -277,31 +269,31 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, if s3Response.StatusCode == 200 && (s3RequestType == PutObject || s3RequestType == CompleteMultiPartUpload) { message, err := p.CreateMessageFromRequest(r.Context(), token.Subject(), s3FilePath) if err != nil { - p.internalServerError(w, err.Error()) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, err.Error()) return } jsonMessage, err := json.Marshal(message) if err != nil { - p.internalServerError(w, fmt.Sprintf("failed to marshal rabbitmq message to json: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("failed to marshal rabbitmq message to json: %v", err)) return } if err = p.checkAndSendMessage(fileID, jsonMessage); err != nil { - p.internalServerError(w, fmt.Sprintf("broker error: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("broker error: %v", err)) return } if err := p.storeObjectSizeInDB(r.Context(), s3FilePath, fileID); err != nil { - p.internalServerError(w, fmt.Sprintf("storeObjectSizeInDB failed because: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("storeObjectSizeInDB failed because: %v", err)) return } if err := p.database.UpdateFileEventLog(fileID, "uploaded", "inbox", "{}", string(jsonMessage)); err != nil { - p.internalServerError(w, fmt.Sprintf("could not connect to db: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("could not connect to db: %v", err)) return } @@ -309,7 +301,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, if isReupload { log.Infof("user: %s, reuploaded file: %s, with id: %s", username, filePath, fileID) if err := p.sendMessageOnOverwrite(username, fileID, s3FilePath); err != nil { - p.internalServerError(w, err.Error()) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, err.Error()) return } @@ -319,7 +311,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, } if err := p.forwardResponseToClient(s3Response, w); err != nil { - p.internalServerError(w, fmt.Sprintf("failed to forward response to client: %v", err)) + p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, fmt.Sprintf("failed to forward response to client: %v", err)) } } From a461255765437fdc940247381bdfb7a28ca6d38b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 15 Apr 2026 11:02:55 +0200 Subject: [PATCH 457/530] fix(s3inbox): update GetFileIDInInbox to check archive_file_path = '' instead stable_id IS NULL to rely on unique_ingested constraint, and update TestGetFileIDInInbox to test additional latest file events --- sda/internal/database/db_functions.go | 2 +- sda/internal/database/db_functions_test.go | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 5c9e494f46..85f96f2cea 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -132,7 +132,7 @@ FROM ( LEFT JOIN sda.file_event_log AS fel ON fel.file_id = f.id WHERE f.submission_user = $1 AND f.submission_file_path = $2 - AND f.stable_id IS NULL + AND f.archive_file_path = '' ORDER BY f.id, fel.started_at DESC LIMIT 1 ) AS id_and_event WHERE id_and_event.event IN ('registered', 'uploaded', 'disabled');` diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 240b9ea58e..4fc097cf18 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1916,6 +1916,18 @@ func (suite *DatabaseTests) TestGetFileIDInInbox() { fileIDFromDB, err := db.GetFileIDInInbox(context.TODO(), "testuser", "TestGetFileIDInInbox.c4gh") assert.NoError(suite.T(), err) assert.Equal(suite.T(), fileID, fileIDFromDB) + + assert.NoError(suite.T(), db.SetArchived("/archive", FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, fileID, fmt.Sprintf("%x", sha256.New()), -1, fmt.Sprintf("%x", sha256.New())}, fileID)) + + fileIDFromDB, err = db.GetFileIDInInbox(context.TODO(), "testuser", "TestGetFileIDInInbox.c4gh") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), "", fileIDFromDB) + + assert.NoError(suite.T(), db.CancelFile(context.TODO(), fileID, "{}")) + + fileIDFromDB, err = db.GetFileIDInInbox(context.TODO(), "testuser", "TestGetFileIDInInbox.c4gh") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), fileID, fileIDFromDB) } func (suite *DatabaseTests) TestGetFileIDInInbox_NotFound() { From 459d9c450c78e0345436fcf325c466905d1b158f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 15 Apr 2026 11:18:49 +0200 Subject: [PATCH 458/530] fix(helper): fix AlwaysDeny comment --- sda/internal/helper/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index dec2370fe7..ff88236da4 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -98,7 +98,7 @@ func (u *AlwaysAllow) Authenticate(_ *http.Request) (jwt.Token, error) { return token, nil } -// AlwaysAllow is an Authenticator that always authenticates +// AlwaysDeny is an Authenticator that always denies authentication type AlwaysDeny struct{} // Authenticate does not authenticate anyone. From 235ba60abcedc94bda7a33d2f9fca00136e3ef6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 15 Apr 2026 11:28:00 +0200 Subject: [PATCH 459/530] fix(s3inbox): update old comment to reduce risk of future confusion --- sda/cmd/s3inbox/proxy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 8edf57f9f2..e6c24a55d5 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -264,8 +264,8 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, _ = s3Response.Body.Close() }() - // Send message to upstream and set file as uploaded in the database - // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections + // Send message to upstream and set file as uploaded in the database when upload is complete(PutObject / CompleteMultipartUpload) + // nolint: nestif if s3Response.StatusCode == 200 && (s3RequestType == PutObject || s3RequestType == CompleteMultiPartUpload) { message, err := p.CreateMessageFromRequest(r.Context(), token.Subject(), s3FilePath) if err != nil { From 3060602cbbe4815c28d5d5ac977003af4f615513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Wed, 15 Apr 2026 13:23:08 +0200 Subject: [PATCH 460/530] fix(s3inbox): add file checksum to upload/reupload log statement --- sda/cmd/s3inbox/proxy.go | 12 ++++++------ sda/cmd/s3inbox/proxy_test.go | 5 +++-- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index e6c24a55d5..d51edeb5e0 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -267,7 +267,7 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, // Send message to upstream and set file as uploaded in the database when upload is complete(PutObject / CompleteMultipartUpload) // nolint: nestif if s3Response.StatusCode == 200 && (s3RequestType == PutObject || s3RequestType == CompleteMultiPartUpload) { - message, err := p.CreateMessageFromRequest(r.Context(), token.Subject(), s3FilePath) + message, checksum, err := p.CreateMessageFromRequest(r.Context(), token.Subject(), s3FilePath) if err != nil { p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, err.Error()) @@ -299,14 +299,14 @@ func (p *Proxy) handleUpload(s3RequestType S3RequestType, w http.ResponseWriter, } if isReupload { - log.Infof("user: %s, reuploaded file: %s, with id: %s", username, filePath, fileID) + log.Infof("user: %s, reuploaded file: %s, with id: %s, checksum: %s", username, filePath, fileID, checksum) if err := p.sendMessageOnOverwrite(username, fileID, s3FilePath); err != nil { p.internalServerError(w, token.Subject(), r.Method, r.URL.Path, r.URL.RawQuery, err.Error()) return } } else { - log.Infof("user: %s, uploaded file: %s, with id: %s", username, filePath, fileID) + log.Infof("user: %s, uploaded file: %s, with id: %s, checksum: %s", username, filePath, fileID, checksum) } } @@ -490,14 +490,14 @@ func detectS3RequestType(r *http.Request) S3RequestType { // CreateMessageFromRequest is a function that can take a http request and // figure out the correct rabbitmq message to send from it. -func (p *Proxy) CreateMessageFromRequest(ctx context.Context, username, s3FilePath string) (Event, error) { +func (p *Proxy) CreateMessageFromRequest(ctx context.Context, username, s3FilePath string) (Event, string, error) { event := Event{} checksum := Checksum{} var err error checksum.Value, event.Filesize, err = p.requestInfo(ctx, s3FilePath) if err != nil { - return event, fmt.Errorf("could not get checksum information: %s", err) + return event, "", fmt.Errorf("could not get checksum information: %s", err) } // Case for simple upload @@ -508,7 +508,7 @@ func (p *Proxy) CreateMessageFromRequest(ctx context.Context, username, s3FilePa checksum.Type = "md5" event.Checksum = []any{checksum} - return event, nil + return event, checksum.Value, nil } // RequestInfo is a function that makes a request to the S3 and collects diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 814413787b..ffa508ad31 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -499,7 +499,7 @@ func (s *ProxyTests) TestMessageFormatting() { proxy := NewProxy(s.s3Fakeconf, s.s3ClientToFake, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" s.fakeServer.headHeaders = map[string]string{"ETag": "\"0a44282bd39178db9680f24813c41aec-1\"", "Content-Length": "1234"} - msg, err := proxy.CreateMessageFromRequest(r.Context(), claims.Subject(), "new_file.txt") + msg, checksumValue, err := proxy.CreateMessageFromRequest(r.Context(), claims.Subject(), "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) @@ -512,10 +512,11 @@ func (s *ProxyTests) TestMessageFormatting() { _ = json.Unmarshal(c, &checksum) assert.Equal(s.T(), "md5", checksum.Type) assert.Equal(s.T(), "0a44282bd39178db9680f24813c41aec-1", checksum.Value) + assert.Equal(s.T(), "0a44282bd39178db9680f24813c41aec-1", checksumValue) // Test single shot upload r.Method = "PUT" - msg, err = proxy.CreateMessageFromRequest(r.Context(), claims.Subject(), "new_file.txt") + msg, _, err = proxy.CreateMessageFromRequest(r.Context(), claims.Subject(), "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), "upload", msg.Operation) From 1b1dcb5db2b5639c1a0e7bd4e59d83abcf5f5f6d Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 15 Apr 2026 17:15:07 +0200 Subject: [PATCH 461/530] test(s3inbox): add concurrent-upload regression guard for #2295 Companion to the race-reproducing test on main. Drives 50 parallel PUT requests per round over 20 rounds through ServeHTTP, exercising the same upload path that used to crash with "concurrent map writes" (proxy.go:182/185/195) before this PR removed the shared fileIDs map. Under `go test -race`: - On main: fails with WARNING: DATA RACE + fatal error: concurrent map writes - On this branch (DB-backed fileID lookup): passes cleanly Acts as a regression guard against reintroducing unsynchronised shared state on the upload path. Addresses the third confirmation criterion in ADR-0003 (#2319). Test setup notes: - Uses real MinIO s3Client (not FakeServer, which has its own unsynchronised `pinged` field and would produce unrelated race reports). - Uses helper.NewAlwaysAllow() which now sets sub="dummy" on this branch. - Sets proxy.s3Conf.Endpoint=":" to make forwardRequestToBackend fail fast after the upload path does its DB lookup and bookkeeping, so the test exercises only the code path of interest. --- sda/cmd/s3inbox/proxy_test.go | 46 +++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index ffa508ad31..61877ba286 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -13,6 +13,7 @@ import ( "net/url" "os" "strings" + "sync" "testing" "github.com/aws/aws-sdk-go-v2/aws" @@ -171,6 +172,51 @@ func (s *ProxyTests) TearDownTest() { s.database.Close() } +// TestServeHTTP_concurrent_put_noRace verifies that after the in-memory +// fileIDs map was replaced with Postgres-backed lookups (#2382), concurrent +// PUT requests no longer trigger the "fatal error: concurrent map writes" +// crash from #2295. The same-shaped test fails on main under -race with +// warnings on proxy.go:182/185/195 and the fatal error. Here it should pass. +func (s *ProxyTests) TestServeHTTP_concurrent_put_noRace() { + db, err := database.NewSDAdb(s.DBConf) + if !assert.NoError(s.T(), err) { + return + } + defer db.Close() + + const workers = 50 + const rounds = 20 + + db.DB.SetMaxOpenConns(workers) + db.DB.SetMaxIdleConns(workers) + + proxy := NewProxy(s.s3Conf, s.s3Client, helper.NewAlwaysAllow(), nil, db, new(tls.Config)) + proxy.s3Conf.Endpoint = ":" // force forwardRequestToBackend to fail fast + + for round := 0; round < rounds; round++ { + start := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(workers) + + for i := 0; i < workers; i++ { + n := round*workers + i + go func(n int) { + defer wg.Done() + req := httptest.NewRequest( + http.MethodPut, + fmt.Sprintf("/dummy/race-%04d.c4gh", n), + http.NoBody, + ) + rec := httptest.NewRecorder() + <-start + proxy.ServeHTTP(rec, req) + }(n) + } + close(start) + wg.Wait() + } +} + type FakeServer struct { ts *httptest.Server headHeaders map[string]string From 653f97fea5ffb70e42ff81d7ba346d89dfee62fd Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 14 Apr 2026 11:35:19 +0200 Subject: [PATCH 462/530] feat(storage): add Ping to Reader interface (#2364) The v2 storage Reader interface had no way to verify a backend is actually reachable, so the download service's readiness probe could only nil-check the reader. Add Ping(ctx) to the interface with: - S3: ListBuckets against every configured endpoint, matching the existing startup verification and failing fast on the first endpoint that does not respond. - POSIX: stat every configured path and confirm each is still a directory, mirroring the constructor's invariants. - Wrapper: delegate to whichever sub-readers are configured and join their errors so the caller sees every failing backend. --- .../storage/v2/posix/reader/reader.go | 16 +++++ .../storage/v2/posix/reader/reader_test.go | 66 +++++++++++++++++++ sda/internal/storage/v2/reader.go | 19 ++++++ sda/internal/storage/v2/s3/reader/reader.go | 16 +++++ .../storage/v2/s3/reader/reader_test.go | 45 +++++++++++++ 5 files changed, 162 insertions(+) diff --git a/sda/internal/storage/v2/posix/reader/reader.go b/sda/internal/storage/v2/posix/reader/reader.go index a01853064f..da62ee350c 100644 --- a/sda/internal/storage/v2/posix/reader/reader.go +++ b/sda/internal/storage/v2/posix/reader/reader.go @@ -1,6 +1,7 @@ package reader import ( + "context" "fmt" "os" @@ -37,3 +38,18 @@ func NewReader(backendName string) (*Reader, error) { return backend, nil } + +// Ping verifies all configured POSIX paths are accessible directories. +func (r *Reader) Ping(_ context.Context) error { + for _, e := range r.configuredEndpoints { + fileInfo, err := os.Stat(e.Path) + if err != nil { + return fmt.Errorf("failed to ping POSIX path: %s, due to: %v", e.Path, err) + } + if !fileInfo.IsDir() { + return fmt.Errorf("failed to ping POSIX path: %s is not a directory", e.Path) + } + } + + return nil +} diff --git a/sda/internal/storage/v2/posix/reader/reader_test.go b/sda/internal/storage/v2/posix/reader/reader_test.go index cdb802bdec..8c2ac74df7 100644 --- a/sda/internal/storage/v2/posix/reader/reader_test.go +++ b/sda/internal/storage/v2/posix/reader/reader_test.go @@ -139,6 +139,72 @@ func (ts *ReaderTestSuite) TestGetFileSize() { ts.Equal(int64(len("file 6 content in dir2")), fileSize) } +func (ts *ReaderTestSuite) TestPing() { + err := ts.reader.Ping(context.TODO()) + ts.NoError(err) +} + +func (ts *ReaderTestSuite) TestPing_PathGone() { + configDir := ts.T().TempDir() + vanishDir := ts.T().TempDir() + + if err := os.WriteFile(filepath.Join(configDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + ping_fail_test: + posix: + - path: %s +`, vanishDir)), 0600); err != nil { + ts.FailNow(err.Error()) + } + + viper.SetConfigFile(filepath.Join(configDir, "config.yaml")) + ts.Require().NoError(viper.ReadInConfig()) + + reader, err := NewReader("ping_fail_test") + ts.Require().NoError(err) + + // Remove the directory after creating the reader + ts.Require().NoError(os.Remove(vanishDir)) + + err = reader.Ping(context.TODO()) + ts.Error(err) + ts.Contains(err.Error(), "failed to ping POSIX path") +} + +func (ts *ReaderTestSuite) TestPing_PathIsFile() { + configDir := ts.T().TempDir() + dir := ts.T().TempDir() + filePath := filepath.Join(dir, "a_file") + + if err := os.WriteFile(filePath, []byte("hello"), 0600); err != nil { + ts.FailNow(err.Error()) + } + + if err := os.WriteFile(filepath.Join(configDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + ping_file_test: + posix: + - path: %s +`, dir)), 0600); err != nil { + ts.FailNow(err.Error()) + } + + viper.SetConfigFile(filepath.Join(configDir, "config.yaml")) + ts.Require().NoError(viper.ReadInConfig()) + + reader, err := NewReader("ping_file_test") + ts.Require().NoError(err) + + // Replace the directory with a file + ts.Require().NoError(os.RemoveAll(dir)) + ts.Require().NoError(os.WriteFile(dir, []byte("now a file"), 0600)) + defer os.Remove(dir) + + err = reader.Ping(context.TODO()) + ts.Error(err) + ts.Contains(err.Error(), "not a directory") +} + func (ts *ReaderTestSuite) TestFindFile() { for _, test := range []struct { testName string diff --git a/sda/internal/storage/v2/reader.go b/sda/internal/storage/v2/reader.go index efe7660d36..88fb479315 100644 --- a/sda/internal/storage/v2/reader.go +++ b/sda/internal/storage/v2/reader.go @@ -21,6 +21,8 @@ type Reader interface { FindFile(ctx context.Context, filePath string) (string, error) // GetFileSize will return the size of the file specified by the file path and location GetFileSize(ctx context.Context, location, filePath string) (int64, error) + // Ping verifies the storage backend is reachable + Ping(ctx context.Context) error } type reader struct { @@ -89,6 +91,23 @@ func (r *reader) GetFileSize(ctx context.Context, location, filePath string) (in return 0, storageerrors.ErrorNoValidReader } +func (r *reader) Ping(ctx context.Context) error { + var errs []error + + if r.s3Reader != nil { + if err := r.s3Reader.Ping(ctx); err != nil { + errs = append(errs, err) + } + } + if r.posixReader != nil { + if err := r.posixReader.Ping(ctx); err != nil { + errs = append(errs, err) + } + } + + return errors.Join(errs...) +} + func (r *reader) FindFile(ctx context.Context, filePath string) (string, error) { if r.s3Reader != nil { loc, err := r.s3Reader.FindFile(ctx, filePath) diff --git a/sda/internal/storage/v2/s3/reader/reader.go b/sda/internal/storage/v2/s3/reader/reader.go index 8d8e77e337..9b6c601e1f 100644 --- a/sda/internal/storage/v2/s3/reader/reader.go +++ b/sda/internal/storage/v2/s3/reader/reader.go @@ -59,6 +59,22 @@ func (reader *Reader) getS3ClientForEndpoint(ctx context.Context, endpoint strin return nil, nil, storageerrors.ErrorNoEndpointConfiguredForLocation } +// Ping verifies all configured S3 endpoints are reachable by calling ListBuckets. +func (reader *Reader) Ping(ctx context.Context) error { + for _, e := range reader.endpoints { + client, err := e.getS3Client(ctx) + if err != nil { + return fmt.Errorf("failed to ping S3 endpoint: %s, due to: %v", e.Endpoint, err) + } + + if _, err = client.ListBuckets(ctx, &s3.ListBucketsInput{}); err != nil { + return fmt.Errorf("failed to ping S3 endpoint: %s, due to: %v", e.Endpoint, err) + } + } + + return nil +} + // parseLocation attempts to parse a location to a s3 endpoint, and a bucket // expected format of location is "${ENDPOINT}/${BUCKET} func parseLocation(location string) (string, string, error) { diff --git a/sda/internal/storage/v2/s3/reader/reader_test.go b/sda/internal/storage/v2/s3/reader/reader_test.go index eb49450c20..3b9521749b 100644 --- a/sda/internal/storage/v2/s3/reader/reader_test.go +++ b/sda/internal/storage/v2/s3/reader/reader_test.go @@ -753,3 +753,48 @@ func (ts *ReaderTestSuite) TestNewFileReaderSeeker_InvalidLocation() { _, err := ts.reader.NewFileReader(context.TODO(), "", "") ts.EqualError(err, storageerrors.ErrorInvalidLocation.Error()) } + +func (ts *ReaderTestSuite) TestPing() { + err := ts.reader.Ping(context.TODO()) + ts.NoError(err) +} + +func (ts *ReaderTestSuite) TestPing_EndpointDown() { + // Create a reader with a stopped server + configDir := ts.T().TempDir() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + if strings.HasSuffix(req.RequestURI, "ListBuckets") { + w.WriteHeader(http.StatusOK) + _, _ = fmt.Fprint(w, ``) + + return + } + w.WriteHeader(http.StatusInternalServerError) + })) + + if err := os.WriteFile(filepath.Join(configDir, "config.yaml"), []byte(fmt.Sprintf(` +storage: + ping_fail_test: + s3: + - endpoint: %s + access_key: ak + secret_key: sk + disable_https: true + region: us-east-1 +`, server.URL)), 0600); err != nil { + ts.FailNow(err.Error()) + } + + viper.SetConfigFile(filepath.Join(configDir, "config.yaml")) + ts.Require().NoError(viper.ReadInConfig()) + + reader, err := NewReader(context.TODO(), "ping_fail_test") + ts.Require().NoError(err) + + // Stop the server, then ping should fail + server.Close() + + err = reader.Ping(context.TODO()) + ts.Error(err) + ts.Contains(err.Error(), "failed to ping S3 endpoint") +} From 15217b348634d41ad34adef603fa533f36335c7a Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 14 Apr 2026 11:35:28 +0200 Subject: [PATCH 463/530] feat(reencrypt): use standard gRPC health check protocol (#2365) The reencrypt client's HealthCheck only called grpc.NewClient, which does no I/O, so it always succeeded even when the remote service was down. Replace it with a real grpc.health.v1.Health/Check RPC against the cached connection. The reencrypt server already registers the standard health service, so no server-side change is required. HealthCheck now takes a context.Context and uses it directly without wrapping: the caller (readiness handler, Kubernetes probe) owns the deadline, keeping the client honest about how long health checks may take. --- sda/cmd/download/reencrypt/reencrypt.go | 28 +++++++-- sda/cmd/download/reencrypt/reencrypt_test.go | 65 ++++++++++++++++++++ 2 files changed, 87 insertions(+), 6 deletions(-) diff --git a/sda/cmd/download/reencrypt/reencrypt.go b/sda/cmd/download/reencrypt/reencrypt.go index 9215da3e98..29f0b72e9d 100644 --- a/sda/cmd/download/reencrypt/reencrypt.go +++ b/sda/cmd/download/reencrypt/reencrypt.go @@ -16,6 +16,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" ) // Client provides methods for re-encrypting crypt4gh headers. @@ -193,12 +194,27 @@ func (c *Client) ReencryptHeaderWithEditList(ctx context.Context, oldHeader []by return res.GetHeader(), nil } -// HealthCheck validates that the gRPC client can be constructed with the -// configured options. Note: grpc.NewClient does no I/O, so this does NOT -// verify that the remote reencrypt service is reachable. -// TODO: perform a real RPC (e.g. grpc health check) to verify reachability. -func (c *Client) HealthCheck() error { - return c.connect() +// HealthCheck verifies the remote reencrypt service is reachable and serving +// using the standard gRPC health check protocol. The caller controls the +// deadline via the provided context; no additional timeout is applied so +// Kubernetes probe timeoutSeconds can bound the total duration. +func (c *Client) HealthCheck(ctx context.Context) error { + if err := c.connect(); err != nil { + return err + } + + client := healthgrpc.NewHealthClient(c.conn) + + resp, err := client.Check(ctx, &healthgrpc.HealthCheckRequest{}) + if err != nil { + return fmt.Errorf("grpc health check failed: %w", err) + } + + if resp.GetStatus() != healthgrpc.HealthCheckResponse_SERVING { + return fmt.Errorf("reencrypt service status: %s", resp.GetStatus().String()) + } + + return nil } // Close closes the gRPC connection. diff --git a/sda/cmd/download/reencrypt/reencrypt_test.go b/sda/cmd/download/reencrypt/reencrypt_test.go index e3685b00d3..cae6f80b28 100644 --- a/sda/cmd/download/reencrypt/reencrypt_test.go +++ b/sda/cmd/download/reencrypt/reencrypt_test.go @@ -1,10 +1,16 @@ package reencrypt import ( + "context" + "net" "testing" "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" ) func TestNewClient(t *testing.T) { @@ -40,5 +46,64 @@ func TestClient_Close_WhenNotConnected(t *testing.T) { assert.NoError(t, err) } +func TestHealthCheck_Serving(t *testing.T) { + // Start a gRPC server with health service + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + srv := grpc.NewServer() + healthSrv := health.NewServer() + healthSrv.SetServingStatus("", healthgrpc.HealthCheckResponse_SERVING) + healthgrpc.RegisterHealthServer(srv, healthSrv) + + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + port := lis.Addr().(*net.TCPAddr).Port + client := NewClient("localhost", port) + defer client.Close() + + err = client.HealthCheck(context.Background()) + assert.NoError(t, err) +} + +func TestHealthCheck_NotServing(t *testing.T) { + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + srv := grpc.NewServer() + healthSrv := health.NewServer() + healthSrv.SetServingStatus("", healthgrpc.HealthCheckResponse_NOT_SERVING) + healthgrpc.RegisterHealthServer(srv, healthSrv) + + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + port := lis.Addr().(*net.TCPAddr).Port + client := NewClient("localhost", port) + defer client.Close() + + err = client.HealthCheck(context.Background()) + assert.Error(t, err) + assert.Contains(t, err.Error(), "NOT_SERVING") +} + +func TestHealthCheck_ServerDown(t *testing.T) { + // Use a port with nothing listening + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + port := lis.Addr().(*net.TCPAddr).Port + lis.Close() + + client := NewClient("localhost", port) + defer client.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + err = client.HealthCheck(ctx) + assert.Error(t, err) +} + // Note: Integration tests for ReencryptHeader would require a running gRPC server // These are better suited as integration tests in a test environment From acb48d7550e1c76c45489cc1be319a17a9614331 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 14 Apr 2026 11:35:36 +0200 Subject: [PATCH 464/530] feat(health): verify storage and reencrypt in readiness probe (#2364, #2365) The /health/ready handler previously reported "ok" for storage and the reencrypt gRPC client based on nil-checks alone, so expired S3 credentials or a dead reencrypt service would not trigger a Kubernetes restart. Wire the new Ping and HealthCheck through the handler, propagating the request's 5s context to every backend check, matching the existing database pattern. Tests cover the healthy 200 path, storage Ping failure, reencrypt NOT_SERVING failure, and the already existing degraded-config case. The gRPC path runs against a real grpc_health_v1 server spun up per test via a helper, so the readiness wiring is exercised end-to-end without introducing a mock interface. --- sda/cmd/download/handlers/handlers_test.go | 135 ++++++++++++++++++ sda/cmd/download/handlers/health.go | 10 +- .../download/handlers/mock_database_test.go | 26 ++++ 3 files changed, 169 insertions(+), 2 deletions(-) diff --git a/sda/cmd/download/handlers/handlers_test.go b/sda/cmd/download/handlers/handlers_test.go index 77a1ae191c..f9b1898fea 100644 --- a/sda/cmd/download/handlers/handlers_test.go +++ b/sda/cmd/download/handlers/handlers_test.go @@ -2,6 +2,8 @@ package handlers import ( "encoding/json" + "fmt" + "net" "net/http" "net/http/httptest" "os" @@ -10,9 +12,13 @@ import ( "github.com/gin-gonic/gin" "github.com/neicnordic/sensitive-data-archive/cmd/download/middleware" + "github.com/neicnordic/sensitive-data-archive/cmd/download/reencrypt" configv2 "github.com/neicnordic/sensitive-data-archive/internal/config/v2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/health" + healthgrpc "google.golang.org/grpc/health/grpc_health_v1" ) var loadConfigOnce sync.Once @@ -115,6 +121,135 @@ func TestHealthReady_Degraded(t *testing.T) { assert.Contains(t, response.Services["grpc"], "not configured") } +func TestHealthReady_StorageAndDBHealthy(t *testing.T) { + router := gin.New() + mockDB := &mockDatabase{} + mockStorage := &mockStorageReader{} + h, err := New(WithDatabase(mockDB), WithStorageReader(mockStorage)) + require.NoError(t, err) + h.RegisterRoutes(router) + + req, _ := http.NewRequest(http.MethodGet, "/health/ready", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + + var response HealthStatus + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + // Still degraded because grpc is not configured + assert.Equal(t, "degraded", response.Status) + assert.Equal(t, "ok", response.Services["database"]) + assert.Equal(t, "ok", response.Services["storage"]) + assert.Contains(t, response.Services["grpc"], "not configured") +} + +func TestHealthReady_StorageUnhealthy(t *testing.T) { + router := gin.New() + mockDB := &mockDatabase{} + mockStorage := &mockStorageReader{pingErr: fmt.Errorf("connection refused")} + h, err := New(WithDatabase(mockDB), WithStorageReader(mockStorage)) + require.NoError(t, err) + h.RegisterRoutes(router) + + req, _ := http.NewRequest(http.MethodGet, "/health/ready", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + + var response HealthStatus + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "degraded", response.Status) + assert.Contains(t, response.Services["storage"], "connection refused") +} + +// startHealthGRPCServer starts a gRPC server with a health service reporting +// the given status. Returns the listening port and a stop function. +func startHealthGRPCServer(t *testing.T, status healthgrpc.HealthCheckResponse_ServingStatus) (int, func()) { + t.Helper() + + lis, err := net.Listen("tcp", "localhost:0") + require.NoError(t, err) + + srv := grpc.NewServer() + healthSrv := health.NewServer() + healthSrv.SetServingStatus("", status) + healthgrpc.RegisterHealthServer(srv, healthSrv) + + go func() { _ = srv.Serve(lis) }() + + return lis.Addr().(*net.TCPAddr).Port, srv.Stop +} + +func TestHealthReady_AllHealthy(t *testing.T) { + port, stopServer := startHealthGRPCServer(t, healthgrpc.HealthCheckResponse_SERVING) + defer stopServer() + + reencryptClient := reencrypt.NewClient("localhost", port) + defer reencryptClient.Close() + + router := gin.New() + h, err := New( + WithDatabase(&mockDatabase{}), + WithStorageReader(&mockStorageReader{}), + WithReencryptClient(reencryptClient), + ) + require.NoError(t, err) + h.RegisterRoutes(router) + + req, _ := http.NewRequest(http.MethodGet, "/health/ready", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + + var response HealthStatus + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "ok", response.Status) + assert.Equal(t, "ok", response.Services["database"]) + assert.Equal(t, "ok", response.Services["storage"]) + assert.Equal(t, "ok", response.Services["grpc"]) +} + +func TestHealthReady_GrpcUnhealthy(t *testing.T) { + port, stopServer := startHealthGRPCServer(t, healthgrpc.HealthCheckResponse_NOT_SERVING) + defer stopServer() + + reencryptClient := reencrypt.NewClient("localhost", port) + defer reencryptClient.Close() + + router := gin.New() + h, err := New( + WithDatabase(&mockDatabase{}), + WithStorageReader(&mockStorageReader{}), + WithReencryptClient(reencryptClient), + ) + require.NoError(t, err) + h.RegisterRoutes(router) + + req, _ := http.NewRequest(http.MethodGet, "/health/ready", nil) + w := httptest.NewRecorder() + + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusServiceUnavailable, w.Code) + + var response HealthStatus + err = json.Unmarshal(w.Body.Bytes(), &response) + assert.NoError(t, err) + assert.Equal(t, "degraded", response.Status) + assert.Equal(t, "ok", response.Services["database"]) + assert.Equal(t, "ok", response.Services["storage"]) + assert.Contains(t, response.Services["grpc"], "NOT_SERVING") +} + // Test that hasDatasetAccess helper works correctly func TestHasDatasetAccess(t *testing.T) { testCases := []struct { diff --git a/sda/cmd/download/handlers/health.go b/sda/cmd/download/handlers/health.go index 0d89767ca3..b958ccd806 100644 --- a/sda/cmd/download/handlers/health.go +++ b/sda/cmd/download/handlers/health.go @@ -40,7 +40,13 @@ func (h *Handlers) HealthReady(c *gin.Context) { // Check storage reader if h.storageReader != nil { - services["storage"] = "ok" + if err := h.storageReader.Ping(ctx); err != nil { + log.Warnf("health check: storage ping failed: %v", err) + services["storage"] = "error: " + err.Error() + allHealthy = false + } else { + services["storage"] = "ok" + } } else { services["storage"] = "error: not configured" allHealthy = false @@ -48,7 +54,7 @@ func (h *Handlers) HealthReady(c *gin.Context) { // Check gRPC reencrypt client if h.reencryptClient != nil { - if err := h.reencryptClient.HealthCheck(); err != nil { + if err := h.reencryptClient.HealthCheck(ctx); err != nil { log.Warnf("health check: grpc connection failed: %v", err) services["grpc"] = "error: " + err.Error() allHealthy = false diff --git a/sda/cmd/download/handlers/mock_database_test.go b/sda/cmd/download/handlers/mock_database_test.go index 55421a0b41..a0657a6011 100644 --- a/sda/cmd/download/handlers/mock_database_test.go +++ b/sda/cmd/download/handlers/mock_database_test.go @@ -2,6 +2,7 @@ package handlers import ( "context" + "io" "sync" "github.com/neicnordic/sensitive-data-archive/cmd/download/audit" @@ -122,3 +123,28 @@ func (m *mockDatabase) GetDatasetFilesPaginated(_ context.Context, _ string, _ d return m.datasetFilesPaged, nil } + +// mockStorageReader is a mock implementation of storage.Reader for testing. +type mockStorageReader struct { + pingErr error +} + +func (m *mockStorageReader) NewFileReader(_ context.Context, _, _ string) (io.ReadCloser, error) { + return nil, nil +} + +func (m *mockStorageReader) NewFileReadSeeker(_ context.Context, _, _ string) (io.ReadSeekCloser, error) { + return nil, nil +} + +func (m *mockStorageReader) FindFile(_ context.Context, _ string) (string, error) { + return "", nil +} + +func (m *mockStorageReader) GetFileSize(_ context.Context, _, _ string) (int64, error) { + return 0, nil +} + +func (m *mockStorageReader) Ping(_ context.Context) error { + return m.pingErr +} From 5de272c619efd44a1c0bfbbb238ca70a6ddac34d Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 14 Apr 2026 15:43:52 +0200 Subject: [PATCH 465/530] fix(health): hide backend error details in readiness response Per review feedback on #2388: the per-service error values in /health/ready previously echoed the raw ping/HealthCheck error from the backend, which can leak implementation details to anyone reaching the endpoint. Replace all three ("database", "storage", "grpc") with a static "error: backend unresponsive". Real errors are still logged at warn level for operators. Also swap a fmt.Errorf("connection refused") for errors.New in TestHealthReady_StorageUnhealthy since no formatting is used, and tighten the storage/grpc unhealthy assertions to check for the exact sanitized message. --- sda/cmd/download/handlers/handlers_test.go | 8 ++++---- sda/cmd/download/handlers/health.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sda/cmd/download/handlers/handlers_test.go b/sda/cmd/download/handlers/handlers_test.go index f9b1898fea..860683ffd8 100644 --- a/sda/cmd/download/handlers/handlers_test.go +++ b/sda/cmd/download/handlers/handlers_test.go @@ -2,7 +2,7 @@ package handlers import ( "encoding/json" - "fmt" + "errors" "net" "net/http" "net/http/httptest" @@ -149,7 +149,7 @@ func TestHealthReady_StorageAndDBHealthy(t *testing.T) { func TestHealthReady_StorageUnhealthy(t *testing.T) { router := gin.New() mockDB := &mockDatabase{} - mockStorage := &mockStorageReader{pingErr: fmt.Errorf("connection refused")} + mockStorage := &mockStorageReader{pingErr: errors.New("connection refused")} h, err := New(WithDatabase(mockDB), WithStorageReader(mockStorage)) require.NoError(t, err) h.RegisterRoutes(router) @@ -165,7 +165,7 @@ func TestHealthReady_StorageUnhealthy(t *testing.T) { err = json.Unmarshal(w.Body.Bytes(), &response) assert.NoError(t, err) assert.Equal(t, "degraded", response.Status) - assert.Contains(t, response.Services["storage"], "connection refused") + assert.Equal(t, "error: backend unresponsive", response.Services["storage"]) } // startHealthGRPCServer starts a gRPC server with a health service reporting @@ -247,7 +247,7 @@ func TestHealthReady_GrpcUnhealthy(t *testing.T) { assert.Equal(t, "degraded", response.Status) assert.Equal(t, "ok", response.Services["database"]) assert.Equal(t, "ok", response.Services["storage"]) - assert.Contains(t, response.Services["grpc"], "NOT_SERVING") + assert.Equal(t, "error: backend unresponsive", response.Services["grpc"]) } // Test that hasDatasetAccess helper works correctly diff --git a/sda/cmd/download/handlers/health.go b/sda/cmd/download/handlers/health.go index b958ccd806..558220673f 100644 --- a/sda/cmd/download/handlers/health.go +++ b/sda/cmd/download/handlers/health.go @@ -28,7 +28,7 @@ func (h *Handlers) HealthReady(c *gin.Context) { if h.db != nil { if err := h.db.Ping(ctx); err != nil { log.Warnf("health check: database ping failed: %v", err) - services["database"] = "error: " + err.Error() + services["database"] = "error: backend unresponsive" allHealthy = false } else { services["database"] = "ok" @@ -42,7 +42,7 @@ func (h *Handlers) HealthReady(c *gin.Context) { if h.storageReader != nil { if err := h.storageReader.Ping(ctx); err != nil { log.Warnf("health check: storage ping failed: %v", err) - services["storage"] = "error: " + err.Error() + services["storage"] = "error: backend unresponsive" allHealthy = false } else { services["storage"] = "ok" @@ -56,7 +56,7 @@ func (h *Handlers) HealthReady(c *gin.Context) { if h.reencryptClient != nil { if err := h.reencryptClient.HealthCheck(ctx); err != nil { log.Warnf("health check: grpc connection failed: %v", err) - services["grpc"] = "error: " + err.Error() + services["grpc"] = "error: backend unresponsive" allHealthy = false } else { services["grpc"] = "ok" From 2861fc6a183c7796232fdb2f8abb693704762f55 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 14 Apr 2026 16:11:43 +0200 Subject: [PATCH 466/530] fix(integration): generate all c4gh keys the reencrypt config expects make_download_credentials.sh only created /shared/c4gh (with the wrong passphrase "c4gh"), but the shared sda/config.yaml used by the reencrypt container lists three private keys (c4gh, sync, rotatekey) and loads c4gh with passphrase "c4ghpass". That left the reencrypt container stuck in a restart loop. With the nil-check readiness probe the failure was silent since /health/ready always returned 200. Now that /health/ready actually pings the gRPC service, Test02_HealthReady caught it. Matches the key set and passphrases in make_sda_credentials.sh. --- .github/integration/scripts/make_download_credentials.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/integration/scripts/make_download_credentials.sh b/.github/integration/scripts/make_download_credentials.sh index 48dac59ae9..2102f95aa0 100644 --- a/.github/integration/scripts/make_download_credentials.sh +++ b/.github/integration/scripts/make_download_credentials.sh @@ -78,12 +78,16 @@ cat > "/shared/trusted-issuers.json" <<'EOF' ] EOF -## create crypt4gh key +## create crypt4gh keys if [ ! -f "/shared/c4gh.pub.pem" ]; then echo "downloading crypt4gh" latest_c4gh=$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name') curl --retry 100 -sL "https://github.com/neicnordic/crypt4gh/releases/download/${latest_c4gh}/crypt4gh_linux_x86_64.tar.gz" | tar -xz -C /shared/ - /shared/crypt4gh generate -n /shared/c4gh -p c4gh + # Generate all keys the shared reencrypt config references; passphrases + # must match sda/config.yaml. + /shared/crypt4gh generate -n /shared/c4gh -p c4ghpass + /shared/crypt4gh generate -n /shared/sync -p syncPass + /shared/crypt4gh generate -n /shared/rotatekey -p rotatekeyPass fi echo "download credentials setup complete" From 1cfe57a5f86e717a79e215bf577ce70aded2f352 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Apr 2026 19:05:11 +0000 Subject: [PATCH 467/530] build(deps): bump softprops/action-gh-release Bumps the github-actions group with 1 update: [softprops/action-gh-release](https://github.com/softprops/action-gh-release). Updates `softprops/action-gh-release` from 2 to 3 - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/v2...v3) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-version: '3' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/release_sda.yml | 2 +- .github/workflows/release_sda_admin.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml index 49346b57c5..10e1173b0f 100644 --- a/.github/workflows/release_sda.yml +++ b/.github/workflows/release_sda.yml @@ -158,7 +158,7 @@ jobs: - name: Checkout uses: actions/checkout@v6 - name: Release - uses: softprops/action-gh-release@v2 + uses: softprops/action-gh-release@v3 with: generate_release_notes: "true" tag_name: "${{ env.VERSION_TAG }}" diff --git a/.github/workflows/release_sda_admin.yaml b/.github/workflows/release_sda_admin.yaml index 5df7953abd..4c24a38049 100644 --- a/.github/workflows/release_sda_admin.yaml +++ b/.github/workflows/release_sda_admin.yaml @@ -17,7 +17,7 @@ jobs: VERSION=$(cat sda-admin/.version) echo "VERSION=$VERSION" >> $GITHUB_ENV - name: Create release - uses : softprops/action-gh-release@v2 + uses : softprops/action-gh-release@v3 with: tag_name: "sda-admin-${{ env.VERSION }}" From 4ae175c5c2d699c57838cc8f42b977fd29fb444e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 20 Apr 2026 15:49:44 +0200 Subject: [PATCH 468/530] fix(storage-v2): do not set ContentEncoding when writing to s3, as it leads to SignatureDoesNotMatch error when talking to ceph --- sda/internal/storage/v2/s3/writer/write_file.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sda/internal/storage/v2/s3/writer/write_file.go b/sda/internal/storage/v2/s3/writer/write_file.go index bbe06eebf0..2441a16950 100644 --- a/sda/internal/storage/v2/s3/writer/write_file.go +++ b/sda/internal/storage/v2/s3/writer/write_file.go @@ -56,10 +56,9 @@ func (writer *Writer) WriteFile(ctx context.Context, filePath string, fileConten }) _, err = uploader.UploadObject(ctx, &transfermanager.UploadObjectInput{ - Body: fileContent, - Bucket: aws.String(activeBucket), - Key: aws.String(filePath), - ContentEncoding: aws.String("application/octet-stream"), + Body: fileContent, + Bucket: aws.String(activeBucket), + Key: aws.String(filePath), }) if err != nil { return "", fmt.Errorf("failed to upload object: %s, bucket: %s, endpoint: %s, due to: %v", filePath, activeBucket, writer.activeEndpoint.Endpoint, err) From 9919ec8dfee21ab66f8d69acaeee7bd1f22d360a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 23 Apr 2026 11:54:35 +0200 Subject: [PATCH 469/530] fix(download / storage/v2): remove SeekableMultiReader as it leaks data from reader into memory, instead update download to just discard from a io.MultiReader up until start and then continue until the requested end --- sda-download/api/sda/sda.go | 40 ++----- .../storage/v2/seekable_multi_reader.go | 107 ------------------ .../storage/v2/seekable_multi_reader_test.go | 59 ---------- sda/cmd/download/streaming/streaming.go | 4 - .../storage/v2/seekable_multi_reader.go | 107 ------------------ .../storage/v2/seekable_multi_reader_test.go | 59 ---------- 6 files changed, 7 insertions(+), 369 deletions(-) delete mode 100644 sda-download/internal/storage/v2/seekable_multi_reader.go delete mode 100644 sda-download/internal/storage/v2/seekable_multi_reader_test.go delete mode 100644 sda/internal/storage/v2/seekable_multi_reader.go delete mode 100644 sda/internal/storage/v2/seekable_multi_reader_test.go diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index f85fcca6ca..be88f5f055 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -401,26 +401,15 @@ func Download(c *gin.Context) { } newHr := bytes.NewReader(newHeader) - - if wholeFile { - fileStream = io.MultiReader(newHr, file) - } else { - seeker, _ := file.(io.ReadSeeker) - seekStream, err := storage.SeekableMultiReader(newHr, seeker) - if err != nil { - log.Errorf("Failed to construct SeekableMultiReader, reason: %v", err) - c.String(http.StatusInternalServerError, "file decoding error") - - return - } - start, end, err = adjustSeekPos(seekStream, start, end) + fileStream = io.MultiReader(newHr, file) + if !wholeFile { + start, end, err = adjustSeekPos(fileStream, start, end) if err != nil { log.Errorf("Could not seek stream: %v", err) c.String(http.StatusInternalServerError, "file decoding error") return } - fileStream = seekStream } default: // Reencrypt header for use with the loaded internal key @@ -433,20 +422,7 @@ func Download(c *gin.Context) { } newHr := bytes.NewReader(newHeader) - - if wholeFile { - fileStream = io.MultiReader(newHr, file) - } else { - seeker, _ := file.(io.ReadSeeker) - fileStream, err = storage.SeekableMultiReader(newHr, seeker) - if err != nil { - log.Errorf("Failed to construct SeekableMultiReader, reason: %v", err) - c.String(http.StatusInternalServerError, "file decoding error") - - return - } - } - + fileStream = io.MultiReader(newHr, file) c4ghfileStream, err := streaming.NewCrypt4GHReader(fileStream, config.Config.C4GH.PrivateKey, nil) defer c4ghfileStream.Close() if err != nil { @@ -474,12 +450,10 @@ func Download(c *gin.Context) { } } -var adjustSeekPos = func(fileStream io.ReadSeeker, start, end int64) (int64, int64, error) { +var adjustSeekPos = func(fileStream io.Reader, start, end int64) (int64, int64, error) { if start != 0 { - // We don't want to read from start, skip ahead to where we should be - _, err := fileStream.Seek(start, 0) - if err != nil { - return 0, 0, fmt.Errorf("error occurred while finding sending start: %v", err) + if _, err := io.CopyN(io.Discard, fileStream, start); err != nil { + return 0, 0, err } // adjust end to reflect that the file start has been moved end -= start diff --git a/sda-download/internal/storage/v2/seekable_multi_reader.go b/sda-download/internal/storage/v2/seekable_multi_reader.go deleted file mode 100644 index 0ecbf17ada..0000000000 --- a/sda-download/internal/storage/v2/seekable_multi_reader.go +++ /dev/null @@ -1,107 +0,0 @@ -package storage - -import ( - "errors" - "fmt" - "io" -) - -// seekableMultiReader is a helper struct to allow io.MultiReader to be used with a seekable reader -type seekableMultiReader struct { - readers []io.Reader - sizes []int64 - currentOffset int64 - totalSize int64 -} - -// SeekableMultiReader constructs a multireader that supports seeking. Requires -// all passed readers to be seekable -func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { - r := make([]io.Reader, len(readers)) - sizes := make([]int64, len(readers)) - - copy(r, readers) - - var totalSize int64 - for i, reader := range readers { - seeker, ok := reader.(io.ReadSeeker) - if !ok { - return nil, fmt.Errorf("reader %d to SeekableMultiReader is not seekable", i) - } - - size, err := seeker.Seek(0, io.SeekEnd) - if err != nil { - return nil, fmt.Errorf("size determination failed for reader %d to SeekableMultiReader: %v", i, err) - } - - sizes[i] = size - totalSize += size - } - - return &seekableMultiReader{r, sizes, 0, totalSize}, nil -} - -func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - r.currentOffset = offset - case io.SeekCurrent: - r.currentOffset += offset - case io.SeekEnd: - r.currentOffset = r.totalSize + offset - default: - return 0, errors.New("unsupported whence") - } - - if r.currentOffset < 0 { - return 0, errors.New("negative seek offset") - } - - return r.currentOffset, nil -} - -func (r *seekableMultiReader) Read(dst []byte) (int, error) { - var readerStartAt int64 - - for i, reader := range r.readers { - if r.currentOffset < readerStartAt { - // We want data from a previous reader - return 0, errors.New("unexpected error: could not find correct reader to read from") - } - - if readerStartAt+r.sizes[i] < r.currentOffset { - // We want data from a later reader - readerStartAt += r.sizes[i] - - continue - } - - // At least part of the data is in this reader - - seekable, ok := reader.(io.ReadSeeker) - if !ok { - return 0, errors.New("expected seekable reader but changed") - } - - _, err := seekable.Seek(r.currentOffset-readerStartAt, 0) - if err != nil { - return 0, fmt.Errorf("unexpected error while seeking: %v", err) - } - - n, err := seekable.Read(dst) - r.currentOffset += int64(n) - - if n > 0 || err != io.EOF { - if err == io.EOF && r.currentOffset < r.totalSize { - // More data left, hold that EOF - err = nil - } - - return n, err - } - - readerStartAt += r.sizes[i] - } - - return 0, io.EOF -} diff --git a/sda-download/internal/storage/v2/seekable_multi_reader_test.go b/sda-download/internal/storage/v2/seekable_multi_reader_test.go deleted file mode 100644 index 1186f9a4ce..0000000000 --- a/sda-download/internal/storage/v2/seekable_multi_reader_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package storage - -import ( - "bytes" - "io" - "os" - "testing" - - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -func TestSeekableMultiReader(t *testing.T) { - writeData := []byte("this is a test") - - readers := make([]io.Reader, 10) - for i := 0; i < 10; i++ { - readers[i] = bytes.NewReader(writeData) - } - - seeker, err := SeekableMultiReader(readers...) - assert.Nil(t, err, "unexpected error from creating SeekableMultiReader") - - var readBackBuffer [4096]byte - - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData, readBackBuffer[:14], "did not read back data as expected") - assert.Nil(t, err, "unexpected error from read") - - offset, err := seeker.Seek(60, io.SeekStart) - - assert.Nil(t, err, "Seek failed") - assert.Equal(t, int64(60), offset, "Seek did not return expected offset") - - // We don't know how many bytes this should return - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData[4:], readBackBuffer[:10], "did not read back data as expected") - assert.Nil(t, err, "Read failed when it should not") - - offset, err = seeker.Seek(0, io.SeekEnd) - assert.Equal(t, int64(140), offset, "Seek did not return expected offset") - assert.Nil(t, err, "Seek failed when it should not") - - n, err := seeker.Read(readBackBuffer[0:4096]) - - assert.Equal(t, 0, n, "Read did not return expected amounts of bytes") - assert.Equal(t, io.EOF, err, "did not get EOF as expected") - - offset, err = seeker.Seek(56, io.SeekStart) - assert.Equal(t, int64(56), offset, "Seek did not return expected offset") - assert.Nil(t, err, "Seek failed unexpectedly") - - largeBuf := make([]byte, 65536) - readLen, err := seeker.Read(largeBuf) - assert.Nil(t, err, "unexpected error when reading back data") - assert.Equal(t, 14, readLen, "did not read back expect amount of data") - - log.SetOutput(os.Stdout) -} diff --git a/sda/cmd/download/streaming/streaming.go b/sda/cmd/download/streaming/streaming.go index a49a38751b..61e5c7178c 100644 --- a/sda/cmd/download/streaming/streaming.go +++ b/sda/cmd/download/streaming/streaming.go @@ -11,7 +11,6 @@ import ( "strconv" "strings" - storage "github.com/neicnordic/sensitive-data-archive/internal/storage/v2" log "github.com/sirupsen/logrus" ) @@ -362,6 +361,3 @@ func StreamBodyOnly(cfg StreamBodyConfig) error { return nil } - -// SeekableMultiReader is a re-export of storage.SeekableMultiReader for convenience. -var SeekableMultiReader = storage.SeekableMultiReader diff --git a/sda/internal/storage/v2/seekable_multi_reader.go b/sda/internal/storage/v2/seekable_multi_reader.go deleted file mode 100644 index 0ecbf17ada..0000000000 --- a/sda/internal/storage/v2/seekable_multi_reader.go +++ /dev/null @@ -1,107 +0,0 @@ -package storage - -import ( - "errors" - "fmt" - "io" -) - -// seekableMultiReader is a helper struct to allow io.MultiReader to be used with a seekable reader -type seekableMultiReader struct { - readers []io.Reader - sizes []int64 - currentOffset int64 - totalSize int64 -} - -// SeekableMultiReader constructs a multireader that supports seeking. Requires -// all passed readers to be seekable -func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { - r := make([]io.Reader, len(readers)) - sizes := make([]int64, len(readers)) - - copy(r, readers) - - var totalSize int64 - for i, reader := range readers { - seeker, ok := reader.(io.ReadSeeker) - if !ok { - return nil, fmt.Errorf("reader %d to SeekableMultiReader is not seekable", i) - } - - size, err := seeker.Seek(0, io.SeekEnd) - if err != nil { - return nil, fmt.Errorf("size determination failed for reader %d to SeekableMultiReader: %v", i, err) - } - - sizes[i] = size - totalSize += size - } - - return &seekableMultiReader{r, sizes, 0, totalSize}, nil -} - -func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { - switch whence { - case io.SeekStart: - r.currentOffset = offset - case io.SeekCurrent: - r.currentOffset += offset - case io.SeekEnd: - r.currentOffset = r.totalSize + offset - default: - return 0, errors.New("unsupported whence") - } - - if r.currentOffset < 0 { - return 0, errors.New("negative seek offset") - } - - return r.currentOffset, nil -} - -func (r *seekableMultiReader) Read(dst []byte) (int, error) { - var readerStartAt int64 - - for i, reader := range r.readers { - if r.currentOffset < readerStartAt { - // We want data from a previous reader - return 0, errors.New("unexpected error: could not find correct reader to read from") - } - - if readerStartAt+r.sizes[i] < r.currentOffset { - // We want data from a later reader - readerStartAt += r.sizes[i] - - continue - } - - // At least part of the data is in this reader - - seekable, ok := reader.(io.ReadSeeker) - if !ok { - return 0, errors.New("expected seekable reader but changed") - } - - _, err := seekable.Seek(r.currentOffset-readerStartAt, 0) - if err != nil { - return 0, fmt.Errorf("unexpected error while seeking: %v", err) - } - - n, err := seekable.Read(dst) - r.currentOffset += int64(n) - - if n > 0 || err != io.EOF { - if err == io.EOF && r.currentOffset < r.totalSize { - // More data left, hold that EOF - err = nil - } - - return n, err - } - - readerStartAt += r.sizes[i] - } - - return 0, io.EOF -} diff --git a/sda/internal/storage/v2/seekable_multi_reader_test.go b/sda/internal/storage/v2/seekable_multi_reader_test.go deleted file mode 100644 index 1186f9a4ce..0000000000 --- a/sda/internal/storage/v2/seekable_multi_reader_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package storage - -import ( - "bytes" - "io" - "os" - "testing" - - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" -) - -func TestSeekableMultiReader(t *testing.T) { - writeData := []byte("this is a test") - - readers := make([]io.Reader, 10) - for i := 0; i < 10; i++ { - readers[i] = bytes.NewReader(writeData) - } - - seeker, err := SeekableMultiReader(readers...) - assert.Nil(t, err, "unexpected error from creating SeekableMultiReader") - - var readBackBuffer [4096]byte - - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData, readBackBuffer[:14], "did not read back data as expected") - assert.Nil(t, err, "unexpected error from read") - - offset, err := seeker.Seek(60, io.SeekStart) - - assert.Nil(t, err, "Seek failed") - assert.Equal(t, int64(60), offset, "Seek did not return expected offset") - - // We don't know how many bytes this should return - _, err = seeker.Read(readBackBuffer[0:4096]) - assert.Equal(t, writeData[4:], readBackBuffer[:10], "did not read back data as expected") - assert.Nil(t, err, "Read failed when it should not") - - offset, err = seeker.Seek(0, io.SeekEnd) - assert.Equal(t, int64(140), offset, "Seek did not return expected offset") - assert.Nil(t, err, "Seek failed when it should not") - - n, err := seeker.Read(readBackBuffer[0:4096]) - - assert.Equal(t, 0, n, "Read did not return expected amounts of bytes") - assert.Equal(t, io.EOF, err, "did not get EOF as expected") - - offset, err = seeker.Seek(56, io.SeekStart) - assert.Equal(t, int64(56), offset, "Seek did not return expected offset") - assert.Nil(t, err, "Seek failed unexpectedly") - - largeBuf := make([]byte, 65536) - readLen, err := seeker.Read(largeBuf) - assert.Nil(t, err, "unexpected error when reading back data") - assert.Equal(t, 14, readLen, "did not read back expect amount of data") - - log.SetOutput(os.Stdout) -} From 9a26ed131d357209646e1ca1cdc01bfb5d82bcfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Thu, 23 Apr 2026 12:26:47 +0200 Subject: [PATCH 470/530] fix(download): dont use ArchiveReader.NewFileReadSeeker as we dont need and ReadSeeker, fix some log messages, rename adjustSeekPos -> adjustToStartPosition and make as func instead of a func variable --- sda-download/api/sda/sda.go | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index be88f5f055..b3bb8c298d 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -326,12 +326,7 @@ func Download(c *gin.Context) { } // Get archive file handle - var file io.ReadCloser - if wholeFile { - file, err = ArchiveReader.NewFileReader(c, fileDetails.ArchiveLocation, fileDetails.ArchivePath) - } else { - file, err = ArchiveReader.NewFileReadSeeker(c, fileDetails.ArchiveLocation, fileDetails.ArchivePath) - } + file, err := ArchiveReader.NewFileReader(c, fileDetails.ArchiveLocation, fileDetails.ArchivePath) if err != nil { log.Errorf("could not find archive file %s, %s", fileDetails.ArchivePath, err) c.String(http.StatusInternalServerError, "archive error") @@ -403,9 +398,9 @@ func Download(c *gin.Context) { newHr := bytes.NewReader(newHeader) fileStream = io.MultiReader(newHr, file) if !wholeFile { - start, end, err = adjustSeekPos(fileStream, start, end) + start, end, err = adjustToStartPosition(fileStream, start, end) if err != nil { - log.Errorf("Could not seek stream: %v", err) + log.Errorf("Could not adjust start position: %v", err) c.String(http.StatusInternalServerError, "file decoding error") return @@ -431,9 +426,9 @@ func Download(c *gin.Context) { return } - start, end, err = adjustSeekPos(c4ghfileStream, start, end) + start, end, err = adjustToStartPosition(c4ghfileStream, start, end) if err != nil { - log.Errorf("Could not seek stream: %v", err) + log.Errorf("Could not adjust start position: %v", err) c.String(http.StatusInternalServerError, "file decoding error") return @@ -450,7 +445,7 @@ func Download(c *gin.Context) { } } -var adjustSeekPos = func(fileStream io.Reader, start, end int64) (int64, int64, error) { +func adjustToStartPosition(fileStream io.Reader, start, end int64) (int64, int64, error) { if start != 0 { if _, err := io.CopyN(io.Discard, fileStream, start); err != nil { return 0, 0, err From ac230c645c623053ed5a34127ac7f0ba3d39fee4 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 21 Apr 2026 16:01:37 +0200 Subject: [PATCH 471/530] docs(adr): propose ADR-0005 introduce RFCs as upstream exploration phase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decision log should record what the system is committed to. Some proposals open an ADR PR long before the team can honestly write "Chosen option: X, because Y", which leaves the log mixing commitments with speculation. Introduce docs/rfcs/ as an upstream exploration phase. RFCs can live there indefinitely; an RFC is promoted to an ADR when the team can commit. The existing ADR status vocabulary is unchanged — proposed still means "under discussion in a PR". The rfcs/ scaffold is added in a follow-up commit. --- ...duce-rfcs-as-upstream-exploration-phase.md | 167 ++++++++++++++++++ docs/decisions/README.md | 22 +++ 2 files changed, 189 insertions(+) create mode 100644 docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md diff --git a/docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md b/docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md new file mode 100644 index 0000000000..aaf6aa7faf --- /dev/null +++ b/docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md @@ -0,0 +1,167 @@ +--- +status: proposed +date: "2026-04-20" +decision-makers: + - "@neicnordic/sensitive-data-development-collaboration" +consulted: [] +informed: [] +--- + +# Introduce RFCs as an Upstream Exploration Phase for ADRs + +## Context and Problem Statement + +The decision log in `docs/decisions/` is intended to record what the system is +committed to. In practice, some proposals open an ADR PR long before the team +can honestly write *"Chosen option: X, because Y"*. The discussion lingers, the +PR stays open, and the decision log ends up mixing committed decisions with +speculative futures. This weakens its value as a trustworthy record of what the +system actually does and intends to do. + +A concrete example: at the time of writing, two ADR PRs +([#2263 ADR-0001][pr-2263], [#2320 ADR-0004][pr-2320]) have been open for +extended periods without convergence. Their implementation horizon is unclear. +They belong to a different conversation than decisions we are ready to commit +to. + +We need a place for exploration that does not add noise to the decision log. + +## Decision Drivers + +* **Decision log trustworthiness** — entries in `docs/decisions/` should reflect + actual commitments, not hopes. +* **Patience for open questions** — exploration needs a venue more patient + than a PR that is expected to merge. +* **Low ceremony** — any new process must be lightweight or it will be ignored. +* **Ease of promotion** — turning a mature exploration into a decision record + should be a near-trivial edit, so the separation does not penalise good + work. +* **No semantic drift** — the existing status vocabulary + (`proposed` → `accepted` \| `rejected` \| `deprecated` \| `superseded`) is + working and should not be redefined. + +## Considered Options + +1. **Keep everything in `docs/decisions/`** — do nothing; accept that some ADRs + will linger in `proposed`. +2. **Parking-lot tag on ADRs** — add a label or front-matter flag (e.g. + `horizon: distant`) to distinguish speculative ADRs from imminent ones. +3. **Introduce `docs/rfcs/` as an upstream exploration phase** — RFCs live + outside the decision log, are promoted to ADRs when the team can commit. + +## Decision Outcome + +Chosen option: **option 3, introduce `docs/rfcs/` as an upstream exploration +phase**. It matches the separation that Rust, Python, Kubernetes, and Go +already use between *"should we?"* and *"we did, because"*. Exploration gets +its own venue, and the existing ADR status vocabulary stays as it is. + +### Consequences + +* **Good**: The decision log reflects real commitments only; `proposed` keeps + its current meaning of *"under discussion in a PR, expected to merge"*. +* **Good**: Exploration has a patient venue. An RFC can live in the repository + indefinitely without cluttering the decision log. +* **Good**: Promotion is a light edit — copy the relevant content into a new + ADR file, fill in `## Decision Outcome`, and freeze the RFC. The RFC template + is a strict subset of MADR, so nothing has to be restructured on the way + across. +* **Good**: The RFC file stays in `docs/rfcs/` after promotion. The exploration + remains a git artifact in its own right, instead of living only in a GitHub + PR thread. +* **Neutral**: Contributors must learn the difference between an RFC and an + ADR. The README files and the promotion rule make this explicit. +* **Bad**: An extra directory in `docs/` to explain. Mitigated by clear README + text and a shared template. +* **Bad**: Risk that RFCs become a *dumping ground* that nobody promotes. + Three things push back on that: an explicit promotion trigger (*"when you + can write 'Chosen option: X, because Y' with a straight face"*), the + expectation that ADR PRs opened from a mature RFC merge quickly, and a + pass over the RFC index at each bi-weekly NeIC SDA-Devs meet-up — every + open RFC either gets touched, withdrawn, or annotated with a reason it is + still open. + +### Confirmation + +Confirmation is delivered in two steps: + +1. The merge of this PR creates `docs/rfcs/` with its README and template, and + adds the promotion rule to `docs/decisions/README.md`. +2. Two follow-up PRs close the currently open ADR PRs + ([#2263][pr-2263], [#2320][pr-2320]) and re-open their content as RFCs. + Successful completion of those follow-ups confirms that the flow works in + practice. + +## Pros and Cons of the Options + +### Option 1 — Keep everything in `docs/decisions/` + +* Good, because there is zero process change. +* Bad, because the decision log continues to mix commitments with speculation. +* Bad, because there is no signal for *"we are exploring this, do not expect a + commitment yet"*. + +### Option 2 — Parking-lot tag on ADRs + +* Good, because it is a one-line change to existing ADRs. +* Bad, because it overloads the decision log with documents that are not + decisions. +* Bad, because it introduces a second orthogonal lifecycle (horizon) on top of + the existing status lifecycle, which is more semantic surface area than + option 3. + +### Option 3 — Introduce `docs/rfcs/` as an upstream exploration phase + +* Good, because it matches the separation used by Rust + ([rust-lang/rfcs][rust-rfcs]), Python ([PEP 1][pep-1]), Kubernetes + ([KEP process][kep]), and Go ([golang/proposal][go-proposal]). +* Good, because the RFC template is a subset of MADR, so promotion is + near-trivial. +* Good, because it leaves the existing ADR status lifecycle untouched. +* Neutral, because it introduces one new directory. +* Bad, because it requires a small amount of new documentation (two READMEs + and a template). + +## More Information + +### Conventions Established by This ADR + +| Convention | Detail | +| --- | --- | +| Directory | `docs/rfcs/` | +| File naming | `NNNN-title-with-dashes.md` — same as ADRs, numbering independent | +| RFC numbering | Starts at 0001; numbers are never reused | +| RFC status lifecycle | `exploring` → `ready-for-decision` → `promoted` \| `withdrawn` | +| RFC template | `docs/rfcs/template.md` (MADR 4.0.0 minus `## Decision Outcome` and `### Confirmation`, plus `## Open Questions`) | +| Promotion trigger | The team can honestly write *"Chosen option: X, because Y"* | +| Promotion mechanics | Create a new ADR file under the next free ADR number from [`docs/decisions/template.md`](template.md), pulling across the relevant content from the RFC. In the RFC: flip `status` to `promoted`, set `promoted-to` to the ADR filename(s), freeze the body. Promotion is only complete once the index in [this README](README.md#index) and the RFC index are both updated. | +| One RFC, multiple ADRs | Supported: `promoted-to` accepts a list of ADR filenames. | +| Reserved ADR numbers | `0001` and `0004` are reserved by open ADR PRs ([#2263][pr-2263], [#2320][pr-2320]). The next free ADR number is `0005` (this ADR); the number after is `0006`. | +| RFC lifetime | Unlimited. An RFC living for a long time without promotion is not a defect. Stale RFCs are surfaced at the bi-weekly NeIC SDA-Devs meet-up. | +| Post-promotion edits | A `promoted` or `withdrawn` RFC is frozen — only metadata and index rows change. Revisions to a decision live in the ADR. | +| ADR lifecycle unchanged | The ADR status lifecycle (`proposed` → `accepted` \| `rejected` \| `deprecated` \| `superseded`) is **not** modified by this ADR. ADR statuses only apply to files in `docs/decisions/`. | + +### Relationship to other ADRs + +This ADR adds to [ADR-0000][adr-0000]; it does not replace it. ADR-0000 +established ADRs; this one adds an exploration phase upstream of them. + +### References + +* [MADR 4.0.0 template][madr] — the ADR template this project uses. +* [Rust RFC process][rust-rfcs] — canonical single-document RFC flow. +* [Python PEP 1][pep-1] — PEP status lifecycle and exploration conventions. +* [Kubernetes KEP process][kep] — provisional vs implementable states. +* [Go proposal template][go-proposal] — proposal-to-design-doc flow. +* [Candost Dagdeviren, *ADRs and RFCs*][candost] — an accepted RFC may spawn + one or more ADRs. + +[pr-2263]: https://github.com/neicnordic/sensitive-data-archive/pull/2263 +[pr-2320]: https://github.com/neicnordic/sensitive-data-archive/pull/2320 +[adr-0000]: 0000-use-markdown-architectural-decision-records.md +[madr]: https://github.com/adr/madr/blob/4.0.0/template/adr-template.md +[rust-rfcs]: https://github.com/rust-lang/rfcs/blob/master/0000-template.md +[pep-1]: https://peps.python.org/pep-0001/ +[kep]: https://github.com/kubernetes/enhancements/tree/master/keps/NNNN-kep-template +[go-proposal]: https://github.com/golang/proposal/blob/master/design/TEMPLATE.md +[candost]: https://candost.blog/adrs-rfcs-differences-when-which/ diff --git a/docs/decisions/README.md b/docs/decisions/README.md index f53eb67ac0..be4da8740e 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -36,6 +36,22 @@ already obvious from the code or commit message. Not every change needs a record — use your judgement. The pull request template includes a checkbox as a reminder. +## RFCs — upstream exploration + +When the team recognises an architectural question but cannot yet commit to a +specific answer, write an [RFC](../rfcs/) instead of an ADR. RFCs live in +`docs/rfcs/` and reuse the MADR structure of this template, minus +`## Decision Outcome` and `### Confirmation`, plus `## Open Questions`. + +An RFC is **promoted** to an ADR when the team can honestly write +*"Chosen option: X, because Y"*. Promotion creates a new ADR file here under +the next free ADR number; the original RFC file stays in `docs/rfcs/` as a +frozen reference to the exploration. ADR statuses (`proposed`, `accepted`, +etc.) only apply to files in this directory. + +See [ADR-0005](0005-introduce-rfcs-as-upstream-exploration-phase.md) for the +rationale and the full promotion procedure. + ## How to create a decision record 1. Copy the template: @@ -95,6 +111,12 @@ superseded-by: "0005-use-new-approach.md" | [0000](0000-use-markdown-architectural-decision-records.md) | Use Markdown Architectural Decision Records | accepted | | [0002](0002-merge-dependabot-package-managers.md) | Merge Dependabot package managers | accepted | | [0003](0003-shared-state-strategy-for-s3inbox-and-caching.md) | Replace s3inbox In-Memory File ID Cache with Database Lookups | proposed | +| [0005](0005-introduce-rfcs-as-upstream-exploration-phase.md) | Introduce RFCs as an upstream exploration phase for ADRs | proposed | + +Numbers `0001` and `0004` are reserved by open ADR PRs +([#2263](https://github.com/neicnordic/sensitive-data-archive/pull/2263), +[#2320](https://github.com/neicnordic/sensitive-data-archive/pull/2320)). The +next free ADR number for a new decision is `0006`. ## More information From c7de4b34e246a6b84d1ea49144a7cf479af7805c Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 21 Apr 2026 16:01:44 +0200 Subject: [PATCH 472/530] docs(rfcs): add folder, README, and template for upstream exploration Scaffolds docs/rfcs/ for architectural questions the team is exploring but not yet ready to commit to. README covers when to write an RFC, the status lifecycle (exploring, ready-for-decision, promoted, withdrawn), and the promotion procedure to an ADR. The template reuses MADR 4.0.0 structure minus Decision Outcome and Confirmation, plus an Open Questions section that captures what the team does not yet know. See ADR-0005 for the rationale. --- docs/rfcs/README.md | 136 ++++++++++++++++++++++++++++++++++++++++++ docs/rfcs/template.md | 104 ++++++++++++++++++++++++++++++++ 2 files changed, 240 insertions(+) create mode 100644 docs/rfcs/README.md create mode 100644 docs/rfcs/template.md diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md new file mode 100644 index 0000000000..33b3cdac2e --- /dev/null +++ b/docs/rfcs/README.md @@ -0,0 +1,136 @@ +# Requests for Comments (RFCs) + +This directory is an **upstream exploration phase** for architectural work. +It is not a decision log. For the decision log, see +[`docs/decisions/`](../decisions/). + +An RFC is the right place to put a question the team is actively exploring but +cannot yet resolve with *"Chosen option: X, because Y"*. When an RFC matures +to that point, it is **promoted** to an ADR. The RFC file stays in this +directory as a frozen reference to the exploration; the ADR is a new file in +`docs/decisions/`. + +For the full rationale, see +[ADR-0005](../decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md). + +## When to write an RFC + +Write an RFC when all of the following are true: + +* The team recognises a question that will benefit from written discussion. +* You can describe the problem, but **cannot yet commit** to one option. +* The discussion will plausibly take longer than a typical ADR PR — days or + weeks, not hours. + +Skip the RFC and open an ADR directly when the team has already converged and +you can write *"Chosen option: X, because Y"* in the initial PR. + +## How to write an RFC + +1. Copy the template: + + ```sh + cp docs/rfcs/template.md docs/rfcs/NNNN-title-with-dashes.md + ``` + + Use the next available RFC number, zero-padded to four digits. RFC and ADR + numbering are independent. + +2. Fill in the sections. The `## Open Questions` section is the heart of an + RFC — not a placeholder, it is the deliverable. + +3. Set `status: exploring` in the front matter. + +4. Open a pull request and label it `rfc`. + +5. The RFC is merged in its current state so the exploration can be versioned. + It does **not** have to reach a conclusion before merging. `exploring` is a + valid end state for a long time. + +## Status lifecycle + +| Status | Meaning | +| --- | --- | +| `exploring` | Under active exploration. Open questions are expected. | +| `ready-for-decision` | The team believes an ADR can now be written from this RFC. | +| `promoted` | One or more ADRs have been created from this RFC. The RFC body is frozen; `promoted-to` lists the resulting ADR filenames. | +| `withdrawn` | No longer pursued. Kept in place for history. | + +RFC numbers are never reused. Once an RFC has any status other than `exploring`, +its body should not be edited further — only `status`, `date`, `promoted-to`, +and index entries change. + +## Promotion — RFC → ADR + +An RFC is ready to be promoted when the team can honestly write +*"Chosen option: X, because Y"*. When that is true: + +1. Create the ADR file at `docs/decisions/NNNN-title-with-dashes.md` using the + next free ADR number (see the [ADR index](../decisions/README.md#index) — + some numbers may already be reserved by open ADR PRs). Start from + [`docs/decisions/template.md`](../decisions/template.md). Pull over the + Context, Considered Options, and the chosen option; drop the Open Questions + that the decision resolves. +2. Add `## Decision Outcome` with *"Chosen option: X, because Y"*, and + `### Confirmation` if applicable. +3. In the original RFC file: flip `status` from `exploring` or + `ready-for-decision` to `promoted`, set `promoted-to` to the ADR filename + (or a list of filenames, if the RFC yielded more than one decision), and + update the `date`. +4. **Do not edit the RFC body after promotion.** The RFC is a frozen artifact + of the exploration that preceded the decision. If the ADR later needs + revision, revise the ADR, not the RFC. +5. Update both indices: + * the RFC row in [this README's index](#index) — mark as `promoted`, link + to the ADR; + * the ADR row in [`docs/decisions/README.md`](../decisions/README.md#index). + +Promotion is complete only when both index updates have been made. + +The ADR PR that follows promotion is expected to merge quickly. If it stalls, +that is a signal that the exploration was not complete — consider reopening +the question as a new RFC rather than leaving the ADR `proposed` indefinitely. + +### Edge cases + +**One RFC, several decisions.** If the exploration reveals more than one +decision the team is ready to make, create one ADR per decision. The RFC's +`promoted-to` field lists all the resulting ADR filenames. No need to split +the RFC. + +**Reviving a `withdrawn` RFC.** Write a new RFC that links back to the +withdrawn one in its `## More Information`. Do not reopen the withdrawn file; +it is part of the historical record. + +## Lifetime + +An RFC can live in this directory indefinitely. That is not a defect. If a +question is worth thinking about but not yet worth committing to, leaving the +RFC in `exploring` is the correct state. Only change the status when something +actually changes. + +To keep the folder from quietly rotting, the RFC index gets a pass at each +bi-weekly NeIC SDA-Devs meet-up. Every `exploring` or `ready-for-decision` +RFC is touched, withdrawn, or left with a short note explaining why it is +still open. The review is a lightweight forcing function, not a deadline. + +## How this relates to ADRs + +| | `docs/rfcs/` | `docs/decisions/` | +| --- | --- | --- | +| Purpose | Exploration | Commitment | +| Typical question | *"Should we?"* | *"We did, because."* | +| Status vocabulary | `exploring`, `ready-for-decision`, `promoted`, `withdrawn` | `proposed`, `accepted`, `rejected`, `deprecated`, `superseded` | +| Template | MADR 4.0.0 structure, minus `## Decision Outcome` and `### Confirmation`, plus `## Open Questions` | Full MADR 4.0.0 | +| Expected PR lifetime | Indefinite | Short — merges when the discussion has already happened | + +ADR statuses only apply after a file lives in `docs/decisions/`. An RFC is not +a `proposed` ADR; it is a separate artifact with its own lifecycle. + +## Index + +| # | RFC | Status | Promoted to | +| --- | --- | --- | --- | + +*(empty at the time this folder is introduced; entries added as RFCs are +written)* diff --git a/docs/rfcs/template.md b/docs/rfcs/template.md new file mode 100644 index 0000000000..6aedec25e8 --- /dev/null +++ b/docs/rfcs/template.md @@ -0,0 +1,104 @@ +--- +# RFC front matter. These are optional unless noted. Remove fields you do not need. +status: exploring # exploring | ready-for-decision | promoted | withdrawn +promoted-to: [] # one or more ADR filenames — set when status is `promoted` +date: "YYYY-MM-DD" # when the RFC was last updated +authors: [] # people actively shaping this RFC +consulted: [] # subject-matter experts consulted (two-way communication) +informed: [] # kept up to date on the discussion (one-way communication) +--- + + + +# {short title, representative of the question the RFC is exploring} + +## Context and Problem Statement + +{Describe the context and the problem in two to three sentences, or as a short +story. Link to collaboration boards, issues, or prior discussions. Unlike an +ADR, an RFC does not have to frame the problem as a question with an imminent +answer — it is fine to describe a tension the team wants to explore.} + + +## Decision Drivers + +* {driver 1 — a force or concern the RFC should account for} +* {driver 2} +* … + +## Considered Options + +* {title of option 1} +* {title of option 2} +* {title of option 3} +* … + + + +## Open Questions + +{List what the team does not yet know or agree on. Unresolved items are not a +gap in the RFC; they are the point of writing one. When the list is short +enough that the team can honestly write "Chosen option: X, because Y", the +RFC is ready for promotion.} + +* {open question 1} +* {open question 2} +* … + + +## Pros and Cons of the Options + +### {title of option 1} + + +{example | description | pointer to more information | …} + +* Good, because {argument a} +* Good, because {argument b} + +* Neutral, because {argument c} +* Bad, because {argument d} +* … + +### {title of other option} + +{example | description | pointer to more information | …} + +* Good, because {argument a} +* Good, because {argument b} +* Neutral, because {argument c} +* Bad, because {argument d} +* … + + +## More Information + +{Provide additional evidence or context. Link to related RFCs, ADRs, prior art +from other projects, and any external discussion threads. If the RFC has a +suspected eventual decision direction, note it here — but do not put a +commitment in the `## Decision Outcome` section until promotion.} From b31ba4f2dabbb0ce0f99d23d0b3df6d16e33c8f7 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 21 Apr 2026 16:23:22 +0200 Subject: [PATCH 473/530] docs(adr): clarify convert-in-place and retired ADR numbers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two open ADR PRs (#2263, #2320) are converted in place to RFCs — the PR threads carry exactly the kind of exploration RFCs are meant to preserve, so closing them would throw away the most useful artefact we have. Updates ADR-0005 Confirmation and the Conventions table, and the matching footnote in docs/decisions/README.md. Also makes explicit that ADR numbers 0001 and 0004 stay retired. The "numbers are never reused" rule from ADR-0000 applies: RFC numbering starts at 0001 in its own namespace; the next free ADR number is 0006. --- ...-introduce-rfcs-as-upstream-exploration-phase.md | 13 ++++++++----- docs/decisions/README.md | 12 ++++++++---- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md b/docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md index aaf6aa7faf..a0f83625a9 100644 --- a/docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md +++ b/docs/decisions/0005-introduce-rfcs-as-upstream-exploration-phase.md @@ -87,10 +87,13 @@ Confirmation is delivered in two steps: 1. The merge of this PR creates `docs/rfcs/` with its README and template, and adds the promotion rule to `docs/decisions/README.md`. -2. Two follow-up PRs close the currently open ADR PRs - ([#2263][pr-2263], [#2320][pr-2320]) and re-open their content as RFCs. - Successful completion of those follow-ups confirms that the flow works in - practice. +2. The two currently open ADR PRs ([#2263][pr-2263], [#2320][pr-2320]) are + converted in place to RFC PRs: the files move from `docs/decisions/` to + `docs/rfcs/`, the front matter and body are reshaped to the RFC template, + and the PRs are relabelled `rfc`. The existing PR discussion threads are + preserved — they are exactly the kind of exploration an RFC should + capture. Successful completion of those conversions confirms the flow + works in practice. ## Pros and Cons of the Options @@ -136,7 +139,7 @@ Confirmation is delivered in two steps: | Promotion trigger | The team can honestly write *"Chosen option: X, because Y"* | | Promotion mechanics | Create a new ADR file under the next free ADR number from [`docs/decisions/template.md`](template.md), pulling across the relevant content from the RFC. In the RFC: flip `status` to `promoted`, set `promoted-to` to the ADR filename(s), freeze the body. Promotion is only complete once the index in [this README](README.md#index) and the RFC index are both updated. | | One RFC, multiple ADRs | Supported: `promoted-to` accepts a list of ADR filenames. | -| Reserved ADR numbers | `0001` and `0004` are reserved by open ADR PRs ([#2263][pr-2263], [#2320][pr-2320]). The next free ADR number is `0005` (this ADR); the number after is `0006`. | +| Retired ADR numbers | `0001` and `0004` were proposed in open PRs ([#2263][pr-2263], [#2320][pr-2320]) but will not be accepted as ADRs; their content is converted to RFCs under this ADR's flow. In line with the *"numbers are never reused"* rule from [ADR-0000][adr-0000], those ADR numbers stay retired. | | RFC lifetime | Unlimited. An RFC living for a long time without promotion is not a defect. Stale RFCs are surfaced at the bi-weekly NeIC SDA-Devs meet-up. | | Post-promotion edits | A `promoted` or `withdrawn` RFC is frozen — only metadata and index rows change. Revisions to a decision live in the ADR. | | ADR lifecycle unchanged | The ADR status lifecycle (`proposed` → `accepted` \| `rejected` \| `deprecated` \| `superseded`) is **not** modified by this ADR. ADR statuses only apply to files in `docs/decisions/`. | diff --git a/docs/decisions/README.md b/docs/decisions/README.md index be4da8740e..457264a42e 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -113,10 +113,14 @@ superseded-by: "0005-use-new-approach.md" | [0003](0003-shared-state-strategy-for-s3inbox-and-caching.md) | Replace s3inbox In-Memory File ID Cache with Database Lookups | proposed | | [0005](0005-introduce-rfcs-as-upstream-exploration-phase.md) | Introduce RFCs as an upstream exploration phase for ADRs | proposed | -Numbers `0001` and `0004` are reserved by open ADR PRs -([#2263](https://github.com/neicnordic/sensitive-data-archive/pull/2263), -[#2320](https://github.com/neicnordic/sensitive-data-archive/pull/2320)). The -next free ADR number for a new decision is `0006`. +Numbers `0001` and `0004` were proposed in PRs +[#2263](https://github.com/neicnordic/sensitive-data-archive/pull/2263) and +[#2320](https://github.com/neicnordic/sensitive-data-archive/pull/2320) but +are being converted to RFCs per +[ADR-0005](0005-introduce-rfcs-as-upstream-exploration-phase.md). Per the +*"numbers are never reused"* rule from +[ADR-0000](0000-use-markdown-architectural-decision-records.md), those ADR +numbers stay retired. ## More information From 6bbdb47c20a6b3e9238cb68319c0907b7a3cbb6f Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Sun, 12 Apr 2026 09:15:30 +0200 Subject: [PATCH 474/530] feat(chart: rotatekey): add template files Update charts/sda-svc/templates/rotatekey-certificate.yaml server auth is not needed Co-authored-by: Joakim Bygdell feat(chart: rotatekey): remove privateKeys def in rotatekey-secrets.yaml feat(chart: rotatekey): fix image policy for rotatekey-deploy.yaml feat(chart: rotatekey): update rotatekey-deploy.yaml feat(chart: rotatekey): update rotatekey-secrets.yaml feat: update rotatekey-deploy.yaml feat: update rotatekey-secrets.yaml feat: update rotatekey-deploy.yaml - remove c4gh section feat: update rotatekey-secrets.yaml - fix yaml format --- .../templates/rotatekey-certificate.yaml | 25 +++++++ .../sda-svc/templates/rotatekey-deploy.yaml | 71 +++++++++++++++++++ .../sda-svc/templates/rotatekey-secrets.yaml | 70 ++++++++++++++++++ 3 files changed, 166 insertions(+) create mode 100644 charts/sda-svc/templates/rotatekey-certificate.yaml create mode 100644 charts/sda-svc/templates/rotatekey-deploy.yaml create mode 100644 charts/sda-svc/templates/rotatekey-secrets.yaml diff --git a/charts/sda-svc/templates/rotatekey-certificate.yaml b/charts/sda-svc/templates/rotatekey-certificate.yaml new file mode 100644 index 0000000000..9a87c1cf03 --- /dev/null +++ b/charts/sda-svc/templates/rotatekey-certificate.yaml @@ -0,0 +1,25 @@ +{{- if .Values.global.tls.enabled }} +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-rotatekey-certs +spec: + secretName: {{ template "sda.fullname" . }}-rotatekey-certs + duration: 2160h # 90d + commonName: {{ template "sda.fullname" . }}-rotatekey + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + dnsNames: + - {{ template "sda.fullname" . }}-rotatekey + - {{ template "sda.fullname" . }}-rotatekey.{{ .Release.Namespace }}.svc + issuerRef: + name: {{ template "TLSissuer" . }} + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + group: cert-manager.io +{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/charts/sda-svc/templates/rotatekey-deploy.yaml b/charts/sda-svc/templates/rotatekey-deploy.yaml new file mode 100644 index 0000000000..675a50a136 --- /dev/null +++ b/charts/sda-svc/templates/rotatekey-deploy.yaml @@ -0,0 +1,71 @@ +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "sda.fullname" . }}-rotatekey + labels: + role: rotatekey + app: {{ template "sda.fullname" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + component: {{ .Release.Name }}-rotatekey + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.rotatekey.replicaCount | default 1 }} + revisionHistoryLimit: {{ default "3" .Values.global.revisionHistory }} + selector: + matchLabels: + app: {{ template "sda.fullname" . }}-rotatekey + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "sda.fullname" . }}-rotatekey + role: rotatekey + release: {{ .Release.Name }} + annotations: + {{- if not .Values.global.vaultSecrets }} + checksum/config: {{ include (print $.Template.BasePath "/rotatekey-secrets.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.global.podAnnotations }} +{{- toYaml .Values.global.podAnnotations | nindent 8 -}} + {{- end }} + {{- if .Values.rotatekey.annotations }} +{{- toYaml .Values.rotatekey.annotations | nindent 8 -}} + {{- end }} + spec: + {{- if .Values.global.rbacEnabled}} + serviceAccountName: {{ .Release.Name }} + {{- end }} + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + containers: + - name: rotatekey + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + command: ["sda-rotatekey"] + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + env: + - name: CONFIGFILE + value: {{ include "confFile" . }} + resources: +{{ toYaml .Values.rotatekey.resources | trim | indent 10 }} + volumeMounts: + - name: config + mountPath: {{ template "configPath" . }} + enableServiceLinks: false + volumes: + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-rotatekey + restartPolicy: Always +{{- end }} diff --git a/charts/sda-svc/templates/rotatekey-secrets.yaml b/charts/sda-svc/templates/rotatekey-secrets.yaml new file mode 100644 index 0000000000..31ba68cdfa --- /dev/null +++ b/charts/sda-svc/templates/rotatekey-secrets.yaml @@ -0,0 +1,70 @@ +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}} +{{- if not .Values.global.vaultSecrets }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "sda.fullname" . }}-rotatekey +type: Opaque +stringData: + # Everything indented under config.yaml is ONE file + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 2 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassRotate" .) }} + queue: {{ default "rotatekey" .Values.global.broker.rotateKeyQueue }} + routingKey: {{ default "" .Values.global.broker.routingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{ .Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserRotate" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB password is required" (include "dbPassRotate" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserRotate" .) }} + + grpc: + host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }} + port: {{ ternary 50443 50051 .Values.global.tls.enabled }} + timeout: {{ default 30 .Values.global.reencrypt.timeout }} + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + + c4gh: + rotatePubKeyPath: {{ template "configPath" . }}/target-key.pub + + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + + target-key.pub: {{ required "global.c4gh.rotatePubKeyData is required" .Values.global.c4gh.rotatePubKeyData | quote }} + +{{- end }} +{{- end }} \ No newline at end of file From 935e412c6115932080335b7afab62bf4a6ed3fdd Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Sun, 12 Apr 2026 09:35:16 +0200 Subject: [PATCH 475/530] feat(chart: rotatekey): update definition of variables in helpers --- charts/sda-svc/templates/_helpers.yaml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 29e483226d..b00f54b20a 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -291,6 +291,23 @@ Create chart name and version as used by the chart label. {{- end -}} {{/**/}} +{{/* +rotatekey credentials +*/}} +{{- define "mqUserRotate" -}} +{{- ternary .Values.global.broker.username .Values.credentials.rotatekey.mqUser (empty .Values.credentials.rotatekey.mqUser) -}} +{{- end -}} +{{- define "mqPassRotate" -}} +{{- ternary .Values.global.broker.password .Values.credentials.rotatekey.mqPassword (empty .Values.credentials.rotatekey.mqPassword) -}} +{{- end -}} +{{- define "dbUserRotate" -}} +{{- ternary .Values.global.db.user .Values.credentials.rotatekey.dbUser (empty .Values.credentials.rotatekey.dbUser) -}} +{{- end -}} +{{- define "dbPassRotate" -}} +{{- ternary .Values.global.db.password .Values.credentials.rotatekey.dbPassword (empty .Values.credentials.rotatekey.dbPassword) -}} +{{- end -}} +{{/**/}} + {{- define "dbSSLmode" -}} {{- if .Values.global.tls.enabled }} {{- printf "%s" .Values.global.db.sslMode }} From b9dd42719932cacd247ea7e4df2343ed01b0057a Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Sun, 12 Apr 2026 10:16:22 +0200 Subject: [PATCH 476/530] feat(chart: rotatekey): update config in values yaml file --- charts/sda-svc/values.yaml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 4304497c73..9784a48e88 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -194,6 +194,7 @@ global: secretName: publicKey: publicKeyData: + rotatePubKeyData: # contains the public key data, not the path to the file privateKeys: - keyName: passphrase: @@ -380,6 +381,12 @@ credentials: mqUser: "" mqPassword: "" + rotatekey: + dbUser: "" + dbPassword: "" + mqUser: "" + mqPassword: "" + ################################## # Service specific settings @@ -560,7 +567,7 @@ ingest: secretName: "" intercept: - name: ingest + name: intercept replicaCount: 1 resources: requests: @@ -754,6 +761,19 @@ verify: tls: secretName: "" +rotatekey: + replicaCount: 1 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "1000m" + annotations: {} + tls: + secretName: "" + jobs: image: postgres:15.4-alpine removeUserName: false From 748b2cdba1851c7e2562a50a7d30811041868c23 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 15 Apr 2026 17:31:33 +0200 Subject: [PATCH 477/530] feat: add smoke test for rotatekey helm chart --- dev-tools/chart-smoke-test/test-rotatekey.sh | 324 +++++++++++++++++++ 1 file changed, 324 insertions(+) create mode 100755 dev-tools/chart-smoke-test/test-rotatekey.sh diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh new file mode 100755 index 0000000000..fb26869e59 --- /dev/null +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -0,0 +1,324 @@ +#!/usr/bin/env bash +# Local k3d smoke test for the rotatekey service chart. +# +# Prerequisites: docker, k3d, kubectl, helm +# +# Usage: +# ./dev-tools/chart-smoke-test/test-rotatekey.sh # full run (build + deploy + test) +# ./dev-tools/chart-smoke-test/test-rotatekey.sh --no-build # skip image build (reuse existing) +# ./dev-tools/chart-smoke-test/test-rotatekey.sh --cleanup # tear down cluster and exit + +set -euo pipefail + +CLUSTER_NAME="sda-rotatekey-test" +IMAGE="ghcr.io/neicnordic/sensitive-data-archive:local-test" +SKIP_BUILD=false + +for arg in "$@"; do + case "$arg" in + --no-build) SKIP_BUILD=true ;; + --cleanup) + echo "Tearing down cluster $CLUSTER_NAME..." + k3d cluster delete "$CLUSTER_NAME" 2>/dev/null || true + echo "Done." + exit 0 + ;; + *) echo "Unknown argument: $arg"; exit 1 ;; + esac +done + +# -- helpers -- +pass=0 +fail=0 + +wait_for_pod() { # wait for a pod with the given label to be ready, with a timeout + local label="$1" timeout="$2" + echo "Waiting for pod $label to be ready (${timeout}s)..." + if ! kubectl wait --for=condition=ready pod -l "$label" --timeout="${timeout}s" 2>/dev/null; then + echo "Pod $label not ready after ${timeout}s" + kubectl logs -l "$label" --tail=20 2>/dev/null + return 1 + fi +} + +echo "=== Step 1: k3d cluster ===" +# Pre-create a dedicated network with the correct MTU (1400) to avoid TLS timeouts +if ! docker network ls | grep -q "sda-test-net"; then + echo "Creating custom Docker network with MTU 1400..." + docker network create sda-test-net --opt com.docker.network.driver.mtu=1400 +fi + +if k3d cluster list 2>/dev/null | grep -q "$CLUSTER_NAME"; then + echo "Cluster $CLUSTER_NAME already exists, reusing." +else + echo "Creating cluster $CLUSTER_NAME..." + # Tell k3d to use our custom network instead of generating its own + k3d cluster create "$CLUSTER_NAME" --network sda-test-net --wait +fi + +echo "=== Step 2: image ===" +if [ "$SKIP_BUILD" = true ]; then + echo "Skipping build (--no-build)." +else + echo "Building sda image..." + docker build -t "$IMAGE" --build-arg GOLANG_VERSION=1.25 --network=host -f sda/Dockerfile sda/ +fi + +echo "Cleaning old image from k3d and importing fresh..." +k3d image delete "$IMAGE" -c "$CLUSTER_NAME" 2>/dev/null || true + +echo "Importing image into cluster..." +k3d image import "$IMAGE" -c "$CLUSTER_NAME" + +echo "=== Step 3: dependencies ===" + +if ! helm status postgres >/dev/null 2>&1; then + echo "Installing postgres..." + helm install postgres charts/sda-db \ + --set global.postgresAdminPassword=rootpass \ + --set global.tls.enabled=false \ + --set persistence.enabled=false \ + --set resources=null \ + --wait --timeout 120s +else + echo "Postgres already installed." +fi + +if ! helm status broker >/dev/null 2>&1; then + echo "Installing rabbitmq..." + helm install broker charts/sda-mq \ + --set global.adminPassword=mqpass \ + --set global.adminUser=admin \ + --set global.tls.enabled=false \ + --set persistence.enabled=false \ + --set resources=null \ + --wait --timeout 120s +else + echo "RabbitMQ already installed." +fi + +echo "=== Step 4: keys and secrets ===" + +# Clean previous run if exists +kubectl delete deploy pipeline-sda-svc-rotatekey 2>/dev/null || true +kubectl delete secret pipeline-sda-svc-rotatekey 2>/dev/null || true + +# OLD_PUB matches the "old" identity +OLD_PUB="-----BEGIN CRYPT4GH PUBLIC KEY----- +miSC7pgHDY3BGLajuGjNz0K3+6TSJ7wEFGpNATZ1DV4= +-----END CRYPT4GH PUBLIC KEY-----" + +# ROTATE_PUB is the "new" destination key +ROTATE_PUB="-----BEGIN CRYPT4GH PUBLIC KEY----- +fFmwrVXywijqMoaLX95CgIXp6klJuo5MOLf/I3+BQ1Q= +-----END CRYPT4GH PUBLIC KEY-----" + +# OLD_PRIV is the encrypted private key corresponding to OLD_PUB, with passphrase "pass" +OLD_PRIV="-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAA9yimIi6tLdrRZim6rlojxQARY2hhY2hhMjBf +cG9seTEzMDUAPPe+btMB/qNtpH1jxIuLonMlOSyrLBFtr7A/9QNyalr33zSA24OB +Eo9Y+poiXK0ECZdEPOPIOvTWewuzSQ== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" + +# Create the C4GH Secret for the reencrypt service +kubectl create secret generic c4gh-secret \ + --namespace default \ + --from-literal=c4gh.key="$OLD_PRIV" \ + --dry-run=client -o yaml | kubectl apply -f - + +OLD_HEX=$(echo "$OLD_PUB" | awk 'NR==2' | base64 -d | xxd -p -c256 | tr -d '\n\r ') +ROTATE_HEX=$(echo "$ROTATE_PUB" | awk 'NR==2' | base64 -d | xxd -p -c256 | tr -d '\n\r ') + +echo "Registering key Hashes in sda.encryption_keys..." + +kubectl exec -i postgres-sda-db-0 -- env PGPASSWORD=rootpass psql -U postgres -d sda < Date: Fri, 17 Apr 2026 09:12:11 +0200 Subject: [PATCH 478/530] feat(chart: rotatekey) bump chart version --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 561e9b8edd..557c8407e8 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.2.2 +version: 3.4.0 appVersion: v3.1.37 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From a3f50d04e90ed05e24e4e3c839c67e85c7ad654c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 22 Apr 2026 11:17:03 +0200 Subject: [PATCH 479/530] feat(chart: rotatekey): update readme Update charts/sda-svc/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Update charts/sda-svc/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- charts/sda-svc/README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 9fda2e30c1..184054072f 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -124,6 +124,7 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.broker.password` | Shared password to the message broker. | `/` | | `global.broker.username` | Shared user to the message broker. | `/` | | `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` | +| `global.broker.rotateKeyQueue` | MQ queue name for key rotation messages. | `"rotatekey"` | | `global.cega.host` | Full URI to the EGA user authentication service. | `""` | | `global.cega.user` | Username for the EGA user authentication service. | `""` | | `global.cega.password` | Password for the EGA user authentication service. | `""` | @@ -133,6 +134,7 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""` | | `global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""` | | `global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""` | +| `global.c4gh.rotatePubKeyData` | The target public key for rotation (base64 encoded string). | `""` | | `global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""` | | `global.db.admin.passKey` | Key in the secret that holds the password. | `""` | | `global.db.admin.userkey` | Key in the secret that holds the username. | `""` | @@ -213,7 +215,7 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.tls.issuer` | Issuer for TLS certificate creation. | `""` | | `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` | | `global.reencrypt.host` | gRPC host for reencryption | `""` | -| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` | +| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` |global.reencrypt.host gRPC host for re-encryption (used by rotatekey) ""global.reencrypt.port Port number of the gRPC host for re-encryption 50051global.reencrypt.timeout Timeout in seconds for re-encryption gRPC calls 30global.c4gh.rotatePubKeyData The target public Crypt4GH key for rotation (base64 or plain string) ""global.broker.rotateKeyQueue MQ queue name for key rotation messages "rotatekey" ### Credentials @@ -258,6 +260,10 @@ of the values file. `credentials.verify.dbPassword` | Database password for verify | `""` `credentials.verify.mqUser` | Broker user for verify | `""` `credentials.verify.mqPassword` | Broker password for verify | `""` + `credentials.rotatekey.dbUser` | Database user for rotatekey | `""` + `credentials.rotatekey.dbPassword` | Database password for rotatekey| `""` + `credentials.rotatekey.mqUser` | Broker user for rotatekey | `""` + `credentials.rotatekey.mqPassword` | Broker password for rotatekey | `""` ### Pod settings @@ -363,6 +369,12 @@ of the values file. `releasetest.repository` | inbox container image repository | `neicnordic/sda-helm-test-support` `releasetest.imageTag` | inbox container image version | `latest` `releasetest.imagePullPolicy` | inbox container image pull policy | `Always` + `rotatekey.replicaCount` | Desired number of replicas. | `1` + `rotatekey.annotations` | Specific annotations for the rotatekey pod. | `{}` + `rotatekey.resources.requests.memory` | Memory request for container. | `128Mi` + `rotatekey.resources.requests.cpu` | CPU request for container. | `100m` + `rotatekey.resources.limits.memory` | Memory limit for container. | `256Mi` + `rotatekey.resources.limits.cpu` | CPU limit for container. | `1000m` ### Jobs From 31d811723152ff79bdb0bf8e103433ce2952818a Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 22 Apr 2026 11:48:57 +0200 Subject: [PATCH 480/530] feat(chart: rotatekey): decode rotatePubKeyData to PEM format --- charts/sda-svc/templates/rotatekey-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/rotatekey-secrets.yaml b/charts/sda-svc/templates/rotatekey-secrets.yaml index 31ba68cdfa..5801ada6ba 100644 --- a/charts/sda-svc/templates/rotatekey-secrets.yaml +++ b/charts/sda-svc/templates/rotatekey-secrets.yaml @@ -64,7 +64,7 @@ stringData: format: {{ .Values.global.log.format }} level: {{ .Values.global.log.level }} - target-key.pub: {{ required "global.c4gh.rotatePubKeyData is required" .Values.global.c4gh.rotatePubKeyData | quote }} + target-key.pub: {{ required "global.c4gh.rotatePubKeyData is required" .Values.global.c4gh.rotatePubKeyData | b64dec | quote }} {{- end }} {{- end }} \ No newline at end of file From 6313f9aa7784623c3baccd0b604272429f7d549c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 22 Apr 2026 11:49:55 +0200 Subject: [PATCH 481/530] refactor(chart: rotatekey): sementic comments --- charts/sda-svc/values.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 9784a48e88..919298d8fa 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -194,7 +194,8 @@ global: secretName: publicKey: publicKeyData: - rotatePubKeyData: # contains the public key data, not the path to the file + ## @param rotatePubKeyData, contains the PEM encoded public key as a base64 encoded string + rotatePubKeyData: privateKeys: - keyName: passphrase: From e513e1711faf2fe8a7a8044eb0374e9ddead7c20 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 22 Apr 2026 11:50:50 +0200 Subject: [PATCH 482/530] feat(chart: rotatekey): use base64 encoding for rotate pub key --- dev-tools/chart-smoke-test/test-rotatekey.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index fb26869e59..6232febd52 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -113,6 +113,8 @@ ROTATE_PUB="-----BEGIN CRYPT4GH PUBLIC KEY----- fFmwrVXywijqMoaLX95CgIXp6klJuo5MOLf/I3+BQ1Q= -----END CRYPT4GH PUBLIC KEY-----" +ROTATE_PUB_BASE64=$(echo "$ROTATE_PUB" | base64 -w0) + # OLD_PRIV is the encrypted private key corresponding to OLD_PUB, with passphrase "pass" OLD_PRIV="-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- YzRnaC12MQAGc2NyeXB0ABQAAAAA9yimIi6tLdrRZim6rlojxQARY2hhY2hhMjBf @@ -150,7 +152,7 @@ HELM_ARGS=( --set global.configPath='/.secrets' # rotatekey - --set "global.c4gh.rotatePubKeyData=$ROTATE_PUB" + --set "global.c4gh.rotatePubKeyData=$ROTATE_PUB_BASE64" # --- database --- --set global.db.host=postgres-sda-db From 593cb898d52e7d77fff92317f088a275ce3acd7c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 22 Apr 2026 23:19:34 +0200 Subject: [PATCH 483/530] feat(chart: rotatekey): generate secretes only when rotatekey is enabled --- charts/sda-svc/templates/rotatekey-secrets.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/rotatekey-secrets.yaml b/charts/sda-svc/templates/rotatekey-secrets.yaml index 5801ada6ba..9597177562 100644 --- a/charts/sda-svc/templates/rotatekey-secrets.yaml +++ b/charts/sda-svc/templates/rotatekey-secrets.yaml @@ -1,3 +1,4 @@ +{{- if .Values.rotatekey.enabled }} {{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}} {{- if not .Values.global.vaultSecrets }} --- @@ -7,7 +8,6 @@ metadata: name: {{ template "sda.fullname" . }}-rotatekey type: Opaque stringData: - # Everything indented under config.yaml is ONE file config.yaml: |- broker: {{- if .Values.global.tls.enabled }} @@ -66,5 +66,6 @@ stringData: target-key.pub: {{ required "global.c4gh.rotatePubKeyData is required" .Values.global.c4gh.rotatePubKeyData | b64dec | quote }} +{{- end }} {{- end }} {{- end }} \ No newline at end of file From d06ae2f73c3a08f3b8bbfa742baff30f0e5813c6 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 10:55:11 +0200 Subject: [PATCH 484/530] feat(chart: rotatekey): add tls in rotatekey-deploy --- charts/sda-svc/templates/rotatekey-deploy.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/rotatekey-deploy.yaml b/charts/sda-svc/templates/rotatekey-deploy.yaml index 675a50a136..416093ac7d 100644 --- a/charts/sda-svc/templates/rotatekey-deploy.yaml +++ b/charts/sda-svc/templates/rotatekey-deploy.yaml @@ -61,11 +61,22 @@ spec: volumeMounts: - name: config mountPath: {{ template "configPath" . }} + {{- if .Values.global.tls.enabled }} + - name: tls + mountPath: {{ template "tlsPath" . }} + readOnly: true + {{- end }} enableServiceLinks: false volumes: - name: config secret: defaultMode: 0440 secretName: {{ template "sda.fullname" . }}-rotatekey + {{- if .Values.global.tls.enabled }} + - name: tls + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-rotatekey-certs + {{- end }} restartPolicy: Always -{{- end }} +{{- end }} \ No newline at end of file From b906c880346d057968262b31c2148ae4b675dc62 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 13:05:41 +0200 Subject: [PATCH 485/530] feat(chart: rotatekey): remove unused routingKey from secrets config --- charts/sda-svc/templates/rotatekey-secrets.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/charts/sda-svc/templates/rotatekey-secrets.yaml b/charts/sda-svc/templates/rotatekey-secrets.yaml index 9597177562..1f1e17f088 100644 --- a/charts/sda-svc/templates/rotatekey-secrets.yaml +++ b/charts/sda-svc/templates/rotatekey-secrets.yaml @@ -23,7 +23,6 @@ stringData: prefetchCount: {{ default 2 .Values.global.broker.prefetchCount }} password: {{ required "MQ password is required" (include "mqPassRotate" .) }} queue: {{ default "rotatekey" .Values.global.broker.rotateKeyQueue }} - routingKey: {{ default "" .Values.global.broker.routingKey }} {{- if ne "" ( default "" .Values.global.broker.serverName ) }} serverName: {{ .Values.global.broker.serverName }} {{- end }} From 58407427e041cba9d6efe1c8a6cf9756bd355b9e Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 13:25:08 +0200 Subject: [PATCH 486/530] feat(chart: rotatekey): update test script --- dev-tools/chart-smoke-test/test-rotatekey.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index 6232febd52..92bb1b514c 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -167,7 +167,6 @@ HELM_ARGS=( --set global.broker.username=admin --set global.broker.password=mqpass --set global.broker.rotateKeyQueue=rotatekey - --set global.broker.routingKey=rotate --set global.broker.prefetchCount=2 # --- archive --- From 884fbab7a7cf9691448095e0a343f0feed962670 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 13:31:36 +0200 Subject: [PATCH 487/530] feat(chart: rotatekey): append rotatekey to app label feat: update test script, remove ingress setting feat: update test script, remove odic setting feat: update test script, remove s3BucketPrefix setting --- charts/sda-svc/templates/rotatekey-deploy.yaml | 2 +- dev-tools/chart-smoke-test/test-rotatekey.sh | 10 +--------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/charts/sda-svc/templates/rotatekey-deploy.yaml b/charts/sda-svc/templates/rotatekey-deploy.yaml index 416093ac7d..fb42a776ab 100644 --- a/charts/sda-svc/templates/rotatekey-deploy.yaml +++ b/charts/sda-svc/templates/rotatekey-deploy.yaml @@ -5,7 +5,7 @@ metadata: name: {{ template "sda.fullname" . }}-rotatekey labels: role: rotatekey - app: {{ template "sda.fullname" . }} + app: {{ template "sda.fullname" . }}-rotatekey chart: {{ .Chart.Name }}-{{ .Chart.Version }} component: {{ .Release.Name }}-rotatekey release: {{ .Release.Name }} diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index 92bb1b514c..5895e98e99 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -175,18 +175,13 @@ HELM_ARGS=( --set global.archive.s3Port=9000 --set global.archive.s3AccessKey=access --set global.archive.s3SecretKey=secretkey - --set global.archive.s3BucketPrefix=sda # --- reencrypt --- --set global.reencrypt.host=pipeline-sda-svc-reencrypt --set global.reencrypt.port=50051 --set reencrypt.readinessProbe.grpc.port=50051 - # --- OIDC & security --- - --set global.oidc.provider=http://oidc - --set global.oidc.id=test - --set global.oidc.secret=test - --set global.api.jwtSecret=jwt-secret + # --- security --- --set global.api.rbacFileSecret=rbac # --- c4gh for reencrypt --- @@ -195,9 +190,6 @@ HELM_ARGS=( --set "global.c4gh.privateKeys[0].passphrase=pass" # --- ingress --- - --set global.ingress.hostName.download="download.local" - --set global.ingress.hostName.ingest="ingest.local" - --set global.ingress.hostName.api="api.local" --set global.ingress.deploy=false # --- logging --- From 119e530b960d8af1b7cb90662f6a5cb8578a12ef Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 15:32:53 +0200 Subject: [PATCH 488/530] feat(chart: rotatekey): ensure rotatekey send the message after key rotation --- dev-tools/chart-smoke-test/test-rotatekey.sh | 26 ++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index 5895e98e99..1cde581477 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -263,6 +263,15 @@ INSERT INTO sda.checksums (file_id, checksum, type, source) VALUES ('$FILE_ID', '$DUMMY_CHECKSUM_UNENCRYPTED', 'SHA256', 'UNENCRYPTED'); EOF +# Create the queue (rotatekey will publish the verification message here after processing) +kubectl run mq-setup --rm -i --restart=Never --image=curlimages/curl -- \ + curl -s -u admin:mqpass -X PUT "http://broker-sda-mq:15672/api/queues/sda/verify" \ + -d '{"durable":true,"auto_delete":false}' + +# Bind the queue to the 'archived' routing key +kubectl run mq-bind --rm -i --restart=Never --image=curlimages/curl -- \ + curl -s -u admin:mqpass -X POST "http://broker-sda-mq:15672/api/bindings/sda/e/sda/q/verify" \ + -d '{"routing_key":"archived"}' # Trigger the rotation via RabbitMQ echo "Publishing rotation message to RabbitMQ..." @@ -310,6 +319,23 @@ else fail=$((fail + 1)) fi +# Peek into the 'verify' queue to see if rotatekey sent the message +VALIDATION_QUEUE="verify" +MQ_RESPONSE=$(kubectl run mq-trigger --rm -i --restart=Never --image=curlimages/curl -- \ + curl -s -u admin:mqpass \ + -X POST "http://broker-sda-mq:15672/api/queues/sda/${VALIDATION_QUEUE}/get" \ + -H "content-type:application/json" \ + -d '{"count":1,"ackmode":"ack_requeue_true","encoding":"auto","truncate":50000}') + +if echo "$MQ_RESPONSE" | grep -q "$FILE_ID"; then + echo " PASS: Outgoing verification message for $FILE_ID found in '${VALIDATION_QUEUE}' queue." + pass=$((pass + 1)) +else + echo " FAIL: No message found for $FILE_ID in '${VALIDATION_QUEUE}' queue." + echo " Hint: Check if rotatekey logs show 'Published message to queue'" + fail=$((fail + 1)) +fi + echo "=== Summary: $pass passed, $fail failed ===" if [ $fail -gt 0 ]; then exit 1 From 8c149fdba16aca77f91b653e677d3fe7dce2bebf Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 22:29:57 +0200 Subject: [PATCH 489/530] feat(chart: rotatekey): update test script - no shared-secrets, no archive config feat(chart: rotatekey): update test script - remove uncessary helm args feat: upate test script - remove unused setting --- dev-tools/chart-smoke-test/test-rotatekey.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index 1cde581477..fe8c56d0e8 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -143,14 +143,9 @@ HELM_ARGS=( # --- basic image/service settings --- --set image.tag=local-test --set image.pullPolicy=IfNotPresent - --set rotatekey.enabled=true --set global.tls.enabled=false --set global.rbacEnabled=false - --set global.secretsPath='/.secrets' - --set global.c4ghPath='/.secrets/c4gh' - --set global.configPath='/.secrets' - # rotatekey --set "global.c4gh.rotatePubKeyData=$ROTATE_PUB_BASE64" @@ -169,13 +164,6 @@ HELM_ARGS=( --set global.broker.rotateKeyQueue=rotatekey --set global.broker.prefetchCount=2 - # --- archive --- - --set global.archive.storageType=s3 - --set global.archive.s3Url=http://minio - --set global.archive.s3Port=9000 - --set global.archive.s3AccessKey=access - --set global.archive.s3SecretKey=secretkey - # --- reencrypt --- --set global.reencrypt.host=pipeline-sda-svc-reencrypt --set global.reencrypt.port=50051 @@ -199,7 +187,7 @@ HELM_ARGS=( echo "=== Step 5: deploy rotatekey ===" -for tmpl in shared-secrets rotatekey-secrets rotatekey-deploy reencrypt-secrets re-encrypt-deploy re-encrypt-service; do +for tmpl in rotatekey-secrets rotatekey-deploy reencrypt-secrets re-encrypt-deploy re-encrypt-service; do helm template pipeline charts/sda-svc "${HELM_ARGS[@]}" \ --show-only "templates/${tmpl}.yaml" done | kubectl apply -f - From 788147334ebf0a8cb873adba8251ef342f5987ac Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 22:51:08 +0200 Subject: [PATCH 490/530] feat(chart: rotatekey): remove condition 'enabled' in secrets yaml --- charts/sda-svc/templates/rotatekey-secrets.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/charts/sda-svc/templates/rotatekey-secrets.yaml b/charts/sda-svc/templates/rotatekey-secrets.yaml index 1f1e17f088..965002d04a 100644 --- a/charts/sda-svc/templates/rotatekey-secrets.yaml +++ b/charts/sda-svc/templates/rotatekey-secrets.yaml @@ -1,4 +1,3 @@ -{{- if .Values.rotatekey.enabled }} {{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}} {{- if not .Values.global.vaultSecrets }} --- @@ -65,6 +64,5 @@ stringData: target-key.pub: {{ required "global.c4gh.rotatePubKeyData is required" .Values.global.c4gh.rotatePubKeyData | b64dec | quote }} -{{- end }} {{- end }} {{- end }} \ No newline at end of file From c64432bd868f891b03c9d6cd80750f91e9ef06a6 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 23 Apr 2026 23:18:47 +0200 Subject: [PATCH 491/530] feat(rotatekey): add rotatePubKeyData in the integration test Update .github/integration/scripts/charts/deploy_charts.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .github/integration/scripts/charts/deploy_charts.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 96e69c9c79..f586946d3d 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -9,6 +9,10 @@ fi MQ_PORT=5672 SCHEME=HTTP GRPC_PORT=50051 +ROTATE_PUB="-----BEGIN CRYPT4GH PUBLIC KEY----- +fFmwrVXywijqMoaLX95CgIXp6klJuo5MOLf/I3+BQ1Q= +-----END CRYPT4GH PUBLIC KEY-----" +ROTATE_PUB_BASE64=$(printf '%s' "$ROTATE_PUB" | base64 -w0) if [ "$3" == "true" ]; then MQ_PORT=5671 SCHEME=HTTPS @@ -81,6 +85,7 @@ if [ "$1" == "sda-svc" ]; then --set s3Inbox.readinessProbe.httpGet.scheme="$SCHEME" \ --set syncAPI.readinessProbe.httpGet.scheme="$SCHEME" \ --set reencrypt.readinessProbe.grpc.port="$GRPC_PORT" \ + --set "global.c4gh.rotatePubKeyData=$ROTATE_PUB_BASE64" \ -f "$dir/values.yaml" \ --wait fi From a5ee80ee60c2205012aae237579d772714daa488 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 09:37:21 +0200 Subject: [PATCH 492/530] feat(chart: rotatekey): just check the archived queue --- dev-tools/chart-smoke-test/test-rotatekey.sh | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index fe8c56d0e8..a38e33a900 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -251,16 +251,6 @@ INSERT INTO sda.checksums (file_id, checksum, type, source) VALUES ('$FILE_ID', '$DUMMY_CHECKSUM_UNENCRYPTED', 'SHA256', 'UNENCRYPTED'); EOF -# Create the queue (rotatekey will publish the verification message here after processing) -kubectl run mq-setup --rm -i --restart=Never --image=curlimages/curl -- \ - curl -s -u admin:mqpass -X PUT "http://broker-sda-mq:15672/api/queues/sda/verify" \ - -d '{"durable":true,"auto_delete":false}' - -# Bind the queue to the 'archived' routing key -kubectl run mq-bind --rm -i --restart=Never --image=curlimages/curl -- \ - curl -s -u admin:mqpass -X POST "http://broker-sda-mq:15672/api/bindings/sda/e/sda/q/verify" \ - -d '{"routing_key":"archived"}' - # Trigger the rotation via RabbitMQ echo "Publishing rotation message to RabbitMQ..." kubectl run mq-trigger --rm -i --restart=Never --image=curlimages/curl -- \ @@ -307,8 +297,8 @@ else fail=$((fail + 1)) fi -# Peek into the 'verify' queue to see if rotatekey sent the message -VALIDATION_QUEUE="verify" +# Peek into the 'archived' queue to see if rotatekey sent the message +VALIDATION_QUEUE="archived" MQ_RESPONSE=$(kubectl run mq-trigger --rm -i --restart=Never --image=curlimages/curl -- \ curl -s -u admin:mqpass \ -X POST "http://broker-sda-mq:15672/api/queues/sda/${VALIDATION_QUEUE}/get" \ From 1ac0be8c509874e205d8521b3c1a792ee218ba39 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 11:16:15 +0200 Subject: [PATCH 493/530] feat(chart: rotatekey): add ip address in certificate --- charts/sda-svc/templates/rotatekey-certificate.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/charts/sda-svc/templates/rotatekey-certificate.yaml b/charts/sda-svc/templates/rotatekey-certificate.yaml index 9a87c1cf03..617a458bfe 100644 --- a/charts/sda-svc/templates/rotatekey-certificate.yaml +++ b/charts/sda-svc/templates/rotatekey-certificate.yaml @@ -6,7 +6,9 @@ metadata: name: {{ template "sda.fullname" . }}-rotatekey-certs spec: secretName: {{ template "sda.fullname" . }}-rotatekey-certs + duration: 2160h # 90d + commonName: {{ template "sda.fullname" . }}-rotatekey isCA: false privateKey: @@ -17,6 +19,8 @@ spec: dnsNames: - {{ template "sda.fullname" . }}-rotatekey - {{ template "sda.fullname" . }}-rotatekey.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 issuerRef: name: {{ template "TLSissuer" . }} kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} From 4fc622bbd7b0945c8a7534e4900bbcc97785e34a Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 11:18:17 +0200 Subject: [PATCH 494/530] feat(chart: rotatekey): include rotatekey in readiness check --- .github/workflows/pr_build_and_test_images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index 8006d388e9..d5d5eb2b3c 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -360,7 +360,7 @@ jobs: - name: Check deployment run: | sleep 30 - for n in api auth download finalize inbox ingest mapper reencrypt sync syncapi verify; do + for n in api auth download finalize inbox ingest mapper reencrypt rotatekey sync syncapi verify; do if [ ${{matrix.storage}} == "posix" ] && [ "$n" == "auth" ] || [ "$n" == "sync" ] || [ "$n" == "syncapi" ]; then continue fi @@ -375,7 +375,7 @@ jobs: run: | kubectl get pods sleep 1 - for svc in api auth finalize inbox ingest mapper reencrypt sync syncapi verify; do + for svc in api auth finalize inbox ingest mapper reencrypt rotatekey sync syncapi verify; do echo "## describe $svc" && kubectl describe pod -l role="$svc" sleep 1 echo "## logs $svc" && kubectl logs -l role="$svc" From 3fadfae9592078b92299ae64bcc6cd8e2569e11d Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 11:33:29 +0200 Subject: [PATCH 495/530] refactor(test): improve base64 encoding portability and correctness Update dev-tools/chart-smoke-test/test-rotatekey.sh Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- dev-tools/chart-smoke-test/test-rotatekey.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index a38e33a900..b622ab8ffb 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -36,7 +36,7 @@ wait_for_pod() { # wait for a pod with the given label to be ready, with a timeo echo "Waiting for pod $label to be ready (${timeout}s)..." if ! kubectl wait --for=condition=ready pod -l "$label" --timeout="${timeout}s" 2>/dev/null; then echo "Pod $label not ready after ${timeout}s" - kubectl logs -l "$label" --tail=20 2>/dev/null + kubectl logs -l "$label" --tail=20 2>/dev/null || true return 1 fi } @@ -113,7 +113,7 @@ ROTATE_PUB="-----BEGIN CRYPT4GH PUBLIC KEY----- fFmwrVXywijqMoaLX95CgIXp6klJuo5MOLf/I3+BQ1Q= -----END CRYPT4GH PUBLIC KEY-----" -ROTATE_PUB_BASE64=$(echo "$ROTATE_PUB" | base64 -w0) +ROTATE_PUB_BASE64=$(printf '%s' "$ROTATE_PUB" | base64 | tr -d '\n') # OLD_PRIV is the encrypted private key corresponding to OLD_PUB, with passphrase "pass" OLD_PRIV="-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- @@ -185,7 +185,6 @@ HELM_ARGS=( --set global.log.format=json ) - echo "=== Step 5: deploy rotatekey ===" for tmpl in rotatekey-secrets rotatekey-deploy reencrypt-secrets re-encrypt-deploy re-encrypt-service; do helm template pipeline charts/sda-svc "${HELM_ARGS[@]}" \ @@ -266,7 +265,6 @@ kubectl run mq-trigger --rm -i --restart=Never --image=curlimages/curl -- \ echo "Waiting for services to react..." sleep 5 - echo "=== Step 8: Verification ===" fail=0 pass=0 From 462626a65ff51fb26a1ce58613c4a6b212a937b8 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 14:04:34 +0200 Subject: [PATCH 496/530] fix(ci): increase timeout and use dynamic wait for readiness fix(ci): logic for skip services fix(ci): better skip logic fix: syntax fix(ci): change back the skip logic --- .github/workflows/pr_build_and_test_images.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index d5d5eb2b3c..0d50a2ae30 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -359,13 +359,19 @@ jobs: - name: Check deployment run: | - sleep 30 + echo "Waiting for pods to be ready..." for n in api auth download finalize inbox ingest mapper reencrypt rotatekey sync syncapi verify; do + # Skip services based on storage type if [ ${{matrix.storage}} == "posix" ] && [ "$n" == "auth" ] || [ "$n" == "sync" ] || [ "$n" == "syncapi" ]; then continue fi - if [ ! $(kubectl get pods -l role="$n" -o=jsonpath='{.items[*].status.containerStatuses[0].ready}' | grep true) ]; then - echo "$n is not ready after 30s, exiting" + + echo "Checking $n..." + # Dynamic wait: up to 90 seconds per service. + if ! kubectl wait --for=condition=ready pod -l role="$n" --timeout=90s; then + echo "ERROR: $n is not ready after 90s" + kubectl describe pod -l role="$n" + kubectl logs -l role="$n" --tail=20 exit 1 fi done From 375338d699c0d38a60b21582d7befa685e1f52c0 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 15:45:39 +0200 Subject: [PATCH 497/530] refactor(chart): align rotatekey TLS logic with PKI service pattern --- charts/sda-svc/templates/rotatekey-deploy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/sda-svc/templates/rotatekey-deploy.yaml b/charts/sda-svc/templates/rotatekey-deploy.yaml index fb42a776ab..c1d5095874 100644 --- a/charts/sda-svc/templates/rotatekey-deploy.yaml +++ b/charts/sda-svc/templates/rotatekey-deploy.yaml @@ -61,7 +61,7 @@ spec: volumeMounts: - name: config mountPath: {{ template "configPath" . }} - {{- if .Values.global.tls.enabled }} + {{- if and .Values.global.tls.enabled (not .Values.global.pkiService) }} - name: tls mountPath: {{ template "tlsPath" . }} readOnly: true @@ -72,7 +72,7 @@ spec: secret: defaultMode: 0440 secretName: {{ template "sda.fullname" . }}-rotatekey - {{- if .Values.global.tls.enabled }} + {{- if and .Values.global.tls.enabled (not .Values.global.pkiService) }} - name: tls secret: defaultMode: 0440 From 2674ab62ecc92deaeb34e73a7beb55649841a989 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 20:49:12 +0200 Subject: [PATCH 498/530] refactor(rotatekey): make component conditional and align TLS logic --- charts/sda-svc/templates/rotatekey-certificate.yaml | 2 +- charts/sda-svc/templates/rotatekey-deploy.yaml | 2 ++ charts/sda-svc/templates/rotatekey-secrets.yaml | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/rotatekey-certificate.yaml b/charts/sda-svc/templates/rotatekey-certificate.yaml index 617a458bfe..f89863b809 100644 --- a/charts/sda-svc/templates/rotatekey-certificate.yaml +++ b/charts/sda-svc/templates/rotatekey-certificate.yaml @@ -1,4 +1,4 @@ -{{- if .Values.global.tls.enabled }} +{{- if and .Values.global.c4gh.rotatePubKeyData .Values.global.tls.enabled (not .Values.global.pkiService) }} {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} apiVersion: cert-manager.io/v1 kind: Certificate diff --git a/charts/sda-svc/templates/rotatekey-deploy.yaml b/charts/sda-svc/templates/rotatekey-deploy.yaml index c1d5095874..9ff2ed256b 100644 --- a/charts/sda-svc/templates/rotatekey-deploy.yaml +++ b/charts/sda-svc/templates/rotatekey-deploy.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.c4gh.rotatePubKeyData }} {{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} apiVersion: apps/v1 kind: Deployment @@ -79,4 +80,5 @@ spec: secretName: {{ template "sda.fullname" . }}-rotatekey-certs {{- end }} restartPolicy: Always +{{- end }} {{- end }} \ No newline at end of file diff --git a/charts/sda-svc/templates/rotatekey-secrets.yaml b/charts/sda-svc/templates/rotatekey-secrets.yaml index 965002d04a..313bbaaf78 100644 --- a/charts/sda-svc/templates/rotatekey-secrets.yaml +++ b/charts/sda-svc/templates/rotatekey-secrets.yaml @@ -1,3 +1,4 @@ +{{- if .Values.global.c4gh.rotatePubKeyData }} {{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}} {{- if not .Values.global.vaultSecrets }} --- @@ -64,5 +65,6 @@ stringData: target-key.pub: {{ required "global.c4gh.rotatePubKeyData is required" .Values.global.c4gh.rotatePubKeyData | b64dec | quote }} +{{- end }} {{- end }} {{- end }} \ No newline at end of file From ecf616f21865d83db32442dc6ba7fafb05c7add9 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 24 Apr 2026 21:55:36 +0200 Subject: [PATCH 499/530] fix(ci): add key hash for the rotation key to db --- .github/integration/scripts/charts/deploy_charts.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index f586946d3d..6b4ef5c8da 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -12,7 +12,7 @@ GRPC_PORT=50051 ROTATE_PUB="-----BEGIN CRYPT4GH PUBLIC KEY----- fFmwrVXywijqMoaLX95CgIXp6klJuo5MOLf/I3+BQ1Q= -----END CRYPT4GH PUBLIC KEY-----" -ROTATE_PUB_BASE64=$(printf '%s' "$ROTATE_PUB" | base64 -w0) +ROTATE_PUB_BASE64=$(printf '%s' "$ROTATE_PUB" | base64 | tr -d '\n') if [ "$3" == "true" ]; then MQ_PORT=5671 SCHEME=HTTPS @@ -35,6 +35,14 @@ if [ "$1" == "sda-db" ]; then --set persistence.enabled=false \ --set resources=null \ --wait + + echo "Registering rotation key in database..." + ROTATE_HEX=$(echo "$ROTATE_PUB" | awk 'NR==2' | base64 -d | xxd -p -c256 | tr -d '\n\r ') + DB_POD=$(kubectl get pods -l app=postgres-sda-db -o jsonpath='{.items[0].metadata.name}') + kubectl exec "$DB_POD" -- env PGPASSWORD="$ROOTPASS" psql -U postgres -d sda -c " + INSERT INTO sda.encryption_keys (key_hash, description) + VALUES ('$ROTATE_HEX', 'rotation key') + ON CONFLICT DO NOTHING;" fi if [ "$1" == "sda-mq" ]; then From 27c99dc5238979adc16ded64f02acbbf00041fee Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 27 Apr 2026 09:58:25 +0200 Subject: [PATCH 500/530] refactor(ci,test): make consistent quoting in shell script --- .github/integration/scripts/charts/deploy_charts.sh | 2 +- dev-tools/chart-smoke-test/test-rotatekey.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 6b4ef5c8da..ec52e2e332 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -93,7 +93,7 @@ if [ "$1" == "sda-svc" ]; then --set s3Inbox.readinessProbe.httpGet.scheme="$SCHEME" \ --set syncAPI.readinessProbe.httpGet.scheme="$SCHEME" \ --set reencrypt.readinessProbe.grpc.port="$GRPC_PORT" \ - --set "global.c4gh.rotatePubKeyData=$ROTATE_PUB_BASE64" \ + --set global.c4gh.rotatePubKeyData="$ROTATE_PUB_BASE64" \ -f "$dir/values.yaml" \ --wait fi diff --git a/dev-tools/chart-smoke-test/test-rotatekey.sh b/dev-tools/chart-smoke-test/test-rotatekey.sh index b622ab8ffb..4556af8e26 100755 --- a/dev-tools/chart-smoke-test/test-rotatekey.sh +++ b/dev-tools/chart-smoke-test/test-rotatekey.sh @@ -147,7 +147,7 @@ HELM_ARGS=( --set global.rbacEnabled=false # rotatekey - --set "global.c4gh.rotatePubKeyData=$ROTATE_PUB_BASE64" + --set global.c4gh.rotatePubKeyData="$ROTATE_PUB_BASE64" # --- database --- --set global.db.host=postgres-sda-db From 8a5c54456253d17962f1643167c272a881d0d400 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 27 Apr 2026 10:21:46 +0200 Subject: [PATCH 501/530] doc: fix formatting for sda-svc README --- charts/sda-svc/README.md | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 184054072f..5e552fffd0 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -124,7 +124,7 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.broker.password` | Shared password to the message broker. | `/` | | `global.broker.username` | Shared user to the message broker. | `/` | | `global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2` | -| `global.broker.rotateKeyQueue` | MQ queue name for key rotation messages. | `"rotatekey"` | +| `global.broker.rotateKeyQueue` | MQ queue name for key rotation messages. | `"rotatekey"` | | `global.cega.host` | Full URI to the EGA user authentication service. | `""` | | `global.cega.user` | Username for the EGA user authentication service. | `""` | | `global.cega.password` | Password for the EGA user authentication service. | `""` | @@ -215,7 +215,7 @@ The following table lists the configurable parameters of the `sda-svc` chart and | `global.tls.issuer` | Issuer for TLS certificate creation. | `""` | | `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""` | | `global.reencrypt.host` | gRPC host for reencryption | `""` | -| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` |global.reencrypt.host gRPC host for re-encryption (used by rotatekey) ""global.reencrypt.port Port number of the gRPC host for re-encryption 50051global.reencrypt.timeout Timeout in seconds for re-encryption gRPC calls 30global.c4gh.rotatePubKeyData The target public Crypt4GH key for rotation (base64 or plain string) ""global.broker.rotateKeyQueue MQ queue name for key rotation messages "rotatekey" +| `global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"` | ### Credentials @@ -260,10 +260,10 @@ of the values file. `credentials.verify.dbPassword` | Database password for verify | `""` `credentials.verify.mqUser` | Broker user for verify | `""` `credentials.verify.mqPassword` | Broker password for verify | `""` - `credentials.rotatekey.dbUser` | Database user for rotatekey | `""` - `credentials.rotatekey.dbPassword` | Database password for rotatekey| `""` - `credentials.rotatekey.mqUser` | Broker user for rotatekey | `""` - `credentials.rotatekey.mqPassword` | Broker password for rotatekey | `""` + `credentials.rotatekey.dbUser` | Database user for rotatekey | `""` + `credentials.rotatekey.dbPassword` | Database password for rotatekey| `""` + `credentials.rotatekey.mqUser` | Broker user for rotatekey | `""` + `credentials.rotatekey.mqPassword` | Broker password for rotatekey | `""` ### Pod settings @@ -369,12 +369,12 @@ of the values file. `releasetest.repository` | inbox container image repository | `neicnordic/sda-helm-test-support` `releasetest.imageTag` | inbox container image version | `latest` `releasetest.imagePullPolicy` | inbox container image pull policy | `Always` - `rotatekey.replicaCount` | Desired number of replicas. | `1` - `rotatekey.annotations` | Specific annotations for the rotatekey pod. | `{}` - `rotatekey.resources.requests.memory` | Memory request for container. | `128Mi` - `rotatekey.resources.requests.cpu` | CPU request for container. | `100m` - `rotatekey.resources.limits.memory` | Memory limit for container. | `256Mi` - `rotatekey.resources.limits.cpu` | CPU limit for container. | `1000m` + `rotatekey.replicaCount` | Desired number of replicas. | `1` + `rotatekey.annotations` | Specific annotations for the rotatekey pod. | `{}` + `rotatekey.resources.requests.memory` | Memory request for container. | `128Mi` + `rotatekey.resources.requests.cpu` | CPU request for container. | `100m` + `rotatekey.resources.limits.memory` | Memory limit for container. | `256Mi` + `rotatekey.resources.limits.cpu` | CPU limit for container. | `1000m` ### Jobs From 5421a25a85a5bd81bde54c36a829bdb74a4fa463 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 27 Apr 2026 10:27:57 +0200 Subject: [PATCH 502/530] chore: bump sda-svc by just one minor version --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 557c8407e8..47ecf16b3b 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.4.0 +version: 3.3.0 appVersion: v3.1.37 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From a8558c9927d022207e63188d63cc92f92d6ee7ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Apr 2026 21:25:24 +0000 Subject: [PATCH 503/530] build(deps): bump pytest Bumps the pip group with 1 update in the /.github/integration/tests/sda/auth directory: [pytest](https://github.com/pytest-dev/pytest). Updates `pytest` from 7.4.3 to 9.0.3 - [Release notes](https://github.com/pytest-dev/pytest/releases) - [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst) - [Commits](https://github.com/pytest-dev/pytest/compare/7.4.3...9.0.3) --- updated-dependencies: - dependency-name: pytest dependency-version: 9.0.3 dependency-type: direct:production dependency-group: pip ... Signed-off-by: dependabot[bot] --- .github/integration/tests/sda/auth/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/auth/requirements.txt b/.github/integration/tests/sda/auth/requirements.txt index d49a6a67ef..47700c7f4e 100644 --- a/.github/integration/tests/sda/auth/requirements.txt +++ b/.github/integration/tests/sda/auth/requirements.txt @@ -1,3 +1,3 @@ requests==2.33.0 tox==4.11.3 -pytest==7.4.3 +pytest==9.0.3 From 735999937bd1ec6a6d444b1d6d49e68a04342527 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Apr 2026 20:41:57 +0000 Subject: [PATCH 504/530] build(deps): bump go.opentelemetry.io/otel from 1.39.0 to 1.41.0 in /sda Bumps [go.opentelemetry.io/otel](https://github.com/open-telemetry/opentelemetry-go) from 1.39.0 to 1.41.0. - [Release notes](https://github.com/open-telemetry/opentelemetry-go/releases) - [Changelog](https://github.com/open-telemetry/opentelemetry-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/open-telemetry/opentelemetry-go/compare/v1.39.0...v1.41.0) --- updated-dependencies: - dependency-name: go.opentelemetry.io/otel dependency-version: 1.41.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- sda/go.mod | 6 +++--- sda/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 1af2657c40..add2bca829 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -168,9 +168,9 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.39.0 // indirect - go.opentelemetry.io/otel/metric v1.39.0 // indirect - go.opentelemetry.io/otel/trace v1.39.0 // indirect + go.opentelemetry.io/otel v1.41.0 // indirect + go.opentelemetry.io/otel/metric v1.41.0 // indirect + go.opentelemetry.io/otel/trace v1.41.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect diff --git a/sda/go.sum b/sda/go.sum index a3c5d829b5..b9b5bdf7b7 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -414,16 +414,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= +go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= +go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= +go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= +go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= From 52046dfb6096c6374970c9c0b4e9224df3614bab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Apr 2026 19:05:32 +0000 Subject: [PATCH 505/530] build(deps): bump aquasecurity/trivy-action in the github-actions group Bumps the github-actions group with 1 update: [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action). Updates `aquasecurity/trivy-action` from 0.35.0 to 0.36.0 - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.35.0...v0.36.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.36.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions ... Signed-off-by: dependabot[bot] --- .github/workflows/pr_build_and_test_images.yml | 4 ++-- .github/workflows/release_sda.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index 0d50a2ae30..c35e04054b 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -81,7 +81,7 @@ jobs: org.opencontainers.image.version=${{ github.sha }} - name: Run Trivy vulnerability scanner on ${{ matrix.name }} sarif - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -115,7 +115,7 @@ jobs: - name: Run Trivy vulnerability scanner on ${{ matrix.name }} markdown output if: steps.changes.outputs.changed == 'true' - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml index 10e1173b0f..331e08394f 100644 --- a/.github/workflows/release_sda.yml +++ b/.github/workflows/release_sda.yml @@ -76,7 +76,7 @@ jobs: - name: Run Trivy vulnerability scanner on ${{ matrix.name }} - uses: aquasecurity/trivy-action@0.35.0 + uses: aquasecurity/trivy-action@v0.36.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From 1ae59e419f652f7c45c6a92dd1ebbf8cc3cc44e6 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 13 Apr 2026 14:05:24 +0200 Subject: [PATCH 506/530] feat(chart): add values and helpers for v2 download service Add Helm values structure for deploying the v2 download service alongside legacy. New values under global.downloadV2 (disabled by default), credentials.downloadV2, and top-level downloadV2 with health probes, resources, and annotations. New helper templates dbUserDownloadV2/dbPassDownloadV2 follow the existing per-service credential pattern. Refs: #2345 --- charts/sda-svc/templates/_helpers.yaml | 8 ++ charts/sda-svc/values.yaml | 105 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index b00f54b20a..2d1388e70c 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -195,6 +195,14 @@ Create chart name and version as used by the chart label. {{- ternary .Values.global.db.password .Values.credentials.download.dbPassword (empty .Values.credentials.download.dbPassword) -}} {{- end -}} +{{/**/}} +{{- define "dbUserDownloadV2" -}} +{{- ternary .Values.global.db.user .Values.credentials.downloadV2.dbUser (empty .Values.credentials.downloadV2.dbUser) -}} +{{- end -}} +{{- define "dbPassDownloadV2" -}} +{{- ternary .Values.global.db.password .Values.credentials.downloadV2.dbPassword (empty .Values.credentials.downloadV2.dbPassword) -}} +{{- end -}} + {{/**/}} {{- define "dbUserFinalize" -}} {{- ternary .Values.global.db.user .Values.credentials.finalize.dbUser (empty .Values.credentials.finalize.dbUser) -}} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 919298d8fa..ad1e47ba47 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -38,6 +38,7 @@ global: auth: "" doa: "" download: "" + downloadV2: "" s3Inbox: "" syncapi: "" # If the certificates is generated by external providers @@ -46,6 +47,7 @@ global: auth: "" doa: "" download: "" + downloadV2: "" s3Inbox: "" syncapi: "" # issuer requires a configured cert-manager @@ -256,6 +258,70 @@ global: c4ghKeyFile: "" secretName: "" + downloadV2: + enabled: false # opt-in during migration + + service: + id: "neicnordic.sda.download" + orgName: "" + orgURL: "" + + app: + environment: "" # set to "production" for production guards + + jwt: + pubkeyPath: "" + pubkeyURL: "" + allowAllData: false + + oidc: + issuer: "" + audience: "" + + session: + expiration: 3600 + domain: "" + secure: true + httpOnly: true + name: "sda_session" + + permission: + model: "combined" + + auth: + allowOpaque: true + + visa: + enabled: false + source: "userinfo" + userinfoURL: "" # optional, auto-discovered from oidc.issuer if empty + trustedIssuers: [] # [{iss: "...", jku: "..."}] + datasetIdMode: "raw" + identityMode: "broker-bound" + validateAsserted: true + allowInsecureJku: false + cache: + tokenTTL: 3600 + jwkTTL: 300 + validationTTL: 120 + userinfoTTL: 60 + limits: + maxVisas: 200 + maxJwksPerRequest: 10 + maxVisaSize: 16384 + + cache: + enabled: true + fileTTL: 300 + permissionTTL: 120 + datasetTTL: 300 + + pagination: + hmacSecret: "" + + audit: + required: false + oidc: provider: "https://login.elixir-czech.org/oidc/" jwkPath: "/jwk" @@ -336,6 +402,10 @@ credentials: dbUser: "" dbPassword: "" + downloadV2: + dbUser: "" + dbPassword: "" + finalize: dbUser: "" dbPassword: "" @@ -533,6 +603,41 @@ download: ## Minimal version of the `sda-cli` to accept, defaults to `v0.3.0` minClientVersion: "" +downloadV2: + name: download-v2 + replicaCount: 2 + livenessProbe: + httpGet: + path: /health/live + port: download-v2 + # scheme set in deploy template via ternary on global.tls.enabled + initialDelaySeconds: 10 + periodSeconds: 30 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /health/ready + port: download-v2 + # scheme set in deploy template via ternary on global.tls.enabled + periodSeconds: 10 + timeoutSeconds: 9 + resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "512Mi" + cpu: "1000m" +# Extra annotations to attach to the service pods +# This should be a multi-line string mapping directly to the a map of +# the annotations to apply to the service pods + annotations: {} +## @param downloadV2.ingressAnnotations +## Ingress annotations specific to the download-v2 service + ingressAnnotations: {} + tls: + secretName: "" + finalize: name: finalize replicaCount: 1 From af1aa1db4da2bce546370e7612a5ef9468c902a9 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 13 Apr 2026 14:05:35 +0200 Subject: [PATCH 507/530] feat(chart): add v2 download service, ingress, certificate and network policy Add four supporting templates for the v2 download deployment: - download-v2-service.yaml: ClusterIP service selecting v2 pods - download-v2-ingress.yaml: separate hostname with required validation to prevent catch-all rules - download-v2-certificate.yaml: cert-manager certificate with -download-v2 DNS names - download-v2-networkpolicy.yaml: DB and reencrypt egress rules, uses sda.fullname consistently (fixes a selector bug in legacy) All gated by global.downloadV2.enabled (default: false). Refs: #2345 --- .../templates/download-v2-certificate.yaml | 41 ++++++++++++ .../templates/download-v2-ingress.yaml | 49 +++++++++++++++ .../templates/download-v2-networkpolicy.yaml | 62 +++++++++++++++++++ .../templates/download-v2-service.yaml | 18 ++++++ 4 files changed, 170 insertions(+) create mode 100644 charts/sda-svc/templates/download-v2-certificate.yaml create mode 100644 charts/sda-svc/templates/download-v2-ingress.yaml create mode 100644 charts/sda-svc/templates/download-v2-networkpolicy.yaml create mode 100644 charts/sda-svc/templates/download-v2-service.yaml diff --git a/charts/sda-svc/templates/download-v2-certificate.yaml b/charts/sda-svc/templates/download-v2-certificate.yaml new file mode 100644 index 0000000000..7d47a81e2d --- /dev/null +++ b/charts/sda-svc/templates/download-v2-certificate.yaml @@ -0,0 +1,41 @@ +{{- if .Values.global.tls.enabled }} +{{- if .Values.global.downloadV2.enabled }} +{{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ template "sda.fullname" . }}-download-v2-certs +spec: + # Secret names are always required. + secretName: {{ template "sda.fullname" . }}-download-v2-certs + + duration: 2160h # 90d + + # The use of the common name field has been deprecated since 2000 and is + # discouraged from being used. + commonName: {{ template "sda.fullname" . }}-download-v2 + isCA: false + privateKey: + algorithm: ECDSA + size: 256 + usages: + - client auth + - server auth + # At least one of a DNS Name, URI, or IP address is required. + dnsNames: + - {{ template "sda.fullname" . }}-download-v2 + - {{ template "sda.fullname" . }}-download-v2.{{ .Release.Namespace }}.svc + ipAddresses: + - 127.0.0.1 + # Issuer references are always required. + issuerRef: + name: {{ template "TLSissuer" . }} + # We can reference ClusterIssuers by changing the kind here. + # The default value is Issuer (i.e. a locally namespaced Issuer) + kind: {{ ternary "Issuer" "ClusterIssuer" (empty .Values.global.tls.clusterIssuer )}} + # This is optional since cert-manager will default to this value however + # if you are using an external issuer, change this to that issuer group. + group: cert-manager.io +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/charts/sda-svc/templates/download-v2-ingress.yaml b/charts/sda-svc/templates/download-v2-ingress.yaml new file mode 100644 index 0000000000..4addd9837e --- /dev/null +++ b/charts/sda-svc/templates/download-v2-ingress.yaml @@ -0,0 +1,49 @@ +{{- if (or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType)) (not .Values.global.deploymentType)) }} +{{- if and .Values.global.ingress.deploy .Values.global.downloadV2.enabled }} +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ template "sda.fullname" . }}-download-v2-ingress + labels: + app: {{ template "sda.fullname" . }} + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +{{- if .Values.global.ingress.labels }} +{{ toYaml .Values.global.ingress.labels | indent 4 }} +{{- end }} + annotations: +{{- if .Values.global.ingress.annotations }} +{{ toYaml .Values.global.ingress.annotations | indent 4 }} +{{- end }} +{{- if .Values.downloadV2.ingressAnnotations }} +{{ toYaml .Values.downloadV2.ingressAnnotations | indent 4 }} +{{- end }} + {{- if .Values.global.ingress.clusterIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }} + {{- else if .Values.global.ingress.issuer }} + cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }} + {{- end }} +spec: +{{- if .Values.global.ingress.ingressClassName }} + ingressClassName: {{ .Values.global.ingress.ingressClassName }} +{{- end }} + rules: + - host: {{ required "An ingress hostname is required for download-v2!" .Values.global.ingress.hostName.downloadV2 }} + http: + paths: + - pathType: Prefix + path: "/" + backend: + service: + name: {{ template "sda.fullname" . }}-download-v2 + port: + number: {{ ternary 443 80 .Values.global.tls.enabled }} +{{- if .Values.global.tls.enabled }} + tls: + - hosts: + - {{ required "An ingress hostname is required for download-v2!" .Values.global.ingress.hostName.downloadV2 }} + secretName: {{ if .Values.global.ingress.secretNames.downloadV2 }}{{ .Values.global.ingress.secretNames.downloadV2 }}{{ else }}"{{ template "sda.fullname" . }}-ingress-download-v2"{{ end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/download-v2-networkpolicy.yaml b/charts/sda-svc/templates/download-v2-networkpolicy.yaml new file mode 100644 index 0000000000..54cb1de956 --- /dev/null +++ b/charts/sda-svc/templates/download-v2-networkpolicy.yaml @@ -0,0 +1,62 @@ +{{- if .Values.global.networkPolicy.create }} +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} +{{- if .Values.global.downloadV2.enabled }} +--- +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: {{ template "sda.fullname" . }}-download-v2 + namespace: {{ .Release.Namespace }} +spec: + podSelector: + matchLabels: + app: {{ template "sda.fullname" . }}-download-v2 + policyTypes: + - Ingress + - Egress + egress: + - to: + {{- if .Values.global.networkPolicy.databaseNamespace }} + - namespaceSelector: + matchLabels: + name: {{ .Values.global.networkPolicy.databaseNamespace }} + {{- end }} + - podSelector: + matchLabels: + role: database + ports: + - protocol: TCP + port: {{ .Values.global.db.port | default 5432 | int }} + - to: + {{- if .Values.global.networkPolicy.internalNamespace }} + - namespaceSelector: + matchLabels: + name: {{ .Values.global.networkPolicy.internalNamespace }} + {{- end }} + - podSelector: + matchLabels: + role: reencrypt + ports: + - protocol: TCP + port: {{ .Values.global.reencrypt.port | int }} + {{- if .Values.global.networkPolicy.ingressNamespace }} + ingress: + - from: + - namespaceSelector: + matchLabels: + name: {{ .Values.global.networkPolicy.ingressNamespace }} + podSelector: + matchLabels: + {{ .Values.global.networkPolicy.ingressLabelMatch | nindent 14 }} + ports: + - protocol: TCP + port: 8080 + - protocol: TCP + port: 8443 + - protocol: TCP + port: 443 + {{- end }} +--- +{{- end }} +{{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/download-v2-service.yaml b/charts/sda-svc/templates/download-v2-service.yaml new file mode 100644 index 0000000000..7c336b672b --- /dev/null +++ b/charts/sda-svc/templates/download-v2-service.yaml @@ -0,0 +1,18 @@ +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} +{{- if .Values.global.downloadV2.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "sda.fullname" . }}-download-v2 + labels: + app: {{ template "sda.fullname" . }}-download-v2 +spec: + ports: + - name: download-v2 + port: {{ ternary 443 80 .Values.global.tls.enabled }} + targetPort: download-v2 + protocol: TCP + selector: + app: {{ template "sda.fullname" . }}-download-v2 +{{- end }} +{{- end }} From 8bd9b91a5a4bb0c207f3c5db34e55bfa710fc8f6 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 13 Apr 2026 14:05:51 +0200 Subject: [PATCH 508/530] feat(chart): add v2 download deployment and secrets templates Add the two main templates for the v2 download service: download-v2-deploy.yaml: - Uses monolith image (no -download suffix) with command sda-download - Config loaded via --config-file flag (not legacy CONFIGFILE env var) - Health probes on /health/live and /health/ready with dynamic scheme - No c4gh-transient volume (v2 removed decrypted serving) - Trusted issuers volume conditionally mounted when visa configured download-v2-secrets.yaml: - Full v2 flat pflag/viper config format with all 50+ keys - Storage section mapped from global.archive.* to v2's storage.archive.s3/posix format - Session domain defaults to ingress hostname when not set - Visa block conditionally rendered with cache/limits/identity config - Trusted issuers Secret generated from values (same pattern as legacy) Verified: helm template renders correctly, k3d deploy confirms the binary loads config, connects to DB, and initializes cache. Refs: #2345 --- .../sda-svc/templates/download-v2-deploy.yaml | 152 ++++++++++++++++++ .../templates/download-v2-secrets.yaml | 126 +++++++++++++++ 2 files changed, 278 insertions(+) create mode 100644 charts/sda-svc/templates/download-v2-deploy.yaml create mode 100644 charts/sda-svc/templates/download-v2-secrets.yaml diff --git a/charts/sda-svc/templates/download-v2-deploy.yaml b/charts/sda-svc/templates/download-v2-deploy.yaml new file mode 100644 index 0000000000..05a1b5a16f --- /dev/null +++ b/charts/sda-svc/templates/download-v2-deploy.yaml @@ -0,0 +1,152 @@ +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} +{{- if .Values.global.downloadV2.enabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "sda.fullname" . }}-download-v2 + labels: + role: download-v2 + app: {{ template "sda.fullname" . }}-download-v2 + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + component: {{ template "sda.fullname" . }}-download-v2 + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.downloadV2.replicaCount }} + revisionHistoryLimit: {{ default "3" .Values.global.revisionHistory }} + selector: + matchLabels: + app: {{ template "sda.fullname" . }}-download-v2 + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "sda.fullname" . }}-download-v2 + role: download-v2 + release: {{ .Release.Name }} + annotations: + {{- if not .Values.global.vaultSecrets }} + checksum/secret: {{ include (print $.Template.BasePath "/download-v2-secrets.yaml") . | sha256sum }} + {{- end }} +{{- if .Values.global.podAnnotations }} +{{- toYaml .Values.global.podAnnotations | nindent 8 -}} +{{- end }} +{{- if .Values.downloadV2.annotations }} +{{- toYaml .Values.downloadV2.annotations | nindent 8 -}} +{{- end }} + spec: + topologySpreadConstraints: + - maxSkew: 1 + whenUnsatisfiable: DoNotSchedule + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app: {{ template "sda.fullname" . }}-download-v2 + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: app + operator: In + values: + - {{ template "sda.fullname" . }}-download-v2 + topologyKey: kubernetes.io/hostname + {{- if .Values.global.rbacEnabled}} + serviceAccountName: {{ .Release.Name }} + {{- end }} + securityContext: + runAsUser: 65534 + runAsGroup: 65534 + fsGroup: 65534 + containers: + - name: download-v2 + image: "{{ .Values.image.repository }}:{{ default .Chart.AppVersion .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy | quote }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + command: ["sda-download"] + args: ["--config-file", "{{ include "confFile" . }}"] + ports: + - name: download-v2 + containerPort: {{ ternary 8443 8080 .Values.global.tls.enabled }} + protocol: TCP + livenessProbe: + httpGet: + path: /health/live + port: download-v2 + scheme: {{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }} + initialDelaySeconds: {{ .Values.downloadV2.livenessProbe.initialDelaySeconds | default 10 }} + periodSeconds: {{ .Values.downloadV2.livenessProbe.periodSeconds | default 30 }} + timeoutSeconds: {{ .Values.downloadV2.livenessProbe.timeoutSeconds | default 5 }} + readinessProbe: + httpGet: + path: /health/ready + port: download-v2 + scheme: {{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }} + periodSeconds: {{ .Values.downloadV2.readinessProbe.periodSeconds | default 10 }} + timeoutSeconds: {{ .Values.downloadV2.readinessProbe.timeoutSeconds | default 9 }} + resources: +{{ toYaml .Values.downloadV2.resources | trim | indent 10 }} + volumeMounts: + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} + {{- if .Values.global.downloadV2.visa.trustedIssuers }} + - name: iss + mountPath: {{ template "trustedIssPath" . }} + {{- end }} + {{- end }} + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + - name: tls + mountPath: {{ template "tlsPath" . }} + {{- end }} + {{- if eq "posix" .Values.global.archive.storageType }} + - name: archive + mountPath: {{ .Values.global.archive.volumePath | quote }} + {{- end }} + enableServiceLinks: false + volumes: + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + - name: tls + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + secret: + defaultMode: 0640 + secretName: {{ template "sda.fullname" . }}-download-v2-certs + {{- else }} + secret: + defaultMode: 0640 + secretName: {{ required "An certificate issuer or a TLS secret name is required for download-v2" .Values.downloadV2.tls.secretName }} + {{- end }} + {{- end }} + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-download-v2 + {{- if .Values.global.downloadV2.visa.trustedIssuers }} + - name: iss + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-download-v2-iss + {{- end }} + {{- end }} + {{- if eq "posix" .Values.global.archive.storageType }} + - name: archive + {{- if .Values.global.archive.existingClaim }} + persistentVolumeClaim: + claimName: {{ .Values.global.archive.existingClaim }} + {{- else }} + nfs: + server: {{ required "An archive NFS server is required" .Values.global.archive.nfsServer | quote }} + path: {{ if .Values.global.archive.nfsPath }}{{ .Values.global.archive.nfsPath | quote }}{{ else }}{{ "/" }}{{ end }} + {{- end }} + {{- end }} + restartPolicy: Always +{{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml new file mode 100644 index 0000000000..3f4941fab8 --- /dev/null +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -0,0 +1,126 @@ +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} +{{- if and .Values.global.downloadV2.enabled (not .Values.global.vaultSecrets) }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "sda.fullname" . }}-download-v2 +type: Opaque +stringData: + config.yaml: |- + api: + host: "0.0.0.0" + port: {{ ternary 8443 8080 .Values.global.tls.enabled }} + {{- if .Values.global.tls.enabled }} + server-cert: {{ template "tlsPath" . }}/tls.crt + server-key: {{ template "tlsPath" . }}/tls.key + {{- end }} + db: + host: {{ .Values.global.db.host }} + port: {{ .Values.global.db.port }} + user: {{ required "DB user is required" (include "dbUserDownloadV2" .) }} + password: {{ required "DB pass is required" (include "dbPassDownloadV2" .) }} + database: {{ .Values.global.db.name }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + {{- if .Values.global.tls.enabled }} + cacert: {{ template "tlsPath" . }}/ca.crt + clientcert: {{ template "tlsPath" . }}/tls.crt + clientkey: {{ template "tlsPath" . }}/tls.key + {{- end }} + grpc: + host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }} + port: {{ ternary 50443 50051 .Values.global.tls.enabled }} + timeout: {{ default 30 .Values.global.reencrypt.timeout }} + {{- if .Values.global.tls.enabled }} + cacert: {{ template "tlsPath" . }}/ca.crt + client-cert: {{ template "tlsPath" . }}/tls.crt + client-key: {{ template "tlsPath" . }}/tls.key + {{- end }} + log: + level: {{ .Values.global.log.level }} + service: + id: {{ .Values.global.downloadV2.service.id }} + org-name: {{ .Values.global.downloadV2.service.orgName }} + org-url: {{ .Values.global.downloadV2.service.orgURL }} + jwt: + {{- if .Values.global.downloadV2.jwt.pubkeyPath }} + pubkey-path: {{ template "jwtPath" . }}/{{ .Values.global.downloadV2.jwt.pubkeyPath }} + {{- end }} + pubkey-url: {{ .Values.global.downloadV2.jwt.pubkeyURL }} + allow-all-data: {{ .Values.global.downloadV2.jwt.allowAllData }} + oidc: + issuer: {{ .Values.global.downloadV2.oidc.issuer }} + audience: {{ .Values.global.downloadV2.oidc.audience }} + session: + expiration: {{ .Values.global.downloadV2.session.expiration }} + domain: {{ default .Values.global.ingress.hostName.downloadV2 .Values.global.downloadV2.session.domain }} + secure: {{ .Values.global.downloadV2.session.secure }} + http-only: {{ .Values.global.downloadV2.session.httpOnly }} + name: {{ .Values.global.downloadV2.session.name }} + permission: + model: {{ .Values.global.downloadV2.permission.model }} + auth: + allow-opaque: {{ .Values.global.downloadV2.auth.allowOpaque }} + {{- if .Values.global.downloadV2.visa.enabled }} + visa: + enabled: true + source: {{ .Values.global.downloadV2.visa.source }} + userinfo-url: {{ .Values.global.downloadV2.visa.userinfoURL }} + trusted-issuers-path: {{ template "trustedIssPath" . }}/iss.json + dataset-id-mode: {{ .Values.global.downloadV2.visa.datasetIdMode }} + identity: + mode: {{ .Values.global.downloadV2.visa.identityMode }} + validate-asserted: {{ .Values.global.downloadV2.visa.validateAsserted }} + allow-insecure-jku: {{ .Values.global.downloadV2.visa.allowInsecureJku }} + cache: + token-ttl: {{ .Values.global.downloadV2.visa.cache.tokenTTL }} + jwk-ttl: {{ .Values.global.downloadV2.visa.cache.jwkTTL }} + validation-ttl: {{ .Values.global.downloadV2.visa.cache.validationTTL }} + userinfo-ttl: {{ .Values.global.downloadV2.visa.cache.userinfoTTL }} + limits: + max-visas: {{ .Values.global.downloadV2.visa.limits.maxVisas }} + max-jwks-per-request: {{ .Values.global.downloadV2.visa.limits.maxJwksPerRequest }} + max-visa-size: {{ .Values.global.downloadV2.visa.limits.maxVisaSize }} + {{- end }} + cache: + enabled: {{ .Values.global.downloadV2.cache.enabled }} + file-ttl: {{ .Values.global.downloadV2.cache.fileTTL }} + permission-ttl: {{ .Values.global.downloadV2.cache.permissionTTL }} + dataset-ttl: {{ .Values.global.downloadV2.cache.datasetTTL }} + pagination: + hmac-secret: {{ .Values.global.downloadV2.pagination.hmacSecret }} + app: + environment: {{ .Values.global.downloadV2.app.environment }} + audit: + required: {{ .Values.global.downloadV2.audit.required }} + storage: + archive: + {{- if eq "s3" .Values.global.archive.storageType }} + s3: + - endpoint: {{ .Values.global.archive.s3Url }}:{{ .Values.global.archive.s3Port }} + access_key: {{ .Values.global.archive.s3AccessKey }} + secret_key: {{ .Values.global.archive.s3SecretKey }} + bucket_prefix: {{ .Values.global.archive.s3BucketPrefix }} + region: {{ .Values.global.archive.s3Region }} + {{- if .Values.global.tls.enabled }} + ca_cert: {{ template "tlsPath" . }}/ca.crt + {{- end }} + {{- if .Values.global.archive.s3ChunkSize }} + chunk_size: {{ .Values.global.archive.s3ChunkSize }} + {{- end }} + disable_https: {{ not .Values.global.tls.enabled }} + {{- else }} + posix: + - path: {{ .Values.global.archive.volumePath }} + {{- end }} +--- +{{- if .Values.global.downloadV2.visa.trustedIssuers }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "sda.fullname" . }}-download-v2-iss +type: Opaque +data: + iss.json: {{ .Values.global.downloadV2.visa.trustedIssuers | toJson | b64enc | quote }} +{{- end }} +{{- end }} +{{- end }} From 31cbc3b2b336765103a6743bc86a4c6f95979994 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 13 Apr 2026 14:37:50 +0200 Subject: [PATCH 509/530] fix(chart): reuse shared OIDC and download DB credentials for v2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the downloadV2-specific oidc block and credentials — the v2 service shares the same OIDC provider (global.oidc) and database user (credentials.download) as legacy since they have identical scope and permissions. --- charts/sda-svc/templates/_helpers.yaml | 8 -------- charts/sda-svc/templates/download-v2-secrets.yaml | 8 ++++---- charts/sda-svc/values.yaml | 8 -------- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 2d1388e70c..b00f54b20a 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -195,14 +195,6 @@ Create chart name and version as used by the chart label. {{- ternary .Values.global.db.password .Values.credentials.download.dbPassword (empty .Values.credentials.download.dbPassword) -}} {{- end -}} -{{/**/}} -{{- define "dbUserDownloadV2" -}} -{{- ternary .Values.global.db.user .Values.credentials.downloadV2.dbUser (empty .Values.credentials.downloadV2.dbUser) -}} -{{- end -}} -{{- define "dbPassDownloadV2" -}} -{{- ternary .Values.global.db.password .Values.credentials.downloadV2.dbPassword (empty .Values.credentials.downloadV2.dbPassword) -}} -{{- end -}} - {{/**/}} {{- define "dbUserFinalize" -}} {{- ternary .Values.global.db.user .Values.credentials.finalize.dbUser (empty .Values.credentials.finalize.dbUser) -}} diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml index 3f4941fab8..fa44a0a930 100644 --- a/charts/sda-svc/templates/download-v2-secrets.yaml +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -17,8 +17,8 @@ stringData: db: host: {{ .Values.global.db.host }} port: {{ .Values.global.db.port }} - user: {{ required "DB user is required" (include "dbUserDownloadV2" .) }} - password: {{ required "DB pass is required" (include "dbPassDownloadV2" .) }} + user: {{ required "DB user is required" (include "dbUserDownload" .) }} + password: {{ required "DB pass is required" (include "dbPassDownload" .) }} database: {{ .Values.global.db.name }} sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} {{- if .Values.global.tls.enabled }} @@ -48,8 +48,8 @@ stringData: pubkey-url: {{ .Values.global.downloadV2.jwt.pubkeyURL }} allow-all-data: {{ .Values.global.downloadV2.jwt.allowAllData }} oidc: - issuer: {{ .Values.global.downloadV2.oidc.issuer }} - audience: {{ .Values.global.downloadV2.oidc.audience }} + issuer: {{ .Values.global.oidc.provider | trimSuffix "/" }} + audience: {{ .Values.global.oidc.id }} session: expiration: {{ .Values.global.downloadV2.session.expiration }} domain: {{ default .Values.global.ingress.hostName.downloadV2 .Values.global.downloadV2.session.domain }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index ad1e47ba47..98def8d251 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -274,10 +274,6 @@ global: pubkeyURL: "" allowAllData: false - oidc: - issuer: "" - audience: "" - session: expiration: 3600 domain: "" @@ -402,10 +398,6 @@ credentials: dbUser: "" dbPassword: "" - downloadV2: - dbUser: "" - dbPassword: "" - finalize: dbUser: "" dbPassword: "" From a1f8ed093725704998fd20e17a331017c6ac3727 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 13 Apr 2026 14:43:49 +0200 Subject: [PATCH 510/530] docs(chart): add @param annotations to downloadV2 values Add helm-docs compatible @param annotations to all global.downloadV2 values. Replaces inline comments with the structured format used elsewhere in the chart for consistent parameter documentation. --- charts/sda-svc/values.yaml | 44 ++++++++++++++++++++++++++++++++++---- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 98def8d251..64b2530761 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -259,63 +259,99 @@ global: secretName: "" downloadV2: - enabled: false # opt-in during migration + # @param global.downloadV2.enabled enable the v2 download service (opt-in during migration) + enabled: false service: + # @param global.downloadV2.service.id GA4GH service-info identifier (reverse domain notation) id: "neicnordic.sda.download" + # @param global.downloadV2.service.orgName organization name for GA4GH service-info orgName: "" + # @param global.downloadV2.service.orgURL organization URL for GA4GH service-info orgURL: "" app: - environment: "" # set to "production" for production guards + # @param global.downloadV2.app.environment set to "production" to enable production guards + environment: "" jwt: + # @param global.downloadV2.jwt.pubkeyPath path to directory containing JWT public key PEM files pubkeyPath: "" + # @param global.downloadV2.jwt.pubkeyURL URL to JWKS endpoint for JWT verification pubkeyURL: "" + # @param global.downloadV2.jwt.allowAllData allow all authenticated users access to all datasets (testing only) allowAllData: false session: + # @param global.downloadV2.session.expiration session expiration time in seconds expiration: 3600 + # @param global.downloadV2.session.domain cookie domain, defaults to ingress hostname if empty domain: "" + # @param global.downloadV2.session.secure use secure cookies (HTTPS only) secure: true + # @param global.downloadV2.session.httpOnly HTTP-only cookies (not accessible via JavaScript) httpOnly: true + # @param global.downloadV2.session.name session cookie name name: "sda_session" permission: + # @param global.downloadV2.permission.model permission model: ownership, visa, or combined model: "combined" auth: + # @param global.downloadV2.auth.allowOpaque allow userinfo-based auth for opaque (non-JWT) tokens allowOpaque: true visa: + # @param global.downloadV2.visa.enabled enable GA4GH visa support for dataset access enabled: false + # @param global.downloadV2.visa.source visa source: userinfo (GA4GH compliant) or token (legacy) source: "userinfo" - userinfoURL: "" # optional, auto-discovered from oidc.issuer if empty - trustedIssuers: [] # [{iss: "...", jku: "..."}] + # @param global.downloadV2.visa.userinfoURL userinfo endpoint URL, auto-discovered from global.oidc.provider if empty + userinfoURL: "" + # @param global.downloadV2.visa.trustedIssuers list of trusted issuer+JKU pairs [{iss: "", jku: ""}] + trustedIssuers: [] + # @param global.downloadV2.visa.datasetIdMode dataset ID extraction: raw (exact visa value) or suffix (last URL/URN segment) datasetIdMode: "raw" + # @param global.downloadV2.visa.identityMode identity binding: broker-bound, strict-sub, or strict-iss-sub identityMode: "broker-bound" + # @param global.downloadV2.visa.validateAsserted reject visas where asserted timestamp is in the future validateAsserted: true + # @param global.downloadV2.visa.allowInsecureJku allow HTTP (non-TLS) JKU URLs in trusted issuers (testing only) allowInsecureJku: false cache: + # @param global.downloadV2.visa.cache.tokenTTL token-keyed cache TTL in seconds tokenTTL: 3600 + # @param global.downloadV2.visa.cache.jwkTTL JWK cache TTL in seconds jwkTTL: 300 + # @param global.downloadV2.visa.cache.validationTTL visa validation cache TTL in seconds validationTTL: 120 + # @param global.downloadV2.visa.cache.userinfoTTL userinfo cache TTL in seconds userinfoTTL: 60 limits: + # @param global.downloadV2.visa.limits.maxVisas maximum visas to process per passport maxVisas: 200 + # @param global.downloadV2.visa.limits.maxJwksPerRequest maximum distinct JKU fetches per request maxJwksPerRequest: 10 + # @param global.downloadV2.visa.limits.maxVisaSize maximum visa JWT size in bytes maxVisaSize: 16384 cache: + # @param global.downloadV2.cache.enabled enable database query caching enabled: true + # @param global.downloadV2.cache.fileTTL file query cache TTL in seconds fileTTL: 300 + # @param global.downloadV2.cache.permissionTTL permission check cache TTL in seconds permissionTTL: 120 + # @param global.downloadV2.cache.datasetTTL dataset query cache TTL in seconds datasetTTL: 300 pagination: + # @param global.downloadV2.pagination.hmacSecret HMAC secret for page tokens (required in production, must be same across replicas) hmacSecret: "" audit: + # @param global.downloadV2.audit.required require a real audit logger (fail startup if noop would be used) required: false oidc: From 8429cb01ac054e238df74792e7e09391fc5e2d00 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 13 Apr 2026 17:04:18 +0200 Subject: [PATCH 511/530] feat(dev-tools): add k3d smoke test for v2 download chart Self-contained script that spins up a k3d cluster, deploys postgres, rabbitmq, and minio, then renders and applies only the download-v2 chart templates. Runs 6 smoke tests against the running service: health/ready (DB, storage, gRPC), service-info, and auth rejection. Usage: ./dev-tools/chart-smoke-test/test-download-v2.sh # full run ./dev-tools/chart-smoke-test/test-download-v2.sh --no-build # skip image build ./dev-tools/chart-smoke-test/test-download-v2.sh --cleanup # tear down Refs: #2345, #2353 --- .../chart-smoke-test/test-download-v2.sh | 233 ++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100755 dev-tools/chart-smoke-test/test-download-v2.sh diff --git a/dev-tools/chart-smoke-test/test-download-v2.sh b/dev-tools/chart-smoke-test/test-download-v2.sh new file mode 100755 index 0000000000..cecabbb105 --- /dev/null +++ b/dev-tools/chart-smoke-test/test-download-v2.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +# Local k3d smoke test for the v2 download service chart. +# +# Prerequisites: docker, k3d, kubectl, helm +# +# Usage: +# ./dev-tools/k3d/test-download-v2.sh # full run (build + deploy + test) +# ./dev-tools/k3d/test-download-v2.sh --no-build # skip image build (reuse existing) +# ./dev-tools/k3d/test-download-v2.sh --cleanup # tear down cluster and exit +set -euo pipefail + +CLUSTER_NAME="sda-download-v2-test" +IMAGE="ghcr.io/neicnordic/sensitive-data-archive:local-test" +SKIP_BUILD=false + +for arg in "$@"; do + case "$arg" in + --no-build) SKIP_BUILD=true ;; + --cleanup) + echo "Tearing down cluster $CLUSTER_NAME..." + k3d cluster delete "$CLUSTER_NAME" 2>/dev/null || true + echo "Done." + exit 0 + ;; + *) echo "Unknown argument: $arg"; exit 1 ;; + esac +done + +# -- helpers -- +pass=0 +fail=0 +check() { + local name="$1" expected="$2" actual="$3" + if echo "$actual" | grep -q "$expected" 2>/dev/null; then + echo " PASS $name" + pass=$((pass + 1)) + else + echo " FAIL $name (expected '$expected', got '$actual')" + fail=$((fail + 1)) + fi +} + +wait_for_pod() { + local label="$1" timeout="$2" + echo "Waiting for pod $label to be ready (${timeout}s)..." + if ! kubectl wait --for=condition=ready pod -l "$label" --timeout="${timeout}s" 2>/dev/null; then + echo "Pod $label not ready after ${timeout}s" + kubectl logs -l "$label" --tail=20 2>/dev/null + return 1 + fi +} + +# -- 1. cluster -- +echo "=== Step 1: k3d cluster ===" +if k3d cluster list 2>/dev/null | grep -q "$CLUSTER_NAME"; then + echo "Cluster $CLUSTER_NAME already exists, reusing." +else + echo "Creating cluster $CLUSTER_NAME..." + k3d cluster create "$CLUSTER_NAME" --wait +fi + +# -- 2. build + import image -- +echo "=== Step 2: image ===" +if [ "$SKIP_BUILD" = true ]; then + echo "Skipping build (--no-build)." +else + echo "Building sda image..." + docker build -t "$IMAGE" --build-arg GOLANG_VERSION=1.25 --network=host -f sda/Dockerfile sda/ +fi +echo "Importing image into cluster..." +k3d image import "$IMAGE" -c "$CLUSTER_NAME" + +# -- 3. dependencies (postgres, rabbitmq, minio) -- +echo "=== Step 3: dependencies ===" + +if ! helm status postgres >/dev/null 2>&1; then + echo "Installing postgres..." + helm install postgres charts/sda-db \ + --set global.postgresAdminPassword=rootpass \ + --set global.tls.enabled=false \ + --set persistence.enabled=false \ + --set resources=null \ + --wait --timeout 120s +else + echo "Postgres already installed." +fi + +if ! helm status broker >/dev/null 2>&1; then + echo "Installing rabbitmq..." + helm install broker charts/sda-mq \ + --set global.adminPassword=mqpass \ + --set global.adminUser=admin \ + --set global.tls.enabled=false \ + --set persistence.enabled=false \ + --set resources=null \ + --wait --timeout 120s +else + echo "RabbitMQ already installed." +fi + +if ! kubectl get deploy minio >/dev/null 2>&1; then + echo "Installing minio..." + kubectl apply -f - <<'MINIO_EOF' +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + containers: + - name: minio + image: minio/minio:latest + command: ["minio", "server", "/data"] + env: + - name: MINIO_ROOT_USER + value: access + - name: MINIO_ROOT_PASSWORD + value: secretkey + ports: + - containerPort: 9000 +--- +apiVersion: v1 +kind: Service +metadata: + name: minio +spec: + ports: + - port: 443 + targetPort: 9000 + selector: + app: minio +MINIO_EOF + wait_for_pod "app=minio" 60 +else + echo "Minio already installed." +fi + +# -- 4. service account -- +kubectl create serviceaccount pipeline 2>/dev/null || true + +# -- 5. deploy download-v2 -- +echo "=== Step 4: deploy download-v2 ===" + +# Clean previous run +kubectl delete deploy pipeline-sda-svc-download-v2 2>/dev/null || true +kubectl delete secret pipeline-sda-svc-download-v2 2>/dev/null || true +kubectl delete svc pipeline-sda-svc-download-v2 2>/dev/null || true + +HELM_ARGS=( + --set image.tag=local-test + --set image.pullPolicy=IfNotPresent + --set global.tls.enabled=false + --set global.db.host=postgres-sda-db + --set global.db.user=postgres + --set global.db.password=rootpass + --set global.broker.host=broker-sda-mq + --set global.broker.username=admin + --set global.broker.password=mqpass + --set global.archive.storageType=s3 + --set global.archive.s3Url=http://minio + --set global.archive.s3Port=443 + --set global.archive.s3AccessKey=access + --set global.archive.s3SecretKey=secretkey + --set global.reencrypt.host=pipeline-sda-svc-reencrypt + --set global.ingress.deploy=false + --set global.oidc.id=test + --set global.oidc.secret=test + --set global.oidc.provider=http://oidc + --set global.api.jwtSecret=jwt-secret + --set global.api.rbacFileSecret=rbac + --set global.c4gh.secretName=c4gh-secret + --set global.c4gh.publicKey=dummy + --set "global.c4gh.privateKeys[0].keyName=c4gh.key" + --set "global.c4gh.privateKeys[0].passphrase=pass" + --set "global.c4gh.privateKeys[0].keyData=dummykey" + --set global.downloadV2.enabled=true + --set global.downloadV2.service.orgName=TestOrg + --set global.downloadV2.service.orgURL=http://test.org + --set global.ingress.hostName.downloadV2=dl-v2.local +) + +# Render only v2 templates and apply directly +for tmpl in download-v2-secrets download-v2-deploy download-v2-service; do + helm template pipeline charts/sda-svc "${HELM_ARGS[@]}" \ + --show-only "templates/${tmpl}.yaml" +done | kubectl apply -f - + +# Scale to 1 replica (single-node cluster can't satisfy topology spread for 2) +kubectl scale deploy pipeline-sda-svc-download-v2 --replicas=1 + +wait_for_pod "app=pipeline-sda-svc-download-v2" 60 + +# -- 6. smoke tests -- +echo "=== Step 5: smoke tests ===" + +SVC="pipeline-sda-svc-download-v2:80" + +run_curl() { + local name="$1"; shift + kubectl run "$name" --rm -i --restart=Never --image=curlimages/curl \ + --quiet -- "$@" 2>/dev/null | grep -v '^pod .* deleted' +} + +health=$(run_curl curl-health curl -sf "http://$SVC/health/ready") +check "/health/ready returns ok" '"status":"ok"' "$health" +check "/health/ready database ok" '"database":"ok"' "$health" +check "/health/ready storage ok" '"storage":"ok"' "$health" + +svcinfo=$(run_curl curl-svcinfo curl -sf "http://$SVC/service-info") +check "/service-info returns org" '"name":"TestOrg"' "$svcinfo" +check "/service-info returns version" '"version":"2.0.0"' "$svcinfo" + +authresp=$(run_curl curl-auth curl -s -o /dev/null -w '%{http_code}' "http://$SVC/datasets") +check "/datasets returns 401 without token" "401" "$authresp" + +# -- summary -- +echo "" +echo "=== Results: $pass passed, $fail failed ===" +if [ "$fail" -gt 0 ]; then + echo "Pod logs:" + kubectl logs -l app=pipeline-sda-svc-download-v2 --tail=30 + exit 1 +fi +echo "" +echo "Cleanup: $0 --cleanup" From 40affaea3f2d1e9317b42cc736d0985590e5a2d7 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Mon, 13 Apr 2026 17:17:28 +0200 Subject: [PATCH 512/530] fix(chart): address review feedback on v2 download templates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes based on review by @jbygdell: - Hardcode container/service port to 8080 (TLS terminates at ingress, not at the application — removes unnecessary ternary logic) - Remove api.server-cert/server-key from config (not needed when TLS is handled by ingress) - Default grpc.host to {{ sda.fullname }}-reencrypt instead of requiring it (only needed when reencrypt is in another namespace) - Remove extra ports (8443, 443) from network policy ingress rules - Todo multi-endpoint Refs: #2345 --- charts/sda-svc/templates/download-v2-deploy.yaml | 2 +- charts/sda-svc/templates/download-v2-ingress.yaml | 2 +- charts/sda-svc/templates/download-v2-networkpolicy.yaml | 4 ---- charts/sda-svc/templates/download-v2-secrets.yaml | 9 +++------ charts/sda-svc/templates/download-v2-service.yaml | 2 +- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/charts/sda-svc/templates/download-v2-deploy.yaml b/charts/sda-svc/templates/download-v2-deploy.yaml index 05a1b5a16f..9fa5c940c3 100644 --- a/charts/sda-svc/templates/download-v2-deploy.yaml +++ b/charts/sda-svc/templates/download-v2-deploy.yaml @@ -74,7 +74,7 @@ spec: args: ["--config-file", "{{ include "confFile" . }}"] ports: - name: download-v2 - containerPort: {{ ternary 8443 8080 .Values.global.tls.enabled }} + containerPort: 8080 protocol: TCP livenessProbe: httpGet: diff --git a/charts/sda-svc/templates/download-v2-ingress.yaml b/charts/sda-svc/templates/download-v2-ingress.yaml index 4addd9837e..3df74b6904 100644 --- a/charts/sda-svc/templates/download-v2-ingress.yaml +++ b/charts/sda-svc/templates/download-v2-ingress.yaml @@ -38,7 +38,7 @@ spec: service: name: {{ template "sda.fullname" . }}-download-v2 port: - number: {{ ternary 443 80 .Values.global.tls.enabled }} + number: 8080 {{- if .Values.global.tls.enabled }} tls: - hosts: diff --git a/charts/sda-svc/templates/download-v2-networkpolicy.yaml b/charts/sda-svc/templates/download-v2-networkpolicy.yaml index 54cb1de956..546604a8f5 100644 --- a/charts/sda-svc/templates/download-v2-networkpolicy.yaml +++ b/charts/sda-svc/templates/download-v2-networkpolicy.yaml @@ -51,10 +51,6 @@ spec: ports: - protocol: TCP port: 8080 - - protocol: TCP - port: 8443 - - protocol: TCP - port: 443 {{- end }} --- {{- end }} diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml index fa44a0a930..8fc764afda 100644 --- a/charts/sda-svc/templates/download-v2-secrets.yaml +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -9,11 +9,7 @@ stringData: config.yaml: |- api: host: "0.0.0.0" - port: {{ ternary 8443 8080 .Values.global.tls.enabled }} - {{- if .Values.global.tls.enabled }} - server-cert: {{ template "tlsPath" . }}/tls.crt - server-key: {{ template "tlsPath" . }}/tls.key - {{- end }} + port: 8080 db: host: {{ .Values.global.db.host }} port: {{ .Values.global.db.port }} @@ -27,7 +23,7 @@ stringData: clientkey: {{ template "tlsPath" . }}/tls.key {{- end }} grpc: - host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }} + host: {{ default (printf "%s-reencrypt" (include "sda.fullname" .)) .Values.global.reencrypt.host }} port: {{ ternary 50443 50051 .Values.global.tls.enabled }} timeout: {{ default 30 .Values.global.reencrypt.timeout }} {{- if .Values.global.tls.enabled }} @@ -95,6 +91,7 @@ stringData: storage: archive: {{- if eq "s3" .Values.global.archive.storageType }} + {{- /* TODO: support multiple archive endpoints (v2 storage format is an array) */}} s3: - endpoint: {{ .Values.global.archive.s3Url }}:{{ .Values.global.archive.s3Port }} access_key: {{ .Values.global.archive.s3AccessKey }} diff --git a/charts/sda-svc/templates/download-v2-service.yaml b/charts/sda-svc/templates/download-v2-service.yaml index 7c336b672b..990e11783d 100644 --- a/charts/sda-svc/templates/download-v2-service.yaml +++ b/charts/sda-svc/templates/download-v2-service.yaml @@ -9,7 +9,7 @@ metadata: spec: ports: - name: download-v2 - port: {{ ternary 443 80 .Values.global.tls.enabled }} + port: 8080 targetPort: download-v2 protocol: TCP selector: From ef8e183a4f8447738e9dfdf34157ca736f13dffb Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 14 Apr 2026 10:46:06 +0200 Subject: [PATCH 513/530] fix(chart): use standard HTTP port on v2 download service Address review feedback from @jbygdell: - Service exposes standard port 80 mapping to targetPort 8080 so clients do not need to specify a non-standard port - Ingress backend port updated to match service port 80 - Smoke test uses --set downloadV2.replicaCount=1 instead of a separate kubectl scale call, and drops the :80 suffix from the test service URL --- charts/sda-svc/templates/download-v2-ingress.yaml | 2 +- charts/sda-svc/templates/download-v2-service.yaml | 2 +- dev-tools/chart-smoke-test/test-download-v2.sh | 7 +++---- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/charts/sda-svc/templates/download-v2-ingress.yaml b/charts/sda-svc/templates/download-v2-ingress.yaml index 3df74b6904..e075f5dac5 100644 --- a/charts/sda-svc/templates/download-v2-ingress.yaml +++ b/charts/sda-svc/templates/download-v2-ingress.yaml @@ -38,7 +38,7 @@ spec: service: name: {{ template "sda.fullname" . }}-download-v2 port: - number: 8080 + number: 80 {{- if .Values.global.tls.enabled }} tls: - hosts: diff --git a/charts/sda-svc/templates/download-v2-service.yaml b/charts/sda-svc/templates/download-v2-service.yaml index 990e11783d..26ec460674 100644 --- a/charts/sda-svc/templates/download-v2-service.yaml +++ b/charts/sda-svc/templates/download-v2-service.yaml @@ -9,7 +9,7 @@ metadata: spec: ports: - name: download-v2 - port: 8080 + port: 80 targetPort: download-v2 protocol: TCP selector: diff --git a/dev-tools/chart-smoke-test/test-download-v2.sh b/dev-tools/chart-smoke-test/test-download-v2.sh index cecabbb105..b0e9782133 100755 --- a/dev-tools/chart-smoke-test/test-download-v2.sh +++ b/dev-tools/chart-smoke-test/test-download-v2.sh @@ -185,6 +185,8 @@ HELM_ARGS=( --set global.downloadV2.service.orgName=TestOrg --set global.downloadV2.service.orgURL=http://test.org --set global.ingress.hostName.downloadV2=dl-v2.local + # Single-node k3d can't satisfy topology spread for the default 2 replicas + --set downloadV2.replicaCount=1 ) # Render only v2 templates and apply directly @@ -193,15 +195,12 @@ for tmpl in download-v2-secrets download-v2-deploy download-v2-service; do --show-only "templates/${tmpl}.yaml" done | kubectl apply -f - -# Scale to 1 replica (single-node cluster can't satisfy topology spread for 2) -kubectl scale deploy pipeline-sda-svc-download-v2 --replicas=1 - wait_for_pod "app=pipeline-sda-svc-download-v2" 60 # -- 6. smoke tests -- echo "=== Step 5: smoke tests ===" -SVC="pipeline-sda-svc-download-v2:80" +SVC="pipeline-sda-svc-download-v2" run_curl() { local name="$1"; shift From 1850c4d673fc307330273adcca813f4a4a4f6b6c Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 15 Apr 2026 15:42:28 +0200 Subject: [PATCH 514/530] fix(chart): mount JWT public keys via global.oidc.jwtSecret Apply review feedback from @jbygdell to reuse the OIDC definition for JWT key material instead of a separate downloadV2.jwt.pubkeyPath knob: - Add global.oidc.jwtSecret as the canonical reference to a k8s secret containing JWT public keys - When set, the v2 download deployment mounts the secret at jwtPath and the config sets jwt.pubkey-path to that mount - Remove the unused global.downloadV2.jwt.pubkeyPath value (it was rendered into config but no volume was mounted, so it could never resolve to a real file) - Keep global.downloadV2.jwt.pubkeyURL as the JWKS-endpoint alternative Also fixes the related Copilot reviewer finding about pubkeyPath pointing at a non-existent file. --- charts/sda-svc/templates/download-v2-deploy.yaml | 10 ++++++++++ charts/sda-svc/templates/download-v2-secrets.yaml | 4 ++-- charts/sda-svc/values.yaml | 6 +++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/charts/sda-svc/templates/download-v2-deploy.yaml b/charts/sda-svc/templates/download-v2-deploy.yaml index 9fa5c940c3..dd5ff24cac 100644 --- a/charts/sda-svc/templates/download-v2-deploy.yaml +++ b/charts/sda-svc/templates/download-v2-deploy.yaml @@ -101,6 +101,10 @@ spec: - name: iss mountPath: {{ template "trustedIssPath" . }} {{- end }} + {{- if .Values.global.oidc.jwtSecret }} + - name: jwt + mountPath: {{ template "jwtPath" . }} + {{- end }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls @@ -135,6 +139,12 @@ spec: defaultMode: 0440 secretName: {{ template "sda.fullname" . }}-download-v2-iss {{- end }} + {{- if .Values.global.oidc.jwtSecret }} + - name: jwt + secret: + defaultMode: 0440 + secretName: {{ .Values.global.oidc.jwtSecret }} + {{- end }} {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml index 8fc764afda..4ef8b3a673 100644 --- a/charts/sda-svc/templates/download-v2-secrets.yaml +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -38,8 +38,8 @@ stringData: org-name: {{ .Values.global.downloadV2.service.orgName }} org-url: {{ .Values.global.downloadV2.service.orgURL }} jwt: - {{- if .Values.global.downloadV2.jwt.pubkeyPath }} - pubkey-path: {{ template "jwtPath" . }}/{{ .Values.global.downloadV2.jwt.pubkeyPath }} + {{- if .Values.global.oidc.jwtSecret }} + pubkey-path: {{ template "jwtPath" . }} {{- end }} pubkey-url: {{ .Values.global.downloadV2.jwt.pubkeyURL }} allow-all-data: {{ .Values.global.downloadV2.jwt.allowAllData }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 64b2530761..f37e6ae189 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -275,9 +275,7 @@ global: environment: "" jwt: - # @param global.downloadV2.jwt.pubkeyPath path to directory containing JWT public key PEM files - pubkeyPath: "" - # @param global.downloadV2.jwt.pubkeyURL URL to JWKS endpoint for JWT verification + # @param global.downloadV2.jwt.pubkeyURL URL to JWKS endpoint for JWT verification (alternative to mounting keys via global.oidc.jwtSecret) pubkeyURL: "" # @param global.downloadV2.jwt.allowAllData allow all authenticated users access to all datasets (testing only) allowAllData: false @@ -359,6 +357,8 @@ global: jwkPath: "/jwk" id: "" secret: "" + # @param global.oidc.jwtSecret name of k8s secret containing JWT public keys for token verification (optional) + jwtSecret: "" s3Inbox: url: "" From 03636f048bdf3ec7b4197ee843a45fc6e2dd872f Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 15 Apr 2026 16:05:26 +0200 Subject: [PATCH 515/530] fix(chart): address Copilot review feedback on v2 download MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Probe scheme: hardcode to HTTP since the v2 API binary serves plain HTTP on 8080 (TLS terminates at the ingress). Previously the probes used HTTPS when global.tls.enabled=true, which would fail every default deployment. - NetworkPolicy: match the grpc.port logic from the secrets template so egress to reencrypt is allowed on 50443 when TLS is enabled (was hardcoded to global.reencrypt.port, blocking TLS deploys). - Secrets: add required guards for global.archive.s3Url and the service.org-name/org-url values to fail fast at helm install/upgrade rather than at pod startup. - Secrets: render s3 endpoint with the legacy pattern (append :port only when s3Port is set) so URLs already containing the port don't get a trailing :0. - Secrets: fail rendering when visa.enabled=true but trustedIssuers is empty — would otherwise produce a config pointing at a file that the chart never creates. - Smoke test: fix stale dev-tools/k3d/ paths in the header usage block after the rename to dev-tools/chart-smoke-test/. --- charts/sda-svc/templates/download-v2-deploy.yaml | 4 ++-- charts/sda-svc/templates/download-v2-networkpolicy.yaml | 2 +- charts/sda-svc/templates/download-v2-secrets.yaml | 9 ++++++--- dev-tools/chart-smoke-test/test-download-v2.sh | 6 +++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/charts/sda-svc/templates/download-v2-deploy.yaml b/charts/sda-svc/templates/download-v2-deploy.yaml index dd5ff24cac..23a9e0e279 100644 --- a/charts/sda-svc/templates/download-v2-deploy.yaml +++ b/charts/sda-svc/templates/download-v2-deploy.yaml @@ -80,7 +80,7 @@ spec: httpGet: path: /health/live port: download-v2 - scheme: {{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }} + scheme: HTTP initialDelaySeconds: {{ .Values.downloadV2.livenessProbe.initialDelaySeconds | default 10 }} periodSeconds: {{ .Values.downloadV2.livenessProbe.periodSeconds | default 30 }} timeoutSeconds: {{ .Values.downloadV2.livenessProbe.timeoutSeconds | default 5 }} @@ -88,7 +88,7 @@ spec: httpGet: path: /health/ready port: download-v2 - scheme: {{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }} + scheme: HTTP periodSeconds: {{ .Values.downloadV2.readinessProbe.periodSeconds | default 10 }} timeoutSeconds: {{ .Values.downloadV2.readinessProbe.timeoutSeconds | default 9 }} resources: diff --git a/charts/sda-svc/templates/download-v2-networkpolicy.yaml b/charts/sda-svc/templates/download-v2-networkpolicy.yaml index 546604a8f5..142697a60a 100644 --- a/charts/sda-svc/templates/download-v2-networkpolicy.yaml +++ b/charts/sda-svc/templates/download-v2-networkpolicy.yaml @@ -38,7 +38,7 @@ spec: role: reencrypt ports: - protocol: TCP - port: {{ .Values.global.reencrypt.port | int }} + port: {{ ternary 50443 (.Values.global.reencrypt.port | default 50051 | int) .Values.global.tls.enabled }} {{- if .Values.global.networkPolicy.ingressNamespace }} ingress: - from: diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml index 4ef8b3a673..251aa52bd6 100644 --- a/charts/sda-svc/templates/download-v2-secrets.yaml +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -35,8 +35,8 @@ stringData: level: {{ .Values.global.log.level }} service: id: {{ .Values.global.downloadV2.service.id }} - org-name: {{ .Values.global.downloadV2.service.orgName }} - org-url: {{ .Values.global.downloadV2.service.orgURL }} + org-name: {{ required "Download v2 service org-name is required" .Values.global.downloadV2.service.orgName }} + org-url: {{ required "Download v2 service org-url is required" .Values.global.downloadV2.service.orgURL }} jwt: {{- if .Values.global.oidc.jwtSecret }} pubkey-path: {{ template "jwtPath" . }} @@ -57,6 +57,9 @@ stringData: auth: allow-opaque: {{ .Values.global.downloadV2.auth.allowOpaque }} {{- if .Values.global.downloadV2.visa.enabled }} + {{- if not .Values.global.downloadV2.visa.trustedIssuers }} + {{- fail "global.downloadV2.visa.trustedIssuers must be set when visa.enabled is true" }} + {{- end }} visa: enabled: true source: {{ .Values.global.downloadV2.visa.source }} @@ -93,7 +96,7 @@ stringData: {{- if eq "s3" .Values.global.archive.storageType }} {{- /* TODO: support multiple archive endpoints (v2 storage format is an array) */}} s3: - - endpoint: {{ .Values.global.archive.s3Url }}:{{ .Values.global.archive.s3Port }} + - endpoint: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}{{- if .Values.global.archive.s3Port }}:{{ .Values.global.archive.s3Port }}{{- end }} access_key: {{ .Values.global.archive.s3AccessKey }} secret_key: {{ .Values.global.archive.s3SecretKey }} bucket_prefix: {{ .Values.global.archive.s3BucketPrefix }} diff --git a/dev-tools/chart-smoke-test/test-download-v2.sh b/dev-tools/chart-smoke-test/test-download-v2.sh index b0e9782133..dc0fc4fb63 100755 --- a/dev-tools/chart-smoke-test/test-download-v2.sh +++ b/dev-tools/chart-smoke-test/test-download-v2.sh @@ -4,9 +4,9 @@ # Prerequisites: docker, k3d, kubectl, helm # # Usage: -# ./dev-tools/k3d/test-download-v2.sh # full run (build + deploy + test) -# ./dev-tools/k3d/test-download-v2.sh --no-build # skip image build (reuse existing) -# ./dev-tools/k3d/test-download-v2.sh --cleanup # tear down cluster and exit +# ./dev-tools/chart-smoke-test/test-download-v2.sh # full run (build + deploy + test) +# ./dev-tools/chart-smoke-test/test-download-v2.sh --no-build # skip image build (reuse existing) +# ./dev-tools/chart-smoke-test/test-download-v2.sh --cleanup # tear down cluster and exit set -euo pipefail CLUSTER_NAME="sda-download-v2-test" From 49674f88a385c01ec62124be06f4f588c9b41e63 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 16 Apr 2026 14:52:02 +0200 Subject: [PATCH 516/530] refactor(chart): use toYaml for v2 download probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Match the pattern used by the other services in the chart (api, auth, download, etc.) — render the liveness/readiness probe from values via toYaml instead of hand-assembling fields in the template. This fixes a drift between values and template: the httpGet.path, port, and scheme fields in values weren't actually wired into the rendered manifest. Now values.yaml is the single source of truth for the probe definitions. Scheme stays HTTP since the v2 API binary always serves plain HTTP on 8080 (TLS terminates at the ingress). Addresses review feedback from @jbygdell. --- charts/sda-svc/templates/download-v2-deploy.yaml | 15 ++------------- charts/sda-svc/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/charts/sda-svc/templates/download-v2-deploy.yaml b/charts/sda-svc/templates/download-v2-deploy.yaml index 23a9e0e279..750707038e 100644 --- a/charts/sda-svc/templates/download-v2-deploy.yaml +++ b/charts/sda-svc/templates/download-v2-deploy.yaml @@ -77,20 +77,9 @@ spec: containerPort: 8080 protocol: TCP livenessProbe: - httpGet: - path: /health/live - port: download-v2 - scheme: HTTP - initialDelaySeconds: {{ .Values.downloadV2.livenessProbe.initialDelaySeconds | default 10 }} - periodSeconds: {{ .Values.downloadV2.livenessProbe.periodSeconds | default 30 }} - timeoutSeconds: {{ .Values.downloadV2.livenessProbe.timeoutSeconds | default 5 }} +{{ toYaml .Values.downloadV2.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - path: /health/ready - port: download-v2 - scheme: HTTP - periodSeconds: {{ .Values.downloadV2.readinessProbe.periodSeconds | default 10 }} - timeoutSeconds: {{ .Values.downloadV2.readinessProbe.timeoutSeconds | default 9 }} +{{ toYaml .Values.downloadV2.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.downloadV2.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index f37e6ae189..6f0cdc84ae 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -638,7 +638,7 @@ downloadV2: httpGet: path: /health/live port: download-v2 - # scheme set in deploy template via ternary on global.tls.enabled + scheme: HTTP initialDelaySeconds: 10 periodSeconds: 30 timeoutSeconds: 5 @@ -646,7 +646,7 @@ downloadV2: httpGet: path: /health/ready port: download-v2 - # scheme set in deploy template via ternary on global.tls.enabled + scheme: HTTP periodSeconds: 10 timeoutSeconds: 9 resources: From 16b42cc61c01b4b84fe73368d68fc82bd93fbc4b Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 16 Apr 2026 15:28:46 +0200 Subject: [PATCH 517/530] fix(chart): add required guards on v2 archive S3 credentials Match the legacy download-secrets hardening pattern: require access_key, secret_key, and bucket_prefix in addition to the endpoint URL, so that an incomplete S3 config fails fast at helm install/upgrade rather than at pod startup. Addresses partial hardening noted in Codex final review. --- charts/sda-svc/templates/download-v2-secrets.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml index 251aa52bd6..1727945911 100644 --- a/charts/sda-svc/templates/download-v2-secrets.yaml +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -97,9 +97,9 @@ stringData: {{- /* TODO: support multiple archive endpoints (v2 storage format is an array) */}} s3: - endpoint: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}{{- if .Values.global.archive.s3Port }}:{{ .Values.global.archive.s3Port }}{{- end }} - access_key: {{ .Values.global.archive.s3AccessKey }} - secret_key: {{ .Values.global.archive.s3SecretKey }} - bucket_prefix: {{ .Values.global.archive.s3BucketPrefix }} + access_key: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + secret_key: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + bucket_prefix: {{ required "S3 archive bucket not set" .Values.global.archive.s3BucketPrefix }} region: {{ .Values.global.archive.s3Region }} {{- if .Values.global.tls.enabled }} ca_cert: {{ template "tlsPath" . }}/ca.crt From 984c995142bf3b89c6c297b09b405bea8eb840d4 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 16 Apr 2026 16:07:04 +0200 Subject: [PATCH 518/530] fix(chart): require hmac-secret when v2 download has >1 replica MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The v2 binary warns (and production-mode forbids) running multiple replicas without a shared pagination.hmac-secret — page tokens are signed with a per-pod random key otherwise, so pagination silently breaks when requests hit different pods. Fail the chart render when downloadV2.replicaCount > 1 but the secret is empty, matching the binary's own production guard. Single-replica deployments still work with an empty secret. Addresses Codex final review concern about unsafe multi-replica defaults. --- charts/sda-svc/templates/download-v2-secrets.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml index 1727945911..d6e8aadfde 100644 --- a/charts/sda-svc/templates/download-v2-secrets.yaml +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -1,5 +1,8 @@ {{- if or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType) ) (not .Values.global.deploymentType) }} {{- if and .Values.global.downloadV2.enabled (not .Values.global.vaultSecrets) }} +{{- if and (gt (int .Values.downloadV2.replicaCount) 1) (not .Values.global.downloadV2.pagination.hmacSecret) }} +{{- fail "global.downloadV2.pagination.hmacSecret must be set when downloadV2.replicaCount > 1 — page tokens won't work across replicas without it" }} +{{- end }} apiVersion: v1 kind: Secret metadata: From 830af64e40039a4319e4410e4630d5fbcfa620de Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Thu, 16 Apr 2026 16:40:22 +0200 Subject: [PATCH 519/530] test(dev-tools): expand v2 smoke test with template render matrix Adds a render_matrix step that runs helm template across all 16 combinations of TLS on/off, S3/POSIX storage, ingress on/off, and networkPolicy on/off. Runs before any cluster work so regressions in ingress, certificate, or networkpolicy templates are caught without needing a full k3d deploy. The deploy + HTTP smoke tests still cover the one slice that k3d can realistically run (no cert-manager, no ingress controller). Addresses the Codex review nit about only covering a single matrix slice. --- .../chart-smoke-test/test-download-v2.sh | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/dev-tools/chart-smoke-test/test-download-v2.sh b/dev-tools/chart-smoke-test/test-download-v2.sh index dc0fc4fb63..fcb73ee0a7 100755 --- a/dev-tools/chart-smoke-test/test-download-v2.sh +++ b/dev-tools/chart-smoke-test/test-download-v2.sh @@ -50,6 +50,93 @@ wait_for_pod() { fi } +# Render the v2 templates across config combinations (TLS, storage, ingress, +# networkPolicy) without deploying — catches regressions in templates that the +# single-slice deploy test doesn't exercise (ingress, certificate, netpol). +render_matrix() { + echo "=== Template render matrix ===" + local matrix_pass=0 matrix_fail=0 + local failed=() + + local base=( + --set global.deploymentType=external + --set global.download.enabled=false + --set global.doa.enabled=false + --set global.db.host=db --set global.db.user=u --set global.db.password=p + --set global.broker.host=mq --set global.broker.username=u --set global.broker.password=p + --set global.archive.s3Url=http://s3 + --set global.archive.s3AccessKey=a --set global.archive.s3SecretKey=s + --set global.reencrypt.host=r + --set global.api.jwtSecret=s --set global.api.rbacFileSecret=r + --set global.oidc.id=id --set global.oidc.secret=s --set global.oidc.provider=http://oidc + --set global.c4gh.secretName=c --set global.c4gh.publicKey=p + --set "global.c4gh.privateKeys[0].keyName=k" + --set "global.c4gh.privateKeys[0].passphrase=p" + --set "global.c4gh.privateKeys[0].keyData=d" + --set global.ingress.hostName.api=api.t + --set global.ingress.hostName.auth=auth.t + --set global.ingress.hostName.download=download.t + --set global.ingress.hostName.syncapi=syncapi.t + --set global.ingress.hostName.downloadV2=dl-v2.t + --set global.downloadV2.enabled=true + --set global.downloadV2.service.orgName=TestOrg + --set global.downloadV2.service.orgURL=http://t + --set downloadV2.replicaCount=1 + ) + + for tls in false true; do + for storage in s3 posix; do + for ingress in false true; do + for netpol in false true; do + local desc="tls=$tls storage=$storage ingress=$ingress netpol=$netpol" + local args=("${base[@]}" + --set "global.tls.enabled=$tls" + --set "global.archive.storageType=$storage" + --set "global.ingress.deploy=$ingress" + --set "global.networkPolicy.create=$netpol" + ) + if [ "$tls" = "true" ]; then + args+=(--set global.tls.issuer=test-issuer) + fi + if [ "$storage" = "posix" ]; then + args+=(--set global.archive.existingClaim=archive-pvc) + fi + if helm template test charts/sda-svc "${args[@]}" >/dev/null 2>&1; then + echo " PASS $desc" + matrix_pass=$((matrix_pass + 1)) + else + echo " FAIL $desc" + matrix_fail=$((matrix_fail + 1)) + failed+=("$desc") + fi + done + done + done + done + + echo " Matrix: $matrix_pass passed, $matrix_fail failed" + if [ "$matrix_fail" -gt 0 ]; then + echo " Failing combinations:" + for c in "${failed[@]}"; do + echo " - $c" + helm template test charts/sda-svc "${base[@]}" \ + --set "global.tls.enabled=$(echo "$c" | sed -n 's/.*tls=\([^ ]*\).*/\1/p')" \ + --set "global.archive.storageType=$(echo "$c" | sed -n 's/.*storage=\([^ ]*\).*/\1/p')" \ + --set "global.ingress.deploy=$(echo "$c" | sed -n 's/.*ingress=\([^ ]*\).*/\1/p')" \ + --set "global.networkPolicy.create=$(echo "$c" | sed -n 's/.*netpol=\([^ ]*\).*/\1/p')" \ + 2>&1 | tail -2 | sed 's/^/ /' + done + return 1 + fi + return 0 +} + +# -- 0. render matrix (fast fail) -- +if ! render_matrix; then + echo "Template matrix failed — aborting before any cluster work" + exit 1 +fi + # -- 1. cluster -- echo "=== Step 1: k3d cluster ===" if k3d cluster list 2>/dev/null | grep -q "$CLUSTER_NAME"; then From 42a98392e9e065aefe0e9c04e4133f64b2289b3e Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Wed, 22 Apr 2026 11:11:55 +0200 Subject: [PATCH 520/530] test(dev-tools): add config parse checks to v2 smoke test Extract config.yaml via yq (.service.org-name, .api.port, storage endpoint, .db.host) and iss.json via base64+jq, so the smoke test catches template regressions that HTTP-level checks would miss. Also switch wait_for_pod to rollout status (avoids race where kubectl wait matched the old pod before it terminated) and bump the wait to 120s to accommodate the ~17s cold-start plus probe delay. --- .../chart-smoke-test/test-download-v2.sh | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/dev-tools/chart-smoke-test/test-download-v2.sh b/dev-tools/chart-smoke-test/test-download-v2.sh index fcb73ee0a7..44b0c1e771 100755 --- a/dev-tools/chart-smoke-test/test-download-v2.sh +++ b/dev-tools/chart-smoke-test/test-download-v2.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # Local k3d smoke test for the v2 download service chart. # -# Prerequisites: docker, k3d, kubectl, helm +# Prerequisites: docker, k3d, kubectl, helm, yq (mikefarah), jq # # Usage: # ./dev-tools/chart-smoke-test/test-download-v2.sh # full run (build + deploy + test) @@ -50,6 +50,16 @@ wait_for_pod() { fi } +wait_for_deploy() { + local name="$1" timeout="$2" + echo "Waiting for deployment $name to be Available (${timeout}s)..." + if ! kubectl rollout status "deploy/$name" --timeout="${timeout}s"; then + echo "Deployment $name not Available after ${timeout}s" + kubectl logs "deploy/$name" --tail=30 2>/dev/null + return 1 + fi +} + # Render the v2 templates across config combinations (TLS, storage, ingress, # networkPolicy) without deploying — catches regressions in templates that the # single-slice deploy test doesn't exercise (ingress, certificate, netpol). @@ -271,6 +281,10 @@ HELM_ARGS=( --set global.downloadV2.enabled=true --set global.downloadV2.service.orgName=TestOrg --set global.downloadV2.service.orgURL=http://test.org + # trustedIssuers set to exercise the iss.json secret (visa.enabled stays false + # so the pod doesn't need a reachable OIDC endpoint). + --set "global.downloadV2.visa.trustedIssuers[0].iss=https://example.com/oidc" + --set "global.downloadV2.visa.trustedIssuers[0].jku=https://example.com/jwks" --set global.ingress.hostName.downloadV2=dl-v2.local # Single-node k3d can't satisfy topology spread for the default 2 replicas --set downloadV2.replicaCount=1 @@ -282,10 +296,37 @@ for tmpl in download-v2-secrets download-v2-deploy download-v2-service; do --show-only "templates/${tmpl}.yaml" done | kubectl apply -f - -wait_for_pod "app=pipeline-sda-svc-download-v2" 60 +wait_for_deploy "pipeline-sda-svc-download-v2" 120 + +# -- 6a. secret content tests -- +echo "=== Step 5a: secret content tests ===" + +SECRET="pipeline-sda-svc-download-v2" +ISS_SECRET="pipeline-sda-svc-download-v2-iss" + +CONFIG_YAML=$(kubectl get secret "$SECRET" -o jsonpath='{.data.config\.yaml}' | base64 -d) + +check "config.yaml .service.org-name" "TestOrg" \ + "$(echo "$CONFIG_YAML" | yq '.service.org-name')" +check "config.yaml .service.org-url" "http://test.org" \ + "$(echo "$CONFIG_YAML" | yq '.service.org-url')" +check "config.yaml .api.port" "8080" \ + "$(echo "$CONFIG_YAML" | yq '.api.port')" +check "config.yaml .storage.archive.s3[0].endpoint" "http://minio:443" \ + "$(echo "$CONFIG_YAML" | yq '.storage.archive.s3[0].endpoint')" +check "config.yaml .db.host" "postgres-sda-db" \ + "$(echo "$CONFIG_YAML" | yq '.db.host')" + +ISS_JSON=$(kubectl get secret "$ISS_SECRET" -o jsonpath='{.data.iss\.json}' | base64 -d) +check "iss.json is valid JSON" "parsed" \ + "$(echo "$ISS_JSON" | jq -e . >/dev/null 2>&1 && echo parsed || echo invalid)" +check "iss.json [0].iss" "https://example.com/oidc" \ + "$(echo "$ISS_JSON" | jq -r '.[0].iss')" +check "iss.json [0].jku" "https://example.com/jwks" \ + "$(echo "$ISS_JSON" | jq -r '.[0].jku')" -# -- 6. smoke tests -- -echo "=== Step 5: smoke tests ===" +# -- 6b. HTTP smoke tests -- +echo "=== Step 5b: HTTP smoke tests ===" SVC="pipeline-sda-svc-download-v2" From 64d366e78be836035c0fa275ca5b3416ee696a14 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Fri, 24 Apr 2026 17:57:00 +0200 Subject: [PATCH 521/530] refactor(chart): use CONFIG_FILE env var for v2 download Match the legacy download deploy and other services which pass the config path via env var rather than --config-file flag. The v2 binary accepts both (viper.AutomaticEnv with a '-' to '_' replacer), so this is a consistency change, not a behavior change. --- charts/sda-svc/templates/download-v2-deploy.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/download-v2-deploy.yaml b/charts/sda-svc/templates/download-v2-deploy.yaml index 750707038e..9e9f6b869b 100644 --- a/charts/sda-svc/templates/download-v2-deploy.yaml +++ b/charts/sda-svc/templates/download-v2-deploy.yaml @@ -71,7 +71,9 @@ spec: seccompProfile: type: "RuntimeDefault" command: ["sda-download"] - args: ["--config-file", "{{ include "confFile" . }}"] + env: + - name: CONFIG_FILE + value: {{ include "confFile" . }} ports: - name: download-v2 containerPort: 8080 From 292d53aed855f5693a0e9033a2bc83f7f8235c1b Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 5 May 2026 09:52:22 +0200 Subject: [PATCH 522/530] refactor(chart): promote downloadV2 environment flag to global scope Moves the production-guard toggle from global.downloadV2.app.environment to global.environment so other services can adopt the same flag without each carrying its own per-service 'app' sub-block. The v2 binary still reads it as app.environment in the rendered config; only the values.yaml shape changes. Per Karl's review feedback on PR #2384. --- charts/sda-svc/templates/download-v2-secrets.yaml | 2 +- charts/sda-svc/values.yaml | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/charts/sda-svc/templates/download-v2-secrets.yaml b/charts/sda-svc/templates/download-v2-secrets.yaml index d6e8aadfde..85087ccbb0 100644 --- a/charts/sda-svc/templates/download-v2-secrets.yaml +++ b/charts/sda-svc/templates/download-v2-secrets.yaml @@ -91,7 +91,7 @@ stringData: pagination: hmac-secret: {{ .Values.global.downloadV2.pagination.hmacSecret }} app: - environment: {{ .Values.global.downloadV2.app.environment }} + environment: {{ .Values.global.environment }} audit: required: {{ .Values.global.downloadV2.audit.required }} storage: diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 6f0cdc84ae..51d1130ca5 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -30,6 +30,12 @@ global: # options are `federated` or `isolated` schemaType: "federated" +# Deployment environment marker. Today only the v2 download service reads it +# (`app.environment: production` enables production-only safety guards), but +# it lives at global scope so other services can adopt the same toggle later. +# @param global.environment deployment environment ("production" enables prod guards) + environment: "" + ingress: deploy: true ingressClassName: "" @@ -270,10 +276,6 @@ global: # @param global.downloadV2.service.orgURL organization URL for GA4GH service-info orgURL: "" - app: - # @param global.downloadV2.app.environment set to "production" to enable production guards - environment: "" - jwt: # @param global.downloadV2.jwt.pubkeyURL URL to JWKS endpoint for JWT verification (alternative to mounting keys via global.oidc.jwtSecret) pubkeyURL: "" From 801db5643ece5a93f6d2e6f0bdb354ff6b079343 Mon Sep 17 00:00:00 2001 From: Jonas Hagberg Date: Tue, 5 May 2026 12:13:16 +0200 Subject: [PATCH 523/530] chore(chart): bump sda-svc chart version to 3.4.0 Minor bump for the new opt-in v2 download service. Main already shipped 3.3.0 (the previous bump in this branch was dropped during rebase since main caught up), so the v2 feature lands as a minor on top of that. --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 47ecf16b3b..557c8407e8 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.3.0 +version: 3.4.0 appVersion: v3.1.37 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From 12d534ff93c10f9eedbaf6a69dd33876c7cf6adb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Feb 2026 20:44:48 +0000 Subject: [PATCH 524/530] build(deps): bump rabbitmq in /rabbitmq Bumps rabbitmq from 3.12.13-management-alpine to 4.2.4-management-alpine. --- updated-dependencies: - dependency-name: rabbitmq dependency-version: 4.2.4-management-alpine dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- rabbitmq/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rabbitmq/Dockerfile b/rabbitmq/Dockerfile index 9077fe73ed..32cec7b572 100644 --- a/rabbitmq/Dockerfile +++ b/rabbitmq/Dockerfile @@ -1,4 +1,4 @@ -FROM rabbitmq:3.12.13-management-alpine +FROM rabbitmq:4.2.4-management-alpine ARG BUILD_DATE ARG SOURCE_COMMIT From e1000c0685efc9a762ea5a1d4b10fe181099b10d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 31 Mar 2026 14:54:20 +0200 Subject: [PATCH 525/530] Change ssl to tls in command arguments --- .github/integration/scripts/charts/deploy_charts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index ec52e2e332..95813a4714 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -61,7 +61,7 @@ if [ "$1" == "sda-mq" ]; then if [ "$4" == "federated" ]; then if [ "$3" = true ] ; then - kubectl exec broker-sda-mq-0 -- rabbitmqadmin --username=admin --port=15671 --ssl --ssl-cert-file=/etc/rabbitmq/tls/tls.crt --ssl-key-file=/etc/rabbitmq/tls/tls.key --ssl-ca-cert-file=/etc/rabbitmq/tls/ca.crt --ssl-disable-hostname-verification --password="$ADMINPASS" declare queue --vhost=sda name=from_cega durable=true + kubectl exec broker-sda-mq-0 -- rabbitmqadmin --username=admin --port=15671 --use-tls --tls-cert-file=/etc/rabbitmq/tls/tls.crt --tls-key-file=/etc/rabbitmq/tls/tls.key --tls-ca-cert-file=/etc/rabbitmq/tls/ca.crt --password="$ADMINPASS" declare queue --vhost=sda name=from_cega durable=true else kubectl exec broker-sda-mq-0 -- rabbitmqadmin --username=admin --password="$ADMINPASS" declare queue --vhost=sda name=from_cega durable=true fi From 2e948572666a76f6f8d863459284d854950b55be Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 1 Apr 2026 07:35:29 +0200 Subject: [PATCH 526/530] Fix rabbitmqadmin command syntax --- .github/integration/scripts/charts/deploy_charts.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 95813a4714..3d15a53445 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -61,9 +61,9 @@ if [ "$1" == "sda-mq" ]; then if [ "$4" == "federated" ]; then if [ "$3" = true ] ; then - kubectl exec broker-sda-mq-0 -- rabbitmqadmin --username=admin --port=15671 --use-tls --tls-cert-file=/etc/rabbitmq/tls/tls.crt --tls-key-file=/etc/rabbitmq/tls/tls.key --tls-ca-cert-file=/etc/rabbitmq/tls/ca.crt --password="$ADMINPASS" declare queue --vhost=sda name=from_cega durable=true + kubectl exec broker-sda-mq-0 -- rabbitmqadmin --username admin --password "$ADMINPASS" --host 127.0.0.1 --port 15671 --use-tls --tls-cert-file /etc/rabbitmq/tls/tls.crt --tls-key-file /etc/rabbitmq/tls/tls.key --tls-ca-cert-file /etc/rabbitmq/tls/ca.crt declare queue --vhost sda --name from_cega --durable true else - kubectl exec broker-sda-mq-0 -- rabbitmqadmin --username=admin --password="$ADMINPASS" declare queue --vhost=sda name=from_cega durable=true + kubectl exec broker-sda-mq-0 -- rabbitmqadmin --username admin --password "$ADMINPASS" --host 127.0.0.1 declare queue --vhost sda --name from_cega --durable true fi fi fi From 7f51ba5ed35008afeb1b7052d0ed3ffba7f0cbed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Mar 2026 08:39:55 +0000 Subject: [PATCH 527/530] build(deps): bump postgres in /postgresql Bumps postgres from 15.16-alpine3.23 to 18.3-alpine3.23. --- updated-dependencies: - dependency-name: postgres dependency-version: 18.3-alpine3.23 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- postgresql/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/postgresql/Dockerfile b/postgresql/Dockerfile index ea936fe9ba..7a840c35e6 100644 --- a/postgresql/Dockerfile +++ b/postgresql/Dockerfile @@ -1,4 +1,4 @@ -FROM postgres:15.16-alpine3.23 +FROM postgres:18.3-alpine3.23 ARG BUILD_DATE ARG SOURCE_COMMIT From de4c2b4a67bd96275612a6069930b2f16e3c2c7a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 31 Mar 2026 14:11:31 +0200 Subject: [PATCH 528/530] Fix test suite --- .github/integration/postgres.yml | 17 ++++++++--------- postgresql/entrypoint.sh | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/integration/postgres.yml b/.github/integration/postgres.yml index dcb98ccfa5..116b2291e7 100644 --- a/.github/integration/postgres.yml +++ b/.github/integration/postgres.yml @@ -22,20 +22,19 @@ services: condition: service_healthy environment: - PGPASSWORD=rootpasswd - image: ubuntu:jammy + image: ubuntu:latest volumes: - ./scripts:/scripts migration_helper: container_name: helper command: - - /usr/bin/tar - - -zxf - - /pgdata.tgz - - -C - - /pgdata/ - image: ubuntu:jammy - user: 70:70 + - /bin/bash + - -c + - "/usr/bin/tar -zxf /pgdata.tgz -C /pgdata/" + - "chmod -R 70:70 /pgdata" + image: ubuntu:latest + user: 0:70 volumes: - pgmigrate:/pgdata - ./tests/postgres/pgdata.tgz:/pgdata.tgz @@ -117,7 +116,7 @@ services: condition: service_healthy environment: - PGPASSWORD=rootpasswd - image: ubuntu:jammy + image: ubuntu:latest profiles: - tests volumes: diff --git a/postgresql/entrypoint.sh b/postgresql/entrypoint.sh index 687542da6d..e771b573a9 100755 --- a/postgresql/entrypoint.sh +++ b/postgresql/entrypoint.sh @@ -7,7 +7,7 @@ if [[ "$1" == postgres* ]] && [ "$(id -u)" = '0' ]; then find /var/lib/postgresql \! -user postgres -exec chown postgres '{}' + fi - exec su-exec postgres "${BASH_SOURCE[0]}" "$@" + exec su postgres "${BASH_SOURCE[0]}" "$@" fi migrate() { From dcd97235e0c994892b3ed4c3a73fba13373333f7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 May 2026 19:21:18 +0000 Subject: [PATCH 529/530] build(deps): bump the go group across 3 directories with 9 updates Bumps the go group with 5 updates in the /sda directory: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.5` | `1.41.7` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.14` | `1.32.17` | | [github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager](https://github.com/aws/aws-sdk-go-v2) | `0.1.15` | `0.1.21` | | [github.com/rabbitmq/amqp091-go](https://github.com/rabbitmq/amqp091-go) | `1.10.0` | `1.11.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.80.0` | `1.81.0` | Bumps the go group with 5 updates in the /sda-download directory: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.41.5` | `1.41.7` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.32.14` | `1.32.17` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.99.0` | `1.101.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.80.0` | `1.81.0` | | [github.com/Masterminds/semver/v3](https://github.com/Masterminds/semver) | `3.4.0` | `3.5.0` | Bumps the go group with 2 updates in the /sda-validator/orchestrator directory: [github.com/rabbitmq/amqp091-go](https://github.com/rabbitmq/amqp091-go) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/aws/aws-sdk-go-v2` from 1.41.5 to 1.41.7 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.5...v1.41.7) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.14 to 1.32.17 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.14...config/v1.32.17) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.14 to 1.19.16 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.14...credentials/v1.19.16) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager` from 0.1.15 to 0.1.21 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/transfermanager/v0.1.15...feature/s3/transfermanager/v0.1.21) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.99.0 to 1.101.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.99.0...service/s3/v1.101.0) Updates `github.com/aws/smithy-go` from 1.24.3 to 1.25.1 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.24.3...v1.25.1) Updates `github.com/rabbitmq/amqp091-go` from 1.10.0 to 1.11.0 - [Release notes](https://github.com/rabbitmq/amqp091-go/releases) - [Changelog](https://github.com/rabbitmq/amqp091-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/rabbitmq/amqp091-go/compare/v1.10.0...v1.11.0) Updates `google.golang.org/grpc` from 1.80.0 to 1.81.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.80.0...v1.81.0) Updates `github.com/aws/aws-sdk-go-v2` from 1.41.5 to 1.41.7 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.41.5...v1.41.7) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.32.14 to 1.32.17 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.32.14...config/v1.32.17) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.19.14 to 1.19.16 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.19.14...credentials/v1.19.16) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.99.0 to 1.101.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.99.0...service/s3/v1.101.0) Updates `github.com/aws/smithy-go` from 1.24.3 to 1.25.1 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.24.3...v1.25.1) Updates `google.golang.org/grpc` from 1.80.0 to 1.81.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.80.0...v1.81.0) Updates `github.com/Masterminds/semver/v3` from 3.4.0 to 3.5.0 - [Release notes](https://github.com/Masterminds/semver/releases) - [Changelog](https://github.com/Masterminds/semver/blob/master/CHANGELOG.md) - [Commits](https://github.com/Masterminds/semver/compare/v3.4.0...v3.5.0) Updates `github.com/rabbitmq/amqp091-go` from 1.10.0 to 1.11.0 - [Release notes](https://github.com/rabbitmq/amqp091-go/releases) - [Changelog](https://github.com/rabbitmq/amqp091-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/rabbitmq/amqp091-go/compare/v1.10.0...v1.11.0) Updates `google.golang.org/grpc` from 1.80.0 to 1.81.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.80.0...v1.81.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.16 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager dependency-version: 0.1.21 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.101.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/smithy-go dependency-version: 1.25.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/rabbitmq/amqp091-go dependency-version: 1.11.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.81.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.41.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.32.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.19.16 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.101.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/aws/smithy-go dependency-version: 1.25.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.81.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/Masterminds/semver/v3 dependency-version: 3.5.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: github.com/rabbitmq/amqp091-go dependency-version: 1.11.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go - dependency-name: google.golang.org/grpc dependency-version: 1.81.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 41 ++++++------ sda-download/go.sum | 102 ++++++++++++++-------------- sda-validator/orchestrator/go.mod | 4 +- sda-validator/orchestrator/go.sum | 28 ++++---- sda/go.mod | 49 +++++++------- sda/go.sum | 106 +++++++++++++++--------------- 6 files changed, 162 insertions(+), 168 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 6936d4c9df..65f259f365 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -4,12 +4,12 @@ go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/Masterminds/semver/v3 v3.4.0 - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.32.14 - github.com/aws/aws-sdk-go-v2/credentials v1.19.14 - github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 - github.com/aws/smithy-go v1.24.3 + github.com/Masterminds/semver/v3 v3.5.0 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/aws/smithy-go v1.25.1 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.12.0 @@ -23,26 +23,25 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.50.0 golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.81.0 google.golang.org/protobuf v1.36.11 ) require ( filippo.io/edwards25519 v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect github.com/bytedance/gopkg v0.1.3 // indirect github.com/bytedance/sonic v1.15.0 // indirect github.com/bytedance/sonic/loader v0.5.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 47fef7121b..4986ded26c 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -2,46 +2,44 @@ filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo= filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc= github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= -github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= -github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= -github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= -github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= +github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= @@ -184,16 +182,16 @@ go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= @@ -217,8 +215,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw= +google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod index c3fa655b96..9a22b6b5f9 100644 --- a/sda-validator/orchestrator/go.mod +++ b/sda-validator/orchestrator/go.mod @@ -10,13 +10,13 @@ require ( github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.12.3 github.com/neicnordic/crypt4gh v1.15.0 - github.com/rabbitmq/amqp091-go v1.10.0 + github.com/rabbitmq/amqp091-go v1.11.0 github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 github.com/spf13/pflag v1.0.10 github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.81.0 ) require ( diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum index 3b5ada3182..c36fd128d3 100644 --- a/sda-validator/orchestrator/go.sum +++ b/sda-validator/orchestrator/go.sum @@ -110,8 +110,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= -github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= -github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8= +github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -155,16 +155,16 @@ go.mongodb.org/mongo-driver/v2 v2.5.0 h1:yXUhImUjjAInNcpTcAlPHiT7bIXhshCTL3jVBkF go.mongodb.org/mongo-driver/v2 v2.5.0/go.mod h1:yOI9kBsufol30iFsl1slpdq1I0eHPzybRWdyYUs8K/0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= -go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= -go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= -go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= -go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= @@ -194,8 +194,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw= +google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sda/go.mod b/sda/go.mod index add2bca829..613b329c59 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,12 +4,12 @@ go 1.25.7 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.32.14 - github.com/aws/aws-sdk-go-v2/credentials v1.19.14 - github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.15 - github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 - github.com/aws/smithy-go v1.24.3 + github.com/aws/aws-sdk-go-v2 v1.41.7 + github.com/aws/aws-sdk-go-v2/config v1.32.17 + github.com/aws/aws-sdk-go-v2/credentials v1.19.16 + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.21 + github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 + github.com/aws/smithy-go v1.25.1 github.com/c2h5oh/datasize v0.0.0-20231215233829-aa82cc1e6500 github.com/casbin/casbin/v2 v2.135.0 github.com/coreos/go-oidc/v3 v3.18.0 @@ -29,7 +29,7 @@ require ( github.com/ory/dockertest v3.3.5+incompatible github.com/ory/dockertest/v3 v3.12.0 github.com/pkg/errors v0.9.1 - github.com/rabbitmq/amqp091-go v1.10.0 + github.com/rabbitmq/amqp091-go v1.11.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.4 github.com/spf13/cobra v1.10.2 @@ -38,7 +38,7 @@ require ( github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.50.0 golang.org/x/oauth2 v0.36.0 - google.golang.org/grpc v1.80.0 + google.golang.org/grpc v1.81.0 google.golang.org/protobuf v1.36.11 ) @@ -54,20 +54,19 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250617153402-88c1d9a79b05 // indirect github.com/andybalholm/brotli v1.2.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.10.0 // indirect github.com/bytedance/gopkg v0.1.3 // indirect @@ -168,9 +167,9 @@ require ( go.mongodb.org/mongo-driver/v2 v2.5.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect - go.opentelemetry.io/otel v1.41.0 // indirect - go.opentelemetry.io/otel/metric v1.41.0 // indirect - go.opentelemetry.io/otel/trace v1.41.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.24.0 // indirect golang.org/x/exp v0.0.0-20260218203240-3dfff04db8fa // indirect diff --git a/sda/go.sum b/sda/go.sum index b9b5bdf7b7..871e39cfbe 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -27,46 +27,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.32.14 h1:opVIRo/ZbbI8OIqSOKmpFaY7IwfFUOCCXBsUpJOwDdI= -github.com/aws/aws-sdk-go-v2/config v1.32.14/go.mod h1:U4/V0uKxh0Tl5sxmCBZ3AecYny4UNlVmObYjKuuaiOo= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14 h1:n+UcGWAIZHkXzYt87uMFBv/l8THYELoX6gVcUvgl6fI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.14/go.mod h1:cJKuyWB59Mqi0jM3nFYQRmnHVQIcgoxjEMAbLkpr62w= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21 h1:NUS3K4BTDArQqNu2ih7yeDLaS3bmHD0YndtA6UP884g= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.21/go.mod h1:YWNWJQNjKigKY1RHVJCuupeWDrrHjRqHm0N9rdrWzYI= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.15 h1:92MfpwB6KjsPIEq9g3DniRPxOe92ew5hUz1h8W8cX7E= -github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.15/go.mod h1:7O129SmOn4acM++3oVfTLAeHmNOsj0y7AA7zmbgnGOk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0 h1:hlSuz394kV0vhv9drL5lhuEFbEOEP1VyQpy15qWh1Pk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.99.0/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9 h1:QKZH0S178gCmFEgst8hN0mCX1KxLgHBKKY/CLqwP8lg= -github.com/aws/aws-sdk-go-v2/service/signin v1.0.9/go.mod h1:7yuQJoT+OoH8aqIxw9vwF+8KpvLZ8AWmvmUWHsGQZvI= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15 h1:lFd1+ZSEYJZYvv9d6kXzhkZu07si3f+GQ1AaYwa2LUM= -github.com/aws/aws-sdk-go-v2/service/sso v1.30.15/go.mod h1:WSvS1NLr7JaPunCXqpJnWk1Bjo7IxzZXrZi1QQCkuqM= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19 h1:dzztQ1YmfPrxdrOiuZRMF6fuOwWlWpD2StNLTceKpys= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.19/go.mod h1:YO8TrYtFdl5w/4vmjL8zaBSsiNp3w0L1FfKVKenZT7w= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10 h1:p8ogvvLugcR/zLBXTXrTkj0RYBUdErbMnAFFp12Lm/U= -github.com/aws/aws-sdk-go-v2/service/sts v1.41.10/go.mod h1:60dv0eZJfeVXfbT1tFJinbHrDfSJ2GZl4Q//OSSNAVw= -github.com/aws/smithy-go v1.24.3 h1:XgOAaUgx+HhVBoP4v8n6HCQoTRDhoMghKqw4LNHsDNg= -github.com/aws/smithy-go v1.24.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.41.7 h1:DWpAJt66FmnnaRIOT/8ASTucrvuDPZASqhhLey6tLY8= +github.com/aws/aws-sdk-go-v2 v1.41.7/go.mod h1:4LAfZOPHNVNQEckOACQx60Y8pSRjIkNZQz1w92xpMJc= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10 h1:gx1AwW1Iyk9Z9dD9F4akX5gnN3QZwUB20GGKH/I+Rho= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.10/go.mod h1:qqY157uZoqm5OXq/amuaBJyC9hgBCBQnsaWnPe905GY= +github.com/aws/aws-sdk-go-v2/config v1.32.17 h1:FpL4/758/diKwqbytU0prpuiu60fgXKUWCpDJtApclU= +github.com/aws/aws-sdk-go-v2/config v1.32.17/go.mod h1:OXqUMzgXytfoF9JaKkhrOYsyh72t9G+MJH8mMRaexOE= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16 h1:r3RJBuU7X9ibt8RHbMjWE6y60QbKBiII6wSrXnapxSU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.16/go.mod h1:6cx7zqDENJDbBIIWX6P8s0h6hqHC8Avbjh9Dseo27ug= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23 h1:UuSfcORqNSz/ey3VPRS8TcVH2Ikf0/sC+Hdj400QI6U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.23/go.mod h1:+G/OSGiOFnSOkYloKj/9M35s74LgVAdJBSD5lsFfqKg= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.21 h1:FwkMh0ChhW75LhLSZKAD6RIM3Qyvf4vVtuX5w2fQp0g= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.1.21/go.mod h1:mu725LPEVyuzXIIkALC+OGjErZ7fuFKSWwsvvgK/eNg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23 h1:GpT/TrnBYuE5gan2cZbTtvP+JlHsutdmlV2YfEyNde0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.23/go.mod h1:xYWD6BS9ywC5bS3sz9Xh04whO/hzK2plt2Zkyrp4JuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23 h1:bpd8vxhlQi2r1hiueOw02f/duEPTMK59Q4QMAoTTtTo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.23/go.mod h1:15DfR2nw+CRHIk0tqNyifu3G1YdAOy68RftkhMDDwYk= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24 h1:OQqn11BtaYv1WLUowvcA30MpzIu8Ti4pcLPIIyoKZrA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.24/go.mod h1:X5ZJyfwVrWA96GzPmUCWFQaEARPR7gCrpq2E92PJwAE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9 h1:FLudkZLt5ci0ozzgkVo8BJGwvqNaZbTWb3UcucAateA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.9/go.mod h1:w7wZ/s9qK7c8g4al+UyoF1Sp/Z45UwMGcqIzLWVQHWk= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15 h1:ieLCO1JxUWuxTZ1cRd0GAaeX7O6cIxnwk7tc1LsQhC4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.15/go.mod h1:e3IzZvQ3kAWNykvE0Tr0RDZCMFInMvhku3qNpcIQXhM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23 h1:pbrxO/kuIwgEsOPLkaHu0O+m4fNgLU8B3vxQ+72jTPw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.23/go.mod h1:/CMNUqoj46HpS3MNRDEDIwcgEnrtZlKRaHNaHxIFpNA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23 h1:03xatSQO4+AM1lTAbnRg5OK528EUg744nW7F73U8DKw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.23/go.mod h1:M8l3mwgx5ToK7wot2sBBce/ojzgnPzZXUV445gTSyE8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0 h1:etqBTKY581iwLL/H/S2sVgk3C9lAsTJFeXWFDsDcWOU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.101.0/go.mod h1:L2dcoOgS2VSgbPLvpak2NyUPsO1TBN7M45Z4H7DlRc4= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11 h1:TdJ+HdzOBhU8+iVAOGUTU63VXopcumCOF1paFulHWZc= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.11/go.mod h1:R82ZRExE/nheo0N+T8zHPcLRTcH8MGsnR3BiVGX0TwI= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17 h1:7byT8HUWrgoRp6sXjxtZwgOKfhss5fW6SkLBtqzgRoE= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.17/go.mod h1:xNWknVi4Ezm1vg1QsB/5EWpAJURq22uqd38U8qKvOJc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21 h1:+1Kl1zx6bWi4X7cKi3VYh29h8BvsCoHQEQ6ST9X8w7w= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.21/go.mod h1:4vIRDq+CJB2xFAXZ+YgGUTiEft7oAQlhIs71xcSeuVg= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1 h1:F/M5Y9I3nwr2IEpshZgh1GeHpOItExNM9L1euNuh/fk= +github.com/aws/aws-sdk-go-v2/service/sts v1.42.1/go.mod h1:mTNxImtovCOEEuD65mKW7DCsL+2gjEH+RPEAexAzAio= +github.com/aws/smithy-go v1.25.1 h1:J8ERsGSU7d+aCmdQur5Txg6bVoYelvQJgtZehD12GkI= +github.com/aws/smithy-go v1.25.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= @@ -321,8 +319,8 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= -github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= -github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rabbitmq/amqp091-go v1.11.0 h1:HxIctVm9Gid/Vtn706necmZ7Wj6pgGI2eqplRbEY8O8= +github.com/rabbitmq/amqp091-go v1.11.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= @@ -414,16 +412,16 @@ go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= -go.opentelemetry.io/otel v1.41.0 h1:YlEwVsGAlCvczDILpUXpIpPSL/VPugt7zHThEMLce1c= -go.opentelemetry.io/otel v1.41.0/go.mod h1:Yt4UwgEKeT05QbLwbyHXEwhnjxNO6D8L5PQP51/46dE= -go.opentelemetry.io/otel/metric v1.41.0 h1:rFnDcs4gRzBcsO9tS8LCpgR0dxg4aaxWlJxCno7JlTQ= -go.opentelemetry.io/otel/metric v1.41.0/go.mod h1:xPvCwd9pU0VN8tPZYzDZV/BMj9CM9vs00GuBjeKhJps= -go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= -go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= -go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= -go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= -go.opentelemetry.io/otel/trace v1.41.0 h1:Vbk2co6bhj8L59ZJ6/xFTskY+tGAbOnCtQGVVa9TIN0= -go.opentelemetry.io/otel/trace v1.41.0/go.mod h1:U1NU4ULCoxeDKc09yCWdWe+3QoyweJcISEVa1RBzOis= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= @@ -511,8 +509,8 @@ gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171 h1:ggcbiqK8WWh6l1dnltU4BgWGIGo+EVYxCaAPih/zQXQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20260226221140-a57be14db171/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM= -google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4= +google.golang.org/grpc v1.81.0 h1:W3G9N3KQf3BU+YuCtGKJk0CmxQNbAISICD/9AORxLIw= +google.golang.org/grpc v1.81.0/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e423d1f33f4d0b58c1e965d592b9e13d7ee73782 Mon Sep 17 00:00:00 2001 From: Krzysztof Kochel Date: Fri, 8 May 2026 23:54:27 +0200 Subject: [PATCH 530/530] fix: use biobank registry --- .../scripts/charts/deploy_charts.sh | 3 +- .github/integration/scripts/charts/k3d.sh | 3 +- .github/integration/sda-benchmark.yml | 2 +- .../integration/sda-download-integration.yml | 6 +- .../sda-download-v2-integration.yml | 6 +- .github/integration/sda-s3-integration.yml | 2 +- .github/workflows/build_pr_container.yaml | 98 +++++++------------ .../workflows/pr_build_and_test_images.yml | 37 ++++--- .github/workflows/release_sda.yml | 28 +++--- .../api/middleware/middleware_test.go | 2 +- sda-download/internal/config/config.go | 2 +- sda/cmd/auth/main.go | 2 +- sda/cmd/rotatekey/rotatekey_test.go | 2 +- 13 files changed, 94 insertions(+), 99 deletions(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 3d15a53445..a814d8917d 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -95,5 +95,6 @@ if [ "$1" == "sda-svc" ]; then --set reencrypt.readinessProbe.grpc.port="$GRPC_PORT" \ --set global.c4gh.rotatePubKeyData="$ROTATE_PUB_BASE64" \ -f "$dir/values.yaml" \ - --wait + --wait \ + --timeout 10m fi diff --git a/.github/integration/scripts/charts/k3d.sh b/.github/integration/scripts/charts/k3d.sh index f93c48347a..204a1db1ea 100755 --- a/.github/integration/scripts/charts/k3d.sh +++ b/.github/integration/scripts/charts/k3d.sh @@ -6,7 +6,8 @@ k8s="$(curl --retry 100 -L -s https://dl.k8s.io/release/stable.txt)" curl --retry 100 -s -L https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | sudo bash if [ -n "$1" ]; then - k8s=$(k3d version list k3s | grep "$1" | head -n 1 | cut -d '-' -f 1) + matched=$(k3d version list k3s | grep "$1" | head -n 1 | cut -d '-' -f 1) || true + [ -n "$matched" ] && k8s="$matched" fi curl --retry 100 -sLO https://dl.k8s.io/release/"$k8s"/bin/linux/amd64/kubectl diff --git a/.github/integration/sda-benchmark.yml b/.github/integration/sda-benchmark.yml index 351b543922..0bb8447424 100644 --- a/.github/integration/sda-benchmark.yml +++ b/.github/integration/sda-benchmark.yml @@ -41,7 +41,7 @@ services: # Old download service (from sda-download module) for comparison download-old: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-download + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-download command: - sda-download - "--config-file=/config.yaml" diff --git a/.github/integration/sda-download-integration.yml b/.github/integration/sda-download-integration.yml index 718571a5f3..574cffcc44 100644 --- a/.github/integration/sda-download-integration.yml +++ b/.github/integration/sda-download-integration.yml @@ -14,7 +14,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres extra_hosts: - "localhost:host-gateway" ports: @@ -187,7 +187,7 @@ services: # Re-encrypt gRPC service reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [sda-reencrypt] container_name: reencrypt depends_on: @@ -202,7 +202,7 @@ services: # Download service under test download: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: - sda-download - "--config-file=/config.yaml" diff --git a/.github/integration/sda-download-v2-integration.yml b/.github/integration/sda-download-v2-integration.yml index 767ce01179..889e4dafc2 100644 --- a/.github/integration/sda-download-v2-integration.yml +++ b/.github/integration/sda-download-v2-integration.yml @@ -14,7 +14,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres extra_hosts: - "localhost:host-gateway" ports: @@ -187,7 +187,7 @@ services: # Re-encrypt gRPC service reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [sda-reencrypt] container_name: reencrypt depends_on: @@ -202,7 +202,7 @@ services: # Download service under test download: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: - sda-download - "--config-file=/config.yaml" diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 0b60fa722c..ab327d5aa8 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -255,7 +255,7 @@ services: # New download service (sda/cmd/download) - feature of this branch download: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: - sda-download - "--config-file=/config.yaml" diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 6aceb117a3..ac32fa309b 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -24,11 +24,8 @@ jobs: packages: write security-events: write steps: - - id: string-github-repo - name: Check String - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.repository }} + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Checkout code uses: actions/checkout@v6 @@ -46,8 +43,8 @@ jobs: context: ./sda-download push: true tags: | - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-download - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-download + ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-download + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}-download labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') @@ -59,8 +56,8 @@ jobs: context: ./sda-validator/orchestrator push: true tags: | - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-validator-orchestrator - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-validator-orchestrator + ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-validator-orchestrator + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}-validator-orchestrator labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') @@ -72,8 +69,8 @@ jobs: context: ./sda push: true tags: | - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }} - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }} + ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }} + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }} labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') @@ -87,11 +84,8 @@ jobs: packages: write security-events: write steps: - - id: string-github-repo - name: Check String - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.repository }} + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Checkout code uses: actions/checkout@v6 @@ -109,8 +103,8 @@ jobs: context: ./postgresql push: true tags: | - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-postgres - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-postgres + ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-postgres + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}-postgres labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') @@ -122,20 +116,20 @@ jobs: context: ./rabbitmq push: true tags: | - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-rabbitmq - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-rabbitmq + ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-rabbitmq + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}-rabbitmq labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@v0.36.0 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false with: - image-ref: ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-postgres + image-ref: ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-postgres format: "sarif" hide-progress: true ignore-unfixed: true @@ -148,12 +142,12 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@v0.36.0 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false with: - image-ref: ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-rabbitmq + image-ref: ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-rabbitmq format: "sarif" hide-progress: true ignore-unfixed: true @@ -173,11 +167,8 @@ jobs: packages: write security-events: write steps: - - id: string-github-repo - name: Check String - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.repository }} + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Checkout code uses: actions/checkout@v6 @@ -195,20 +186,20 @@ jobs: context: ./sda-sftp-inbox push: true tags: | - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-sftp-inbox - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-sftp-inbox + ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-sftp-inbox + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}-sftp-inbox labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.33.1 + uses: aquasecurity/trivy-action@v0.36.0 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false with: - image-ref: ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-sftp-inbox + image-ref: ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-sftp-inbox format: "sarif" hide-progress: true ignore-unfixed: true @@ -226,20 +217,17 @@ jobs: context: ./sda-doa push: true tags: | - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-doa - ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-doa + ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-doa + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}-doa labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.33.1 - env: - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true + uses: aquasecurity/trivy-action@v0.36.0 with: - image-ref: ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-doa + image-ref: ghcr.io/${{ env.REPO_LC }}:sha-${{ github.sha }}-doa format: "sarif" hide-progress: true ignore-unfixed: true @@ -253,11 +241,8 @@ jobs: name: rabbitmq-federation-test runs-on: ubuntu-latest steps: - - id: string-github-repo - name: Check String - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.repository }} + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Check out code uses: actions/checkout@v6 @@ -271,11 +256,8 @@ jobs: name: postgres-test-suite runs-on: ubuntu-latest steps: - - id: string-github-repo - name: Check String - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.repository }} + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Check out code uses: actions/checkout@v6 @@ -292,11 +274,8 @@ jobs: matrix: storage: ["posix", "s3"] steps: - - id: string-github-repo - name: Check String - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.repository }} + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Check out code uses: actions/checkout@v6 @@ -371,11 +350,8 @@ jobs: tls: "false" storage: "posix" steps: - - id: string-github-repo - name: Check String - uses: ASzc/change-string-case-action@v6 - with: - string: ${{ github.repository }} + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v6 @@ -393,7 +369,7 @@ jobs: shell: bash - name: Deploy external services - run: bash .github/integration/scripts/charts/dependencies.sh ${{matrix.deployment}} ${{matrix.storage}} + run: bash .github/integration/scripts/charts/dependencies.sh ${{matrix.deployment}} ${{matrix.storage}} ${{matrix.tls}} shell: bash - name: Deploy DB diff --git a/.github/workflows/pr_build_and_test_images.yml b/.github/workflows/pr_build_and_test_images.yml index c35e04054b..cde5b605f4 100644 --- a/.github/workflows/pr_build_and_test_images.yml +++ b/.github/workflows/pr_build_and_test_images.yml @@ -54,6 +54,9 @@ jobs: image-tag-suffix: "-sftp-inbox" steps: + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + - name: Checkout code uses: actions/checkout@v6 @@ -70,23 +73,33 @@ jobs: context: ./${{ matrix.path }} push: true tags: | - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} labels: | # As this is a PR we set the version by the git sha org.opencontainers.image.title=${{ github.repository }} org.opencontainers.image.description=Sensitive Data Archive - ${{ matrix.name }} org.opencontainers.image.authors=NeIC System Developers org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }}/tree/${{ github.sha }}$/{{ matrix.path }} - org.opencontainers.image.created=${{ github.event.pull_request.updated_at }} - org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=${{ github.event.pull_request.updated_at }} + org.opencontainers.image.revision=${{ github.sha }} org.opencontainers.image.version=${{ github.sha }} + - name: Get current date for Trivy cache key + id: date + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + + - name: Restore Trivy DB cache + uses: actions/cache/restore@v5 + with: + path: ${{ github.workspace }}/.cache/trivy + key: cache-trivy-${{ steps.date.outputs.date }} + restore-keys: | + cache-trivy- + - name: Run Trivy vulnerability scanner on ${{ matrix.name }} sarif uses: aquasecurity/trivy-action@v0.36.0 - env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true with: - image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + image-ref: ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + cache-dir: ${{ github.workspace }}/.cache/trivy format: "sarif" hide-progress: true ignore-unfixed: true @@ -96,14 +109,14 @@ jobs: # We do this step such that the report in the Github code scanning is more clear for which image a vulnerability originates from - name: Append image info to sarif file run: | - jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri |= "ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }}/" + .' \ + jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri |= "ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }}/" + .' \ results.sarif > annotated_results.sarif - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v4 with: sarif_file: annotated_results.sarif - category: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + category: ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} # Only run trivy scan with markdown output and add PR comment if PR changes the relevant file - uses: dorny/paths-filter@v4 @@ -116,11 +129,9 @@ jobs: - name: Run Trivy vulnerability scanner on ${{ matrix.name }} markdown output if: steps.changes.outputs.changed == 'true' uses: aquasecurity/trivy-action@v0.36.0 - env: # As we have the Update Trivy Cache cronjob which updates the cache daily we do not it per scan - TRIVY_SKIP_DB_UPDATE: true - TRIVY_SKIP_JAVA_DB_UPDATE: true with: - image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + image-ref: ghcr.io/${{ env.REPO_LC }}:PR${{ github.event.number }}${{ matrix.image-tag-suffix }} + cache-dir: ${{ github.workspace }}/.cache/trivy format: template template: '@.github/trivy_report_template_markdown.tpl' output: trivy-results.md diff --git a/.github/workflows/release_sda.yml b/.github/workflows/release_sda.yml index 331e08394f..27e6bd4af3 100644 --- a/.github/workflows/release_sda.yml +++ b/.github/workflows/release_sda.yml @@ -48,6 +48,9 @@ jobs: image-tag-suffix: "-sftp-inbox" steps: + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + - name: Checkout code uses: actions/checkout@v6 @@ -64,7 +67,7 @@ jobs: context: ./${{ matrix.path }} push: true tags: | - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} + ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} labels: | org.opencontainers.image.title=${{ github.repository }} org.opencontainers.image.description=Sensitive Data Archive - ${{ matrix.name }} @@ -81,7 +84,7 @@ jobs: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true with: - image-ref: ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} + image-ref: ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} format: "sarif" hide-progress: true ignore-unfixed: true @@ -91,14 +94,14 @@ jobs: # We do this step such that the report in the Github code scanning is more clear for which image a vulnerability originates from - name: Append image info to sarif file run: | - jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri |= "ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }}/" + .' \ + jq '.runs[].results[].locations[].physicalLocation.artifactLocation.uri |= "ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }}/" + .' \ results.sarif > annotated_results.sarif - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v4 with: sarif_file: annotated_results.sarif - category: ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} + category: ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}${{ matrix.image-tag-suffix }} code_ql: name: Run CodeQL @@ -155,6 +158,9 @@ jobs: name: Create SDA release runs-on: ubuntu-latest steps: + - name: Set lowercase repo name + run: echo "REPO_LC=${GITHUB_REPOSITORY,,}" >> $GITHUB_ENV + - name: Checkout uses: actions/checkout@v6 - name: Release @@ -164,10 +170,10 @@ jobs: tag_name: "${{ env.VERSION_TAG }}" body: | # Docker images: - - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }} - - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-download - - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-doa - - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-validator-orchestrator - - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-stfp-inbox - - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-postgres - - ghcr.io/${{ github.repository }}:${{ env.VERSION_TAG }}-rabbitmq \ No newline at end of file + - ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }} + - ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}-download + - ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}-doa + - ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}-validator-orchestrator + - ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}-stfp-inbox + - ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}-postgres + - ghcr.io/${{ env.REPO_LC }}:${{ env.VERSION_TAG }}-rabbitmq \ No newline at end of file diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index ffdcd35c0a..01026360ef 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -233,7 +233,7 @@ func TestTokenMiddleware_Success_FromCache(t *testing.T) { r := httptest.NewRequest("GET", "/", nil) _, router := gin.CreateTestContext(w) - r.AddCookie(&http.Cookie{ + r.AddCookie(&http.Cookie{ //nolint:gosec Name: "sda_session_key", Value: "key", }) diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 156fad1c65..56eccec275 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -349,7 +349,7 @@ func (c *Map) appConfig() error { log.Infoln("Internal c4gh key-pair loaded") } - if !slices.Contains(availableMiddlewares, c.App.Middleware) { + if !slices.Contains(availableMiddlewares, c.App.Middleware) { //nolint:govet err := fmt.Errorf("app.middleware value=%v is not one of allowed values=%v", c.App.Middleware, availableMiddlewares) return err diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index eb8fa6448c..1485746146 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -222,7 +222,7 @@ func (auth AuthHandler) getEGAConf(ctx iris.Context) { // getOIDC redirects to the oidc page defined in auth.Config func (auth AuthHandler) getOIDC(ctx iris.Context) { state := uuid.New() - ctx.SetCookie(&http.Cookie{Name: "state", Value: state.String(), Secure: true}) + ctx.SetCookie(&http.Cookie{Name: "state", Value: state.String(), Secure: true}) //nolint:gosec redirectURI := ctx.Request().URL.Query().Get("redirect_uri") if redirectURI != "" { diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go index 271f66bd0d..dc1a1906c6 100644 --- a/sda/cmd/rotatekey/rotatekey_test.go +++ b/sda/cmd/rotatekey/rotatekey_test.go @@ -96,7 +96,7 @@ func TestMain(m *testing.M) { // pulls an image, creates a container based on it and runs it rabbitmq, err := pool.RunWithOptions(&dockertest.RunOptions{ - Repository: "ghcr.io/biobanklab/sensitive-data-archive", + Repository: "ghcr.io/neicnordic/sensitive-data-archive", Tag: "v3.0.0-rabbitmq", }, func(config *docker.HostConfig) { // set AutoRemove to true so that stopped container goes away by itself