From 86602cfc25104f5280c542e56f423f7eb60965da Mon Sep 17 00:00:00 2001 From: Parisa Date: Fri, 16 May 2025 12:53:57 +0200 Subject: [PATCH 001/262] add info on how to build sda-doa --- README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/README.md b/README.md index 9a75c7f33b..07df3ee1d1 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,35 @@ To build the `sda-admin` CLI tool: $ make build-sda-admin ``` +To build the `sda-doa`: + +```sh +$ make build-sda-doa +``` + +SDA-DOA uses two JARs hosted on Github Packages. To build the Docker image successfully , you need to provide authentication credentials to access the GitHub package registry. +#### Required Setup: `settings.xml` + +Create a `settings.xml` file in the root directory (next to the Dockerfile in /sda-doa), and include the following content: + +```xml + + + + github-fega-norway + YOUR_GITHUB_USERNAME + YOUR_GITHUB_PERSONAL_ACCESS_TOKEN + + + +``` +Replace `YOUR_GITHUB_USERNAME` and `YOUR_GITHUB_PERSONAL_ACCESS_TOKEN` with your actual GitHub username and personal access token, respectively. The personal access token should have the `read:packages` scope. +```sh +$ make build-sda-doa +``` + ### Running the services #### Start services with Docker Compose From 1b880dd454a232d130d490fd11ac1c4ec437aa21 Mon Sep 17 00:00:00 2001 From: Parisa Date: Fri, 16 May 2025 12:55:31 +0200 Subject: [PATCH 002/262] add settings.xml to gitignore to avoid leaking credentials accidentally --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e3ca9a9c37..d62acf38b3 100644 --- a/.gitignore +++ b/.gitignore @@ -185,7 +185,7 @@ buildNumber.properties .mvn/timing.properties .mvn/wrapper/maven-wrapper.jar .flattened-pom.xml - +settings.xml # End of https://www.gitignore.io/api/java,maven,eclipse,intellij+all db/ From c0de14d74f11bfee26ad40b119e9cbce9a85cf26 Mon Sep 17 00:00:00 2001 From: Parisa Date: Fri, 16 May 2025 13:25:50 +0200 Subject: [PATCH 003/262] Update README.md Co-authored-by: Joakim Bygdell --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 07df3ee1d1..d5e54fa426 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,7 @@ Create a `settings.xml` file in the root directory (next to the Dockerfile in /s ``` Replace `YOUR_GITHUB_USERNAME` and `YOUR_GITHUB_PERSONAL_ACCESS_TOKEN` with your actual GitHub username and personal access token, respectively. The personal access token should have the `read:packages` scope. + ```sh $ make build-sda-doa ``` From 0f35dfa912fdbde23e96f690347d15fe01ff1d0a Mon Sep 17 00:00:00 2001 From: Parisa Date: Fri, 16 May 2025 13:26:14 +0200 Subject: [PATCH 004/262] Update README.md Co-authored-by: Joakim Bygdell --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index d5e54fa426..6b139e9f7e 100644 --- a/README.md +++ b/README.md @@ -49,10 +49,6 @@ $ make build-sda-admin To build the `sda-doa`: -```sh -$ make build-sda-doa -``` - SDA-DOA uses two JARs hosted on Github Packages. To build the Docker image successfully , you need to provide authentication credentials to access the GitHub package registry. #### Required Setup: `settings.xml` From 7e9d5e97e44a74872f00f543bd492cc34a96bf80 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 19:13:17 +0000 Subject: [PATCH 005/262] Bump codecov/codecov-action from 5.4.2 to 5.4.3 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.4.2 to 5.4.3. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.4.2...v5.4.3) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 5.4.3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c488d2a90b..e51e4db8fe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.2 + uses: codecov/codecov-action@v5.4.3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.2 + uses: codecov/codecov-action@v5.4.3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.2 + uses: codecov/codecov-action@v5.4.3 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt From 72d566e5b121eee91ff6de76c641ec6ff983591c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 19:43:31 +0000 Subject: [PATCH 006/262] Bump google.golang.org/grpc in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `google.golang.org/grpc` from 1.72.0 to 1.72.1 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.72.0...v1.72.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.72.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 73eae28d65..7aabc6d4ce 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -19,7 +19,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.38.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - google.golang.org/grpc v1.72.0 + google.golang.org/grpc v1.72.1 google.golang.org/protobuf v1.36.6 ) diff --git a/sda-download/go.sum b/sda-download/go.sum index e4383bafcb..fe997dd69d 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -233,8 +233,8 @@ golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= -google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= -google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From db45d8018271af0b105bc5b3e40142c46ec6aeb9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 19:11:29 +0000 Subject: [PATCH 007/262] Bump google.golang.org/grpc in /sda in the all-modules group Bumps the all-modules group in /sda with 1 update: [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `google.golang.org/grpc` from 1.72.0 to 1.72.1 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.72.0...v1.72.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.72.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index d41bb2cefc..c320513176 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -35,7 +35,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.38.0 golang.org/x/oauth2 v0.30.0 - google.golang.org/grpc v1.72.0 + google.golang.org/grpc v1.72.1 google.golang.org/protobuf v1.36.6 ) diff --git a/sda/go.sum b/sda/go.sum index 6933adbf9a..25c9e695bd 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -513,8 +513,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= -google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= -google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= +google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 205ac74dfa125b6d9eabab63e4eb237ea7be7aed Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 20 May 2025 09:13:22 +0000 Subject: [PATCH 008/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index e2cb47ecdf..10348d29ba 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.84 -appVersion: v0.5.14 +version: 0.8.85 +appVersion: v0.5.16 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 15080c15d3..4338581df0 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.85 -appVersion: v0.5.14 +version: 0.7.86 +appVersion: v0.5.16 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 4103b3095f..b40545bbe5 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.15 -appVersion: v0.5.14 +version: 0.30.16 +appVersion: v0.5.16 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 80d4277a80181e0cbb741ccae63f88db41bfd523 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 08:35:20 +0200 Subject: [PATCH 009/262] [ingest] ensure C4GH keylist is not empty on startup. --- sda/cmd/ingest/ingest.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 6c66889fb6..c864df6083 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -8,6 +8,7 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" @@ -75,10 +76,9 @@ func main() { panic(err) } app.ArchiveKeyList, err = config.GetC4GHprivateKeys() - if err != nil { - log.Error(err) + if err != nil || len(app.ArchiveKeyList) == 0 { sigc <- syscall.SIGINT - panic(err) + panic(errors.New("no C4GH private keys configured")) } app.Archive, err = storage.NewBackend(app.Conf.Archive) if err != nil { From 01851d96ed64333483fb83e6c98499dd3f733fbc Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 08:44:57 +0200 Subject: [PATCH 010/262] [ingest-test] check for empty C4GH key list --- sda/cmd/ingest/ingest_test.go | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index bd647a2e93..c2e391e408 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -169,13 +169,18 @@ type TestSuite struct { filePath string pubKeyList [][32]byte ingest Ingest + tempDir string } func (ts *TestSuite) SetupSuite() { + var err error viper.Set("log.level", "debug") - tempDir := ts.T().TempDir() - keyFile1 := fmt.Sprintf("%s/c4gh1.key", tempDir) - keyFile2 := fmt.Sprintf("%s/c4gh2.key", tempDir) + ts.tempDir, err = os.MkdirTemp("", "c4gh-keys") + if err != nil { + ts.FailNow("Failed to create temp directory") + } + keyFile1 := fmt.Sprintf("%s/c4gh1.key", ts.tempDir) + keyFile2 := fmt.Sprintf("%s/c4gh2.key", ts.tempDir) publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test") if err != nil { @@ -237,6 +242,7 @@ func (ts *TestSuite) SetupSuite() { func (ts *TestSuite) TearDownSuite() { _ = os.RemoveAll(ts.ingest.Conf.Archive.Posix.Location) _ = os.RemoveAll(ts.ingest.Conf.Inbox.Posix.Location) + _ = os.RemoveAll(ts.tempDir) } func (ts *TestSuite) SetupTest() { @@ -295,6 +301,11 @@ func (ts *TestSuite) SetupTest() { if err != nil { ts.FailNow("failed to setup inbox backend") } + + viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ + {FilePath: filepath.Join(ts.tempDir, "c4gh1.key"), Passphrase: "test"}, + {FilePath: filepath.Join(ts.tempDir, "c4gh2.key"), Passphrase: "test"}, + }) } func (ts *TestSuite) TestTryDecrypt_wrongFile() { tempDir := ts.T().TempDir() @@ -345,6 +356,7 @@ func (ts *TestSuite) TestTryDecrypt() { privateKeys, err := config.GetC4GHprivateKeys() assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(privateKeys)) for i, key := range privateKeys { header, err := tryDecrypt(key, buf) @@ -353,7 +365,7 @@ func (ts *TestSuite) TestTryDecrypt() { assert.NoError(ts.T(), err) assert.NotNil(ts.T(), header) default: - assert.Contains(ts.T(), err.Error(), "could not find matching public key heade") + assert.Contains(ts.T(), err.Error(), "could not find matching public key header") assert.Nil(ts.T(), header) } } @@ -723,3 +735,9 @@ func (ts *TestSuite) TestIngestFile_missingFile() { assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } +func (ts *TestSuite) TestDetectMisingC4GHKeys() { + viper.Set("c4gh.privateKeys", "") + privateKeys, err := config.GetC4GHprivateKeys() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 0, len(privateKeys)) +} From 134913d89ea72ae970c46f7fe458cb3c5dc23542 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 08:46:15 +0200 Subject: [PATCH 011/262] [verify] ensure C4GH keylist is not empty on startup. --- sda/cmd/verify/verify.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 908a2546a6..53c3623f62 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -41,8 +41,8 @@ func main() { log.Fatal(err) } archiveKeyList, err := config.GetC4GHprivateKeys() - if err != nil { - log.Fatal(err) + if err != nil || len(archiveKeyList) == 0 { + log.Fatal("no C4GH private keys configured") } defer mq.Channel.Close() From 9c2da4a752b3b901f20cf9c7a42b21c1c6f5877a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 May 2025 19:44:05 +0000 Subject: [PATCH 012/262] Bump com.hierynomus:sshj in /sda-sftp-inbox in the all-modules group Bumps the all-modules group in /sda-sftp-inbox with 1 update: [com.hierynomus:sshj](https://github.com/hierynomus/sshj). Updates `com.hierynomus:sshj` from 0.39.0 to 0.40.0 - [Commits](https://github.com/hierynomus/sshj/compare/v0.39.0...v0.40.0) --- updated-dependencies: - dependency-name: com.hierynomus:sshj dependency-version: 0.40.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index e52ee1ebbd..48931ef309 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -144,7 +144,7 @@ com.hierynomus sshj - 0.39.0 + 0.40.0 test From 15fb7c97d4ed3ec641000b3575d982d698b99519 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 20 May 2025 12:21:33 +0200 Subject: [PATCH 013/262] Use 4096 bit RSA key --- sda-sftp-inbox/src/test/resources/rsa.sec | 104 +++++++++++----------- sda-sftp-inbox/src/test/resources/rsa.ssh | 2 +- 2 files changed, 51 insertions(+), 55 deletions(-) diff --git a/sda-sftp-inbox/src/test/resources/rsa.sec b/sda-sftp-inbox/src/test/resources/rsa.sec index 55bb962c4c..55a3c8c154 100644 --- a/sda-sftp-inbox/src/test/resources/rsa.sec +++ b/sda-sftp-inbox/src/test/resources/rsa.sec @@ -1,54 +1,50 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-256-CBC,D0F80CA3EA0ACF3698A232F9F2D82B54 - -8BqfqBi5JjcuC8d01M+zGmyH8402V6yN+tui6kUIdKPyJvIq3nyaczXpngfp9oay -G+8RW2QlgLpouoAXQ354cCj/V5mpTVDRRKjZGbo3oaIjARuuRRfXJz4M95zR5iMh -RlOUoTAtpumoHStd0TNwhsBQtZwdjBxw8vHeAmA/hWpD82tTOhTtFmWmfMmZrilu -T4roC8jEsyfqoBGkhKPm0RM34xkP/tgJ63QqRLhUd2wbyvF4I9jtQumNAxLUbRi+ -sBeLJteLXg7z3JiaKTdYCYdecoNPFEzo9vHF0HKEVBGvbhnUCJTyqdqMzc0SohT/ -NG89RvdD75ryBVVKVsNj/fbTvC5Fg0XYHMW4DQ8qmnVO2MpHojpj+RWM7chIPwEX -ndBGLUJk3aCdz/0qMxN06v7NdT/bQglApVMTOXNMz5TjmtSMTGlXH7DJeY8vwwQa -t+RPnZSLCA4IoXGXH/ODlEilPQuGIQ458HLYBJjjBaMWGF3j80vZXFQyq752JrNH -3fbEfwEw08TNIcFIeuXBa1gx7cWhSqArSh/wdx0ke9FfbBTitue6GBFOOeABdCjd -gt188i+XAGxJQfUz4GCug3t7AFiYNgG4hizhTdGgpuGEYgkMxm5FK8RkeMyBof6A -ESODLfgbuG1GGU6ohOL/lQQ/1PC/a2Fm3H1VWsdMGthPyfJ+B/CHD0W8RKUa3fpQ -I9qqOLpRD2eEWA+FVZXC3xkPRkhXEId5O5jyKAsU2FNhZqL64UkEpWlkd+nQhdJE -/Gkd+wxrTu2dO//AJVFZLGLTKBqipebEolR4uOWlc+AoQaxZtdedl87jFNa5AdQa -tNQ6P5uy/X13EkkOYlFYWxwDmj9hlCcCaAUbExVy77lMBRmBjAqSm6IYRnMjZWfj -x2zh0ilCpoEuetcQc0+EaLBqzrBBOr4vVopr4rKfZ4SSYq13HWIFZre5XHxx3ddl -FJXIoKUniUCB/0O7krbTtPbKrv6bgl+77VJPN8G2t0Pz7DPUzEOxFH12D/LxvMqh -2EslcN/pRLWQLC26loybmSq0UXL1BW6UbtoTeGidLvmalSFLpkFkZhu/lgvM5ith -bwABkBfIaFoAismpCkYIyEK5mscYi1E4ziiwu+U83yaOILOUK66M6XaNMElWVVLq -fe68BwBUl0GTHgQmEYcPntQUiHNsAhc0HG6qrSYcD/c2uIC8aTYiPo8VbLz/qKhn -qDeDvI9Y/IVapB/tZ6c7mEiKiW2o4GvDA/647yOv1qMXjsU8ME12/VciV+1KL4AZ -eeCas/4G2FtLTDvzJZhJtbncFiSl9Q/RitvQBkVWc16vApPpMUHVokBl3ZsbiT8c -RBKo3V1h6ekSr6YM3wsNiD65LnQr4HP+PkDbbAOYSo1a4EAnuAg6sD08alN96ajm -bXucuXmxoiUdNSWH/mWCXo48C7bVFcpimXJnExf7E/ebccmGeQe78Zu4d4W3uofh -HfNGBcZLFRgPPEDwf0GZqjhuMA7AyWXckZtEGU7Hd03GwLz8Q/spknpontTfd1cH -xeu6hMvtMz934vPyyXSMrXWUK2wnQSWh2wQPXC2tPr0HVsZdwlwhIftNFWlTlHqL -aednAapHGIA1vJxn2dT1cvJympy+f+nJFvQYi6rO54Fj0AfwCASDiIUge+G/CVKA -OdDd46pcmzJr4peRiAJ4smTTWFLK+sJ1hAy4wmocsdhn6+6QRxSv/ayXOZ9ZxMRi -yOVHTIIq/c9b8GNvMZt75sQr8eh12VSWyWskwat1GqKNpx/AAhgGxnxxVGVYRVI+ -dgPLaioKKQGtlEhefxFGUg4NLc1AH70smzPn/44YqSbcMC5zDgObNaGI62X9ELgE -7v8BBgsH0i9HFEqaDd2jk1na0tXBg6nRpu4Jf/Va0DhQ2/aT0td4jtMckCw4lfHB -/xfcRwN6j7smorGMd4DT8rgrhTFK2qcQYYtaFF8i92+OTBE0byIXi4JNhxQoz6Vr -VeApj94au3KLG3L4+nkZp+Pmk5ayPQr2t+Ni1B75cdfKHnO0+yfkEBR7j+n9v4aA -hEjAOr/e6FMcuk0/oT1j0svJcWAvPlneFgfmBv8eEby0E7jmBinafqn5MktjMpDd -OccHpC4/P2c6z5w787t52X99UNWaUahoPZGkQzexm+A1XQc3s8JS5TFpoH9fO0OW -buiUXAurpSE7UUJ4ygWAZiHCHsfQJbd585m+3Xd5B188m4okXvvXF6sWksdrUMS2 -01tj29xMm2g0TTXro9ZsbEMpOIQOP/NMZY9+iDqbQ7+lpkHUcDaQVndJsbsJGbZA -7+b+OgSv1mMQvrbcbSSInsDEP6vkdrOgWvVpYDSbhIMnEMpzQLMfGymiN/IAOvAj -xfUz24wnhhdUmk7we5yrqActuWDxE64W9nabo6Le1lNpVg1NY2ppAylcomc27rzh -97NzIPDZSWpHVr5l6LADlb6yw/II55vtmRKAq3fM6NZUyyM4oZ3FQOx+3N/z1JML -lMwKwzqGOqplS4KN1hurB/BGqAhqiKGXjTWfDWMBtESLkRZZp3c2U7wZbbOaN++y -3pUO0HenycRivYcThh6Qlc6jwBubDCV0oqXb1JCyfnfYoQ8G9hfxPOPbXdhvdGnI -wLx0KLDUrxArk1goachtA/ldTbktz5oAlViUDc1Rv195jR1sqIScCgcad9e0fFdN -azF6ZJVKy4/MZOEXDOgZPYg3rmuxAssthOR+kZRoSAvxXfK39iYc6LCLFRJVfFz1 -EhwIRDds7GmW/JY9VNUwO4iDCrwsVX9ZClWx600itJMqM5lPIhB1hLZZkzHex/uo -NeeJkelUXakkCul5W8aCTnNhvggdxwCx74uZWGKqMGbrQm3bUdUIJKWrwPd6UYEY -i7hlapy4w4j87z2hiAxNyP2hK/neL4E6y2qERKsxl4/UpcPOzP/R1bh1ZRnNlMEq -pwzQ1o6BrQX4nxuNudaglGEpExeaGZxb555CqHFXswM3ft+5J8F3NG2SdOzt+k/y -rmessOyro/b9XWdt9+XJBo0FKS/DuKwMoXetoAMhiFvifEMdhz8ZOdV6nCRpjFw6 -/X4X0iQ/vxaKAWyrPApU9YUY/Y+IZTaBHVyCgYqWds4Ri12sqK1lRjhm0ycgwGlQ ------END RSA PRIVATE KEY----- +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABDlggJkmc +9n/V4c070nKIL8AAAAGAAAAAEAAAIXAAAAB3NzaC1yc2EAAAADAQABAAACAQC/MfPjl2bE +Y860QjfNC9phNeZ8FqSeJoP8WnWfO05AsOV/xC2RV6HlYCC/7QQ8Sa9ulrLse/7EUo3qtO +F3J7/xNxEugVCInA2GvSWrI2fHIPOfjl6TdqjwzT3RwfNFl7RJPHeO40MlC83O1o3MdxXN +YrFQUCbQsoLOdeBsqKsQc0b8DaGxUhAcle45M4cppbu1omhQgsJyLmiYUu7vZvjLF267wE +AuysUgAi3m4aE1egN+N896XasWdZjzWWUVshBLJU5NuaGO9PkSv/i1Lkj5syZOXCgrmTfW +UbVz3GMzokUG+rZoWqnZNpbBzxqqYaGIZssSGwjzpFWPAgfVsfDW0Qq0bLqZsszallx28e +8r7fd8O068U0E7u/gMa6ct+nqW7TMuetPUap0SKdAsevgW6KcUJUnyb91Nrq3P552HqYYM +8RcDapxvKvPCZSnHNFTF48shA0AcNGFuYVFichbFdttP+P/Cxx8aqyt4as77JwwW4/JdO9 +393kKUwKjk5gCl1yMLS8iq2eQcXGKHztKfBh4yxl4ZaEXl+1Ceqj4SBm1/pJRgP0PQfi/I +kvUMA+W90oJ/XUdrbV7+k2jzWMR3yD5SX+3F2CFXjIj228UX07Gfgh/UkPHErIaity8lQ2 +Qk+czrVgqdER9hOWELhcmMpx3ZBgxywhGeUjgbs6v8SQAAB1BWtRJsI0i/MF5cJaX3y7Gc +9kcG5XavsDuZQjp7DYQ91So7BRX7Jp4aCJD3DXsT/4uQKE+9feLfAKB1nxBrKuDN6mo+F7 +vsfHUAZM1nXBZGe02Y8sX02eer9w/jz2aEV1ovbFEpMbrZhjRy4HYDUAdLvhO3GFhAjtEe +p9pj0W5URgGOvEG/QxR1xskecPb9apYsDKjCGe+ROtbmiPMu7apdgU+IsBw1Dpe8MOYu/W +FW6xLhgx88V4QmbztjtwfoZDRqX9fJNBu8PJqaVAFVxVgBGn726K1a9VPljflqHsMLef2i +m/d/t8c8xY01oKWQ9cypg+BnYcZQ/N+T7vSbhH6/SvcXaR2AwfbRUOkpjiKQ3Oh9sX724c +TtoOHHB24a7sU3XeooogoieGFjJW0caMIjOJ6SjXtaEzmjy7W4pQaI8wFl/Pi7mm5xkI7l +1wsLhS4l4GuPooxRcTk02fj/97PM7dbhA0zDG1PGJx1teCZOx0ca29bAsU7pyoez1/TU5i +r/nXQNnMpNM4bzBNqy+tqEIRDbzg2MSErJeG4q+ov4GJLSAp6YoU0OsWgxEpM6So4vXq2m +DVyP25TqJMjjemqI4E0vTqFYVbI9HsPmNLyPbDXm5dVj/IMqEFcB+N18rr1TvNggH62qIX +m/I0JqWtfuIDUHan9+sA0HL8wCdGUxiyZ153v/nFRIqUWY35AxtyGmi0UJFF9yd5PNz3iM +/MklE0s3dYBRTo3/XejWokzTnNeVkh3IDx+oBvqEGPNTOV0r2qZN1TUGrsV9ahgiR4Cn1U +J88OUkfkshH7kqZs+WnvOvIhj9wFNmWEGv0oPBhz7ew7lK+4xp+p1mwCYgJhS+eae3sbcw +VM9Dg1BxC7xNpHY0c2C4LsUIdrfW/LVJRRePjWBAzmFFRRrMZKd/ojWiYUTQad6F6+obh/ +NeJ3PPfBGmg9merJbZSf2w+R7PTd/49Npid2XGTidWieUMylQYCMHMz1g1q2SXHR/HUCLM +3ezM81f0CNm9QRVuRZ1Z+L7H9/X/ULwoog5+IHnNa71lboi/DKPSENW08MV5Nhv8I1yF6V +2zvkevGUSSs9AtPlVWDRBXPr+rgGKcR4MY8p4oMEJwSbMTiuhLU1Scq4/uggZu6Gb0qyAG +59ReewfAG7y6ZzwiNGvcOWsUSgKkoVLWksCbUX2IbwojhP63ZX/8fide1Ls5dmEVbVKp6Y +n/eu/FumDM50bWm6Zwy/krFU64eVfYD5YRgJeDbC0cjG8M2o/UASYarINFlmHWVkVa1/zl +VEMd1ZnEDHXlrcPndIt4m+XXuuUiNbV/jmsbEWl7T2SM0ijunBZtZzjvZD0XxEu9xno0MB ++0tWLHK+WtXR3uk53vwFZgaJfYjb8scWf81o5pjAfNzDzR+2z0f4UlTi+xVnCWfa8is3yQ +jg3/7QR+KD6PIg2Zlf/B4A0oX8RwyeGEsYt3bmITJiQganPR3gc6zs1AL4dbBseC38V4Fr +UuROgiG6TDcgBsfJdoTPeESZK808fDO3ywylMRjQZ9GL6UJJfe7iP6FQrwR6SjFcPEYdO5 +a63ylDVdS+kY0IL56b25U0whakJj+QIEEdGB6e0iLUGlZ9ggaAufLOc4lbPtHwSpZCu5u5 +3BVjc/b3xTRFym4N+qLR+jPatRfkMeIdFKNItgXT15wK9UdTJNT4gTASLGZvIRzXMjUH0k +LVDru6h52yD9bU8OjOBAMS6eM83SXItXKy1OhQ3ESqADXjoD7hHLfo00kvOd9VHEVIc4zZ +FSCVMSFOUFo99ulLJoUJlF8gh/q7VSHGCPeq/rUlNvfRIrt+ieW68dOuyQKxxFZQ8Hzv8U +7EZT7Bx5OEMSXhnR+GC+m2OKamBjjJgZTG+t43y+n696h1jJu8hW7WdecrHeVdQe2rDKeP +I2vzTK0dlOPXXZcxN2L2V4GfO6LTm7wl3gJX9lAZ/AVEZHkbgGEbOoM+1hzXJWIo7NMiDk +MjVqSejabEtz5HU43wYvjpHhifbNClD5qaKi5L3q+f/5C1FOZjuWcuwgXMcZnYPtmDIWdl +C5Ir5MUWdYZdHvhoVq+320jSyHNinCr9qXZVRLH7CX3bc4n78YZ4qZQNlrp0850Fyrod7x +qNuut2hzx/s9CkMT6HvOIEPb4BjWRU9WQJRo+h4KIXOEXgsEnnLwQ75dnPi8cQlywnHAd7 +DnicsM5P6kcisWAe5qjmgu3s7j6PdV9fz4OWJdB+B9xTIdYyijQh8dlfpPIsTeYkV2ZLS7 +hKZRld4Fv1ESdf8TzwkGp1l2wn2zD8ZfL3opVb98S3E7CxLzHSkcIbT+NNe/3X8L2uLjxR +As427SYRR5Z3aqRVYtktWKRKZ8lsMJZqfVKIEU/+70FccDA/+mdETXDK2rQ8wlVtEjA3l2 +Tlsw5aQXriNFA19I4nlQC18oD/nijAQYNyro5goRkB41ZCO6G54MN2k8ETYRVCUf5L5vDH +0nP6FOt+47Rri8u2UUS2m278M= +-----END OPENSSH PRIVATE KEY----- diff --git a/sda-sftp-inbox/src/test/resources/rsa.ssh b/sda-sftp-inbox/src/test/resources/rsa.ssh index fe47a6cd92..a29f900c1d 100644 --- a/sda-sftp-inbox/src/test/resources/rsa.ssh +++ b/sda-sftp-inbox/src/test/resources/rsa.ssh @@ -1 +1 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqSXigbq5S/aPGE412BmHOYaLEQilVzXvjKRWVLOcZOGrElTjtiu7/Ezqdg4B3bVvKGC4TLUKDflO7qM+QnmpSbEFLz7suGz+0XzdEi6En1ihdoPOKDEvRlqZhNHlf3qjoatX9ScQa6vMuKhDwfneZNfhOzUzxJ8UihXYGHzoVvt4CGkSVSSFwj9dUmg2bG3hGL6i6w/F9rjCO4t9u3Rh/TFMBzcGIq+ft0BOM8XqSWmUDn/hYdt1Xkxnz1D0QHS5SjVmqnzb4r60pkgKSzQ3M9ECIz/ktVFCVZxH61dAeqko2196WHfe+zn0PpYfnMjhqtJlV4gstdz5C2g3IUKL2jxP0APDXib6lZuNSSiXT/1wiWE4jJDLtGVnauKDMerpJlcwl0zawMcCxqIxjY4OrfP+/k35JfDM1o3VJ8EZMtd4bCdG6w9iFzu/Yyjl91ryFHvPY/J4wuzErbzBUDfri18dKXbgH7EES7J6jbKJcmgVRWaEHpwogGcPHvJHxMDTOuwLxaNHkMxm2WTaFG4Q5uC2ZJhPPX40k/sn/paE3zgX0u7VIwczOr+cBqNNW4lqbGqooTn5RhHnXfpKRFZNz8fAdL+EPg0Y3hoPvZtRl04bvc4d5Ajj86Do5B/Wq2qAzDZoTtspPhcf7RnO3L+EzikshZcjES6I8IxCCppkMrQ== \ No newline at end of file +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/MfPjl2bEY860QjfNC9phNeZ8FqSeJoP8WnWfO05AsOV/xC2RV6HlYCC/7QQ8Sa9ulrLse/7EUo3qtOF3J7/xNxEugVCInA2GvSWrI2fHIPOfjl6TdqjwzT3RwfNFl7RJPHeO40MlC83O1o3MdxXNYrFQUCbQsoLOdeBsqKsQc0b8DaGxUhAcle45M4cppbu1omhQgsJyLmiYUu7vZvjLF267wEAuysUgAi3m4aE1egN+N896XasWdZjzWWUVshBLJU5NuaGO9PkSv/i1Lkj5syZOXCgrmTfWUbVz3GMzokUG+rZoWqnZNpbBzxqqYaGIZssSGwjzpFWPAgfVsfDW0Qq0bLqZsszallx28e8r7fd8O068U0E7u/gMa6ct+nqW7TMuetPUap0SKdAsevgW6KcUJUnyb91Nrq3P552HqYYM8RcDapxvKvPCZSnHNFTF48shA0AcNGFuYVFichbFdttP+P/Cxx8aqyt4as77JwwW4/JdO9393kKUwKjk5gCl1yMLS8iq2eQcXGKHztKfBh4yxl4ZaEXl+1Ceqj4SBm1/pJRgP0PQfi/IkvUMA+W90oJ/XUdrbV7+k2jzWMR3yD5SX+3F2CFXjIj228UX07Gfgh/UkPHErIaity8lQ2Qk+czrVgqdER9hOWELhcmMpx3ZBgxywhGeUjgbs6v8SQ== \ No newline at end of file From 0a5ce86387d981b5fcaca8c974ae0156e0635a62 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 21 May 2025 12:46:39 +0200 Subject: [PATCH 014/262] [sda] update dependencies --- sda/go.mod | 78 ++++++++++++------------- sda/go.sum | 166 +++++++++++++++++++++++++++-------------------------- 2 files changed, 121 insertions(+), 123 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index c320513176..3cfe465262 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -1,8 +1,6 @@ module github.com/neicnordic/sensitive-data-archive -go 1.24.0 - -toolchain go1.24.1 +go 1.24.1 require ( github.com/aws/aws-sdk-go-v2 v1.36.3 @@ -13,10 +11,10 @@ require ( github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.105.0 github.com/coreos/go-oidc/v3 v3.14.1 - github.com/gin-gonic/gin v1.10.0 + github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 - github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 + github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c github.com/kataras/iris/v12 v12.2.11 github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.10.9 @@ -43,14 +41,14 @@ require ( dario.cat/mergo v1.0.1 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/BurntSushi/toml v1.4.0 // indirect + github.com/BurntSushi/toml v1.5.0 // indirect github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect - github.com/CloudyKit/jet/v6 v6.2.0 // indirect + github.com/CloudyKit/jet/v6 v6.3.1 // indirect github.com/Joker/jade v1.1.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect + github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect @@ -65,14 +63,13 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect - github.com/bytedance/sonic v1.12.2 // indirect - github.com/bytedance/sonic/loader v0.2.0 // indirect - github.com/casbin/govaluate v1.3.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect + github.com/bytedance/sonic v1.13.2 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/casbin/govaluate v1.6.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/containerd/continuity v0.4.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect @@ -83,33 +80,33 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.5 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.4 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-jose/go-jose/v4 v4.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.22.1 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kataras/blocks v0.0.8 // indirect - github.com/kataras/golog v0.1.12 // indirect - github.com/kataras/pio v0.0.13 // indirect + github.com/kataras/blocks v0.0.11 // indirect + github.com/kataras/golog v0.1.13 // indirect + github.com/kataras/pio v0.0.14 // indirect github.com/kataras/sitemap v0.0.6 // indirect github.com/kataras/tunnel v0.0.4 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.3 // indirect @@ -118,7 +115,7 @@ require ( github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/mailgun/raymond/v2 v2.0.48 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -130,20 +127,20 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.2.3 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.12.0 // indirect - github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/cast v1.8.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tdewolff/minify/v2 v2.20.37 // indirect - github.com/tdewolff/parse/v2 v2.7.15 // indirect + github.com/tdewolff/minify/v2 v2.23.6 // indirect + github.com/tdewolff/parse/v2 v2.8.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -154,14 +151,13 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yosssi/ace v0.0.5 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.10.0 // indirect - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/arch v0.17.0 // indirect + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect + golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect - golang.org/x/time v0.8.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index 25c9e695bd..7a5d49fe16 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -4,12 +4,12 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= -github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= -github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oME= -github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= +github.com/CloudyKit/jet/v6 v6.3.1 h1:6IAo5Cx21xrHVaR8zzXN5gJatKV/wO7Nf6bfCnCSbUw= +github.com/CloudyKit/jet/v6 v6.3.1/go.mod h1:lf8ksdNsxZt7/yH/3n4vJQWA9RUq4wpaHtArHhGVMOw= github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk= @@ -18,12 +18,12 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 h1:gjbp60h8IZQbN/TpDaYJedWbbD1h1aDPEwWnYWaDaUY= -github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= +github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b h1:3QuwXO2TnDssBmmNKfRlO3ur9zOupJRdM7evq61Oi84= +github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= @@ -64,24 +64,25 @@ github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= -github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= +github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/casbin/casbin/v2 v2.105.0 h1:dLj5P6pLApBRat9SADGiLxLZjiDPvA1bsPkyV4PGx6I= github.com/casbin/casbin/v2 v2.105.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= -github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= +github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= @@ -106,26 +107,26 @@ github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6 github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw= github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= -github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= +github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY= +github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -136,16 +137,16 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= @@ -154,13 +155,14 @@ github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 h1:ZPy+2XJ8u0bB3sNFi+I72gMEMS7MTg7aZCCXPOjV8iw= -github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60qA17CptGXhwfZ+k1sFNJIUNR8DdbcuUk= +github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -173,16 +175,16 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE= -github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 h1:BuEwEv6gPS9pSX0+O5FIxxKUoRziXBFdWI1S4ivKQtY= -github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72/go.mod h1:DQJ0KlNPppOfMC+0x0ADeFQk0WmQMVU9rJQzFY4nUfA= +github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c h1:7gyBIX75xKttBJo35lyVLUb/OQh7jY2wcEIVUygmsL8= +github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c/go.mod h1:SvRi2dE8S9rgUvgDrGg7hpb/1gt9zVnvLnWjMQJUONo= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -191,31 +193,30 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kataras/blocks v0.0.8 h1:MrpVhoFTCR2v1iOOfGng5VJSILKeZZI+7NGfxEh3SUM= -github.com/kataras/blocks v0.0.8/go.mod h1:9Jm5zx6BB+06NwA+OhTbHW1xkMOYxahnqTN5DveZ2Yg= -github.com/kataras/golog v0.1.12 h1:Bu7I/G4ilJlbfzjmU39O9N+2uO1pBcMK045fzZ4ytNg= -github.com/kataras/golog v0.1.12/go.mod h1:wrGSbOiBqbQSQznleVNX4epWM8rl9SJ/rmEacl0yqy4= +github.com/kataras/blocks v0.0.11 h1:JJdYW0AUaJKLx5kEWs/oRVCvKVXo+6CAAeaVAiJf7wE= +github.com/kataras/blocks v0.0.11/go.mod h1:b4UySrJySEOq6drKH9U3bOpMI+dRH148mayYfS3RFb8= +github.com/kataras/golog v0.1.13 h1:bGbPglTdCutekqwOUf8L1jq3tZ5ADG9gfPBd5p5SzKA= +github.com/kataras/golog v0.1.13/go.mod h1:oQmzBTCv/35TetBosjJl/k+LPdlJEblaTupkNwJlwj8= github.com/kataras/iris/v12 v12.2.11 h1:sGgo43rMPfzDft8rjVhPs6L3qDJy3TbBrMD/zGL1pzk= github.com/kataras/iris/v12 v12.2.11/go.mod h1:uMAeX8OqG9vqdhyrIPv8Lajo/wXTtAF43wchP9WHt2w= -github.com/kataras/pio v0.0.13 h1:x0rXVX0fviDTXOOLOmr4MUxOabu1InVSTu5itF8CXCM= -github.com/kataras/pio v0.0.13/go.mod h1:k3HNuSw+eJ8Pm2lA4lRhg3DiCjVgHlP8hmXApSej3oM= +github.com/kataras/pio v0.0.14 h1:VGBHOmhwrMMrZeuRqoSfOrFwG+v1JxQge8N50DhmRYQ= +github.com/kataras/pio v0.0.14/go.mod h1:ZIlcw5+5Zyb/kOlU7X4uosZ8dbnXmA4GcGKt1XyyTY0= github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY= github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA= github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -240,8 +241,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -286,8 +287,8 @@ github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4 github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= @@ -301,8 +302,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= -github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= +github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -311,8 +312,8 @@ github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiy github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -321,10 +322,10 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= +github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= @@ -345,13 +346,12 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tdewolff/minify/v2 v2.20.37 h1:Q97cx4STXCh1dlWDlNHZniE8BJ2EBL0+2b0n92BJQhw= -github.com/tdewolff/minify/v2 v2.20.37/go.mod h1:L1VYef/jwKw6Wwyk5A+T0mBjjn3mMPgmjjA688RNsxU= -github.com/tdewolff/parse/v2 v2.7.15 h1:hysDXtdGZIRF5UZXwpfn3ZWRbm+ru4l53/ajBRGpCTw= -github.com/tdewolff/parse/v2 v2.7.15/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA= -github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= -github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo= -github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= +github.com/tdewolff/minify/v2 v2.23.6 h1:s0ktOnrlqKfhignkv9M/rh9wCfRlIPiIpjPOq3xUJ84= +github.com/tdewolff/minify/v2 v2.23.6/go.mod h1:VW3ISUd3gDOZuQ/jwZr4sCzsuX+Qvsx87FDMjk6Rvno= +github.com/tdewolff/parse/v2 v2.8.1 h1:J5GSHru6o3jF1uLlEKVXkDxxcVx6yzOlIVIotK4w2po= +github.com/tdewolff/parse/v2 v2.8.1/go.mod h1:Hwlni2tiVNKyzR1o6nUs4FOF07URA+JLBLd6dlIXYqo= +github.com/tdewolff/test v1.0.11 h1:FdLbwQVHxqG16SlkGveC0JVyrJN62COWTRyUFzfbtBE= +github.com/tdewolff/test v1.0.11/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= @@ -369,6 +369,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA= @@ -397,8 +399,8 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= -golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= +golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -410,8 +412,8 @@ golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= @@ -435,8 +437,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -494,8 +496,8 @@ golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= -golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -511,8 +513,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= From 6b76e35757424a61059176070cc029288ce327f1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 21 May 2025 12:46:48 +0200 Subject: [PATCH 015/262] [sda-download] update dependencies #major --- sda-download/go.mod | 43 ++++++++++++------------- sda-download/go.sum | 77 ++++++++++++++++++++++----------------------- 2 files changed, 58 insertions(+), 62 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 7aabc6d4ce..c52966f926 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -1,14 +1,12 @@ module github.com/neicnordic/sda-download -go 1.24.0 - -toolchain go1.24.1 +go 1.24.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/aws/aws-sdk-go v1.55.7 github.com/dgraph-io/ristretto v1.0.0 - github.com/gin-gonic/gin v1.10.0 + github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 github.com/lestrrat-go/jwx/v2 v2.1.6 @@ -18,33 +16,32 @@ require ( github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.38.0 - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.72.1 google.golang.org/protobuf v1.36.6 ) require ( filippo.io/edwards25519 v1.1.0 // indirect - github.com/bytedance/sonic v1.12.2 // indirect - github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/bytedance/sonic v1.13.2 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.5 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.22.1 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.3 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -54,27 +51,27 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/locafero v0.9.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.12.0 // indirect - github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/afero v1.14.0 // indirect + github.com/spf13/cast v1.8.0 // indirect github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.10.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/arch v0.17.0 // indirect + golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect golang.org/x/text v0.25.0 // indirect - golang.org/x/tools v0.25.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect + golang.org/x/tools v0.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index fe997dd69d..98643b916b 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -5,18 +5,17 @@ github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= -github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -34,14 +33,14 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= -github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -52,12 +51,12 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= @@ -75,8 +74,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -107,8 +106,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U= github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -118,8 +117,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= -github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= +github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= @@ -130,10 +129,10 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= +github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= +github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= +github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= @@ -172,14 +171,14 @@ go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= -golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= +golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -189,8 +188,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= +golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -228,11 +227,11 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= -golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= +golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= From f826dc8723e441be7faaf4e2bcc32d9cc2849780 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 9 May 2025 09:53:40 +0200 Subject: [PATCH 016/262] [ingest] use config file instead of ENVs This allows for configuring multiple crypt4gh keys --- charts/sda-svc/templates/_helpers.yaml | 4 + charts/sda-svc/templates/ingest-deploy.yaml | 162 +------------------ charts/sda-svc/templates/ingest-secrets.yaml | 87 +++++++++- charts/sda-svc/values.yaml | 6 +- 4 files changed, 95 insertions(+), 164 deletions(-) diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 7079c71768..0d66679185 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -101,6 +101,10 @@ Create chart name and version as used by the chart label. {{- .Values.global.secretsPath -}} {{- end -}} +{{- define "configPath" -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.confFilePath) (empty .Values.global.confFilePath) -}} +{{- end -}} + {{- define "c4ghPath" -}} {{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.c4ghPath) (empty .Values.global.c4ghPath) -}} {{- end -}} diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index 9f2fb49c2c..e6be859a6d 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -72,168 +72,14 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}/" - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: "ingest" - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_ROUTINGKEY - value: "archived" - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: C4GH_FILEPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - - name: INBOX_TYPE - {{- if eq "s3" .Values.global.inbox.storageType }} - value: "s3" - - name: INBOX_BUCKET - value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - - name: INBOX_REGION - value: {{ default "us-east-1" .Values.global.inbox.s3Region }} - - name: INBOX_URL - value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- else }} - value: "posix" - - name: INBOX_LOCATION - value: "{{ .Values.global.inbox.path }}/" - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: dbUser - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - {{- if eq "s3" .Values.global.inbox.storageType }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - {{- end }} - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.ingest.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} @@ -251,6 +97,10 @@ spec: {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-ingest - name: c4gh secret: defaultMode: 0440 diff --git a/charts/sda-svc/templates/ingest-secrets.yaml b/charts/sda-svc/templates/ingest-secrets.yaml index 74bb2d085f..d08426f88b 100644 --- a/charts/sda-svc/templates/ingest-secrets.yaml +++ b/charts/sda-svc/templates/ingest-secrets.yaml @@ -6,11 +6,86 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-ingest type: Opaque -data: - c4ghPassphrase: {{ .Values.global.c4gh.passphrase | b64enc }} - dbPassword: {{ required "DB password is required" (include "dbPassIngest" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserIngest" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassIngest" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserIngest" .) | b64enc }} +stringData: + config.yaml: |- + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassIngest" .) }} + queue: {{ default "ingest" .Values.global.broker.ingestQueue }} + routingKey: {{ default "archived" .Values.global.broker.routingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserIngest" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + c4gh: + privateKeys: + {{- range $k := .Values.global.c4gh.privateKeys }} + - filePath: {{ template "c4ghPath" $ }}/{{ $k.keyName }} + passphrase: {{ $k.passphrase }} + {{- end }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassIngest" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserIngest" .) }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} {{- end }} {{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 6c2e83ff44..40494fd036 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -9,7 +9,7 @@ global: # Path where the sensitive files can be found, default is "/.secrets". # TLS certificates or C4GH key locations can be set using global.tlsPath or global.c4ghPath respectively, # this path will be a subpath to the secretsPath. - secretsPath: /.secrets + secretsPath: "/.secrets" c4ghPath: "c4gh" tlsPath: "tls" jwtPath: "jwt" @@ -195,7 +195,9 @@ global: publicFile: "" passphrase: "" backupPubKey: "" - + privateKeys: + - keyName: + passphrase: db: host: "" name: "sda" From 4f4a7a7a290bec215c984190d2d08378b1c099a8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 9 May 2025 09:59:49 +0200 Subject: [PATCH 017/262] [verify] use config file instead of ENVs This allows for configuring multiple crypt4gh keys --- charts/sda-svc/templates/verify-deploy.yaml | 130 +------------------ charts/sda-svc/templates/verify-secrets.yaml | 68 +++++++++- 2 files changed, 69 insertions(+), 129 deletions(-) diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index fed66d48a0..e008c76181 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -55,136 +55,14 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_QUEUE - value: "archived" - - name: BROKER_ROUTINGKEY - value: "verified" - {{- if .Values.global.broker.serverName }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.serverName | quote }} - {{- end }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: C4GH_FILEPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: mqUser - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: dbUser - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.verify.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} @@ -210,6 +88,10 @@ spec: {{- end }} {{- end }} {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-ingest - name: c4gh secret: defaultMode: 0440 diff --git a/charts/sda-svc/templates/verify-secrets.yaml b/charts/sda-svc/templates/verify-secrets.yaml index cc3b1fe0bc..9b7dd30de1 100644 --- a/charts/sda-svc/templates/verify-secrets.yaml +++ b/charts/sda-svc/templates/verify-secrets.yaml @@ -7,10 +7,68 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-verify type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassVerify" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserVerify" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassVerify" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserVerify" .) | b64enc }} +stringData: + config.yaml: |- + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassIngest" .) }} + queue: {{ default "archived" .Values.global.broker.verifyQueue }} + routingKey: {{ default "verified" .Values.global.broker.routingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserIngest" .) }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + vhost: {{ include "brokerVhost" . }} + c4gh: + privateKeys: + {{- range $k := .Values.global.c4gh.privateKeys }} + - filePath: {{ template "c4ghPath" $ }}/{{ $k.keyName }} + passphrase: {{ $k.passphrase }} + {{- end }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassVerify" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserVerify" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} {{- end }} {{- end }} From 1d0ac15e06ce6f06b7ad1f4e9a88063e5b8421db Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 13:25:55 +0200 Subject: [PATCH 018/262] [API] use config file instead of ENVs --- charts/sda-svc/templates/api-deploy.yaml | 152 ++-------------------- charts/sda-svc/templates/api-secrets.yaml | 78 ++++++++++- 2 files changed, 87 insertions(+), 143 deletions(-) diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index ae607d6011..b616940b50 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -5,7 +5,7 @@ metadata: name: {{ template "sda.fullname" . }}-api labels: role: api - app: {{ template "sda.fullname" . }} + app: {{ template "sda.fullname" . }}-api chart: {{ .Chart.Name }}-{{ .Chart.Version }} component: {{ template "sda.fullname" . }}-api release: {{ .Release.Name }} @@ -71,137 +71,8 @@ spec: {{- end }} command: ["sda-api"] env: -{{- if not .Values.global.vaultSecrets }} - - name: API_RBACFILE - value: {{ template "secretsPath" . }}/rbac.json - {{- if .Values.global.tls.enabled }} - - name: API_SERVERCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: API_SERVERKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: mqUser - {{- if .Values.global.tls.enabled }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: DB_SSLMODE - value: {{ .Values.global.db.sslMode | quote }} - {{- else }} - - name: DB_SSLMODE - value: "disable" - {{- end }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbUser - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: INBOX_TYPE - {{- if eq "s3" .Values.global.inbox.storageType }} - value: "s3" - - name: INBOX_BUCKET - value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - - name: INBOX_REGION - value: {{ default "us-east-1" .Values.global.inbox.s3Region }} - - name: INBOX_URL - value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- else }} - value: "posix" - - name: INBOX_LOCATION - value: "{{ .Values.global.inbox.path }}/" - {{- end }} - {{- if eq "s3" .Values.global.inbox.storageType }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "federated" .Values.global.schemaType }} - {{- if .Values.global.api.jwtPubKeyName }} - - name: SERVER_JWTPUBKEYPATH - value: {{ include "jwtPath" . }} - {{- else }} - - name: SERVER_JWTPUBKEYURL - value: {{ required "A oidc provider is required" .Values.global.oidc.provider }}{{ .Values.global.oidc.jwkPath }} - {{- end }} -{{- else }} - - name: SERVER_CONFFILE + - name: CONFIGFILE value: {{ include "confFile" . }} -{{- end }} ports: - name: api containerPort: 8080 @@ -229,13 +100,13 @@ spec: resources: {{ toYaml .Values.api.resources | trim | indent 10 }} volumeMounts: - {{- if .Values.global.api.jwtPubKeyName }} - - name: jwt - mountPath: {{ include "jwtPath" . }} - {{- end }} {{- if not .Values.global.vaultSecrets }} - - name: rbac + - name: config mountPath: {{ template "secretsPath" . }} + {{- if .Values.global.api.jwtPubKeyName }} + - name: jwt + mountPath: {{ include "jwtPath" . }} + {{- end }} {{- end }} {{- if .Values.global.tls.enabled }} - name: tls @@ -243,13 +114,18 @@ spec: {{- end }} {{- if eq "posix" .Values.global.inbox.storageType }} - name: inbox - mountPath: {{ .Values.global.inbox.path | quote }} + mountPath: "/inbox" {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} - - name: rbac + - name: config projected: sources: + - secret: + name: {{ template "sda.fullname" . }}-api + items: + - key: config.yaml + path: config.yaml - secret: name: {{ required "a secret containing the RBAC policy is required" .Values.global.api.rbacFileSecret }} items: diff --git a/charts/sda-svc/templates/api-secrets.yaml b/charts/sda-svc/templates/api-secrets.yaml index a2a1d387a9..1a06e50a18 100644 --- a/charts/sda-svc/templates/api-secrets.yaml +++ b/charts/sda-svc/templates/api-secrets.yaml @@ -6,11 +6,79 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-api type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassAPI" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserAPI" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassAPI" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserAPI" .) | b64enc }} +stringData: + config.yaml: |- + api: + rbacFile: {{ template "secretsPath" . }}/rbac.json + {{- if .Values.global.tls.enabled }} + serverCert: {{ template "tlsPath" . }}/tls.crt + serverKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassAPI" .) }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserAPI" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if eq "verify-full" .Values.global.db.sslMode}} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassAPI" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserAPI" .) }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} + server: + {{- if .Values.global.api.jwtPubKeyName }} + jwtpubkeypath: {{ include "jwtPath" . }} + {{- else }} + jwtpubkeyurl: "{{ required "A oidc provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/{{ .Values.global.oidc.jwkPath | trimPrefix "/" }}" + {{- end }} --- {{- if not .Values.global.api.adminsFileSecret }} apiVersion: v1 From 7a4998d80b4461343e8fad6b268e3c410c95281f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 13:50:57 +0200 Subject: [PATCH 019/262] [auth] use config file instead of ENVs --- charts/sda-svc/templates/auth-deploy.yaml | 130 ++------------------- charts/sda-svc/templates/auth-secrets.yaml | 66 +++++++++-- 2 files changed, 65 insertions(+), 131 deletions(-) diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index b7d2542f0a..5df2c4cfea 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -72,124 +72,8 @@ spec: {{- end }} command: ["sda-auth"] env: - {{- if not .Values.global.vaultSecrets }} - {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} - - name: OIDC_ID - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: oidcID - - name: OIDC_SECRET - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: oidcSecret - {{- end }} - {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - - name: AUTH_CEGA_ID - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: cegaID - - name: AUTH_CEGA_SECRET - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: cegaSecret - {{- end }} - {{- else }} - - name: SERVER_CONFFILE + - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} - {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - - name: AUTH_CEGA_AUTHURL - value: {{ .Values.global.cega.host | quote }} - {{- end }} - {{- if .Values.global.auth.corsOrigins }} - - name: AUTH_CORS_ORIGINS - value: {{ .Values.global.auth.corsOrigins | quote }} - - name: AUTH_CORS_METHODS - value: {{ .Values.global.auth.corsMethods | quote }} - - name: AUTH_CORS_CREDENTIALS - value: {{ .Values.global.auth.corsCreds | quote }} - {{- end }} - - name: AUTH_INFOTEXT - value: {{ .Values.global.auth.infoText }} - - name: AUTH_INFOURL - value: {{ .Values.global.auth.inforURL }} - {{- if .Values.global.auth.resignJwt }} - - name: AUTH_JWT_ISSUER - {{- if .Values.global.tls.enabled }} - value: "https://{{ .Values.global.ingress.hostName.auth }}" - {{- else }} - value: "http://{{ .Values.global.ingress.hostName.auth }}" - {{- end }} - - name: AUTH_JWT_PRIVATEKEY - value: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}" - - name: AUTH_JWT_SIGNATUREALG - value: {{ .Values.global.auth.jwtAlg }} - - name: AUTH_JWT_TOKENTTL - value: {{ .Values.global.auth.jwtTTL | quote }} - {{- end }} - - name: AUTH_PUBLICFILE - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.publicFile }}" - - name: AUTH_RESIGNJWT - value: {{ .Values.global.auth.resignJwt | quote }} - - name: AUTH_S3INBOX - value: {{ .Values.global.ingress.hostName.s3Inbox }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} - - name: OIDC_REDIRECTURL - value: {{ template "authRedirect" .}} - - name: OIDC_PROVIDER - value: "{{ .Values.global.oidc.provider }}" - - name: OIDC_JWKPATH - value: {{ .Values.global.oidc.jwkPath | quote }} - {{- end }} - {{- if .Values.global.tls.enabled}} - - name: SERVER_CERT - value: {{ template "tlsPath" . }}/tls.crt - - name: SERVER_KEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: DB_SSLMODE - value: {{ .Values.global.db.sslMode | quote }} - {{- else }} - - name: DB_SSLMODE - value: "disable" - {{- end }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbUser - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} ports: - name: auth containerPort: 8080 @@ -226,11 +110,14 @@ spec: mountPath: {{ template "jwtPath" . }} {{- end }} {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} volumes: - {{- if and (.Values.global.auth.resignJwt) (not .Values.global.vaultSecrets) }} + {{- if not .Values.global.vaultSecrets }} + {{- if and (.Values.global.auth.resignJwt) (not .Values.global.vaultSecrets) }} - name: jwt projected: defaultMode: 0440 @@ -240,8 +127,11 @@ spec: items: - key: {{ required "The name of the JWT signing key is needed" .Values.global.auth.jwtKey }} path: {{ .Values.global.auth.jwtKey }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} + {{- end }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-auth - name: c4gh secret: defaultMode: 0440 diff --git a/charts/sda-svc/templates/auth-secrets.yaml b/charts/sda-svc/templates/auth-secrets.yaml index 81de9f9e67..80310d7a8b 100644 --- a/charts/sda-svc/templates/auth-secrets.yaml +++ b/charts/sda-svc/templates/auth-secrets.yaml @@ -6,17 +6,61 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-auth type: Opaque -data: - {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} - oidcID: {{ .Values.global.oidc.id | quote | trimall "\"" | b64enc }} - oidcSecret: {{ .Values.global.oidc.secret | quote | trimall "\"" | b64enc }} - {{- end }} - {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - cegaID: {{ .Values.global.cega.user | quote | trimall "\"" | b64enc }} - cegaSecret: {{ .Values.global.cega.password | quote | trimall "\"" | b64enc }} - {{- end }} - dbPassword: {{ required "DB password is required" (include "dbPassAuth" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserAuth" .) | b64enc }} +stringData: + config.yaml: |- + auth: + {{- if eq "federated" .Values.global.schemaType }} + cega: + authURL: {{ required "C-EGA NSS host is required" .Values.global.cega.host }} + id: {{ required "C-EGA NSS user is required" .Values.global.cega.user }} + secret: {{ required "C-EGA NSS password is required" .Values.global.cega.password }} + {{- end }} + {{- if .Values.global.auth.corsOrigins }} + cors: + credentials: {{ .Values.global.auth.corsCreds }} + methods: {{ .Values.global.auth.corsMethods }} + orogins: {{ .Values.global.auth.corsOrigins }} + {{- end }} + infotext: {{ .Values.global.auth.infoText }} + infoURL: {{ .Values.global.auth.infoURL }} + {{- if .Values.global.auth.resignJwt }} + jwt: + issuer: {{ ternary "https://" "http://" .Values.global.tls.enabled}}{{ .Values.global.ingress.hostName.auth }} + privatekey: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}" + signaturealg: {{ .Values.global.auth.jwtAlg }} + tokenttl: {{ .Values.global.auth.jwtTTL }} + {{- end }} + publicfile: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.publicFile }}" + resignjwt: {{ .Values.global.auth.resignJwt }} + s3Inbox: {{ .Values.global.ingress.hostName.s3Inbox }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassAuth" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserAuth" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} + oidc: + id: {{ .Values.global.oidc.id }} + jwkPath: {{ .Values.global.oidc.jwkPath }} + provider: {{ .Values.global.oidc.provider }} + redirectURL: {{ template "authRedirect" .}} + secret: {{ .Values.global.oidc.secret }} + {{- end }} + {{- if .Values.global.tls.enabled}} + server: + cert: {{ template "tlsPath" . }}/tls.crt + key: {{ template "tlsPath" . }}/tls.key + {{- end }} {{- end }} {{- end }} {{- end }} From fa61d262bd793ee4dc78be6e921a6ea9c8de4411 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 15:35:58 +0200 Subject: [PATCH 020/262] [finalize] use config file instead of ENVs --- charts/sda-svc/templates/finalize-deploy.yaml | 165 ++---------------- .../sda-svc/templates/finalize-secrets.yaml | 86 ++++++++- 2 files changed, 91 insertions(+), 160 deletions(-) diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml index 77530022ef..068f8752ad 100644 --- a/charts/sda-svc/templates/finalize-deploy.yaml +++ b/charts/sda-svc/templates/finalize-deploy.yaml @@ -71,166 +71,15 @@ spec: seccompProfile: type: "RuntimeDefault" env: -{{- if .Values.global.backupArchive.storageType }} - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: BACKUP_TYPE - {{- if eq "s3" .Values.global.backupArchive.storageType }} - value: "s3" - - name: BACKUP_URL - value: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }} - {{- if .Values.global.backupArchive.s3Port }} - - name: BACKUP_PORT - value: {{ .Values.global.backupArchive.s3Port | quote }} - {{- end }} - - name: BACKUP_BUCKET - value: {{ required "S3 backup archive bucket missing" .Values.global.backupArchive.s3Bucket }} - - name: BACKUP_REGION - value: {{ default "us-east-1" .Values.global.backupArchive.s3Region }} - - name: BACKUP_CHUNKSIZE - value: {{ .Values.global.backupArchive.s3ChunkSize | quote }} - {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} - - name: BACKUP_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: BACKUP_LOCATION - value: "{{ .Values.global.backupArchive.volumePath }}" - {{- end }} -{{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: "accession" - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_ROUTINGKEY - value: "completed" - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "sda" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "federated" .Values.global.schemaType }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - {{- if eq "s3" .Values.global.backupArchive.storageType }} - - name: BACKUP_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3backup-keys - key: s3BackupAccessKey - - name: BACKUP_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3backup-keys - key: s3BackupSecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: dbUser - {{ else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.finalize.resources | trim | indent 10 }} volumeMounts: + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} @@ -244,6 +93,12 @@ spec: mountPath: {{ .Values.global.backupArchive.volumePath | quote }} {{- end }} volumes: + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-finalize + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} diff --git a/charts/sda-svc/templates/finalize-secrets.yaml b/charts/sda-svc/templates/finalize-secrets.yaml index 6fb4d350bb..c8d92e3a2a 100644 --- a/charts/sda-svc/templates/finalize-secrets.yaml +++ b/charts/sda-svc/templates/finalize-secrets.yaml @@ -6,10 +6,86 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-finalize type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassFinalize" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserFinalize" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassFinalize" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserFinalize" .) | b64enc }} +stringData: + config.yaml: |- + {{- if .Values.global.backupArchive.storageType }} + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + backup: + type: {{ .Values.global.backupArchive.storageType }} + {{- if eq "s3" .Values.global.backupArchive.storageType }} + accessKey: {{ required "Accesskey required for backup archive" .Values.global.backupArchive.s3AccessKey }} + bucket: {{ required "S3 backup archive bucket not set" .Values.global.backupArchive.s3Bucket }} + {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.backupArchive.s3ChunkSize }} + {{- if .Values.global.backupArchive.s3Port }} + port: {{ .Values.global.backupArchive.s3Port }} + {{- end }} + readypath: {{ .Values.global.backupArchive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for backup archive" .Values.global.backupArchive.s3SecretKey }} + url: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }} + {{- else }} + location: /backup/ + {{- end }} + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassFinalize" .) }} + queue: {{ default "accession" .Values.global.broker.finalizeQueue }} + routingKey: {{ default "completed" .Values.global.broker.finalizeRoutingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserFinalize" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassFinalize" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserFinalize" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} {{- end }} {{- end }} From f4080ee8a0b16f6714e17198f35f982b0ed41f8a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 15:58:07 +0200 Subject: [PATCH 021/262] [intercept] use config file instead of ENVs --- .../sda-svc/templates/intercept-deploy.yaml | 66 ++++--------------- .../sda-svc/templates/intercept-secrets.yaml | 31 ++++++++- 2 files changed, 42 insertions(+), 55 deletions(-) diff --git a/charts/sda-svc/templates/intercept-deploy.yaml b/charts/sda-svc/templates/intercept-deploy.yaml index 5c18e9d188..062ae95fb5 100644 --- a/charts/sda-svc/templates/intercept-deploy.yaml +++ b/charts/sda-svc/templates/intercept-deploy.yaml @@ -58,74 +58,36 @@ spec: {{- toYaml .Values.global.extraSecurityContext | nindent 10 -}} {{- end }} env: - {{- if not .Values.global.vaultSecrets }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-intercept - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-intercept - key: mqUser - {{- else }} - name: CONFIGFILE value: {{ template "confFile" . }} - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange | quote}} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_QUEUE - value: "from_cega" - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} resources: {{ toYaml .Values.intercept.resources | trim | indent 10 }} - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + {{- if not .Values.global.vaultSecrets }} volumeMounts: + - name: config + mountPath: {{ template "configPath" . }} + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} + {{- end }} volumes: - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + - name: config + projected: + sources: + - secret: + name: {{ template "sda.fullname" . }}-intercept + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls - {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: defaultMode: 0440 secretName: {{ template "sda.fullname" . }}-intercept-certs - {{- else }} + {{- else }} secret: defaultMode: 0440 secretName: {{ required "An certificate issuer or a TLS secret name is required for intercept" .Values.intercept.tls.secretName }} + {{- end }} {{- end }} - {{- end }} {{- end }} restartPolicy: Always {{- end }} diff --git a/charts/sda-svc/templates/intercept-secrets.yaml b/charts/sda-svc/templates/intercept-secrets.yaml index bd9295972e..71d65ad89f 100644 --- a/charts/sda-svc/templates/intercept-secrets.yaml +++ b/charts/sda-svc/templates/intercept-secrets.yaml @@ -7,9 +7,34 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-intercept type: Opaque -data: - mqPassword: {{ required "MQ password is required" (include "mqPassInterceptor" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserInterceptor" .) | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassInterceptor" .) }} + queue: "from_cega" + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserInterceptor" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} {{- end }} {{- end }} {{- end }} From fb8766d2164f9cdce57585d1cb7340fbb8323915 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 16:09:22 +0200 Subject: [PATCH 022/262] [mapper] use config file instead of ENVs --- charts/sda-svc/templates/mapper-deploy.yaml | 127 ++----------------- charts/sda-svc/templates/mapper-secrets.yaml | 65 +++++++++- 2 files changed, 72 insertions(+), 120 deletions(-) diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml index 91e41c2de1..7c89f09343 100644 --- a/charts/sda-svc/templates/mapper-deploy.yaml +++ b/charts/sda-svc/templates/mapper-deploy.yaml @@ -54,133 +54,30 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: "mappings" - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - - name: INBOX_TYPE - {{- if eq "s3" .Values.global.inbox.storageType }} - value: "s3" - - name: INBOX_BUCKET - value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - - name: INBOX_REGION - value: {{ default "us-east-1" .Values.global.inbox.s3Region }} - - name: INBOX_URL - value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- else }} - value: "posix" - - name: INBOX_LOCATION - value: "{{ .Values.global.inbox.path }}/" - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "federated" .Values.global.schemaType }} - {{- if not .Values.global.vaultSecrets }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: dbUser - {{- if eq "s3" .Values.global.inbox.storageType }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - {{- end }} - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.mapper.resources | trim | indent 10 }} volumeMounts: + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} {{- end }} - {{- if eq "posix" .Values.global.inbox.storageType }} + {{- end }} + {{- if eq "posix" .Values.global.inbox.storageType }} - name: inbox mountPath: {{ .Values.global.inbox.path | quote }} - {{- end }} + {{- end }} volumes: + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-mapper + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} diff --git a/charts/sda-svc/templates/mapper-secrets.yaml b/charts/sda-svc/templates/mapper-secrets.yaml index 3252c43f9c..c66ed0dc15 100644 --- a/charts/sda-svc/templates/mapper-secrets.yaml +++ b/charts/sda-svc/templates/mapper-secrets.yaml @@ -6,10 +6,65 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-mapper type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassMapper" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserMapper" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassMapper" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserMapper" .) | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassMapper" .) }} + queue: {{ default "mappings" .Values.global.broker.finalizeQueue }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserMapper" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassMapper" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserMapper" .) }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} {{- end }} {{- end }} From fa545e790aca7f93c911035feb93d4a89cab790e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 17:16:06 +0200 Subject: [PATCH 023/262] [s3inbox] use config file instead of ENVs --- charts/sda-svc/templates/s3-inbox-deploy.yaml | 143 +++--------------- .../sda-svc/templates/s3-inbox-secrets.yaml | 78 +++++++++- 2 files changed, 90 insertions(+), 131 deletions(-) diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index bf97b7faa4..39ae8ea13b 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -90,124 +90,8 @@ spec: {{- toYaml .Values.global.extraSecurityContext | nindent 10 -}} {{- end }} env: - {{- if not .Values.global.vaultSecrets }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-inbox - key: mqPassword - - name: BROKER_USER - value: {{ include "mqUserInbox" . | quote }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-inbox - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-inbox - key: dbUser - {{- else }} - - name: SERVER_CONFFILE + - name: CONFIGFILE value: {{ include "confFile" .}} - {{- end }} - - name: INBOX_URL - value: {{ .Values.global.inbox.s3Url | quote }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: "{{ include "tlsPath" . }}/ca.crt" - {{- end }} - {{- if .Values.global.inbox.s3Region }} - - name: INBOX_REGION - value: {{ .Values.global.inbox.s3Region | quote }} - {{- end }} - - name: INBOX_BUCKET - value: {{ .Values.global.inbox.s3Bucket | quote }} - {{- if .Values.global.inbox.s3ReadyPath }} - - name: INBOX_READYPATH - value: {{ .Values.global.inbox.s3ReadyPath }} - {{- end }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_VHOST - value: {{ include "brokerVhost" . | quote }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_ROUTINGKEY - value: "inbox" - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.tls.enabled }} - - name: SERVER_CERT - value: {{ include "tlsPath" . }}/tls.crt - - name: SERVER_KEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- if .Values.global.auth.jwtPub }} - - name: SERVER_JWTPUBKEYPATH - value: {{ include "jwtPath" . }} - {{- end }} - {{- if .Values.global.auth.resignJwt }} - {{- $_ := required "The signing pub key is required for s3inbox when auth resigns tokens" .Values.global.auth.jwtPub }} - {{- else }} - - name: SERVER_JWTPUBKEYURL - value: {{ .Values.global.oidc.provider }}{{ .Values.global.oidc.jwkPath }} - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} ports: - name: inbox containerPort: 8000 @@ -233,9 +117,13 @@ spec: - name: tls mountPath: {{ include "tlsPath" . }} {{- end }} - {{- if .Values.global.auth.jwtPub }} + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} + {{- if or .Values.global.auth.jwtPub .Values.global.oidc.jwtPub }} - name: jwt mountPath: {{ include "jwtPath" . }} + {{- end }} {{- end }} volumes: {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} @@ -250,15 +138,18 @@ spec: secretName: {{ required "An certificate issuer or a TLS secret name is required for s3inbox" .Values.s3Ibox.tls.secretName }} {{- end }} {{- end }} - {{- if .Values.global.auth.jwtPub }} + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-inbox + {{- if or .Values.global.auth.jwtPub .Values.global.oidc.jwtPub }} - name: jwt - projected: - sources: - - secret: - name: {{ .Values.global.auth.jwtSecret }} - items: - - key: {{ required "The name of the JWT signing key is needed" .Values.global.auth.jwtPub }} - path: "{{ .Values.global.ingress.hostName.auth }}.pub" + secret: + defaultMode: 0440 + secretName: {{ ternary .Values.global.auth.jwtSecret .Values.global.oidc.jwtSecret .Values.global.auth.resignJwt }} + + {{- end }} {{- end }} restartPolicy: Always {{- end }} diff --git a/charts/sda-svc/templates/s3-inbox-secrets.yaml b/charts/sda-svc/templates/s3-inbox-secrets.yaml index ab17a02507..4afbd48c70 100644 --- a/charts/sda-svc/templates/s3-inbox-secrets.yaml +++ b/charts/sda-svc/templates/s3-inbox-secrets.yaml @@ -6,11 +6,79 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-inbox type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassInbox" .) | quote | trimall "\"" | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserInbox" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassInbox" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserInbox" .) | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassInbox" .) }} + routingKey: {{ default "inbox" .Values.global.broker.inboxRoutingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserInbox" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassInbox" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserInbox" .) }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} + server: + {{- if .Values.global.tls.enabled }} + cert: {{ include "tlsPath" . }}/tls.crt + key: {{ include "tlsPath" . }}/tls.key + {{- end }} + {{- if or .Values.global.oidc.jwtPub .Values.global.auth.resignJwt }} + jwtpubkeypath: {{ include "jwtPath" . }} + {{- if .Values.global.auth.resignJwt }} + {{- $_ := required "The signing pub key is required for s3inbox when auth resigns tokens" .Values.global.auth.jwtPub }} + {{- end }} + {{- else }} + jwtpubkeyurl: {{ required "A oidc provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/{{ required "A oidc jwk path is required" .Values.global.oidc.jwkPath | trimPrefix "/" }} + {{- end }} {{- end }} {{- end }} {{- end }} From 768629603eaf36f2ffd6f05b4edbb3e01c6a8172 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 21:08:23 +0200 Subject: [PATCH 024/262] [download] use config file instead of ENVs --- charts/sda-svc/templates/download-deploy.yaml | 142 ++---------------- .../sda-svc/templates/download-secrets.yaml | 68 ++++++++- 2 files changed, 78 insertions(+), 132 deletions(-) diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index 9b1a2134eb..8bf2e580b0 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -69,128 +69,8 @@ spec: type: "RuntimeDefault" command: ["sda-download"] env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if .Values.global.archive.s3CaFile }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/{{ .Values.global.archive.s3CaFile }} - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: OIDC_CONFIGURATION_URL - value: "{{ .Values.global.oidc.provider | trimSuffix "/" }}/.well-known/openid-configuration" - {{- if .Values.global.download.trusted.iss }} - - name: OIDC_TRUSTED_ISS - value: {{ include "trustedIssPath" . }}/{{ default "iss.json" .Values.global.download.trusted.configFile }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: DB_SSLMODE - value: {{ .Values.global.db.sslMode | quote }} - {{- else }} - - name: DB_SSLMODE - value: "disable" - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - {{- if .Values.global.tls.enabled }} - - name: GRPC_CACERT - value: {{ include "tlsPath" . }}/ca.crt - - name: GRPC_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: GRPC_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: GRPC_HOST - value: {{ required "A valid grpc host is required" .Values.global.reencrypt.host | quote }} - - name: GRPC_PORT - value: {{ .Values.global.reencrypt.port | quote }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} - - name: C4GH_TRANSIENTKEYPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }} - - name: C4GH_TRANSIENTPASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the transient c4gh key is required" .Values.global.download.serveDecrypted.secretName }} - key: passphrase - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: APP_PORT - value: "8443" - - name: APP_SERVERCERT - value: "{{ template "tlsPath" . }}/tls.crt" - - name: APP_SERVERKEY - value: "{{ template "tlsPath" . }}/tls.key" - {{- else }} - - name: SESSION_SECURE - value: "false" - {{- end }} - - name: SESSION_DOMAIN - value: {{ .Values.global.ingress.hostName.download | quote }} - - name: SESSION_EXPIRATION - value: {{ .Values.global.download.sessionExpiration | quote }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-download - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-download - key: dbUser - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} ports: - name: download containerPort: {{ ternary 8443 8080 .Values.global.tls.enabled }} @@ -228,22 +108,24 @@ spec: resources: {{ toYaml .Values.download.resources | trim | indent 10 }} volumeMounts: - {{- if not .Values.global.vaultSecrets }} + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} - name: c4gh-transient mountPath: {{ template "c4ghPath" . }} {{- end }} - name: iss mountPath: {{ template "trustedIssPath" . }} - {{- end }} - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + {{- end }} + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} - {{- end }} - {{- if eq "posix" .Values.global.archive.storageType }} + {{- end }} + {{- if eq "posix" .Values.global.archive.storageType }} - name: archive mountPath: {{ .Values.global.archive.volumePath | quote }} - {{- end }} + {{- end }} volumes: {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls @@ -258,7 +140,11 @@ spec: {{- end }} {{- end }} {{- if not .Values.global.vaultSecrets }} - {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-download + {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} - name: c4gh-transient secret: defaultMode: 0440 @@ -266,7 +152,7 @@ spec: items: - key: {{ .Values.global.download.serveDecrypted.c4ghKeyFile }} path: {{ .Values.global.download.serveDecrypted.c4ghKeyFile }} - {{- end }} + {{- end }} - name: iss secret: defaultMode: 0440 diff --git a/charts/sda-svc/templates/download-secrets.yaml b/charts/sda-svc/templates/download-secrets.yaml index 463811cfe4..f649ae85ee 100644 --- a/charts/sda-svc/templates/download-secrets.yaml +++ b/charts/sda-svc/templates/download-secrets.yaml @@ -5,10 +5,70 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-download type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassDownload" .) | b64enc }} - c4ghPassphrase: {{ .Values.global.c4gh.passphrase | b64enc }} - dbUser: {{ required "MQ user is required" (include "dbUserDownload" .) | b64enc }} +stringData: + config.yaml: |- + app: + {{- if .Values.global.tls.enabled }} + port: 8443 + serverCert: {{ template "tlsPath" . }}/tls.crt + serverKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} + c4gh: + transientKeyPath: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }} + transientKeyPassphrase: {{ .Values.global.download.serveDecrypted.passphrade }} + {{- end }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB pass is required" (include "dbPassDownload" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserDownload" .) }} + grpc: + host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }} + port: {{ ternary 50443 50051 .Values.global.tls.enabled }} + timeout: {{ default 30 .Values.global.reencrypt.timeout }} + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + oidc: + configuration: + url: {{ required "A oidc provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/.well-known/openid-configuration + trusted: + iss: {{ include "trustedIssPath" . }}/{{ default "iss.json" .Values.global.download.trusted.configFile }} + session: + domain: {{ .Values.global.ingress.hostName.download }} + expiration: {{ .Values.global.download.sessionExpiration }} + secure: {{ ternary true false .Values.global.tls.enabled }} --- apiVersion: v1 kind: Secret From 09b0c3f884cb46f4ebd721643d02a02d7ee33e9a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 12 May 2025 16:42:02 +0200 Subject: [PATCH 025/262] [re-encrypt] use config file instead of ENVs --- .../sda-svc/templates/re-encrypt-deploy.yaml | 24 +++++------------ .../sda-svc/templates/reencrypt-secrets.yaml | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+), 18 deletions(-) create mode 100644 charts/sda-svc/templates/reencrypt-secrets.yaml diff --git a/charts/sda-svc/templates/re-encrypt-deploy.yaml b/charts/sda-svc/templates/re-encrypt-deploy.yaml index a5d30a185d..9b9a0a56bd 100644 --- a/charts/sda-svc/templates/re-encrypt-deploy.yaml +++ b/charts/sda-svc/templates/re-encrypt-deploy.yaml @@ -51,26 +51,8 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: C4GH_FILEPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} - {{- if not .Values.global.vaultSecrets }} - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} - {{- if .Values.global.tls.enabled}} - - name: GRPC_CACERT - value: {{ template "tlsPath" . }}/ca.crt - - name: GRPC_SERVERCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: GRPC_SERVERKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} ports: - name: grpc containerPort: {{ ternary 50443 50051 ( .Values.global.tls.enabled ) }} @@ -86,6 +68,8 @@ spec: {{ toYaml .Values.reencrypt.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} @@ -95,6 +79,10 @@ spec: {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-reencrypt - name: c4gh secret: defaultMode: 0440 diff --git a/charts/sda-svc/templates/reencrypt-secrets.yaml b/charts/sda-svc/templates/reencrypt-secrets.yaml new file mode 100644 index 0000000000..de613fc92e --- /dev/null +++ b/charts/sda-svc/templates/reencrypt-secrets.yaml @@ -0,0 +1,26 @@ +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}} +{{- if not .Values.global.vaultSecrets }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "sda.fullname" . }}-reencrypt +type: Opaque +stringData: + config.yaml: |- + c4gh: + {{- with (first .Values.global.c4gh.privateKeys) }} + filePath: {{ template "c4ghPath" $ }}/{{ .keyName }} + passphrase: {{ required "a passphrase for the c4gh key is required" .passphrase }} + {{- end }} + {{- if .Values.global.tls.enabled }} + grpc: + caCert: {{ template "tlsPath" . }}/ca.crt + serverCert: {{ template "tlsPath" . }}/tls.crt + serverKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} +{{- end }} +{{- end }} From f9ee377005e38292cab0a184e3f3bba0757bad3e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:00:22 +0200 Subject: [PATCH 026/262] [sync] use config file instead of ENVs --- charts/sda-svc/templates/sync-deploy.yaml | 180 +-------------------- charts/sda-svc/templates/sync-secrets.yaml | 96 +++++++++-- 2 files changed, 92 insertions(+), 184 deletions(-) diff --git a/charts/sda-svc/templates/sync-deploy.yaml b/charts/sda-svc/templates/sync-deploy.yaml index 6e4bec1ddb..a96e230f45 100644 --- a/charts/sda-svc/templates/sync-deploy.yaml +++ b/charts/sda-svc/templates/sync-deploy.yaml @@ -73,184 +73,16 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: SYNC_CENTERPREFIX - value: {{ .Values.global.sync.centerPrefix}} - - name: SYNC_DESTINATION_TYPE - {{- if eq "s3" .Values.global.sync.destination.storageType }} - value: "s3" - - name: SYNC_DESTINATION_URL - value: {{ required "S3 sync archive URL missing" .Values.global.sync.destination.url }} - {{- if .Values.global.sync.destination.port }} - - name: SYNC_DESTINATION_PORT - value: {{ .Values.global.sync.destination.port | quote }} - {{- end }} - - name: SYNC_DESTINATION_BUCKET - value: {{ required "S3 sync archive bucket missing" .Values.global.sync.destination.bucket }} - - name: SYNC_DESTINATION_REGION - value: {{ default "us-east-1" .Values.global.sync.destination.region }} - - name: SYNC_DESTINATION_CHUNKSIZE - value: {{ .Values.global.sync.destination.s3ChunkSize | quote }} - {{- if and .Values.global.sync.destination.s3CaFile .Values.global.tls.enabled }} - - name: SYNC_DESTINATION_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- end }} - - name: C4GH_FILEPATH - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}" - - name: C4GH_SYNCPUBKEYPATH - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.syncPubKey }}" - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: {{ default "sync" .Values.global.sync.brokerQueue }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "sda" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "isolated" .Values.global.schemaType }} - - name: SYNC_REMOTE_HOST - value: {{ .Values.global.sync.remote.host }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - {{- if eq "s3" .Values.global.sync.destination.storageType }} - - name: SYNC_DESTINATION_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: s3AccessKey - - name: SYNC_DESTINATION_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: s3SecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: mqUser - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: dbUser - - name: SYNC_REMOTE_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: restUser - - name: SYNC_REMOTE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: restPassword - {{ else }} - - name: CONFIGFILE - value: {{ include "confFile" . }} - {{- end }} + - name: CONFIGFILE + value: {{ include "confFile" . }} resources: {{ toYaml .Values.sync.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} - name: c4gh mountPath: {{ template "c4ghPath" . }} + - name: config + mountPath: {{ template "configPath" . }} {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive @@ -283,6 +115,10 @@ spec: path: {{ .Values.global.c4gh.keyFile }} - key: {{ .Values.global.c4gh.syncPubKey }} path: {{ .Values.global.c4gh.syncPubKey }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-sync {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index c8f9154aa3..cc4c50751b 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -8,18 +8,90 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-sync type: Opaque -data: - c4ghPassphrase: {{ required "Crypt4gh passphrase is required" .Values.global.c4gh.passphrase | b64enc }} - dbPassword: {{ required "DB password is required" (include "dbPassSync" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserSync" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassSync" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserSync" .) | b64enc }} - restPassword: {{ required "REST password is required" .Values.global.sync.remote.password | b64enc }} - restUser: {{ required "REST user is required" .Values.global.sync.remote.user | b64enc }} - {{- if eq "s3" .Values.global.sync.destination.storageType }} - s3AccessKey: {{ required "Accesskey required for sync destination" .Values.global.sync.destination.accessKey | quote | trimall "\"" | b64enc }} - s3SecretKey: {{ required "Secretkey required for sync destination" .Values.global.sync.destination.secretKey | quote | trimall "\"" | b64enc }} - {{- end }} +stringData: + config.yaml: |- + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassMapper" .) }} + queue: {{ default "mappings" .Values.global.broker.finalizeQueue }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserMapper" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + c4gh: + filePath: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} + passphrase: {{ .Values.global.c4gh.passphrase }} + syncPubKeyPath: {{ template "c4ghPath" . }}/{{ required "remote sync public c4gh key is missing" .Values.global.c4gh.syncPubKey }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassMapper" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserMapper" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + sync: + centerPrefix: {{ .Values.global.sync.centerPrefix }} + destination: + type: s3 + url: {{ required "S3 sync archive URL missing" .Values.global.sync.destination.url }} + {{- if not (empty .Values.global.sync.destination.port) }} + port: {{ .Values.global.sync.destination.port }} + {{- end }} + {{- if not (empty .Values.global.sync.destination.readypath) }} + readypath: {{ .Values.global.sync.destination.readypath }} + {{- end }} + accessKey: {{ required "Accesskey required for sync destination" .Values.global.sync.destination.accessKey }} + secretKey: {{ required "SecretKey required for sync destination" .Values.global.sync.destination.secretKey }} + bucket: {{ required "S3 sync destination bucket missing" .Values.global.sync.destination.bucket }} + region: {{ default "us-east-1" .Values.global.sync.destination.region }} + remote: + host: {{ required "remote sync API host is required" .Values.global.sync.remote.host }} + password: {{ required "remote sync API password is required" .Values.global.sync.remote.password }} + {{- if not (empty .Values.global.sync.remote.port) }} + port: {{ .Values.global.sync.remote.port }} + {{- end }} + user: {{ required "remote sync API user is required" .Values.global.sync.remote.user }} {{- end }} {{- end }} {{- end }} From 502b1c8b383dc8a4ea2f6dba5095ebdc582c2192 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:00:32 +0200 Subject: [PATCH 027/262] [sync-api] use config file instead of ENVs --- charts/sda-svc/templates/syncapi-deploy.yaml | 85 +++---------------- charts/sda-svc/templates/syncapi-secrets.yaml | 49 +++++++++-- 2 files changed, 54 insertions(+), 80 deletions(-) diff --git a/charts/sda-svc/templates/syncapi-deploy.yaml b/charts/sda-svc/templates/syncapi-deploy.yaml index 2a9cd16a43..970f69f9d0 100644 --- a/charts/sda-svc/templates/syncapi-deploy.yaml +++ b/charts/sda-svc/templates/syncapi-deploy.yaml @@ -73,83 +73,8 @@ spec: {{- toYaml .Values.global.extraSecurityContext | nindent 10 -}} {{- end }} env: - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: mqUser - - name: SYNC_API_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: password - - name: SYNC_API_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: user - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: SERVER_CERT - value: {{ template "tlsPath" . }}/tls.crt - - name: SERVER_KEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- if .Values.global.sync.api.accessionRouting }} - - name: SYNC_API_ACCESSIONROUTING - value: {{ .Values.global.sync.api.accessionRouting | quote }} - {{- end }} - {{- if .Values.global.sync.api.ingestRouting }} - - name: SYNC_API_INGESTROUTING - value: {{ .Values.global.sync.api.ingestRouting | quote }} - {{- end }} - {{- if .Values.global.sync.api.mappingRouting }} - - name: SYNC_API_MAPPINGROUTING - value: {{ .Values.global.sync.api.mappingRouting | quote }} - {{- end }} ports: - name: syncapi containerPort: 8080 @@ -162,11 +87,21 @@ spec: resources: {{ toYaml .Values.syncapi.resources | trim | indent 10 }} volumeMounts: + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} + {{- end }} {{- if .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} {{- end }} volumes: + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-syncapi + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} diff --git a/charts/sda-svc/templates/syncapi-secrets.yaml b/charts/sda-svc/templates/syncapi-secrets.yaml index 78a5bfcf63..2c55c2345c 100644 --- a/charts/sda-svc/templates/syncapi-secrets.yaml +++ b/charts/sda-svc/templates/syncapi-secrets.yaml @@ -8,11 +8,50 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-syncapi type: Opaque -data: - mqPassword: {{ required "MQ password is required" (include "mqPassSync" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserSync" .) | b64enc }} - password: {{ required "Sync API password is required" .Values.global.sync.api.password | b64enc }} - user: {{ required "Sync API username is required" .Values.global.sync.api.user | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassMapper" .) }} + queue: {{ default "mappings" .Values.global.broker.finalizeQueue }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserMapper" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + {{- if .Values.global.tls.enabled }} + server: + cert: {{ template "tlsPath" . }}/tls.crt + key: {{ template "tlsPath" . }}/tls.key + {{- end }} + sync: + api: + password: {{ required "sync API password is required" .Values.global.sync.api.password }} + user: {{ required "sync API user is required" .Values.global.sync.api.user }} + brokerQueue: {{ default "mappings" .Values.global.sync.brokerQueue }} + centerPrefix: {{ .Values.global.sync.centerPrefix }} + remote: + host: {{ required "remote sync API host is required" .Values.global.sync.remote.host }} + password: {{ required "remote sync API password is required" .Values.global.sync.remote.password }} + port: {{ .Values.global.sync.remote.port }} + user: {{ required "remote sync API user is required" .Values.global.sync.remote.user }} {{- end }} {{- end }} {{- end }} From ceadbf6d5f446734a1be91b0556be70b091dad88 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:01:16 +0200 Subject: [PATCH 028/262] [helpers] ensure there is no double slashes in paths --- charts/sda-svc/templates/_helpers.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 0d66679185..29e483226d 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -102,28 +102,28 @@ Create chart name and version as used by the chart label. {{- end -}} {{- define "configPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.confFilePath) (empty .Values.global.confFilePath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.confFilePath | trimPrefix "/")) (empty .Values.global.confFilePath) -}} {{- end -}} {{- define "c4ghPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.c4ghPath) (empty .Values.global.c4ghPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.c4ghPath | trimPrefix "/")) (empty .Values.global.c4ghPath) -}} {{- end -}} {{- define "trustedIssPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.download.trusted.configPath) (empty .Values.global.download.trusted.configPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.download.trusted.configPath | trimPrefix "/")) (empty .Values.global.download.trusted.configPath) -}} {{- end -}} {{- define "tlsPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.tlsPath) (empty .Values.global.tlsPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.tlsPath | trimPrefix "/")) (empty .Values.global.tlsPath) -}} {{- end -}} {{- define "jwtPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.jwtPath) (empty .Values.global.jwtPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.jwtPath | trimPrefix "/")) (empty .Values.global.jwtPath) -}} {{- end -}} {{- define "confFile" -}} {{- if .Values.global.confFilePath -}} - {{- printf "%s/%s/%s" .Values.global.secretsPath .Values.global.confFilePath .Values.global.confFile }} + {{- printf "%s/%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.confFilePath | trimPrefix "/") .Values.global.confFile }} {{- else }} {{- printf "%s/%s" .Values.global.secretsPath .Values.global.confFile -}} {{ end }} From c95d278d876fe1a8bdad336e505042146a805321 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:04:13 +0200 Subject: [PATCH 029/262] [sda-svc] Remove obsolete entries from the values file --- charts/sda-svc/values.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 40494fd036..c0db7ebb96 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -401,8 +401,6 @@ auth: doa: name: doa - repository: ghcr.io/neicnordic/sda-doa - imageTag: v1.6.62 imagePullPolicy: IfNotPresent replicaCount: 2 resources: @@ -474,7 +472,6 @@ ingest: secretName: "" intercept: - deploy: true name: ingest replicaCount: 1 resources: @@ -606,7 +603,6 @@ releasetest: verify: replicaCount: 1 repository: ghcr.io/neicnordic/sda-pipeline - imageTag: v0.4.27 imagePullPolicy: IfNotPresent resources: requests: From 59c139f7d14809686255e02092ff15d9d9454b25 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:07:12 +0200 Subject: [PATCH 030/262] [sda-svc] update readme --- charts/sda-svc/README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 7ca4df4bba..9915059936 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -236,8 +236,6 @@ Parameter | Description | Default `sync.resources.limits.cpu` | CPU limit for sync container. |`250m` `sync.deploy` | Set to true if the sync service should be active | `false` `doa.replicaCount` | desired number of replicas | `2` -`doa.repository` | dataedge container image repository | `neicnordic/sda-doa` -`doa.imageTag` | dataedge container image version | `"latest"` `doa.keystorePass` | keystore password | `changeit` `doa.annotations` | Specific annotation for the doa pod | `{}` `doa.resources.requests.memory` | Memory request for dataedge container. |`128Mi` @@ -264,7 +262,6 @@ Parameter | Description | Default `ingest.resources.limits.cpu` | CPU limit for ingest container. |`2000m` `intercept.replicaCount` | desired number of intercept workers | `1` `intercept.annotations` | Specific annotation for the intercept pod | `{}` -`intercept.deploy` | Set to false in a non federated deployment | `true` `intercept.resources.requests.memory` | Memory request for intercept container. |`32Mi` `intercept.resources.requests.cpu` | CPU request for intercept container. |`100m` `intercept.resources.limits.memory` | Memory limit for intercept container. |`128Mi` From 1de2ab83d19d872348d736226d098f6af07e4a24 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:10:14 +0200 Subject: [PATCH 031/262] [sda-svc] bump chart version --- charts/sda-svc/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index b40545bbe5..35a5a9a75a 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.16 -appVersion: v0.5.16 +version: 2.0.0 +appVersion: v2.0.0 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From a2a74f9e203bc2b282e271b40c0a63c73584b481 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 21 May 2025 12:53:48 +0200 Subject: [PATCH 032/262] [sda-db] bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 10348d29ba..e66cfb67c4 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.85 -appVersion: v0.5.16 +version: 2.0.0 +appVersion: v2.0.0 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 736ea18a3496200c435146a7146e337b4d36fd10 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 21 May 2025 12:53:58 +0200 Subject: [PATCH 033/262] [sda-mq] bump chart version --- charts/sda-mq/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 4338581df0..9564f1a438 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.86 -appVersion: v0.5.16 +version: 2.0.0 +appVersion: v2.0.0 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From fd38a74de04c7220f7c07c1598dfa4ff1d79d38d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 22 May 2025 14:11:48 +0200 Subject: [PATCH 034/262] Update charts/sda-svc/templates/s3-inbox-secrets.yaml Co-authored-by: Panos Chatzopoulos --- charts/sda-svc/templates/s3-inbox-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/s3-inbox-secrets.yaml b/charts/sda-svc/templates/s3-inbox-secrets.yaml index 4afbd48c70..340a6a7f86 100644 --- a/charts/sda-svc/templates/s3-inbox-secrets.yaml +++ b/charts/sda-svc/templates/s3-inbox-secrets.yaml @@ -77,7 +77,7 @@ stringData: {{- $_ := required "The signing pub key is required for s3inbox when auth resigns tokens" .Values.global.auth.jwtPub }} {{- end }} {{- else }} - jwtpubkeyurl: {{ required "A oidc provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/{{ required "A oidc jwk path is required" .Values.global.oidc.jwkPath | trimPrefix "/" }} + jwtpubkeyurl: {{ required "An OIDC provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/{{ required "An OIDC JWK path is required" .Values.global.oidc.jwkPath | trimPrefix "/" }} {{- end }} {{- end }} {{- end }} From b237aedbb4fcb1c51514e868c87311d2ec44ee23 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:23:37 +0200 Subject: [PATCH 035/262] [integration tests] update chart values file --- .github/integration/scripts/charts/values.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index 7c119d3cde..6847412194 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -25,6 +25,7 @@ global: s3Url: "http://minio.minio" s3Bucket: "archive" s3Port: 9000 + s3ReadyPath: "/minio/health/ready" existingClaim: archive-pvc backupArchive: storageType: "s3" @@ -33,6 +34,7 @@ global: s3Url: "http://minio.minio" s3Bucket: "backup" s3Port: 9000 + s3ReadyPath: "/minio/health/ready" existingClaim: backup-pvc auth: jwtSecret: jwk @@ -49,6 +51,7 @@ global: username: "admin" password: PLACEHOLDER_VALUE cega: + host: https://cega-nss:8443/username password: PLACEHOLDER_VALUE user: PLACEHOLDER_VALUE c4gh: @@ -57,6 +60,9 @@ global: publicFile: c4gh.pub.pem passphrase: PLACEHOLDER_VALUE syncPubKey: c4gh.pub.pem + privateKeys: + - keyName: c4gh.sec.pem + passphrase: PLACEHOLDER_VALUE db: host: "postgres-sda-db" user: "postgres" @@ -90,8 +96,8 @@ global: port: 50443 sync: api: - password: "" - user: "" + password: "apiuser" + user: "apipass" brokerQueue: "mapping_stream" centerPrefix: "SYNC" destination: @@ -104,7 +110,7 @@ global: bucket: "sync" region: "us-east-1" remote: - host: "" + host: "http://remote-sync" port: "8080" password: "pass" user: "user" @@ -127,7 +133,7 @@ finalize: ingest: resources: null intercept: - deploy: false + resources: null mapper: resources: null releasetest: From d2a28078433778b01e58b60d1c376d468af9f9d7 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 19 May 2025 13:24:22 +0200 Subject: [PATCH 036/262] [integration tests] add option to deploy charts as federated or isolated --- .github/integration/scripts/charts/cleanup.sh | 4 +- .../scripts/charts/dependencies.sh | 19 ++++++ .../scripts/charts/dependencies.yaml | 61 ++++++++++++++++++- .../scripts/charts/deploy_charts.sh | 7 +++ .github/integration/scripts/charts/k3d.sh | 6 +- .github/workflows/build_pr_container.yaml | 13 +++- Makefile | 20 +++--- README.md | 15 +++-- 8 files changed, 125 insertions(+), 20 deletions(-) diff --git a/.github/integration/scripts/charts/cleanup.sh b/.github/integration/scripts/charts/cleanup.sh index bbc6c3eeaa..d063854c4e 100644 --- a/.github/integration/scripts/charts/cleanup.sh +++ b/.github/integration/scripts/charts/cleanup.sh @@ -16,4 +16,6 @@ helm uninstall broker || true helm uninstall postgres || true kubectl delete secrets api-rbac broker-sda-mq-certs broker-sda-mq-test-certs postgres-sda-db-certs postgres-sda-db-test-certs || true -rm /tmp/values.yaml /tmp/c4gh.pub.pem /tmp/c4gh.sec.pem /tmp/jwt.key /tmp/jwt.pub \ No newline at end of file +kubectl delete deployment.apps/cega-nss || true +kubectl delete cm cega-nss || true +rm /tmp/values.yaml /tmp/c4gh.pub.pem /tmp/c4gh.sec.pem /tmp/jwt.key /tmp/jwt.pub /tmp/users.json || true \ No newline at end of file diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index 1b885de96b..f9d6f47a4e 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -99,6 +99,10 @@ if [ "$1" == "local" ]; then fi ## update values file with all credentials +if [ "$2" == "federated" ]; then + yq -i '.global.schemaType = federated' "$values_file" +fi + yq -i ' .global.archive.s3AccessKey = strenv(MINIO_ACCESS) | .global.archive.s3SecretKey = strenv(MINIO_SECRET) | @@ -106,6 +110,7 @@ yq -i ' .global.backupArchive.s3SecretKey = strenv(MINIO_SECRET) | .global.broker.password = strenv(MQPASSWORD) | .global.c4gh.passphrase = strenv(C4GHPASSPHRASE) | +.global.c4gh.privateKeys[0].passphrase = strenv(C4GHPASSPHRASE) | .global.db.password = strenv(PGPASSWORD) | .global.inbox.s3AccessKey = strenv(MINIO_ACCESS) | .global.inbox.s3SecretKey = strenv(MINIO_SECRET) | @@ -115,3 +120,17 @@ yq -i ' ' "$values_file" kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json" + +cat >/tmp/users.json < Date: Thu, 24 Apr 2025 14:14:59 +0200 Subject: [PATCH 037/262] Fix config file for running admin-API with go run --- sda/config_local.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 8355143fdb..d2eb9f0885 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -3,6 +3,11 @@ log: level: "debug" api: rbacFile: ../.github/integration/sda/rbac.json + port: 8090 + +grpc: + host: "localhost" + port: 50051 archive: type: s3 @@ -86,7 +91,7 @@ server: cert: "" key: "" jwtpubkeypath: "/tmp/shared/keys/pub/" - jwtpubkeyurl: "http://oidc:8080/jwk" + jwtpubkeyurl: "http://localhost:8800/oidc/jwk" sync: api: From 8b9427b11fc48f4d9ec1a9abb0084215b8787db0 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 25 Apr 2025 13:52:05 +0200 Subject: [PATCH 038/262] [api] Add endpoint for downloading file from the inbox --- sda/cmd/api/api.go | 230 ++++++++++++++++++++++++++++++++-- sda/internal/config/config.go | 10 ++ 2 files changed, 229 insertions(+), 11 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index b57981b1a7..a8599e71cb 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -4,11 +4,13 @@ import ( "bytes" "context" "crypto/tls" + "crypto/x509" "encoding/base64" "encoding/hex" "encoding/json" "errors" "fmt" + "io" "math" "net/http" "os" @@ -22,14 +24,19 @@ import ( "github.com/gin-gonic/gin" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/neicnordic/crypt4gh/keys" + "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" + "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" ) type dataset struct { @@ -104,16 +111,18 @@ func setup(conf *config.Config) *http.Server { r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash r.DELETE("/file/:username/:fileid", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here - r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file - r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file - r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file - r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset - r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible - r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset - r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status - r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specififc user - r.GET("/users", rbac(e), listActiveUsers) // Lists all users - r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user + r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file + r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file + r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file + r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset + r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible + r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset + r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status + r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specififc user + r.GET("/users", rbac(e), listActiveUsers) // Lists all users + r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user + r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file for a user + cfg := &tls.Config{MinVersion: tls.VersionTLS12} srv := &http.Server{ @@ -311,7 +320,7 @@ func deleteFile(c *gin.Context) { } // Get the file path from the fileID and submission user - filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) + filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID, true) if err != nil { log.Errorf("getting file from fileID failed, reason: (%v)", err) c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") @@ -344,6 +353,205 @@ func deleteFile(c *gin.Context) { c.Status(http.StatusOK) } +// reencryptHeader re-encrypts the header of a file using the public key +// provided in the request header and returns the new header. The function uses +// gRPC to communicate with the re-encrypt service and handles TLS configuration +// if needed. The function also handles the case where the CA certificate is +// provided for secure communication. +func reencryptHeader(oldHeader []byte, reEncKey string) ([]byte, error) { + var opts []grpc.DialOption + switch { + case Conf.ReEncrypt.ServerKey != "" && Conf.ReEncrypt.ServerCert != "": + rootCAs, err := x509.SystemCertPool() + if err != nil { + log.Errorf("failed to read system CAs: %v, using an empty pool as base", err) + rootCAs = x509.NewCertPool() + } + if Conf.ReEncrypt.CACert != "" { + cacertByte, err := os.ReadFile(Conf.ReEncrypt.CACert) + if err != nil { + log.Errorf("Failed to read CA certificate file, reason: %s", err) + + return nil, err + } + ok := rootCAs.AppendCertsFromPEM(cacertByte) + if !ok { + log.Errorf("Failed to append CA certificate to rootCAs") + + return nil, errors.New("failed to append CA certificate to cert pool") + } + } + + certs, err := tls.LoadX509KeyPair(Conf.ReEncrypt.ServerCert, Conf.ReEncrypt.ServerKey) + if err != nil { + log.Errorf("Failed to load client key pair for reencrypt, reason: %s", err) + + return nil, err + } + clientCreds := credentials.NewTLS( + &tls.Config{ + Certificates: []tls.Certificate{certs}, + MinVersion: tls.VersionTLS13, + RootCAs: rootCAs, + }, + ) + + opts = append(opts, grpc.WithTransportCredentials(clientCreds)) + default: + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + address := fmt.Sprintf("%s:%d", Conf.ReEncrypt.Host, Conf.ReEncrypt.Port) + log.Debugf("Address of the reencrypt service: %s", address) + + conn, err := grpc.NewClient(address, opts...) + if err != nil { + log.Errorf("Failed to connect to the reencrypt service, reason: %s", err) + + return nil, err + } + defer conn.Close() + + timeoutDuration := time.Duration(Conf.ReEncrypt.Timeout) * time.Second + ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + defer cancel() + + c := reencrypt.NewReencryptClient(conn) + res, err := c.ReencryptHeader(ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: reEncKey}) + if err != nil { + log.Errorf("Failed response from the reencrypt service, reason: %s", err) + + return nil, err + } + + return res.Header, nil +} + +// The downloadFile function download a file re-encrypted with the public key +// provided in the request header from the inbox. It retrieves the file path +// from the database using the file ID and user ID. +func downloadFile(c *gin.Context) { + inbox, err := storage.NewBackend(Conf.Inbox) + if err != nil { + log.Error(err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + + // Extract the username and file ID from the URL parameters. + submissionUser := c.Param("username") + fileID := c.Param("fileid") + submissionUser = strings.TrimPrefix(submissionUser, "/") + fileID = strings.TrimPrefix(fileID, "/") + + if fileID == "" { + c.AbortWithStatusJSON(http.StatusBadRequest, "file ID is required") + + return + } + + // Retrieve the actual file path for the user's file. + filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID, false) + if err != nil { + log.Errorf("getting file path from fileID (%s) failed, reason: (%v)", fileID, err) + c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in the inbox") + + return + } + + // Get inbox file handle + file, err := inbox.NewFileReader(filePath) + if err != nil { + log.Errorf("could not find inbox file %s, %s", filePath, err) + c.String(http.StatusInternalServerError, "inbox error") + + return + } + + // get the header of the crypt4gh file + headerConsumed := false + header, err := Conf.API.DB.GetHeader(fileID) + if err != nil { + log.Debugf("failed to get header for fileID %s from the database, reason: %v", fileID, err) + log.Debugln("trying to read header from the file") + + header, err = headers.ReadHeader(file) + headerConsumed = true + if err != nil { + log.Debugf("failed to read header for fileID %s, reason: %v", fileID, err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + + return + } + } + + decryptedChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileID) + if err != nil { + log.Debugf("Warning: failed to get decrypted checksum for fileID %s, reason: %v", fileID, err) + } + + // Get the public key from the request header. + reEncKey := c.GetHeader("Client-Public-Key") + if reEncKey == "" { + c.String(http.StatusBadRequest, "client public key is not provided in the header") + + return + } + + // Set the headers for the response. + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("filename: %v", fileID)) + c.Header("ETag", fmt.Sprintf("decryptedChecksum: %s", decryptedChecksum)) + + newHeader, err := reencryptHeader(header, reEncKey) + if err != nil { + log.Errorf("failed to reencrypt header, reason: %v", err) + c.String(http.StatusInternalServerError, "failed to reencrypt header") + + return + } + + // Skip the original header in the file if it hasn't been consumed yet. + if !headerConsumed { + headerLength := int64(len(header)) + _, err = io.CopyN(io.Discard, file, headerLength) + if err != nil { + log.Errorf("failed to skip original header, reason: %v", err) + c.String(http.StatusInternalServerError, "failed to skip original header") + + return + } + } + + fileStream := io.MultiReader(bytes.NewReader(newHeader), file) + + err = sendStream(c.Writer, fileStream) + if err != nil { + log.Errorf("error occurred while sending stream: %v", err) + c.String(http.StatusInternalServerError, "an error occurred") + + return + } + + c.Status(http.StatusOK) +} + +// sendStream streams file contents from a reader +var sendStream = func(w http.ResponseWriter, file io.Reader) error { + n, err := io.Copy(w, file) + if err != nil { + log.Errorf("file streaming failed, reason: %v", err) + http.Error(w, "file streaming failed", http.StatusInternalServerError) + + return err + } + + log.Debugf("Sent %d bytes", n) + + return nil +} + func setAccession(c *gin.Context) { var accession schema.IngestionAccession if err := c.BindJSON(&accession); err != nil { diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 27de641f69..a023ee4592 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -54,6 +54,7 @@ type Config struct { type ReEncConfig struct { APIConf Crypt4GHKey *[32]byte + Timeout int } type Sync struct { @@ -493,6 +494,11 @@ func NewConfig(app string) (*Config, error) { return nil, err } c.configSchemas() + + err = c.configReEncryptServer() + if err != nil { + return nil, err + } case "auth": c.Auth.Cega.AuthURL = viper.GetString("auth.cega.authUrl") c.Auth.Cega.ID = viper.GetString("auth.cega.id") @@ -901,6 +907,7 @@ func (c *Config) configOrchestrator() { } func (c *Config) configReEncryptServer() (err error) { + viper.SetDefault("grpc.timeout", 5) // set default to 5 seconds if viper.IsSet("grpc.host") { c.ReEncrypt.Host = viper.GetString("grpc.host") } @@ -916,6 +923,9 @@ func (c *Config) configReEncryptServer() (err error) { if viper.IsSet("grpc.serverkey") { c.ReEncrypt.ServerKey = viper.GetString("grpc.serverkey") } + if viper.IsSet("grpc.timeout") { + c.ReEncrypt.Timeout = viper.GetInt("grpc.timeout") + } if c.ReEncrypt.ServerCert != "" && c.ReEncrypt.ServerKey != "" { c.ReEncrypt.Port = 50443 From 11728f16da59c28899b227e37df9029047e0c10e Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 25 Apr 2025 13:55:36 +0200 Subject: [PATCH 039/262] Add reencrypt parameters to the yaml config of the integration test --- .github/integration/sda/config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index c14fd719e1..e5473a023f 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -13,6 +13,10 @@ archive: bucket: "archive" region: "us-east-1" +grpc: + host: "reencrypt" + port: 50051 + auth: cega: authUrl: "http://cega-nss:8443/username/" From 8df6918f44d56b33bd4aaaae8a42a61adf114e4a Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 25 Apr 2025 13:56:54 +0200 Subject: [PATCH 040/262] Enable permission for downloading files from the inbox in rBAC --- .github/integration/sda/rbac.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 7d37530658..8e49ee7472 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -45,6 +45,11 @@ "path": "/users/:username/files", "action": "GET" }, + { + "role": "submission", + "path": "/users/:username/file/*", + "action": "GET" + }, { "role": "*", "path": "/datasets", From 1b30aa6212042feffcac05f03c18e5e70b85b7b3 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 25 Apr 2025 16:18:35 +0200 Subject: [PATCH 041/262] Fix unit test for API config --- sda/internal/config/config_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 4c33844ff2..8ce67166e8 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -226,8 +226,14 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { assert.Error(ts.T(), err) assert.Nil(ts.T(), config) - // testing deafult values + // testing default values ts.SetupTest() + keyPath, _ := os.MkdirTemp("", "key") + keyFile := keyPath + "/c4gh.key" + _, err = helper.CreatePrivateKeyFile(keyFile, "test") + viper.Set("c4gh.filepath", keyPath+"/c4gh.key") + viper.Set("c4gh.passphrase", "test") + assert.NoError(ts.T(), err) config, err = NewConfig("api") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) @@ -248,6 +254,8 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { viper.Set("api.session.secure", false) viper.Set("api.session.domain", "test") viper.Set("api.session.expiration", 60) + viper.Set("c4gh.filepath", keyPath+"/c4gh.key") + viper.Set("c4gh.passphrase", "test") config, err = NewConfig("api") assert.NotNil(ts.T(), config) @@ -258,6 +266,8 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { assert.Equal(ts.T(), false, config.API.Session.Secure) assert.Equal(ts.T(), "test", config.API.Session.Domain) assert.Equal(ts.T(), 60*time.Second, config.API.Session.Expiration) + + defer os.RemoveAll(keyPath) } func (ts *ConfigTestSuite) TestNotifyConfiguration() { From dc26e1480e1f4fdaee2d044f1968546533228443 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 28 Apr 2025 14:26:42 +0200 Subject: [PATCH 042/262] [api] add unit tests for downloadFile --- sda/cmd/api/api.go | 9 +- sda/cmd/api/api_test.go | 196 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 201 insertions(+), 4 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index a8599e71cb..7bbc82e724 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -46,9 +46,10 @@ type dataset struct { } var ( - Conf *config.Config - err error - auth *userauth.ValidateFromToken + Conf *config.Config + err error + auth *userauth.ValidateFromToken + reencryptHeaderFunc = reencryptHeader // for testing purposes ) func main() { @@ -504,7 +505,7 @@ func downloadFile(c *gin.Context) { c.Header("Content-Disposition", fmt.Sprintf("filename: %v", fileID)) c.Header("ETag", fmt.Sprintf("decryptedChecksum: %s", decryptedChecksum)) - newHeader, err := reencryptHeader(header, reEncKey) + newHeader, err := reencryptHeaderFunc(header, reEncKey) if err != nil { log.Errorf("failed to reencrypt header, reason: %v", err) c.String(http.StatusInternalServerError, "failed to reencrypt header") diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index c8a38b4387..6004234e8b 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/sha256" "database/sql" + "encoding/base64" "encoding/json" "fmt" "io" @@ -22,6 +23,8 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" _ "github.com/lib/pq" + "github.com/neicnordic/crypt4gh/keys" + "github.com/neicnordic/crypt4gh/streaming" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" @@ -31,6 +34,7 @@ import ( "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" log "github.com/sirupsen/logrus" + "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -229,6 +233,12 @@ func TestMain(m *testing.M) { os.Exit(code) } +type ClientKey struct { + PubKey [32]byte + PrivKeyPath string + PubKeyBase64 string +} + type TestSuite struct { suite.Suite Path string @@ -238,6 +248,8 @@ type TestSuite struct { RBAC []byte Token string User string + PubKeyList [][32]byte + ClientKey ClientKey } func (s *TestSuite) TestShutdown() { @@ -388,12 +400,50 @@ func (s *TestSuite) SetupSuite() { {"role":"submission","path":"/file/accession","action":"POST"}, {"role":"submission","path":"/users","action":"GET"}, {"role":"submission","path":"/users/:username/files","action":"GET"}, + {"role":"submission","path":"/users/:username/file/*","action":"GET"}, {"role":"*","path":"/files","action":"GET"}], "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"admin"}]}`) + + c.ReEncrypt.Host = "localhost" + c.ReEncrypt.Port = 50051 + c.ReEncrypt.Crypt4GHKey, _ = config.GetC4GHKey() + + Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") + if err != nil { + s.FailNow("failed to create temp folder") + } + + keyFile1 := fmt.Sprintf("%s/c4gh1.key", s.Path) + keyFile2 := fmt.Sprintf("%s/c4gh2.key", s.Path) + + publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test") + if err != nil { + s.FailNow("Failed to create c4gh key") + } + // Add only the first public key to the list + s.PubKeyList = append(s.PubKeyList, publicKey) + + _, err = helper.CreatePrivateKeyFile(keyFile2, "test") + if err != nil { + s.FailNow("Failed to create c4gh key") + } + + viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ + {FilePath: keyFile1, Passphrase: "test"}, + {FilePath: keyFile2, Passphrase: "test"}, + }) + + s.ClientKey.PrivKeyPath = fmt.Sprintf("%s/client.key", s.Path) + s.ClientKey.PubKey, err = helper.CreatePrivateKeyFile(s.ClientKey.PrivKeyPath, "test") + if err != nil { + s.FailNow("Failed to create c4gh key") + } + s.ClientKey.PubKeyBase64 = base64.StdEncoding.EncodeToString(s.ClientKey.PubKey[:]) } func (s *TestSuite) TearDownSuite() { assert.NoError(s.T(), os.RemoveAll(s.Path)) + assert.NoError(s.T(), os.RemoveAll(Conf.Inbox.Posix.Location)) } func (s *TestSuite) SetupTest() { Conf.Database = database.DBConf{ @@ -2152,3 +2202,149 @@ func (s *TestSuite) TestReVerifyDataset_wrongDatasetName() { defer okResponse.Body.Close() assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } + +func (s *TestSuite) TestDownloadFileSuccess() { + origReencryptHeaderFunc := reencryptHeaderFunc + reencryptHeaderFunc = func(oldHeader []byte, reEncKey string) ([]byte, error) { + return oldHeader, nil + } + defer func() { reencryptHeaderFunc = origReencryptHeaderFunc }() + + gin.SetMode(gin.ReleaseMode) + r := gin.Default() + m, err := model.NewModelFromString(jsonadapter.Model) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") + } + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") + } + + r.GET("/users/:username/file/:fileid", rbac(e), func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + assert.NoError(s.T(), setupJwtAuth()) + + err = os.MkdirAll(path.Join(Conf.Inbox.Posix.Location, s.User), 0750) + assert.NoError(s.T(), err, "failed to create inbox directory") + + // Create a test file in the inbox directory + fileContent := []byte("This is the content of the test file.") + contentReader := bytes.NewReader(fileContent) + testFilePath := path.Join(Conf.Inbox.Posix.Location, s.User, "test_download.c4gh") + + outFile, err := os.Create(testFilePath) + if err != nil { + s.FailNow("failed to create encrypted test file") + } + defer outFile.Close() + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + s.FailNow("failed to create private c4gh key") + } + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, s.PubKeyList, nil) + if err != nil { + s.FailNow("failed to create c4gh writer") + } + + _, err = io.Copy(crypt4GHWriter, contentReader) + if err != nil { + s.FailNow("failed to write predefined data to encrypted test file") + } + crypt4GHWriter.Close() + + defer os.Remove(testFilePath) + log.Debugf("test file created: %s", testFilePath) + + // Now read back the entire encrypted file and store it as expectedContent. + encryptedFile, err := os.Open(testFilePath) + if err != nil { + s.FailNow("failed to open encrypted test file for reading") + } + defer encryptedFile.Close() + + expectedContent, err := io.ReadAll(encryptedFile) + if err != nil { + s.FailNow("failed to read encrypted test file") + } + encryptedFile.Close() + + // Register the file in the database + filePath := fmt.Sprintf("/%v/test_download.c4gh", s.User) + fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + // Mock request to download the file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("Authorization", "Bearer "+s.Token) + req.Header.Set("Client-Public-Key", s.ClientKey.PubKeyBase64) + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) + assert.Equal(s.T(), "application/octet-stream", resp.Header.Get("Content-Type")) + + // Check the content of the response + actualContent, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err, "failed to read response body") + assert.Equal(s.T(), expectedContent, actualContent) +} + +func (s *TestSuite) TestDownloadFile_FileNotExist() { + gin.SetMode(gin.ReleaseMode) + r := gin.Default() + m, err := model.NewModelFromString(jsonadapter.Model) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") + } + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") + } + + r.GET("/users/:username/file/:fileid", rbac(e), func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + assert.NoError(s.T(), setupJwtAuth()) + + // Register a file in the database (but don't create the actual file) + filePath := fmt.Sprintf("/%v/nonexistent.c4gh", s.User) + fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + // Mock request to download the file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("Authorization", "Bearer "+s.Token) + req.Header.Set("Client-Public-Key", "base64-key-string") + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + assert.Equal(s.T(), http.StatusInternalServerError, resp.StatusCode) +} From c8ab96f3c59982b3593d500392996d1abd702ce6 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 28 Apr 2025 16:13:12 +0200 Subject: [PATCH 043/262] [api] add integration test for file downloading --- .../scripts/make_sda_credentials.sh | 5 +++ .../tests/sda/60_api_admin_test.sh | 43 +++++++++++++++++++ 2 files changed, 48 insertions(+) diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 83bc347d0e..9beaec98d6 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -111,6 +111,11 @@ if [ ! -f "/shared/c4gh1.sec.pem" ]; then /shared/crypt4gh generate -n /shared/c4gh1 -p c4ghpass fi +if [ ! -f "/shared/client.sec.pem" ]; then # client key for re-encryption + echo "creating client crypth4gh key" + /shared/crypt4gh generate -n /shared/client -p c4ghpass +fi + if [ ! -f "/shared/sync.sec.pem" ]; then echo "creating sync crypth4gh key" /shared/crypt4gh generate -n /shared/sync -p syncPass diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 3e2bc143bc..f17be07784 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -66,6 +66,42 @@ if [ -z "$output" ] ; then echo "Uploaded file not in inbox" exit 1 fi + +# download the file, re-encrypted with the client key +clientPubKey="$(base64 -w0 /shared/client.pub.pem)" +outFile="download_reenc_NA12878.bam.c4gh" +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" +if [ "$resp" != "200" ]; then + echo "Error when downloading the file, expected 200 got: $resp" + exit 1 +fi + +# decrypt the downloaded file +export C4GH_PASSPHRASE=c4ghpass +if [ ! -f "$outFile" ]; then + echo "downloaded file $outFile not found" + exit 1 +fi +if [ ! -f "/shared/client.sec.pem" ]; then + echo "client key not found" + exit 1 +fi +if ! /shared/crypt4gh decrypt -f "$outFile" -s "/shared/client.sec.pem"; then + echo "decrypting file $outFile failed" + exit 1 +fi + +# check the file content +decryptedFile="${outFile%.*}" +if [ ! -f "$decryptedFile" ]; then + echo "decrypted file ${decryptedFile} not found" + exit 1 +fi +if ! cmp -s "$decryptedFile" "NA12878.bam" ; then + echo "downloaded file is different from the original one" + exit 1 +fi + # delete it resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "200" ]; then @@ -85,6 +121,13 @@ if [ -n "$output" ] ; then exit 1 fi +# Try to download the file that has been deleted +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" +if [ "$resp" != "500" ]; then + echo "Trying to download a non existing file, expected 500 got: $resp" + exit 1 +fi + # Try to delete an unknown file resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/badfileid")" if [ "$resp" != "404" ]; then From dfec650a2fad8f5d4242f543922555c1888200dd Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 28 Apr 2025 16:13:41 +0200 Subject: [PATCH 044/262] [api] update readme for file downloading --- sda/cmd/api/api.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 0501f3c583..dbcf617c60 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -220,6 +220,24 @@ Admin endpoints are only available to a set of whitelisted users specified in th - `401` Token user is not in the list of admins. - `500` Internal error due to DB failure. +- `/users/:username/file/:fileid` + - accepts `GET` requests. + - download a file from the inbox, re-encrypted with the client’s public key provided in the request header. + - the public key provided in the header should be `base64` encoded. + + - Error codes + - `200` Query execute ok. + - `400` Client public key not provided or not valid. + - `401` Token user is not in the list of admins. + - `404` File not found + - `500` Internal error due to Inbox, Reencrypt, DB, MQ or streaming failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -H "Client-Public-Key: $base64_encoded_public_key" -X GET https://HOSTNAME/users/submitter@example.org/file/123abc + ``` + - `/c4gh-keys/add` - accepts `POST` requests with the hex hash of the key and its description - registers the key hash in the database. From 775eb8d9be5fc2348e12351b038a80ba4cb5cd29 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 29 Apr 2025 11:01:27 +0200 Subject: [PATCH 045/262] [rbac] use more specific rbac setting --- .github/integration/sda/rbac.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 8e49ee7472..2176e704a2 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -47,7 +47,7 @@ }, { "role": "submission", - "path": "/users/:username/file/*", + "path": "/users/:username/file/:fileid", "action": "GET" }, { From 33969158404a5d4516a18aedc2e8576997321bdd Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 29 Apr 2025 11:47:33 +0200 Subject: [PATCH 046/262] [api] apply reviewer's comments for api and api_test --- sda/cmd/api/api.go | 66 +++++++++++------------------------------ sda/cmd/api/api_test.go | 6 ++-- 2 files changed, 21 insertions(+), 51 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 7bbc82e724..45f8508fa4 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -321,7 +321,7 @@ func deleteFile(c *gin.Context) { } // Get the file path from the fileID and submission user - filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID, true) + filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) if err != nil { log.Errorf("getting file from fileID failed, reason: (%v)", err) c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in inbox") @@ -432,6 +432,14 @@ func reencryptHeader(oldHeader []byte, reEncKey string) ([]byte, error) { // provided in the request header from the inbox. It retrieves the file path // from the database using the file ID and user ID. func downloadFile(c *gin.Context) { + // Get the public key from the request header. + c4ghPubKey := c.GetHeader("Client-Public-Key") + if c4ghPubKey == "" { + c.String(http.StatusBadRequest, "client public key is not provided in the header") + + return + } + inbox, err := storage.NewBackend(Conf.Inbox) if err != nil { log.Error(err) @@ -453,7 +461,7 @@ func downloadFile(c *gin.Context) { } // Retrieve the actual file path for the user's file. - filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID, false) + filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) if err != nil { log.Errorf("getting file path from fileID (%s) failed, reason: (%v)", fileID, err) c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in the inbox") @@ -471,41 +479,15 @@ func downloadFile(c *gin.Context) { } // get the header of the crypt4gh file - headerConsumed := false - header, err := Conf.API.DB.GetHeader(fileID) - if err != nil { - log.Debugf("failed to get header for fileID %s from the database, reason: %v", fileID, err) - log.Debugln("trying to read header from the file") - - header, err = headers.ReadHeader(file) - headerConsumed = true - if err != nil { - log.Debugf("failed to read header for fileID %s, reason: %v", fileID, err) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) - - return - } - } - - decryptedChecksum, err := Conf.API.DB.GetDecryptedChecksum(fileID) + header, err := headers.ReadHeader(file) if err != nil { - log.Debugf("Warning: failed to get decrypted checksum for fileID %s, reason: %v", fileID, err) - } - - // Get the public key from the request header. - reEncKey := c.GetHeader("Client-Public-Key") - if reEncKey == "" { - c.String(http.StatusBadRequest, "client public key is not provided in the header") + log.Debugf("failed to read header for fileID %s, reason: %v", fileID, err) + c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) return } - // Set the headers for the response. - c.Header("Content-Type", "application/octet-stream") - c.Header("Content-Disposition", fmt.Sprintf("filename: %v", fileID)) - c.Header("ETag", fmt.Sprintf("decryptedChecksum: %s", decryptedChecksum)) - - newHeader, err := reencryptHeaderFunc(header, reEncKey) + newHeader, err := reencryptHeaderFunc(header, c4ghPubKey) if err != nil { log.Errorf("failed to reencrypt header, reason: %v", err) c.String(http.StatusInternalServerError, "failed to reencrypt header") @@ -513,21 +495,10 @@ func downloadFile(c *gin.Context) { return } - // Skip the original header in the file if it hasn't been consumed yet. - if !headerConsumed { - headerLength := int64(len(header)) - _, err = io.CopyN(io.Discard, file, headerLength) - if err != nil { - log.Errorf("failed to skip original header, reason: %v", err) - c.String(http.StatusInternalServerError, "failed to skip original header") - - return - } - } - - fileStream := io.MultiReader(bytes.NewReader(newHeader), file) - - err = sendStream(c.Writer, fileStream) + // Set the headers for the response. + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("filename: %v", fileID)) + err = sendStream(c.Writer, io.MultiReader(bytes.NewReader(newHeader), file)) if err != nil { log.Errorf("error occurred while sending stream: %v", err) c.String(http.StatusInternalServerError, "an error occurred") @@ -547,7 +518,6 @@ var sendStream = func(w http.ResponseWriter, file io.Reader) error { return err } - log.Debugf("Sent %d bytes", n) return nil diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 6004234e8b..1440228232 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -400,7 +400,7 @@ func (s *TestSuite) SetupSuite() { {"role":"submission","path":"/file/accession","action":"POST"}, {"role":"submission","path":"/users","action":"GET"}, {"role":"submission","path":"/users/:username/files","action":"GET"}, - {"role":"submission","path":"/users/:username/file/*","action":"GET"}, + {"role":"submission","path":"/users/:username/file/:fileid","action":"GET"}, {"role":"*","path":"/files","action":"GET"}], "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"admin"}]}`) @@ -2203,7 +2203,7 @@ func (s *TestSuite) TestReVerifyDataset_wrongDatasetName() { assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } -func (s *TestSuite) TestDownloadFileSuccess() { +func (s *TestSuite) TestDownloadFile() { origReencryptHeaderFunc := reencryptHeaderFunc reencryptHeaderFunc = func(oldHeader []byte, reEncKey string) ([]byte, error) { return oldHeader, nil @@ -2305,7 +2305,7 @@ func (s *TestSuite) TestDownloadFileSuccess() { assert.Equal(s.T(), expectedContent, actualContent) } -func (s *TestSuite) TestDownloadFile_FileNotExist() { +func (s *TestSuite) TestDownloadFile_fileNotExist() { gin.SetMode(gin.ReleaseMode) r := gin.Default() m, err := model.NewModelFromString(jsonadapter.Model) From ed157507f382fd28f6fc70b4f5d03c7de52e0159 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 29 Apr 2025 12:02:58 +0200 Subject: [PATCH 047/262] [api] fix integraiton test --- .github/integration/tests/sda/60_api_admin_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index f17be07784..450b88a525 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -123,8 +123,8 @@ fi # Try to download the file that has been deleted resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" -if [ "$resp" != "500" ]; then - echo "Trying to download a non existing file, expected 500 got: $resp" +if [ "$resp" != "404" ]; then + echo "Trying to download a non existing file, expected 404 got: $resp" exit 1 fi From f5ebd1ced669251bcf4e5fc71970c39b98fd270b Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 29 Apr 2025 12:59:58 +0200 Subject: [PATCH 048/262] [api] not checking fileID in downloadFile --- sda/cmd/api/api.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 45f8508fa4..fa85530ead 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -454,12 +454,6 @@ func downloadFile(c *gin.Context) { submissionUser = strings.TrimPrefix(submissionUser, "/") fileID = strings.TrimPrefix(fileID, "/") - if fileID == "" { - c.AbortWithStatusJSON(http.StatusBadRequest, "file ID is required") - - return - } - // Retrieve the actual file path for the user's file. filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) if err != nil { From 7d44988a0ab7705b69c65cc3dbf8b4a5502a8278 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 29 Apr 2025 14:30:46 +0200 Subject: [PATCH 049/262] [api] add c4gh pubkey validation in downloadFile --- sda/cmd/api/api.go | 7 +++++++ sda/cmd/api/api_test.go | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index fa85530ead..ce09084e76 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -28,6 +28,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" @@ -440,6 +441,12 @@ func downloadFile(c *gin.Context) { return } + if !helper.IsValidC4GHPubKey(c4ghPubKey) { + c.String(http.StatusBadRequest, "client public key is not valid") + + return + } + inbox, err := storage.NewBackend(Conf.Inbox) if err != nil { log.Error(err) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 1440228232..0c09f119e6 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -2339,7 +2339,8 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { req, err := http.NewRequest("GET", downloadURL, nil) assert.NoError(s.T(), err) req.Header.Set("Authorization", "Bearer "+s.Token) - req.Header.Set("Client-Public-Key", "base64-key-string") + req.Header.Set("Client-Public-Key", s.ClientKey.PubKeyBase64) + log.Debugf("public key: %s", s.ClientKey.PubKeyBase64) client := &http.Client{} resp, err := client.Do(req) From 9cf528e1082484ba0c47496a5432ed73cb734253 Mon Sep 17 00:00:00 2001 From: nanjiangshu Date: Tue, 29 Apr 2025 14:41:48 +0200 Subject: [PATCH 050/262] [api] add unit test for sendStream --- sda/cmd/api/api_test.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 0c09f119e6..367751ac03 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -42,6 +42,13 @@ import ( var dbPort, mqPort, OIDCport int var BrokerAPI string +// Mock Reader that returns an error +type errorReader struct{} + +func (e *errorReader) Read(p []byte) (n int, err error) { + return 0, io.ErrUnexpectedEOF +} + func TestMain(m *testing.M) { if _, err := os.Stat("/.dockerenv"); err == nil { m.Run() @@ -2349,3 +2356,31 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { assert.Equal(s.T(), http.StatusInternalServerError, resp.StatusCode) } + +func (s *TestSuite) TestSendStream() { + // Create a mock ResponseWriter + recorder := httptest.NewRecorder() + + data := []byte("This is some test data.") + reader := bytes.NewReader(data) + + err := sendStream(recorder, reader) + + s.NoError(err, "sendStream should not return an error on success") + s.Equal(http.StatusOK, recorder.Code, "Response status code should be OK") + s.Equal(data, recorder.Body.Bytes(), "Response body should match the streamed data") +} + +func (s *TestSuite) TestSendStream_copyError() { + // Create a mock ResponseWriter + recorder := httptest.NewRecorder() + + // Create a mock Reader that will return an error on Read + errReader := &errorReader{} + + err := sendStream(recorder, errReader) + + s.Error(err, "sendStream should return an error if io.Copy fails") + s.Equal(http.StatusInternalServerError, recorder.Code, "Response status code should be Internal Server Error") + s.Equal("file streaming failed\n", recorder.Body.String(), "Response body should contain the error message") +} From f5ad57c861c4bbb7f9a0cc54cdcb44fed36754ca Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 29 Apr 2025 23:28:55 +0200 Subject: [PATCH 051/262] [api] add unit tests for reencryptHeader --- sda/cmd/api/api.go | 15 ++++++++----- sda/cmd/api/api_test.go | 50 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 58 insertions(+), 7 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index ce09084e76..95b19267c3 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -47,10 +47,13 @@ type dataset struct { } var ( - Conf *config.Config - err error - auth *userauth.ValidateFromToken - reencryptHeaderFunc = reencryptHeader // for testing purposes + Conf *config.Config + err error + auth *userauth.ValidateFromToken + reencryptHeaderFunc = reencryptHeader // for testing purposes + grpcReencryptHeaderClient = func(c reencrypt.ReencryptClient, ctx context.Context, in *reencrypt.ReencryptRequest, opts ...grpc.CallOption) (*reencrypt.ReencryptResponse, error) { + return c.ReencryptHeader(ctx, in, opts...) // for testing purposes + } ) func main() { @@ -360,7 +363,7 @@ func deleteFile(c *gin.Context) { // gRPC to communicate with the re-encrypt service and handles TLS configuration // if needed. The function also handles the case where the CA certificate is // provided for secure communication. -func reencryptHeader(oldHeader []byte, reEncKey string) ([]byte, error) { +func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { var opts []grpc.DialOption switch { case Conf.ReEncrypt.ServerKey != "" && Conf.ReEncrypt.ServerCert != "": @@ -419,7 +422,7 @@ func reencryptHeader(oldHeader []byte, reEncKey string) ([]byte, error) { defer cancel() c := reencrypt.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: reEncKey}) + res, err := grpcReencryptHeaderClient(c, ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: c4ghPubKey}) if err != nil { log.Errorf("Failed response from the reencrypt service, reason: %s", err) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 367751ac03..8322b21640 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -2,10 +2,12 @@ package main import ( "bytes" + "context" "crypto/sha256" "database/sql" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -30,6 +32,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" + "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" @@ -37,6 +40,7 @@ import ( "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "google.golang.org/grpc" ) var dbPort, mqPort, OIDCport int @@ -2212,7 +2216,7 @@ func (s *TestSuite) TestReVerifyDataset_wrongDatasetName() { func (s *TestSuite) TestDownloadFile() { origReencryptHeaderFunc := reencryptHeaderFunc - reencryptHeaderFunc = func(oldHeader []byte, reEncKey string) ([]byte, error) { + reencryptHeaderFunc = func(oldHeader []byte, c4ghPubKey string) ([]byte, error) { return oldHeader, nil } defer func() { reencryptHeaderFunc = origReencryptHeaderFunc }() @@ -2384,3 +2388,47 @@ func (s *TestSuite) TestSendStream_copyError() { s.Equal(http.StatusInternalServerError, recorder.Code, "Response status code should be Internal Server Error") s.Equal("file streaming failed\n", recorder.Body.String(), "Response body should contain the error message") } + +func (s *TestSuite) TestReencryptHeader() { + expectedNewHeader := []byte("expected reencrypted header data") + + origGrpcReencryptHeaderClient := grpcReencryptHeaderClient + defer func() { grpcReencryptHeaderClient = origGrpcReencryptHeaderClient }() + + // Mock the function to simulate the gRPC response + grpcReencryptHeaderClient = func(c reencrypt.ReencryptClient, ctx context.Context, in *reencrypt.ReencryptRequest, opts ...grpc.CallOption) (*reencrypt.ReencryptResponse, error) { + mockResp := &reencrypt.ReencryptResponse{ + Header: expectedNewHeader, + } + + return mockResp, nil + } + + oldHeader := []byte("some header data") + c4ghPubKey := "some public key" + + newHeader, err := reencryptHeader(oldHeader, c4ghPubKey) + s.Require().NoError(err) + s.Require().NotNil(newHeader) + + s.Assert().Equal(expectedNewHeader, newHeader, "The reencrypted header does not match the expected value.") +} + +func (s *TestSuite) TestReencryptHeader_gRPCServiceError() { + expectedError := errors.New("failed to reencrypt header") + + // Mock the grpcReencryptHeaderClient function to return the error + origGrpcReencryptHeaderClient := grpcReencryptHeaderClient + grpcReencryptHeaderClient = func(c reencrypt.ReencryptClient, ctx context.Context, in *reencrypt.ReencryptRequest, opts ...grpc.CallOption) (*reencrypt.ReencryptResponse, error) { + return nil, expectedError + } + defer func() { grpcReencryptHeaderClient = origGrpcReencryptHeaderClient }() + + oldHeader := []byte("some header data") + c4ghPubKey := "some public key" + newHeader, err := reencryptHeader(oldHeader, c4ghPubKey) + + s.Require().Error(err) + s.Require().Nil(newHeader) + s.Require().EqualError(err, expectedError.Error()) +} From 0657fcc55c3ef1c1665f810dea74ffa791923ca1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 6 May 2025 14:34:17 +0200 Subject: [PATCH 052/262] [internal/config] Add configuration option for GRPC client --- sda/internal/config/config.go | 77 ++++++++++++++++++++++++++++-- sda/internal/config/config_test.go | 18 +++++++ 2 files changed, 90 insertions(+), 5 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index a023ee4592..a2dfbce11d 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" + "google.golang.org/grpc/credentials" ) const ( @@ -51,8 +52,20 @@ type Config struct { Auth AuthConf } +type Grpc struct { + CACert string + ClientCert string + ClientKey string + ClientCreds credentials.TransportCredentials + ServerCert string + ServerKey string + Host string + Port int + Timeout int +} + type ReEncConfig struct { - APIConf + Grpc Crypt4GHKey *[32]byte Timeout int } @@ -906,11 +919,65 @@ func (c *Config) configOrchestrator() { } } -func (c *Config) configReEncryptServer() (err error) { - viper.SetDefault("grpc.timeout", 5) // set default to 5 seconds - if viper.IsSet("grpc.host") { - c.ReEncrypt.Host = viper.GetString("grpc.host") +func configReEncryptClient() (Grpc, error) { + var grpc Grpc + grpc.Host = viper.GetString("grpc.host") + grpc.Port = 50051 + if viper.IsSet("grpc.port") { + grpc.Port = viper.GetInt("grpc.port") + } + + grpc.Timeout = 30 + if viper.IsSet("grpc.timeout") { + grpc.Timeout = viper.GetInt("grpc.timeout") + } + + if viper.IsSet("grpc.clientcert") && viper.IsSet("grpc.clientkey") { + grpc.Port = 50443 + switch { + case viper.IsSet("grpc.cacert"): + cacertByte, err := os.ReadFile(viper.GetString("grpc.cacert")) + if err != nil { + return Grpc{}, fmt.Errorf("failed to read CA certificate file, reason: %s", err.Error()) + } + + caCert := x509.NewCertPool() + if !caCert.AppendCertsFromPEM(cacertByte) { + return Grpc{}, errors.New("failed to append CA certificate to cert pool") + } + + certs, err := tls.LoadX509KeyPair(viper.GetString("grpc.clientcert"), viper.GetString("grpc.clientkey")) + if err != nil { + return Grpc{}, errors.New("Failed to load client key pair for reencrypt") + } + + grpc.ClientCreds = credentials.NewTLS( + &tls.Config{ + Certificates: []tls.Certificate{certs}, + MinVersion: tls.VersionTLS13, + RootCAs: caCert, + }, + ) + default: + certs, err := tls.LoadX509KeyPair(viper.GetString("grpc.clientcert"), viper.GetString("grpc.clientkey")) + if err != nil { + return Grpc{}, errors.New("Failed to load client key pair for reencrypt") + } + + grpc.ClientCreds = credentials.NewTLS( + &tls.Config{ + Certificates: []tls.Certificate{certs}, + MinVersion: tls.VersionTLS13, + }, + ) + } } + + return grpc, nil +} + +func (c *Config) configReEncryptServer() (err error) { + c.ReEncrypt.Port = 50051 if viper.IsSet("grpc.port") { c.ReEncrypt.Port = viper.GetInt("grpc.port") } diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 8ce67166e8..75eb6d0cc5 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -59,6 +59,7 @@ func (ts *ConfigTestSuite) SetupTest() { viper.Set("db.user", "test") viper.Set("db.password", "test") viper.Set("db.database", "test") + viper.Set("grpc.host", "reencrypt") viper.Set("inbox.url", "testurl") viper.Set("inbox.accesskey", "testaccess") viper.Set("inbox.secretkey", "testsecret") @@ -491,6 +492,23 @@ func (ts *ConfigTestSuite) TestConfigReEncryptServer() { assert.Equal(ts.T(), certPath+"/tls.crt", config.ReEncrypt.ServerCert) } +func (ts *ConfigTestSuite) TestConfigReEncryptClient() { + ts.SetupTest() + conf, err := configReEncryptClient() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "reencrypt", conf.Host) + assert.Nil(ts.T(), conf.ClientCreds) +} + +func (ts *ConfigTestSuite) TestConfigReEncryptClient_withTLS() { + viper.Set("grpc.CACert", certPath+"/ca.crt") + viper.Set("grpc.clientCert", certPath+"/tls.crt") + viper.Set("grpc.clientKey", certPath+"/tls.key") + conf, err := configReEncryptClient() + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), conf.ClientCreds) + assert.Equal(ts.T(), 50443, conf.Port) +} func (ts *ConfigTestSuite) TestConfigAuth_CEGA() { ts.SetupTest() From 15c59b9fce82dd42547c209ab991a61a80eba0c0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 6 May 2025 14:34:52 +0200 Subject: [PATCH 053/262] [internal/config] add grpc client config to the API --- sda/internal/config/config.go | 4 +++- sda/internal/config/config_test.go | 11 +---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index a2dfbce11d..489f7d4e44 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -98,6 +98,7 @@ type APIConf struct { DB *database.SDAdb MQ *broker.AMQPBroker INBOX storage.Backend + Grpc Grpc } type SessionConfig struct { @@ -233,6 +234,7 @@ func NewConfig(app string) (*Config, error) { "db.user", "db.password", "db.database", + "grpc.host", } switch viper.GetString("inbox.type") { case S3: @@ -508,7 +510,7 @@ func NewConfig(app string) (*Config, error) { } c.configSchemas() - err = c.configReEncryptServer() + c.API.Grpc, err = configReEncryptClient() if err != nil { return nil, err } diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 75eb6d0cc5..2752f8ff70 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -229,12 +229,6 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { // testing default values ts.SetupTest() - keyPath, _ := os.MkdirTemp("", "key") - keyFile := keyPath + "/c4gh.key" - _, err = helper.CreatePrivateKeyFile(keyFile, "test") - viper.Set("c4gh.filepath", keyPath+"/c4gh.key") - viper.Set("c4gh.passphrase", "test") - assert.NoError(ts.T(), err) config, err = NewConfig("api") assert.NotNil(ts.T(), config) assert.NoError(ts.T(), err) @@ -247,6 +241,7 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { assert.Equal(ts.T(), -1*time.Second, config.API.Session.Expiration) rbac, _ := os.ReadFile(viper.GetString("api.rbacFile")) assert.Equal(ts.T(), rbac, config.API.RBACpolicy) + assert.Equal(ts.T(), "reencrypt", config.API.Grpc.Host) viper.Reset() ts.SetupTest() @@ -255,8 +250,6 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { viper.Set("api.session.secure", false) viper.Set("api.session.domain", "test") viper.Set("api.session.expiration", 60) - viper.Set("c4gh.filepath", keyPath+"/c4gh.key") - viper.Set("c4gh.passphrase", "test") config, err = NewConfig("api") assert.NotNil(ts.T(), config) @@ -267,8 +260,6 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { assert.Equal(ts.T(), false, config.API.Session.Secure) assert.Equal(ts.T(), "test", config.API.Session.Domain) assert.Equal(ts.T(), 60*time.Second, config.API.Session.Expiration) - - defer os.RemoveAll(keyPath) } func (ts *ConfigTestSuite) TestNotifyConfiguration() { From 6248f1c1de1dd3338df66d5d9235dcdb1ad27bbf Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 6 May 2025 14:36:05 +0200 Subject: [PATCH 054/262] [API] rework reencryptHeader function --- sda/cmd/api/api.go | 52 +++++----------------------------------------- 1 file changed, 5 insertions(+), 47 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 95b19267c3..debdbb3366 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/tls" - "crypto/x509" "encoding/base64" "encoding/hex" "encoding/json" @@ -36,7 +35,6 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" "google.golang.org/grpc" - "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" ) @@ -366,50 +364,13 @@ func deleteFile(c *gin.Context) { func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { var opts []grpc.DialOption switch { - case Conf.ReEncrypt.ServerKey != "" && Conf.ReEncrypt.ServerCert != "": - rootCAs, err := x509.SystemCertPool() - if err != nil { - log.Errorf("failed to read system CAs: %v, using an empty pool as base", err) - rootCAs = x509.NewCertPool() - } - if Conf.ReEncrypt.CACert != "" { - cacertByte, err := os.ReadFile(Conf.ReEncrypt.CACert) - if err != nil { - log.Errorf("Failed to read CA certificate file, reason: %s", err) - - return nil, err - } - ok := rootCAs.AppendCertsFromPEM(cacertByte) - if !ok { - log.Errorf("Failed to append CA certificate to rootCAs") - - return nil, errors.New("failed to append CA certificate to cert pool") - } - } - - certs, err := tls.LoadX509KeyPair(Conf.ReEncrypt.ServerCert, Conf.ReEncrypt.ServerKey) - if err != nil { - log.Errorf("Failed to load client key pair for reencrypt, reason: %s", err) - - return nil, err - } - clientCreds := credentials.NewTLS( - &tls.Config{ - Certificates: []tls.Certificate{certs}, - MinVersion: tls.VersionTLS13, - RootCAs: rootCAs, - }, - ) - - opts = append(opts, grpc.WithTransportCredentials(clientCreds)) + case Conf.API.Grpc.ClientCreds != nil: + opts = append(opts, grpc.WithTransportCredentials(Conf.API.Grpc.ClientCreds)) default: opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) } - address := fmt.Sprintf("%s:%d", Conf.ReEncrypt.Host, Conf.ReEncrypt.Port) - log.Debugf("Address of the reencrypt service: %s", address) - - conn, err := grpc.NewClient(address, opts...) + conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", Conf.API.Grpc.Host, Conf.API.Grpc.Port), opts...) if err != nil { log.Errorf("Failed to connect to the reencrypt service, reason: %s", err) @@ -417,15 +378,12 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { } defer conn.Close() - timeoutDuration := time.Duration(Conf.ReEncrypt.Timeout) * time.Second - ctx, cancel := context.WithTimeout(context.Background(), timeoutDuration) + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(Conf.API.Grpc.Timeout)*time.Second) defer cancel() c := reencrypt.NewReencryptClient(conn) - res, err := grpcReencryptHeaderClient(c, ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: c4ghPubKey}) + res, err := c.ReencryptHeader(ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: c4ghPubKey}) if err != nil { - log.Errorf("Failed response from the reencrypt service, reason: %s", err) - return nil, err } From 4f37d3a47ccd81ea0dab224675e2c1185cd7899d Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 7 May 2025 14:32:27 +0200 Subject: [PATCH 055/262] [api] rework pubkey validation in downloadFile --- sda/cmd/api/api.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index debdbb3366..b4604ba2a0 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -27,7 +27,6 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" - "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" @@ -396,14 +395,11 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { func downloadFile(c *gin.Context) { // Get the public key from the request header. c4ghPubKey := c.GetHeader("Client-Public-Key") - if c4ghPubKey == "" { - c.String(http.StatusBadRequest, "client public key is not provided in the header") - return - } - - if !helper.IsValidC4GHPubKey(c4ghPubKey) { - c.String(http.StatusBadRequest, "client public key is not valid") + pubKey, err := base64.StdEncoding.DecodeString(c4ghPubKey) + if err != nil || len(pubKey) == 0 { + c.AbortWithStatusJSON(http.StatusBadRequest, "bad public key") + log.Errorf("bad public key, error: %v", err) return } From 0f3069e0e0b54e829aec51fe0231ff33ce599f5f Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 7 May 2025 15:22:07 +0200 Subject: [PATCH 056/262] [api] rework the downloadFile function --- sda/cmd/api/api.go | 49 +++++++++++++++++------------------------ sda/cmd/api/api_test.go | 30 +------------------------ 2 files changed, 21 insertions(+), 58 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index b4604ba2a0..78cd6cc3d2 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -14,6 +14,7 @@ import ( "net/http" "os" "os/signal" + "path" "strings" "syscall" "time" @@ -394,7 +395,7 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { // from the database using the file ID and user ID. func downloadFile(c *gin.Context) { // Get the public key from the request header. - c4ghPubKey := c.GetHeader("Client-Public-Key") + c4ghPubKey := c.GetHeader("C4GH-Public-Key") pubKey, err := base64.StdEncoding.DecodeString(c4ghPubKey) if err != nil || len(pubKey) == 0 { @@ -412,17 +413,15 @@ func downloadFile(c *gin.Context) { return } - // Extract the username and file ID from the URL parameters. - submissionUser := c.Param("username") - fileID := c.Param("fileid") - submissionUser = strings.TrimPrefix(submissionUser, "/") - fileID = strings.TrimPrefix(fileID, "/") - // Retrieve the actual file path for the user's file. - filePath, err := Conf.API.DB.GetInboxFilePathFromID(submissionUser, fileID) + fileID := strings.TrimPrefix(c.Param("fileid"), "/") + filePath, err := Conf.API.DB.GetInboxFilePathFromID( + strings.TrimPrefix(c.Param("username"), "/"), + fileID, + ) if err != nil { log.Errorf("getting file path from fileID (%s) failed, reason: (%v)", fileID, err) - c.AbortWithStatusJSON(http.StatusNotFound, "File could not be found in the inbox") + c.AbortWithStatusJSON(http.StatusNotFound, "failed to retrieve inbox file path") return } @@ -431,7 +430,7 @@ func downloadFile(c *gin.Context) { file, err := inbox.NewFileReader(filePath) if err != nil { log.Errorf("could not find inbox file %s, %s", filePath, err) - c.String(http.StatusInternalServerError, "inbox error") + c.AbortWithStatusJSON(http.StatusInternalServerError, "storage backend error") return } @@ -440,7 +439,7 @@ func downloadFile(c *gin.Context) { header, err := headers.ReadHeader(file) if err != nil { log.Debugf("failed to read header for fileID %s, reason: %v", fileID, err) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to read the start of the file") return } @@ -448,37 +447,29 @@ func downloadFile(c *gin.Context) { newHeader, err := reencryptHeaderFunc(header, c4ghPubKey) if err != nil { log.Errorf("failed to reencrypt header, reason: %v", err) - c.String(http.StatusInternalServerError, "failed to reencrypt header") + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to reencrypt header") return } // Set the headers for the response. c.Header("Content-Type", "application/octet-stream") - c.Header("Content-Disposition", fmt.Sprintf("filename: %v", fileID)) - err = sendStream(c.Writer, io.MultiReader(bytes.NewReader(newHeader), file)) + c.Header("Content-Disposition", fmt.Sprintf("filename: %v", path.Base(filePath))) + + reader := io.MultiReader(bytes.NewReader(newHeader), file) + n, err := io.Copy(c.Writer, reader) if err != nil { log.Errorf("error occurred while sending stream: %v", err) - c.String(http.StatusInternalServerError, "an error occurred") + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ + "error": "failed to stream data to client", + "details": err.Error(), + }) return } - - c.Status(http.StatusOK) -} - -// sendStream streams file contents from a reader -var sendStream = func(w http.ResponseWriter, file io.Reader) error { - n, err := io.Copy(w, file) - if err != nil { - log.Errorf("file streaming failed, reason: %v", err) - http.Error(w, "file streaming failed", http.StatusInternalServerError) - - return err - } log.Debugf("Sent %d bytes", n) - return nil + c.Status(http.StatusOK) } func setAccession(c *gin.Context) { diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 8322b21640..b6aadc764e 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -2300,7 +2300,7 @@ func (s *TestSuite) TestDownloadFile() { req, err := http.NewRequest("GET", downloadURL, nil) assert.NoError(s.T(), err) req.Header.Set("Authorization", "Bearer "+s.Token) - req.Header.Set("Client-Public-Key", s.ClientKey.PubKeyBase64) + req.Header.Set("C4GH-Public-Key", s.ClientKey.PubKeyBase64) client := &http.Client{} resp, err := client.Do(req) @@ -2361,34 +2361,6 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { assert.Equal(s.T(), http.StatusInternalServerError, resp.StatusCode) } -func (s *TestSuite) TestSendStream() { - // Create a mock ResponseWriter - recorder := httptest.NewRecorder() - - data := []byte("This is some test data.") - reader := bytes.NewReader(data) - - err := sendStream(recorder, reader) - - s.NoError(err, "sendStream should not return an error on success") - s.Equal(http.StatusOK, recorder.Code, "Response status code should be OK") - s.Equal(data, recorder.Body.Bytes(), "Response body should match the streamed data") -} - -func (s *TestSuite) TestSendStream_copyError() { - // Create a mock ResponseWriter - recorder := httptest.NewRecorder() - - // Create a mock Reader that will return an error on Read - errReader := &errorReader{} - - err := sendStream(recorder, errReader) - - s.Error(err, "sendStream should return an error if io.Copy fails") - s.Equal(http.StatusInternalServerError, recorder.Code, "Response status code should be Internal Server Error") - s.Equal("file streaming failed\n", recorder.Body.String(), "Response body should contain the error message") -} - func (s *TestSuite) TestReencryptHeader() { expectedNewHeader := []byte("expected reencrypted header data") From 73a673c94102a988e7921162e5545f9cb26db8d2 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 7 May 2025 15:36:06 +0200 Subject: [PATCH 057/262] minor text fixing for comments and error messages --- sda/cmd/api/api.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 78cd6cc3d2..a678051be9 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -121,10 +121,10 @@ func setup(conf *config.Config) *http.Server { r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status - r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specififc user + r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user r.GET("/users", rbac(e), listActiveUsers) // Lists all users r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user - r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file for a user + r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox cfg := &tls.Config{MinVersion: tls.VersionTLS12} @@ -372,7 +372,7 @@ func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", Conf.API.Grpc.Host, Conf.API.Grpc.Port), opts...) if err != nil { - log.Errorf("Failed to connect to the reencrypt service, reason: %s", err) + log.Errorf("failed to connect to the reencrypt service, reason: %s", err) return nil, err } @@ -541,7 +541,7 @@ func createDataset(c *gin.Context) { } if len(dataset.AccessionIDs) == 0 { - c.AbortWithStatusJSON(http.StatusBadRequest, "at least one accessionID is reqired") + c.AbortWithStatusJSON(http.StatusBadRequest, "at least one accessionID is required") return } From b848866722637a36c3ae921099f7f5c64b0c2621 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 7 May 2025 15:36:28 +0200 Subject: [PATCH 058/262] [docs] update readme for admin API --- sda/cmd/api/api.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index dbcf617c60..91e127a210 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -235,7 +235,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th Example: ```bash - curl -H "Authorization: Bearer $token" -H "Client-Public-Key: $base64_encoded_public_key" -X GET https://HOSTNAME/users/submitter@example.org/file/123abc + curl -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $base64_encoded_public_key" -X GET https://HOSTNAME/users/submitter@example.org/file/c2acecc6-f208-441c-877a-2670e4cbb040 ``` - `/c4gh-keys/add` From 072df0a5d9ba17d7e0741760ee7507a3bd566ad9 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 7 May 2025 18:29:39 +0200 Subject: [PATCH 059/262] [api] update swagger --- sda/cmd/api/swagger_v1.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index f0efc433de..abc66ff8aa 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -315,6 +315,31 @@ paths: description: Authentication failure "500": description: Internal application error + /users/{userName}/file/{fileID}: + get: + description: Download a file from the inbox re-encrypted with the user public key. + parameters: + - in: path + name: userName + schema: + type: string + required: true + - in: path + name: fileID + schema: + type: string + required: true + responses: + "200": + description: Successful operation. + "400": + description: User public key not provided or not valid. + "401": + description: Authentication failure. + "404": + description: File not found. + "500": + description: Internal application error. components: schemas: C4ghKeyAdd: From 8da6d5cee61b9d3e8e170032d3d291299915f2d4 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 7 May 2025 18:31:55 +0200 Subject: [PATCH 060/262] update integration test for admin-API --- .github/integration/tests/sda/60_api_admin_test.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 450b88a525..f85c9a6d0f 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -70,7 +70,7 @@ fi # download the file, re-encrypted with the client key clientPubKey="$(base64 -w0 /shared/client.pub.pem)" outFile="download_reenc_NA12878.bam.c4gh" -resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" if [ "$resp" != "200" ]; then echo "Error when downloading the file, expected 200 got: $resp" exit 1 @@ -122,7 +122,7 @@ if [ -n "$output" ] ; then fi # Try to download the file that has been deleted -resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "Client-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" if [ "$resp" != "404" ]; then echo "Trying to download a non existing file, expected 404 got: $resp" exit 1 From d9a3761723b559f883c66bfb3530bec75a0ac989 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 8 May 2025 16:45:09 +0200 Subject: [PATCH 061/262] [api] rework unit tests --- sda/cmd/api/api.go | 36 ++-- sda/cmd/api/api_test.go | 402 ++++++++++++++++++++++++---------------- 2 files changed, 260 insertions(+), 178 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index a678051be9..ddd68d952c 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -45,13 +45,9 @@ type dataset struct { } var ( - Conf *config.Config - err error - auth *userauth.ValidateFromToken - reencryptHeaderFunc = reencryptHeader // for testing purposes - grpcReencryptHeaderClient = func(c reencrypt.ReencryptClient, ctx context.Context, in *reencrypt.ReencryptRequest, opts ...grpc.CallOption) (*reencrypt.ReencryptResponse, error) { - return c.ReencryptHeader(ctx, in, opts...) // for testing purposes - } + Conf *config.Config + err error + auth *userauth.ValidateFromToken ) func main() { @@ -399,16 +395,16 @@ func downloadFile(c *gin.Context) { pubKey, err := base64.StdEncoding.DecodeString(c4ghPubKey) if err != nil || len(pubKey) == 0 { - c.AbortWithStatusJSON(http.StatusBadRequest, "bad public key") log.Errorf("bad public key, error: %v", err) + c.AbortWithStatusJSON(http.StatusBadRequest, "bad public key") return } inbox, err := storage.NewBackend(Conf.Inbox) if err != nil { - log.Error(err) - c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) + log.Errorf("failed to initialize inbox backend, reason: %v", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, "storage backend error") return } @@ -420,7 +416,7 @@ func downloadFile(c *gin.Context) { fileID, ) if err != nil { - log.Errorf("getting file path from fileID (%s) failed, reason: (%v)", fileID, err) + log.Errorf("getting file path from fileID (%s) failed, reason: %v", fileID, err) c.AbortWithStatusJSON(http.StatusNotFound, "failed to retrieve inbox file path") return @@ -429,8 +425,8 @@ func downloadFile(c *gin.Context) { // Get inbox file handle file, err := inbox.NewFileReader(filePath) if err != nil { - log.Errorf("could not find inbox file %s, %s", filePath, err) - c.AbortWithStatusJSON(http.StatusInternalServerError, "storage backend error") + log.Errorf("inbox file %s not found or failed to read, %s", filePath, err) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to read inbox file") return } @@ -438,13 +434,13 @@ func downloadFile(c *gin.Context) { // get the header of the crypt4gh file header, err := headers.ReadHeader(file) if err != nil { - log.Debugf("failed to read header for fileID %s, reason: %v", fileID, err) + log.Errorf("failed to read header for fileID %s, reason: %v", fileID, err) c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to read the start of the file") return } - newHeader, err := reencryptHeaderFunc(header, c4ghPubKey) + newHeader, err := reencryptHeader(header, c4ghPubKey) if err != nil { log.Errorf("failed to reencrypt header, reason: %v", err) c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to reencrypt header") @@ -457,17 +453,13 @@ func downloadFile(c *gin.Context) { c.Header("Content-Disposition", fmt.Sprintf("filename: %v", path.Base(filePath))) reader := io.MultiReader(bytes.NewReader(newHeader), file) - n, err := io.Copy(c.Writer, reader) + _, err = io.Copy(c.Writer, reader) if err != nil { - log.Errorf("error occurred while sending stream: %v", err) - c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ - "error": "failed to stream data to client", - "details": err.Error(), - }) + log.Errorf("error occurred while sending stream, reason: %v", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to stream data to client") return } - log.Debugf("Sent %d bytes", n) c.Status(http.StatusOK) } diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index b6aadc764e..04394b6fab 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -6,14 +6,16 @@ import ( "crypto/sha256" "database/sql" "encoding/base64" + "encoding/hex" "encoding/json" - "errors" "fmt" "io" + "net" "net/http" "net/http/httptest" "os" "path" + "path/filepath" "runtime" "strconv" "strings" @@ -37,7 +39,6 @@ import ( "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" log "github.com/sirupsen/logrus" - "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" "google.golang.org/grpc" @@ -46,11 +47,31 @@ import ( var dbPort, mqPort, OIDCport int var BrokerAPI string -// Mock Reader that returns an error -type errorReader struct{} +type server struct { + reencrypt.UnimplementedReencryptServer + c4ghPrivateKey *[32]byte +} + +func (s *server) ReencryptHeader(ctx context.Context, req *reencrypt.ReencryptRequest) (*reencrypt.ReencryptResponse, error) { + // Mock response based on your needs + mockedResponse := &reencrypt.ReencryptResponse{ + Header: []byte("predefined header response"), + } + + return mockedResponse, nil +} + +func splitHostPort(addr string) (string, int, error) { + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return "", 0, fmt.Errorf("invalid address format: %w", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("invalid port in address '%s': %w", addr, err) + } -func (e *errorReader) Read(p []byte) (n int, err error) { - return 0, io.ErrUnexpectedEOF + return host, port, nil } func TestMain(m *testing.M) { @@ -244,23 +265,36 @@ func TestMain(m *testing.M) { os.Exit(code) } -type ClientKey struct { - PubKey [32]byte - PrivKeyPath string - PubKeyBase64 string +type UserKey struct { + PublicKey [32]byte + PrivateKey [32]byte + PrivateKeyPath string + PubKeyBase64 string +} + +type GrpcListener struct { + Listener net.Listener + gs *grpc.Server + serverOpts []grpc.ServerOption } type TestSuite struct { suite.Suite - Path string - PublicPath string - PrivatePath string - KeyName string - RBAC []byte - Token string - User string - PubKeyList [][32]byte - ClientKey ClientKey + Path string + PublicPath string + PrivatePath string + KeyName string + RBAC []byte + Token string + User string + PubKeyList [][32]byte + PrivateKey *[32]byte + UserKey UserKey + FileData []byte + FileHeader []byte + GoodC4ghFile string + BadC4ghFile string + GrpcListener GrpcListener } func (s *TestSuite) TestShutdown() { @@ -416,45 +450,100 @@ func (s *TestSuite) SetupSuite() { "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"admin"}]}`) - c.ReEncrypt.Host = "localhost" - c.ReEncrypt.Port = 50051 - c.ReEncrypt.Crypt4GHKey, _ = config.GetC4GHKey() - Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") if err != nil { s.FailNow("failed to create temp folder") } - keyFile1 := fmt.Sprintf("%s/c4gh1.key", s.Path) - keyFile2 := fmt.Sprintf("%s/c4gh2.key", s.Path) + s.UserKey.PublicKey, s.UserKey.PrivateKey, err = keys.GenerateKeyPair() + if err != nil { + s.T().FailNow() + } + + s.UserKey.PrivateKeyPath = s.Path + "/user.key" + key, err := os.Create(s.UserKey.PrivateKeyPath) + if err != nil { + s.T().FailNow() + } + if err := keys.WriteCrypt4GHX25519PrivateKey(key, s.UserKey.PrivateKey, []byte("password")); err != nil { + s.T().FailNow() + } + + buf := new(bytes.Buffer) + if err := keys.WriteCrypt4GHX25519PublicKey(buf, s.UserKey.PublicKey); err != nil { + s.T().FailNow() + } + s.UserKey.PubKeyBase64 = base64.StdEncoding.EncodeToString(buf.Bytes()) + + Conf.API.Grpc = config.Grpc{ + Host: "localhost", + Port: 50051, + Timeout: 30, + } + + s.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") + s.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") + + err = os.MkdirAll(path.Join(Conf.Inbox.Posix.Location, s.User), 0750) + assert.NoError(s.T(), err, "failed to create inbox directory") + + // Create test files in the inbox + fileContent := []byte("This is the content of the test file.") + contentReader := bytes.NewReader(fileContent) + s.GoodC4ghFile = path.Join(Conf.Inbox.Posix.Location, s.User, "test_download.c4gh") + s.BadC4ghFile = path.Join(Conf.Inbox.Posix.Location, s.User, "badc4ghfile.c4gh") - publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test") + outFile, err := os.Create(s.GoodC4ghFile) if err != nil { - s.FailNow("Failed to create c4gh key") + s.FailNow("failed to create encrypted test file") } - // Add only the first public key to the list - s.PubKeyList = append(s.PubKeyList, publicKey) + defer outFile.Close() - _, err = helper.CreatePrivateKeyFile(keyFile2, "test") + _, privateKey, err := keys.GenerateKeyPair() if err != nil { - s.FailNow("Failed to create c4gh key") + s.FailNow("failed to create private c4gh key") } - viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ - {FilePath: keyFile1, Passphrase: "test"}, - {FilePath: keyFile2, Passphrase: "test"}, - }) + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, s.PubKeyList, nil) + if err != nil { + s.FailNow("failed to create c4gh writer") + } - s.ClientKey.PrivKeyPath = fmt.Sprintf("%s/client.key", s.Path) - s.ClientKey.PubKey, err = helper.CreatePrivateKeyFile(s.ClientKey.PrivKeyPath, "test") + _, err = io.Copy(crypt4GHWriter, contentReader) if err != nil { - s.FailNow("Failed to create c4gh key") + s.FailNow("failed to write predefined data to encrypted test file") } - s.ClientKey.PubKeyBase64 = base64.StdEncoding.EncodeToString(s.ClientKey.PubKey[:]) + crypt4GHWriter.Close() + + err = os.WriteFile(s.BadC4ghFile, []byte("bad c4gh content"), 0600) + if err != nil { + s.FailNow("failed to write content to file") + } + + // create a gPRC listener for tests using the reencrypt service + s.GrpcListener.Listener, err = net.Listen("tcp", "localhost:0") + if err != nil { + s.T().FailNow() + } + + go func() { + var opts []grpc.ServerOption + s.GrpcListener.gs = grpc.NewServer(opts...) + reencrypt.RegisterReencryptServer(s.GrpcListener.gs, &server{c4ghPrivateKey: Conf.ReEncrypt.Crypt4GHKey}) + if err := s.GrpcListener.gs.Serve(s.GrpcListener.Listener); err != nil { + s.T().Fail() + } + }() } func (s *TestSuite) TearDownSuite() { assert.NoError(s.T(), os.RemoveAll(s.Path)) assert.NoError(s.T(), os.RemoveAll(Conf.Inbox.Posix.Location)) + if s.GrpcListener.gs != nil { + s.GrpcListener.gs.GracefulStop() + } + if s.GrpcListener.Listener != nil { + s.GrpcListener.Listener.Close() + } } func (s *TestSuite) SetupTest() { Conf.Database = database.DBConf{ @@ -1223,7 +1312,7 @@ func (s *TestSuite) TestCreateDataset_MissingAccessionIDs() { response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(s.T(), string(body), "at least one accessionID is reqired") + assert.Contains(s.T(), string(body), "at least one accessionID is required") } func (s *TestSuite) TestCreateDataset_WrongIDs() { @@ -2215,81 +2304,23 @@ func (s *TestSuite) TestReVerifyDataset_wrongDatasetName() { } func (s *TestSuite) TestDownloadFile() { - origReencryptHeaderFunc := reencryptHeaderFunc - reencryptHeaderFunc = func(oldHeader []byte, c4ghPubKey string) ([]byte, error) { - return oldHeader, nil + mockServerAddress := s.GrpcListener.Listener.Addr().String() + Conf.API.Grpc.Host, Conf.API.Grpc.Port, err = splitHostPort(mockServerAddress) + if err != nil { + s.T().Fatal("failed to split host and port:", err) } - defer func() { reencryptHeaderFunc = origReencryptHeaderFunc }() gin.SetMode(gin.ReleaseMode) r := gin.Default() - m, err := model.NewModelFromString(jsonadapter.Model) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC model") - } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC enforcer") - } - r.GET("/users/:username/file/:fileid", rbac(e), func(c *gin.Context) { + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { downloadFile(c) }) ts := httptest.NewServer(r) defer ts.Close() - assert.NoError(s.T(), setupJwtAuth()) - - err = os.MkdirAll(path.Join(Conf.Inbox.Posix.Location, s.User), 0750) - assert.NoError(s.T(), err, "failed to create inbox directory") - - // Create a test file in the inbox directory - fileContent := []byte("This is the content of the test file.") - contentReader := bytes.NewReader(fileContent) - testFilePath := path.Join(Conf.Inbox.Posix.Location, s.User, "test_download.c4gh") - - outFile, err := os.Create(testFilePath) - if err != nil { - s.FailNow("failed to create encrypted test file") - } - defer outFile.Close() - - _, privateKey, err := keys.GenerateKeyPair() - if err != nil { - s.FailNow("failed to create private c4gh key") - } - - crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, s.PubKeyList, nil) - if err != nil { - s.FailNow("failed to create c4gh writer") - } - - _, err = io.Copy(crypt4GHWriter, contentReader) - if err != nil { - s.FailNow("failed to write predefined data to encrypted test file") - } - crypt4GHWriter.Close() - - defer os.Remove(testFilePath) - log.Debugf("test file created: %s", testFilePath) - - // Now read back the entire encrypted file and store it as expectedContent. - encryptedFile, err := os.Open(testFilePath) - if err != nil { - s.FailNow("failed to open encrypted test file for reading") - } - defer encryptedFile.Close() - - expectedContent, err := io.ReadAll(encryptedFile) - if err != nil { - s.FailNow("failed to read encrypted test file") - } - encryptedFile.Close() - // Register the file in the database - filePath := fmt.Sprintf("/%v/test_download.c4gh", s.User) + filePath := fmt.Sprintf("/%s/%s", s.User, filepath.Base(s.GoodC4ghFile)) fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") @@ -2299,8 +2330,7 @@ func (s *TestSuite) TestDownloadFile() { downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) req, err := http.NewRequest("GET", downloadURL, nil) assert.NoError(s.T(), err) - req.Header.Set("Authorization", "Bearer "+s.Token) - req.Header.Set("C4GH-Public-Key", s.ClientKey.PubKeyBase64) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) client := &http.Client{} resp, err := client.Do(req) @@ -2311,32 +2341,70 @@ func (s *TestSuite) TestDownloadFile() { assert.Equal(s.T(), "application/octet-stream", resp.Header.Get("Content-Type")) // Check the content of the response - actualContent, err := io.ReadAll(resp.Body) + body, err := io.ReadAll(resp.Body) assert.NoError(s.T(), err, "failed to read response body") - assert.Equal(s.T(), expectedContent, actualContent) + assert.Contains(s.T(), string(body), "predefined header respons") } -func (s *TestSuite) TestDownloadFile_fileNotExist() { - gin.SetMode(gin.ReleaseMode) +func (s *TestSuite) TestDownloadFile_badPublicKey() { r := gin.Default() - m, err := model.NewModelFromString(jsonadapter.Model) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC model") - } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) - if err != nil { - s.T().Logf("failure: %v", err) - s.FailNow("failed to setup RBAC enforcer") - } + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + // Mock request to download a file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, "d35a6b53-a5d8-4b1b-921f-0a2829b8d2f2") + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", "invalid_key") + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) - r.GET("/users/:username/file/:fileid", rbac(e), func(c *gin.Context) { + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(body), "bad public key") +} + +func (s *TestSuite) TestDownloadFile_fileIDNotExist() { + r := gin.Default() + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { downloadFile(c) }) ts := httptest.NewServer(r) defer ts.Close() - assert.NoError(s.T(), setupJwtAuth()) + // Mock a request to download a file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, "d35a6b53-a5d8-4b1b-921f-0a2829b8d2f2") + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) + + assert.Equal(s.T(), http.StatusNotFound, resp.StatusCode) + assert.Contains(s.T(), string(body), "failed to retrieve inbox file path") +} + +func (s *TestSuite) TestDownloadFile_fileNotExist() { + r := gin.Default() + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() // Register a file in the database (but don't create the actual file) filePath := fmt.Sprintf("/%v/nonexistent.c4gh", s.User) @@ -2349,58 +2417,80 @@ func (s *TestSuite) TestDownloadFile_fileNotExist() { downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) req, err := http.NewRequest("GET", downloadURL, nil) assert.NoError(s.T(), err) - req.Header.Set("Authorization", "Bearer "+s.Token) - req.Header.Set("Client-Public-Key", s.ClientKey.PubKeyBase64) - log.Debugf("public key: %s", s.ClientKey.PubKeyBase64) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) client := &http.Client{} resp, err := client.Do(req) assert.NoError(s.T(), err) defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusInternalServerError, resp.StatusCode) + assert.Contains(s.T(), string(body), "failed to read inbox file") } -func (s *TestSuite) TestReencryptHeader() { - expectedNewHeader := []byte("expected reencrypted header data") - - origGrpcReencryptHeaderClient := grpcReencryptHeaderClient - defer func() { grpcReencryptHeaderClient = origGrpcReencryptHeaderClient }() +func (s *TestSuite) TestDownloadFile_badC4ghFile() { + r := gin.Default() + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() - // Mock the function to simulate the gRPC response - grpcReencryptHeaderClient = func(c reencrypt.ReencryptClient, ctx context.Context, in *reencrypt.ReencryptRequest, opts ...grpc.CallOption) (*reencrypt.ReencryptResponse, error) { - mockResp := &reencrypt.ReencryptResponse{ - Header: expectedNewHeader, - } + // Register a file in the database (but don't create the actual file) + filePath := fmt.Sprintf("/%s/%s", s.User, filepath.Base(s.BadC4ghFile)) + fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") - return mockResp, nil - } + // Mock request to download the file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) - oldHeader := []byte("some header data") - c4ghPubKey := "some public key" + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() - newHeader, err := reencryptHeader(oldHeader, c4ghPubKey) - s.Require().NoError(err) - s.Require().NotNil(newHeader) + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) - s.Assert().Equal(expectedNewHeader, newHeader, "The reencrypted header does not match the expected value.") + assert.Equal(s.T(), http.StatusInternalServerError, resp.StatusCode) + assert.Contains(s.T(), string(body), "failed to read the start of the file") } -func (s *TestSuite) TestReencryptHeader_gRPCServiceError() { - expectedError := errors.New("failed to reencrypt header") +func (s *TestSuite) TestReencryptHeader() { + mockServerAddress := s.GrpcListener.Listener.Addr().String() + log.Debugf("Mock server address: %s", mockServerAddress) + Conf.API.Grpc.Host, Conf.API.Grpc.Port, err = splitHostPort(mockServerAddress) + if err != nil { + s.T().Fatal("failed to split host and port:", err) + } - // Mock the grpcReencryptHeaderClient function to return the error - origGrpcReencryptHeaderClient := grpcReencryptHeaderClient - grpcReencryptHeaderClient = func(c reencrypt.ReencryptClient, ctx context.Context, in *reencrypt.ReencryptRequest, opts ...grpc.CallOption) (*reencrypt.ReencryptResponse, error) { - return nil, expectedError + // Call the function under test + newHeader, err := reencryptHeader(s.FileHeader, s.UserKey.PubKeyBase64) + if err != nil { + s.T().Fatal("reencryptHeader failed:", err) } - defer func() { grpcReencryptHeaderClient = origGrpcReencryptHeaderClient }() - oldHeader := []byte("some header data") - c4ghPubKey := "some public key" - newHeader, err := reencryptHeader(oldHeader, c4ghPubKey) + // Validate the returned header + expectedHeader := []byte("predefined header response") + assert.Equal(s.T(), expectedHeader, newHeader, "returned header does not match the expected header") +} - s.Require().Error(err) - s.Require().Nil(newHeader) - s.Require().EqualError(err, expectedError.Error()) +func (s *TestSuite) TestReencryptHeader_failedToConnect() { + // Mock the server address to an invalid one + Conf.API.Grpc.Host, Conf.API.Grpc.Port, _ = splitHostPort("localhost:9999") + + newHeader, err := reencryptHeader(s.FileHeader, s.UserKey.PubKeyBase64) + if err == nil { + s.T().Fatal("Expected an error due to failed connection, but got nil") + } + assert.Equal(s.T(), newHeader, []uint8([]byte(nil)), "expected header to be nil") + assert.ErrorContains(s.T(), err, "connection refused") } From c4fd58ca1cfabc1ac49afb47a22a72419ec7dc77 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 8 May 2025 18:01:39 +0200 Subject: [PATCH 062/262] [API] Update helm chart, resolve merge conflicts --- charts/sda-svc/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 9915059936..5ba3fe87e0 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -168,6 +168,8 @@ Parameter | Description | Default `global.tls.enabled` | Use TLS for all connections. |`true` `global.tls.issuer` | Issuer for TLS certificate creation. |`""` `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` +`global.reencrypt.host` | gRPC host for reencryption |`""` +`global.reencrypt.port` | port number of the gRPC host for reencryption |`"50051"` ### Credentials From ca79a833c393f54d04e3b6c94906b9737cba27ec Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 9 May 2025 15:04:00 +0200 Subject: [PATCH 063/262] Apply reviewers comments Co-authored-by: Joakim Bygdell Update sda/cmd/api/api.md Co-authored-by: Malin Klang Update sda/cmd/api/api.go Co-authored-by: Joakim Bygdell --- sda/cmd/api/api.go | 2 +- sda/cmd/api/api.md | 2 +- sda/cmd/api/swagger_v1.yml | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index ddd68d952c..6fb66f94ce 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -425,7 +425,7 @@ func downloadFile(c *gin.Context) { // Get inbox file handle file, err := inbox.NewFileReader(filePath) if err != nil { - log.Errorf("inbox file %s not found or failed to read, %s", filePath, err) + log.Errorf("inbox file %s not found or failed to read, %s", filePath, err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to read inbox file") return diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 91e127a210..59a410df1c 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -222,7 +222,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th - `/users/:username/file/:fileid` - accepts `GET` requests. - - download a file from the inbox, re-encrypted with the client’s public key provided in the request header. + - downloads a file from the inbox, re-encrypted with the client’s public key provided in the request header. - the public key provided in the header should be `base64` encoded. - Error codes diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index abc66ff8aa..fefbb0e96a 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -331,7 +331,12 @@ paths: required: true responses: "200": - description: Successful operation. + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successful operation "400": description: User public key not provided or not valid. "401": From cd390bff8ad89d42e4ef2871096eaa7001d8fc2a Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 9 May 2025 16:13:12 +0200 Subject: [PATCH 064/262] Add integration test to download a file as a non admin user --- .github/integration/tests/sda/60_api_admin_test.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index f85c9a6d0f..331f432c05 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -102,6 +102,14 @@ if ! cmp -s "$decryptedFile" "NA12878.bam" ; then exit 1 fi +# download file as a non admin user should fail +token_nonAdmin=$(grep 'access_token' /shared/s3cfg | sed -E 's/access_token="?([^"]+)"?/\1/') +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token_nonAdmin" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" +if [ "$resp" != "401" ]; then + echo "Error when downloading the file, expected 401 got: $resp" + exit 1 +fi + # delete it resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "200" ]; then From 8dd332d8154ba9d810b66f15fcb20b3433b71db6 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 9 May 2025 17:54:34 +0200 Subject: [PATCH 065/262] [API] Correct Content-Disposition header format in response --- sda/cmd/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 6fb66f94ce..66af761c55 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -450,7 +450,7 @@ func downloadFile(c *gin.Context) { // Set the headers for the response. c.Header("Content-Type", "application/octet-stream") - c.Header("Content-Disposition", fmt.Sprintf("filename: %v", path.Base(filePath))) + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(filePath))) reader := io.MultiReader(bytes.NewReader(newHeader), file) _, err = io.Copy(c.Writer, reader) From b193461b5f87c7ce1956459541e299f3609e9283 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 9 May 2025 18:02:24 +0200 Subject: [PATCH 066/262] [Integration test] Ensure correct downloaded filename when no output is set --- .../tests/sda/60_api_admin_test.sh | 115 ++++++++++-------- 1 file changed, 63 insertions(+), 52 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 331f432c05..f68ef3ce9d 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -66,50 +66,6 @@ if [ -z "$output" ] ; then echo "Uploaded file not in inbox" exit 1 fi - -# download the file, re-encrypted with the client key -clientPubKey="$(base64 -w0 /shared/client.pub.pem)" -outFile="download_reenc_NA12878.bam.c4gh" -resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" -if [ "$resp" != "200" ]; then - echo "Error when downloading the file, expected 200 got: $resp" - exit 1 -fi - -# decrypt the downloaded file -export C4GH_PASSPHRASE=c4ghpass -if [ ! -f "$outFile" ]; then - echo "downloaded file $outFile not found" - exit 1 -fi -if [ ! -f "/shared/client.sec.pem" ]; then - echo "client key not found" - exit 1 -fi -if ! /shared/crypt4gh decrypt -f "$outFile" -s "/shared/client.sec.pem"; then - echo "decrypting file $outFile failed" - exit 1 -fi - -# check the file content -decryptedFile="${outFile%.*}" -if [ ! -f "$decryptedFile" ]; then - echo "decrypted file ${decryptedFile} not found" - exit 1 -fi -if ! cmp -s "$decryptedFile" "NA12878.bam" ; then - echo "downloaded file is different from the original one" - exit 1 -fi - -# download file as a non admin user should fail -token_nonAdmin=$(grep 'access_token' /shared/s3cfg | sed -E 's/access_token="?([^"]+)"?/\1/') -resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token_nonAdmin" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" -if [ "$resp" != "401" ]; then - echo "Error when downloading the file, expected 401 got: $resp" - exit 1 -fi - # delete it resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "200" ]; then @@ -129,13 +85,6 @@ if [ -n "$output" ] ; then exit 1 fi -# Try to download the file that has been deleted -resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -o $outFile)" -if [ "$resp" != "404" ]; then - echo "Trying to download a non existing file, expected 404 got: $resp" - exit 1 -fi - # Try to delete an unknown file resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/badfileid")" if [ "$resp" != "404" ]; then @@ -206,4 +155,66 @@ resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer if [ "$resp" != "404" ]; then echo "Error when deleting the file, expected 404 got: $resp" exit 1 -fi \ No newline at end of file +fi + +# Test downloading the file from the inbox +# Reupload a file under a different name, test to download it +s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" + +output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) +if [ -z "$output" ] ; then + echo "Uploaded file NC12878.bam.c4gh not found in inbox" + exit 1 +fi + +# download the file, re-encrypted with the client key +clientPubKey="$(base64 -w0 /shared/client.pub.pem)" +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -JO)" +if [ "$resp" != "200" ]; then + echo "Error when downloading the file, expected 200 got: $resp" + exit 1 +fi + +# decrypt the downloaded file +export C4GH_PASSPHRASE=c4ghpass +if [ ! -f "NC12878.bam.c4gh" ]; then + echo "downloaded file NC12878.bam.c4gh not found" + exit 1 +fi +if [ ! -f "/shared/client.sec.pem" ]; then + echo "client key not found" + exit 1 +fi + +# copy the file to avoid overwriting the original +if ! /shared/crypt4gh decrypt -f "NC12878.bam.c4gh" -s "/shared/client.sec.pem" ; then + echo "decrypting file NC12878.bam.c4gh failed" + exit 1 +fi + +# check the file content +if [ ! -f "NC12878.bam" ]; then + echo "decrypted file NC12878.bam not found" + exit 1 +fi +if ! cmp -s "NC12878.bam" "NA12878.bam" ; then + echo "downloaded file is different from the original one" + exit 1 +fi + +# download file as a non admin user should fail +token_nonAdmin=$(grep 'access_token' /shared/s3cfg | sed -E 's/access_token="?([^"]+)"?/\1/') +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token_nonAdmin" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -JO)" +if [ "$resp" != "401" ]; then + echo "Error when downloading the file, expected 401 got: $resp" + exit 1 +fi + +# download a non existing file should fail +badfileid="badfileid" +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$badfileid" -JO)" +if [ "$resp" != "404" ]; then + echo "Trying to download a non existing file, expected 404 got: $resp" + exit 1 +fi From aaed5b9362a60ebdb8f2f200fbfcc35bac20fdfe Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 23 May 2025 15:04:57 +0200 Subject: [PATCH 067/262] [charts] define grpc in the API helm chart template [chart] Apply reviewers comment --- charts/sda-svc/templates/api-secrets.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/charts/sda-svc/templates/api-secrets.yaml b/charts/sda-svc/templates/api-secrets.yaml index 1a06e50a18..f29d39b30c 100644 --- a/charts/sda-svc/templates/api-secrets.yaml +++ b/charts/sda-svc/templates/api-secrets.yaml @@ -50,6 +50,11 @@ stringData: port: {{ .Values.global.db.port }} sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} user: {{ required "DB user is required" (include "dbUserAPI" .) }} + grpc: + host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }} + {{- if .Values.global.reencrypt.port}} + port: {{ .Values.global.reencrypt.port }} + {{- end }} inbox: type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} {{- if eq "s3" .Values.global.inbox.storageType }} From c022f698a716819524ccfa08f1628fb5df283f8d Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 23 May 2025 16:37:12 +0200 Subject: [PATCH 068/262] Remove necessary Timeout config setting for reencrypt Co-authored-by: Joakim Bygdell --- sda/internal/config/config.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 489f7d4e44..f761d663f8 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -992,9 +992,6 @@ func (c *Config) configReEncryptServer() (err error) { if viper.IsSet("grpc.serverkey") { c.ReEncrypt.ServerKey = viper.GetString("grpc.serverkey") } - if viper.IsSet("grpc.timeout") { - c.ReEncrypt.Timeout = viper.GetInt("grpc.timeout") - } if c.ReEncrypt.ServerCert != "" && c.ReEncrypt.ServerKey != "" { c.ReEncrypt.Port = 50443 From e7bc302230e64a7d7241efe3714eb2785ba8b42b Mon Sep 17 00:00:00 2001 From: neicnordic Date: Mon, 26 May 2025 13:37:10 +0000 Subject: [PATCH 069/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index e66cfb67c4..329e8f57a3 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.0 -appVersion: v2.0.0 +version: 2.0.1 +appVersion: v2.0.1 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 9564f1a438..4ccdffd420 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.0 -appVersion: v2.0.0 +version: 2.0.1 +appVersion: v2.0.1 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 35a5a9a75a..0cf4000b29 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.0 -appVersion: v2.0.0 +version: 2.0.1 +appVersion: v2.0.1 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 66472279622326434081d0bb72763ee2e3488cd6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 26 May 2025 11:52:37 +0200 Subject: [PATCH 070/262] Roll back minio-mc version The mc command syntax has changed, rolling back until someone have time to look into it in detail. --- .github/integration/sda-doa-s3-outbox.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index ada7422362..5c6af4f6d0 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -99,7 +99,7 @@ services: init-bucket: container_name: init-bucket - image: minio/mc + image: minio/mc:RELEASE.2025-04-16T18-13-26Z depends_on: outbox: condition: service_healthy @@ -107,7 +107,6 @@ services: /bin/sh -c " /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage; /usr/bin/mc mb s3/lega; - exit 0; " doa: @@ -138,7 +137,6 @@ services: - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass - OUTBOX_TYPE=S3 - - SSL_ENABLED=false - ROOT_CERT_PATH=/certs/ca.crt - CERT_PATH=/certs/client.crt - CERT_KEY=/certs/client.der @@ -165,9 +163,12 @@ services: - client_certs:/certs depends_on: - - doa - - mockauth - - init-bucket + doa: + condition: service_started + mockauth: + condition: service_started + init-bucket: + condition: service_completed_successfully environment: - OUTBOX_TYPE=S3 - DOA_URL=http://doa:8080 From 8ab396dbf405cfe4b9c0175ada45b74ff76ae75d Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 27 May 2025 11:05:29 +0200 Subject: [PATCH 071/262] change deprecated 'mc config host add' to 'mc alias set' for s3 configuration --- .github/integration/sda-doa-s3-outbox.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 5c6af4f6d0..8ca9c558db 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -99,13 +99,13 @@ services: init-bucket: container_name: init-bucket - image: minio/mc:RELEASE.2025-04-16T18-13-26Z + image: minio/mc depends_on: outbox: condition: service_healthy entrypoint: > /bin/sh -c " - /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage; + /usr/bin/mc alias set s3 http://outbox:9000 minio miniostorage; /usr/bin/mc mb s3/lega; " From aaff5f35bd4eed8e5e7cbe8a8e445a22310a99eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 May 2025 11:03:51 +0000 Subject: [PATCH 072/262] Bump google.golang.org/grpc in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `google.golang.org/grpc` from 1.72.1 to 1.72.2 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.72.1...v1.72.2) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.72.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index c52966f926..d4e1dfe863 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -17,7 +17,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.38.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 - google.golang.org/grpc v1.72.1 + google.golang.org/grpc v1.72.2 google.golang.org/protobuf v1.36.6 ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 98643b916b..a5a6eaf614 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -232,8 +232,8 @@ golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= -google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 4f2464507422ed251e02d84c50c0234aa42b5a51 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 May 2025 11:04:00 +0000 Subject: [PATCH 073/262] Bump the all-modules group in /sda with 3 updates Bumps the all-modules group in /sda with 3 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.75 to 1.17.76 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.75...feature/s3/manager/v1.17.76) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.79.3 to 1.79.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.79.3...service/s3/v1.79.4) Updates `google.golang.org/grpc` from 1.72.1 to 1.72.2 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.72.1...v1.72.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.76 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.79.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.72.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 8 ++++---- sda/go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 3cfe465262..56c4750e0c 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,8 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 - github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.76 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.105.0 github.com/coreos/go-oidc/v3 v3.14.1 @@ -33,7 +33,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.38.0 golang.org/x/oauth2 v0.30.0 - google.golang.org/grpc v1.72.1 + google.golang.org/grpc v1.72.2 google.golang.org/protobuf v1.36.6 ) @@ -56,7 +56,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect diff --git a/sda/go.sum b/sda/go.sum index 7a5d49fe16..8c12a31b1d 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9 github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.76 h1:TZEAZHyLeRbSvETr20mAoJDUPhIMuFZ9ZwjkftWongU= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.76/go.mod h1:7h7z0FVKk7IYXuIZ8bWI58Afwc3kPMHqVIdczGgU3wc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -46,14 +46,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcu github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 h1:BCG7DCXEXpNCcpwCxg1oi9pkJWH2+eZzTn9MY56MbVw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 h1:4yxno6bNHkekkfqG/a1nz/gC2gBwhJSojV1+oTE7K+4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= @@ -515,8 +515,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= -google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= +google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From d534ba1d5bf7bd9881eb6cfe577069fc2d67892a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 May 2025 11:04:12 +0000 Subject: [PATCH 074/262] Bump org.springframework.boot:spring-boot-starter-parent Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.5 to 3.5.0 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.5...v3.5.0) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 48931ef309..51dce96397 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.5 + 3.5.0 From 53db7d4dd8f11b6991f8302ad24a28433d34b81d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 May 2025 11:05:14 +0000 Subject: [PATCH 075/262] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.5 to 3.5.0 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.5...v3.5.0) Updates `no.elixir:clearinghouse` from 2.0.18 to 2.0.19 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-2.0.18...clearinghouse-2.0.19) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 2.0.19 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 5b42709df8..6de51bb58a 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.5 + 3.5.0 no.uio.ifi @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.18 + 2.0.19 org.slf4j From c1cb804e85f8792787d4bf847fafa8b722c413a9 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 27 May 2025 14:42:35 +0200 Subject: [PATCH 076/262] change deprecated 'mc config host add' to 'mc alias set' for s3 configuration in sda-download workflows --- sda-download/dev_utils/compose-no-tls.yml | 2 +- sda-download/dev_utils/compose-sda.yml | 2 +- sda-download/dev_utils/compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/dev_utils/compose-no-tls.yml b/sda-download/dev_utils/compose-no-tls.yml index 3e7ee3ea79..25eed5d4db 100644 --- a/sda-download/dev_utils/compose-no-tls.yml +++ b/sda-download/dev_utils/compose-no-tls.yml @@ -42,7 +42,7 @@ services: condition: service_healthy entrypoint: > /bin/sh -c " - /usr/bin/mc config host add s3 http://s3:9000 access secretkey; + /usr/bin/mc alias set add s3 http://s3:9000 access secretkey; /usr/bin/mc mb s3/archive; exit 0; " diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 370de0a6ca..482556f7c0 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -66,7 +66,7 @@ services: entrypoint: > /bin/sh -c " sleep 10; - /usr/bin/mc config host add s3 https://s3:9000 access secretkey; + /usr/bin/mc alias set s3 https://s3:9000 access secretkey; /usr/bin/mc mb s3/archive; exit 0; " diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 3a491beddc..75b4b699e9 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -70,7 +70,7 @@ services: condition: service_healthy entrypoint: > /bin/sh -c " - /usr/bin/mc config host add s3 https://s3:9000 access secretkey; + /usr/bin/mc alias set s3 https://s3:9000 access secretkey; /usr/bin/mc mb s3/archive; exit 0; " From 2305bc69e3846da36ba62483059f466992e5fb12 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 27 May 2025 15:15:46 +0200 Subject: [PATCH 077/262] update minio/minio images to latest tag --- .github/integration/sda-s3-integration.yml | 2 +- .github/integration/sda-sync-integration.yml | 2 +- sda-download/dev_utils/compose-no-tls.yml | 2 +- sda-download/dev_utils/compose-sda.yml | 2 +- sda-download/dev_utils/compose.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 5ae0a4391c..d47f11c9ad 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -58,7 +58,7 @@ services: - rabbitmq_data:/var/lib/rabbitmq minio: - image: minio/minio:RELEASE.2023-05-18T00-05-36Z + image: minio/minio command: server /data --console-address ":9001" container_name: s3 environment: diff --git a/.github/integration/sda-sync-integration.yml b/.github/integration/sda-sync-integration.yml index 2af84742e1..673b617fc5 100644 --- a/.github/integration/sda-sync-integration.yml +++ b/.github/integration/sda-sync-integration.yml @@ -57,7 +57,7 @@ services: - rabbitmq_data:/var/lib/rabbitmq minio: - image: minio/minio:RELEASE.2023-05-18T00-05-36Z + image: minio/minio command: server /data --console-address ":9001" container_name: s3 environment: diff --git a/sda-download/dev_utils/compose-no-tls.yml b/sda-download/dev_utils/compose-no-tls.yml index 25eed5d4db..af1aebb55f 100644 --- a/sda-download/dev_utils/compose-no-tls.yml +++ b/sda-download/dev_utils/compose-no-tls.yml @@ -30,7 +30,7 @@ services: interval: 5s timeout: 20s retries: 3 - image: minio/minio:RELEASE.2021-11-24T23-19-33Z + image: minio/minio ports: - "9000:9000" diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 482556f7c0..2d1a92b22f 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -51,7 +51,7 @@ services: interval: 5s timeout: 20s retries: 3 - image: minio/minio:RELEASE.2020-06-03T22-13-49Z + image: minio/minio ports: - "9000:9000" volumes: diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 75b4b699e9..0f84f7fc2e 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -54,7 +54,7 @@ services: interval: 5s timeout: 20s retries: 3 - image: minio/minio:RELEASE.2020-06-03T22-13-49Z + image: minio/minio ports: - "9000:9000" volumes: From ed0abaecda3b977d9be82cce65c12360715c59df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 19:25:56 +0000 Subject: [PATCH 078/262] Bump the all-modules group in /sda with 2 updates Bumps the all-modules group in /sda with 2 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.76 to 1.17.77 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.76...feature/s3/manager/v1.17.77) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.79.4 to 1.80.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.79.4...service/s3/v1.80.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.77 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.80.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 4 ++-- sda/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 56c4750e0c..8602dd7258 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,8 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.76 - github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77 + github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.105.0 github.com/coreos/go-oidc/v3 v3.14.1 diff --git a/sda/go.sum b/sda/go.sum index 8c12a31b1d..487cd90098 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9 github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.76 h1:TZEAZHyLeRbSvETr20mAoJDUPhIMuFZ9ZwjkftWongU= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.76/go.mod h1:7h7z0FVKk7IYXuIZ8bWI58Afwc3kPMHqVIdczGgU3wc= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77 h1:xaRN9fags7iJznsMEjtcEuON1hGfCZ0y5MVfEMKtrx8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77/go.mod h1:lolsiGkT47AZ3DWqtxgEQM/wVMpayi7YWNjl3wHSRx8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4 h1:4yxno6bNHkekkfqG/a1nz/gC2gBwhJSojV1+oTE7K+4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.4/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0 h1:fV4XIU5sn/x8gjRouoJpDVHj+ExJaUk4prYF+eb6qTs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= From a8f6de4a727b5492a843ee7e9b5f1978e8fdbcab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 19:50:05 +0000 Subject: [PATCH 079/262] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.24 to 3.0.25 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.24...crypt4gh-3.0.25) Updates `no.elixir:clearinghouse` from 2.0.19 to 2.0.20 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-2.0.19...clearinghouse-2.0.20) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.25 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 2.0.20 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 6de51bb58a..da33086bab 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.24 + 3.0.25 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.19 + 2.0.20 org.slf4j From e9b99a676e2ef1b14bd63cd7a9e5321260a93d56 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 3 Jun 2025 10:23:39 +0000 Subject: [PATCH 080/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 329e8f57a3..9667a809b5 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.1 -appVersion: v2.0.1 +version: 2.0.2 +appVersion: v2.0.8 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 4ccdffd420..5eb6077e5f 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.1 -appVersion: v2.0.1 +version: 2.0.2 +appVersion: v2.0.8 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 0cf4000b29..0838128154 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.1 -appVersion: v2.0.1 +version: 2.0.2 +appVersion: v2.0.8 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From ad18a0545852dc65829c93483fe09e0aebb64642 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Jun 2025 19:12:00 +0000 Subject: [PATCH 081/262] Bump org.junit:junit-bom in /sda-sftp-inbox in the all-modules group Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.junit:junit-bom](https://github.com/junit-team/junit5). Updates `org.junit:junit-bom` from 5.12.2 to 5.13.0 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.12.2...r5.13.0) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 5.13.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 51dce96397..7cf45ea78f 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -29,7 +29,7 @@ org.junit junit-bom - 5.12.2 + 5.13.0 pom import From 164e82bb58e49ca7f35539bd489afcf7a534bf85 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 3 Jun 2025 11:46:47 +0000 Subject: [PATCH 082/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 329e8f57a3..7598f4450f 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.1 -appVersion: v2.0.1 +version: 2.0.2 +appVersion: v2.0.9 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 4ccdffd420..560a70d523 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.1 -appVersion: v2.0.1 +version: 2.0.2 +appVersion: v2.0.9 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 0cf4000b29..86deaf8576 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.1 -appVersion: v2.0.1 +version: 2.0.2 +appVersion: v2.0.9 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 75497ac15f0024d827eae2156f5a30344938cd5f Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 5 Jun 2025 10:51:07 +0200 Subject: [PATCH 083/262] replace WITH_V: true with TAG_PREFIX: v cause WITH_V is deprecated --- .github/workflows/publish_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index 4dbd4f8bf9..839aa0b01b 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -32,7 +32,7 @@ jobs: uses: anothrNick/github-tag-action@1.73.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true + TAG_PREFIX: v DEFAULT_BUMP: patch TAG_CONTEXT: branch From 9173489011699358f1ae46d1d479f1d8f200e8c3 Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 5 Jun 2025 12:03:44 +0200 Subject: [PATCH 084/262] add a workflow for scanning images --- .github/workflows/scan-images.yml | 72 +++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 .github/workflows/scan-images.yml diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml new file mode 100644 index 0000000000..4f286b5944 --- /dev/null +++ b/.github/workflows/scan-images.yml @@ -0,0 +1,72 @@ +name: Scan Images +on: + merge_group: + pull_request: + branches: + - main + paths-ignore: + - ".github/**" + - ".gitignore" + - "**/*.md" + - "charts/**" + - "Makefile" + - "sda-admin/**" + - "tools/*" + types: [ closed ] + +jobs: + scan-all: + if: github.event.pull_request.merged == true + name: Scan ${{ matrix.image-name }} Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + security-events: write + strategy: + matrix: + image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] + steps: + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.30.0 + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-${{ matrix.image-name }} + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: '${{ matrix.image-name }}-results.sarif' + severity: "CRITICAL,HIGH" + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: '${{ matrix.image-name }}-results.sarif' + category: ${{ matrix.image-name }} + scan-sda: + if: github.event.pull_request.merged == true + name: Scan SDA Docker image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + security-events: write + steps: + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.30.0 + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }} + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: 'sda-results.sarif' + severity: "CRITICAL,HIGH" + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'sda-results.sarif' + category: sda \ No newline at end of file From 201cf607a540975ad9ddc1ebcbebe6c3d1292e64 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 19:21:01 +0000 Subject: [PATCH 085/262] Bump aquasecurity/trivy-action from 0.30.0 to 0.31.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.30.0 to 0.31.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.30.0...0.31.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.31.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 5403523a70..61c90c9344 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -105,7 +105,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.31.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -123,7 +123,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.31.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -172,7 +172,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.31.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -218,7 +218,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.31.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From 36479337590d23a096fe6b9fd69db751c09ea0f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 19:23:58 +0000 Subject: [PATCH 086/262] Bump the all-modules group in /sda-sftp-inbox with 3 updates Bumps the all-modules group in /sda-sftp-inbox with 3 updates: [org.junit:junit-bom](https://github.com/junit-team/junit5), [com.amazonaws:aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) and [org.bouncycastle:bcprov-jdk18on](https://github.com/bcgit/bc-java). Updates `org.junit:junit-bom` from 5.13.0 to 5.13.1 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.13.0...r5.13.1) Updates `com.amazonaws:aws-java-sdk-s3` from 1.12.783 to 1.12.785 - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.783...1.12.785) Updates `org.bouncycastle:bcprov-jdk18on` from 1.80 to 1.81 - [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html) - [Commits](https://github.com/bcgit/bc-java/commits) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 5.13.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-version: 1.12.785 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: org.bouncycastle:bcprov-jdk18on dependency-version: '1.81' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 7cf45ea78f..68c74cab42 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -29,7 +29,7 @@ org.junit junit-bom - 5.13.0 + 5.13.1 pom import @@ -108,7 +108,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.783 + 1.12.785 com.google.guava @@ -123,7 +123,7 @@ org.bouncycastle bcprov-jdk18on - 1.80 + 1.81 From 1a52cfd2ceecb6d3a04517ab3278f0fc2031365c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 19:40:37 +0000 Subject: [PATCH 087/262] Bump the all-modules group in /sda with 7 updates Bumps the all-modules group in /sda with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.14` | `1.29.15` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.67` | `1.17.68` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.77` | `1.17.78` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.80.0` | `1.80.1` | | [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) | `2.105.0` | `2.107.0` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.38.0` | `0.39.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.72.2` | `1.73.0` | Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.14 to 1.29.15 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.14...config/v1.29.15) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.67 to 1.17.68 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.67...credentials/v1.17.68) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.77 to 1.17.78 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.77...feature/s3/manager/v1.17.78) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.80.0 to 1.80.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.80.0...service/s3/v1.80.1) Updates `github.com/casbin/casbin/v2` from 2.105.0 to 2.107.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.105.0...v2.107.0) Updates `golang.org/x/crypto` from 0.38.0 to 0.39.0 - [Commits](https://github.com/golang/crypto/compare/v0.38.0...v0.39.0) Updates `google.golang.org/grpc` from 1.72.2 to 1.73.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.72.2...v1.73.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.29.15 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.17.68 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.78 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.80.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.107.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.39.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.73.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 18 +++++++++--------- sda/go.sum | 56 +++++++++++++++++++++++++++--------------------------- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 8602dd7258..b80360a493 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,12 +4,12 @@ go 1.24.1 require ( github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.14 - github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77 - github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0 + github.com/aws/aws-sdk-go-v2/config v1.29.15 + github.com/aws/aws-sdk-go-v2/credentials v1.17.68 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.78 + github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1 github.com/aws/smithy-go v1.22.3 - github.com/casbin/casbin/v2 v2.105.0 + github.com/casbin/casbin/v2 v2.107.0 github.com/coreos/go-oidc/v3 v3.14.1 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -31,9 +31,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.38.0 + golang.org/x/crypto v0.39.0 golang.org/x/oauth2 v0.30.0 - google.golang.org/grpc v1.72.2 + google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 ) @@ -61,7 +61,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect @@ -155,7 +155,7 @@ require ( golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/text v0.26.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 487cd90098..ffa0c04c70 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -28,14 +28,14 @@ github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38y github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= -github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/config v1.29.15 h1:I5XjesVMpDZXZEZonVfjI12VNMrYa38LtLnw4NtY5Ss= +github.com/aws/aws-sdk-go-v2/config v1.29.15/go.mod h1:tNIp4JIPonlsgaO5hxO372a6gjhN63aSWl2GVl5QoBQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.68 h1:cFb9yjI02/sWHBSYXAtkamjzCuRymvmeFmt0TC0MbYY= +github.com/aws/aws-sdk-go-v2/credentials v1.17.68/go.mod h1:H6E+jBzyqUu8u0vGaU6POkK3P0NylYEeRZ6ynBpMqIk= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77 h1:xaRN9fags7iJznsMEjtcEuON1hGfCZ0y5MVfEMKtrx8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.77/go.mod h1:lolsiGkT47AZ3DWqtxgEQM/wVMpayi7YWNjl3wHSRx8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.78 h1:tvUv5jdxr+6zPiRA4I5GN+q2g7Ls9pxXmO7nK6jLqic= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.78/go.mod h1:MbNrCDTndc0qvjKSL+bY8wae5xVWlkoXlgFCCYVw03g= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -52,14 +52,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0 h1:fV4XIU5sn/x8gjRouoJpDVHj+ExJaUk4prYF+eb6qTs= -github.com/aws/aws-sdk-go-v2/service/s3 v1.80.0/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1 h1:xYEAf/6QHiTZDccKnPMbsMwlau13GsDsTgdue3wmHGw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 h1:oIaQ1e17CSKaWmUTu62MtraRWVIosn/iONMuZt0gbqc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.20/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.105.0 h1:dLj5P6pLApBRat9SADGiLxLZjiDPvA1bsPkyV4PGx6I= -github.com/casbin/casbin/v2 v2.105.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.107.0 h1:Kk1+9S2ou8aTTQd30L+vRvFBNf5YvbN65N5uCzdren8= +github.com/casbin/casbin/v2 v2.107.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= @@ -385,16 +385,16 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -410,8 +410,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -494,8 +494,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -515,8 +515,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 22dd588946e2b41e32d77e6d9f5393c5000ed138 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Jun 2025 20:10:05 +0000 Subject: [PATCH 088/262] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [golang.org/x/crypto](https://github.com/golang/crypto) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `golang.org/x/crypto` from 0.38.0 to 0.39.0 - [Commits](https://github.com/golang/crypto/compare/v0.38.0...v0.39.0) Updates `google.golang.org/grpc` from 1.72.2 to 1.73.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.72.2...v1.73.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.39.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.73.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 6 +++--- sda-download/go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index d4e1dfe863..82d0e0feb2 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -15,9 +15,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.38.0 + golang.org/x/crypto v0.39.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 - google.golang.org/grpc v1.72.2 + google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 ) @@ -69,7 +69,7 @@ require ( golang.org/x/arch v0.17.0 // indirect golang.org/x/net v0.40.0 // indirect golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect + golang.org/x/text v0.26.0 // indirect golang.org/x/tools v0.33.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index a5a6eaf614..4ef7ddc24d 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -59,8 +59,8 @@ github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -159,24 +159,24 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= +golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= +golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -219,8 +219,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= +golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -232,8 +232,8 @@ golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.72.2 h1:TdbGzwb82ty4OusHWepvFWGLgIbNo1/SUynEN0ssqv8= -google.golang.org/grpc v1.72.2/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= +google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 300f249bfe6e5b24ec7f791aaf1f1a8c4a200090 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 10 Jun 2025 08:11:11 +0000 Subject: [PATCH 089/262] Bump requests Bumps the pip group with 1 update in the /.github/integration/tests/sda/auth directory: [requests](https://github.com/psf/requests). Updates `requests` from 2.32.0 to 2.32.4 - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.32.0...v2.32.4) --- updated-dependencies: - dependency-name: requests dependency-version: 2.32.4 dependency-type: direct:production dependency-group: pip ... Signed-off-by: dependabot[bot] --- .github/integration/tests/sda/auth/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/tests/sda/auth/requirements.txt b/.github/integration/tests/sda/auth/requirements.txt index 1cca1d478f..8073666c6d 100644 --- a/.github/integration/tests/sda/auth/requirements.txt +++ b/.github/integration/tests/sda/auth/requirements.txt @@ -1,3 +1,3 @@ -requests==2.32.0 +requests==2.32.4 tox==4.11.3 pytest==7.4.3 From ae49f6bcdac169b1801b28b8ca1842f39dde5203 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 10 Jun 2025 11:28:01 +0000 Subject: [PATCH 090/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 3a92cf99a5..bec835449e 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.3 -appVersion: v2.0.9 +version: 2.0.4 +appVersion: v2.0.12 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 1471dd90dc..112644347d 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.3 -appVersion: v2.0.9 +version: 2.0.4 +appVersion: v2.0.12 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 94bd5218ac..066cbc3892 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.3 -appVersion: v2.0.9 +version: 2.0.4 +appVersion: v2.0.12 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From f22ee0b2c41dbfe77b9b7a691b5f2592410feba0 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 10 Jun 2025 13:56:58 +0200 Subject: [PATCH 091/262] add general exception to endpoints --- .../localega/doa/rest/MetadataController.java | 29 +++++++++++++------ .../doa/rest/StreamingController.java | 3 ++ 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java index 0ca74c911d..944a2ee073 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java @@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.RestController; import jakarta.servlet.http.HttpServletRequest; + import java.util.Set; /** @@ -38,9 +39,14 @@ public class MetadataController { @SuppressWarnings("unchecked") @GetMapping("/datasets") public ResponseEntity datasets() { - log.info("User has permissions to list datasets"); - Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); - return ResponseEntity.ok(metadataService.datasets(datasetIds)); + try { + log.info("User has permissions to list datasets"); + Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); + return ResponseEntity.ok(metadataService.datasets(datasetIds)); + } catch (Exception e) { + log.error("Error in listing datasets", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error while listing datasets: " + e.getMessage()); + } } /** @@ -52,13 +58,18 @@ public ResponseEntity datasets() { @SuppressWarnings("unchecked") @GetMapping("/datasets/{datasetId}/files") public ResponseEntity files(@PathVariable(value = "datasetId") String datasetId) { - Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); - if (!datasetIds.contains(datasetId)) { - log.info("User doesn't have permissions to list files in the requested dataset: {}", datasetId); - return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); + try { + Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); + if (!datasetIds.contains(datasetId)) { + log.info("User doesn't have permissions to list files in the requested dataset: {}", datasetId); + return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); + } + log.info("User has permissions to list files in the requested dataset: {}", datasetId); + return ResponseEntity.ok(metadataService.files(datasetId)); + } catch (Exception e) { + log.error("Error in listing files in the dataset: {}", datasetId, e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error while listing files in the dataset: " + e.getMessage()); } - log.info("User has permissions to list files in the requested dataset: {}", datasetId); - return ResponseEntity.ok(metadataService.files(datasetId)); } } diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java index 6123f13acb..e4aceccf06 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java @@ -59,6 +59,9 @@ public ResponseEntity files(@RequestHeader(value = "Public-Key", required = f } catch (AuthException e) { log.info("User doesn't have permissions to download requested file: {}", fileId); return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); + } catch (Exception e) { + log.error("Error in streaming file: {}", fileId, e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error in streaming file: " + e.getMessage()); } } From 5a67c0214d005584b17f5033bac0b32fe3bb8645 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Fri, 13 Jun 2025 11:00:13 +0000 Subject: [PATCH 092/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index bec835449e..125beaea63 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.4 -appVersion: v2.0.12 +version: 2.0.5 +appVersion: v2.0.13 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 112644347d..4e6ddfe4a7 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.4 -appVersion: v2.0.12 +version: 2.0.5 +appVersion: v2.0.13 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 066cbc3892..65c2a1ca54 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.4 -appVersion: v2.0.12 +version: 2.0.5 +appVersion: v2.0.13 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 86f0230c29c963803d02db4c2002fd93daf6cba5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 20:02:08 +0000 Subject: [PATCH 093/262] Bump aquasecurity/trivy-action from 0.30.0 to 0.31.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.30.0 to 0.31.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.30.0...0.31.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.31.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/scan-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index 4f286b5944..465f087701 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -28,7 +28,7 @@ jobs: image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.31.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -54,7 +54,7 @@ jobs: security-events: write steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.31.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From f0c6567fe8791146a3a40c2bca9eac6031fb228f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 20:13:04 +0000 Subject: [PATCH 094/262] Bump no.elixir:clearinghouse in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:clearinghouse` from 2.0.20 to 3.0.0 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-2.0.20...clearinghouse-3.0.0) --- updated-dependencies: - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.0 dependency-type: direct:production update-type: version-update:semver-major dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index da33086bab..9a73de9cda 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.20 + 3.0.0 org.slf4j From b0fdcf75b249dcf71fe51714cb0e104315d4f560 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 20:18:59 +0000 Subject: [PATCH 095/262] Bump com.amazonaws:aws-java-sdk-s3 Bumps the all-modules group in /sda-sftp-inbox with 1 update: [com.amazonaws:aws-java-sdk-s3](https://github.com/aws/aws-sdk-java). Updates `com.amazonaws:aws-java-sdk-s3` from 1.12.785 to 1.12.787 - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.785...1.12.787) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-version: 1.12.787 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 68c74cab42..211a21cf73 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -108,7 +108,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.785 + 1.12.787 com.google.guava From 6770c0db70881478ff10f669fd17bb616d0d5def Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Jun 2025 20:59:45 +0000 Subject: [PATCH 096/262] Bump the all-modules group in /sda with 5 updates Bumps the all-modules group in /sda with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.36.3` | `1.36.4` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.15` | `1.29.16` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.68` | `1.17.69` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.78` | `1.17.79` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.80.1` | `1.80.2` | Updates `github.com/aws/aws-sdk-go-v2` from 1.36.3 to 1.36.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.36.3...v1.36.4) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.15 to 1.29.16 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.15...config/v1.29.16) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.68 to 1.17.69 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.68...credentials/v1.17.69) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.78 to 1.17.79 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.78...feature/s3/manager/v1.17.79) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.80.1 to 1.80.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.80.1...service/s3/v1.80.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.36.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.29.16 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.17.69 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.79 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.80.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 30 +++++++++++++-------------- sda/go.sum | 60 +++++++++++++++++++++++++++--------------------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index b80360a493..4064d9ce55 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.15 - github.com/aws/aws-sdk-go-v2/credentials v1.17.68 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.78 - github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1 + github.com/aws/aws-sdk-go-v2 v1.36.4 + github.com/aws/aws-sdk-go-v2/config v1.29.16 + github.com/aws/aws-sdk-go-v2/credentials v1.17.69 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79 + github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.107.0 github.com/coreos/go-oidc/v3 v3.14.1 @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index ffa0c04c70..af03d29031 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= -github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2 v1.36.4 h1:GySzjhVvx0ERP6eyfAbAuAXLtAda5TEy19E5q5W8I9E= +github.com/aws/aws-sdk-go-v2 v1.36.4/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.15 h1:I5XjesVMpDZXZEZonVfjI12VNMrYa38LtLnw4NtY5Ss= -github.com/aws/aws-sdk-go-v2/config v1.29.15/go.mod h1:tNIp4JIPonlsgaO5hxO372a6gjhN63aSWl2GVl5QoBQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.68 h1:cFb9yjI02/sWHBSYXAtkamjzCuRymvmeFmt0TC0MbYY= -github.com/aws/aws-sdk-go-v2/credentials v1.17.68/go.mod h1:H6E+jBzyqUu8u0vGaU6POkK3P0NylYEeRZ6ynBpMqIk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.78 h1:tvUv5jdxr+6zPiRA4I5GN+q2g7Ls9pxXmO7nK6jLqic= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.78/go.mod h1:MbNrCDTndc0qvjKSL+bY8wae5xVWlkoXlgFCCYVw03g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/config v1.29.16 h1:XkruGnXX1nEZ+Nyo9v84TzsX+nj86icbFAeust6uo8A= +github.com/aws/aws-sdk-go-v2/config v1.29.16/go.mod h1:uCW7PNjGwZ5cOGZ5jr8vCWrYkGIhPoTNV23Q/tpHKzg= +github.com/aws/aws-sdk-go-v2/credentials v1.17.69 h1:8B8ZQboRc3uaIKjshve/XlvJ570R7BKNy3gftSbS178= +github.com/aws/aws-sdk-go-v2/credentials v1.17.69/go.mod h1:gPME6I8grR1jCqBFEGthULiolzf/Sexq/Wy42ibKK9c= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 h1:oQWSGexYasNpYp4epLGZxxjsDo8BMBh6iNWkTXQvkwk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31/go.mod h1:nc332eGUU+djP3vrMI6blS0woaCfHTe3KiSQUVTMRq0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79 h1:mGo6WGWry+s5GEf2GLfw3zkHad109FQmtvBV3VYQ8mA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79/go.mod h1:siwnpWxHYFSSge7Euw9lGMgQBgvRyym352mCuGNHsMQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 h1:o1v1VFfPcDVlK3ll1L5xHsaQAFdNtZ5GXnNR7SwueC4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35/go.mod h1:rZUQNYMNG+8uZxz9FOerQJ+FceCiodXvixpeRtdESrU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 h1:R5b82ubO2NntENm3SAm0ADME+H630HomNJdgv+yZ3xw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35/go.mod h1:FuA+nmgMRfkzVKYDNEqQadvEMxtxl9+RLT9ribCwEMs= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 h1:th/m+Q18CkajTw1iqx2cKkLCij/uz8NMwJFPK91p2ug= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35/go.mod h1:dkJuf0a1Bc8HAA0Zm2MoTGm/WDC18Td9vSbrQ1+VqE8= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2 h1:BCG7DCXEXpNCcpwCxg1oi9pkJWH2+eZzTn9MY56MbVw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1 h1:xYEAf/6QHiTZDccKnPMbsMwlau13GsDsTgdue3wmHGw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.80.1/go.mod h1:qbn305Je/IofWBJ4bJz/Q7pDEtnnoInw/dGt71v6rHE= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 h1:oIaQ1e17CSKaWmUTu62MtraRWVIosn/iONMuZt0gbqc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.20/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 h1:VHPZakq2L7w+RLzV54LmQavbvheFaR2u1NomJRSEfcU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3/go.mod h1:DX1e/lkbsAt0MkY3NgLYuH4jQvRfw8MYxTe9feR7aXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 h1:/ldKrPPXTC421bTNWrUIpq3CxwHwRI/kpc+jPUTJocM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16/go.mod h1:5vkf/Ws0/wgIMJDQbjI4p2op86hNW6Hie5QtebrDgT8= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 h1:2HuI7vWKhFWsBhIr2Zq8KfFZT6xqaId2XXnXZjkbEuc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16/go.mod h1:BrwWnsfbFtFeRjdx0iM1ymvlqDX1Oz68JsQaibX/wG8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 h1:T6Wu+8E2LeTUqzqQ/Bh1EoFNj1u4jUyveMgmTlu9fDU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2/go.mod h1:chSY8zfqmS0OnhZoO/hpPx/BHfAIL80m77HwhRLYScY= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 h1:EU58LP8ozQDVroOEyAfcq0cGc5R/FTZjVoYJ6tvby3w= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.4/go.mod h1:CrtOgCcysxMvrCoHnvNAD7PHWclmoFG78Q2xLK0KKcs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 h1:XB4z0hbQtpmBnb1FQYvKaCM7UsS6Y/u8jVBwIUGeCTk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2/go.mod h1:hwRpqkRxnQ58J9blRDrB4IanlXCpcKmsC83EhG77upg= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 h1:nyLjs8sYJShFYj6aiyjCBI3EcLn1udWrQTjEF+SOXB0= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.21/go.mod h1:EhdxtZ+g84MSGrSrHzZiUm9PYiZkrADNja15wtRJSJo= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= From 4c16c78e202b76bbb59713bd1473800bb13b882a Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 17 Jun 2025 08:27:58 +0200 Subject: [PATCH 097/262] update imports due to change in clearing house --- .../java/no/uio/ifi/localega/doa/services/AAIService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java index e22b223c7e..916aa88bf6 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java @@ -3,9 +3,9 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; -import no.uio.ifi.clearinghouse.Clearinghouse; -import no.uio.ifi.clearinghouse.model.Visa; -import no.uio.ifi.clearinghouse.model.VisaType; +import no.elixir.clearinghouse.Clearinghouse; +import no.elixir.clearinghouse.model.Visa; +import no.elixir.clearinghouse.model.VisaType; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; From 64764059e70d790f8e125d00bf84c162916ef425 Mon Sep 17 00:00:00 2001 From: zeidlitz Date: Tue, 17 Jun 2025 10:33:57 +0200 Subject: [PATCH 098/262] [config] Adjust log levels --- sda/internal/config/config.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index f761d663f8..29787c4d38 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -183,7 +183,7 @@ func NewConfig(app string) (*Config, error) { if viper.IsSet("configPath") { cp := viper.GetString("configPath") - log.Infof("configPath: %s", cp) + log.Debugf("configPath: %s", cp) if !strings.HasSuffix(cp, "/") { cp += "/" } @@ -192,21 +192,21 @@ func NewConfig(app string) (*Config, error) { if viper.IsSet("configFile") { viper.SetConfigFile(viper.GetString("configFile")) } - log.Infoln("reading config") + log.Debugf("Attempting to read configuration...") if err := viper.ReadInConfig(); err != nil { - log.Infoln(err.Error()) + log.Debugf("Error reading config : %v", err.Error()) if _, ok := err.(viper.ConfigFileNotFoundError); !ok { - log.Infoln("ReadInConfig Error") + log.Warnf("ReadInConfig Error : %v", err.Error()) return nil, err } - log.Infoln("No config file found, using ENVs only") + log.Debugf("No config file found, using ENVs only") } if viper.IsSet("log.format") { if viper.GetString("log.format") == "json" { log.SetFormatter(&log.JSONFormatter{}) - log.Info("The logs format is set to JSON") + log.Debugf("The logs format is set to JSON") } } @@ -214,7 +214,7 @@ func NewConfig(app string) (*Config, error) { stringLevel := viper.GetString("log.level") intLevel, err := log.ParseLevel(stringLevel) if err != nil { - log.Infof("Log level '%s' not supported, setting to 'trace'", stringLevel) + log.Debugf("Log level '%s' not supported, setting to 'trace'", stringLevel) intLevel = log.TraceLevel } log.SetLevel(intLevel) From 462c756ca5b1bf1db3e30d80aa11c4fa3f7368fe Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 17 Jun 2025 12:47:02 +0000 Subject: [PATCH 099/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 125beaea63..492e858144 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.5 -appVersion: v2.0.13 +version: 2.0.6 +appVersion: v2.0.16 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 4e6ddfe4a7..568dd66ac1 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.5 -appVersion: v2.0.13 +version: 2.0.6 +appVersion: v2.0.16 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 65c2a1ca54..bb42a01b72 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.5 -appVersion: v2.0.13 +version: 2.0.6 +appVersion: v2.0.16 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 09ce310fc3e94dfc0f28ecf14f6a1d53e751b0d6 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 16 Jun 2025 14:30:37 +0200 Subject: [PATCH 100/262] increase mock ls-aai token ttl --- .github/integration/sda/aai-mock/clients/aai-auth.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/integration/sda/aai-mock/clients/aai-auth.yaml b/.github/integration/sda/aai-mock/clients/aai-auth.yaml index b6ec2ed326..184c76203b 100644 --- a/.github/integration/sda/aai-mock/clients/aai-auth.yaml +++ b/.github/integration/sda/aai-mock/clients/aai-auth.yaml @@ -5,4 +5,7 @@ redirect-uris: ["http://localhost:8801/oidc/login"] token-endpoint-auth-method: "client_secret_basic" scope: ["openid", "profile", "email", "ga4gh_passport_v1", "eduperson_entitlement"] grant-types: ["authorization_code"] -post-logout-redirect-uris: ["http://localhost:8801/oidc/login"] \ No newline at end of file +post-logout-redirect-uris: ["http://localhost:8801/oidc/login"] + +# How many seconds is the access token valid #30days +access-token-validity-seconds: 2592000 \ No newline at end of file From 1e8d0dd252f3a7d616b5b013f8b2175916348986 Mon Sep 17 00:00:00 2001 From: zeidlitz Date: Mon, 23 Jun 2025 10:59:46 +0200 Subject: [PATCH 101/262] [config] Adjust log levels --- sda/internal/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 29787c4d38..1795ef6db3 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -194,9 +194,9 @@ func NewConfig(app string) (*Config, error) { } log.Debugf("Attempting to read configuration...") if err := viper.ReadInConfig(); err != nil { - log.Debugf("Error reading config : %v", err.Error()) + log.Debugf("Error reading config : %v", err) if _, ok := err.(viper.ConfigFileNotFoundError); !ok { - log.Warnf("ReadInConfig Error : %v", err.Error()) + log.Warnf("ReadInConfig Error : %v", err) return nil, err } From 27dbdd1bf8a5a3433025912d7bb8d51916caacc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:49:48 +0000 Subject: [PATCH 102/262] Bump org.springframework.boot:spring-boot-starter-parent Bumps the all-modules group in /sda-doa with 1 update: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.0 to 3.5.3 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.0...v3.5.3) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 9a73de9cda..5afd46ff0f 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.0 + 3.5.3 no.uio.ifi From e7821d2dc6c163fa3daa75159d1ce6474088c526 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 25 Jun 2025 12:41:57 +0000 Subject: [PATCH 103/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 492e858144..b82fc46e13 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.6 -appVersion: v2.0.16 +version: 2.0.7 +appVersion: v2.0.18 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 568dd66ac1..8b20c32077 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.6 -appVersion: v2.0.16 +version: 2.0.7 +appVersion: v2.0.18 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index bb42a01b72..835c13a950 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.6 -appVersion: v2.0.16 +version: 2.0.7 +appVersion: v2.0.18 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From c5278d1efbe1de566f20448909db2b8db6d61bf1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 21:32:16 +0000 Subject: [PATCH 104/262] Bump org.springframework.boot:spring-boot-starter-parent Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.0 to 3.5.3 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.0...v3.5.3) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 211a21cf73..59121c67a0 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.0 + 3.5.3 From 1cfb0b80b9a172f427381af027ffbd08b41ae0ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 23 Jun 2025 20:51:31 +0000 Subject: [PATCH 105/262] Bump the all-modules group in /sda with 7 updates Bumps the all-modules group in /sda with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.36.4` | `1.36.5` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.16` | `1.29.17` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.69` | `1.17.70` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.79` | `1.17.81` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.80.2` | `1.81.0` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.22.3` | `1.22.4` | | [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) | `2.107.0` | `2.108.0` | Updates `github.com/aws/aws-sdk-go-v2` from 1.36.4 to 1.36.5 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.36.4...v1.36.5) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.16 to 1.29.17 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.16...config/v1.29.17) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.69 to 1.17.70 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.69...credentials/v1.17.70) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.79 to 1.17.81 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.79...feature/s3/manager/v1.17.81) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.80.2 to 1.81.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.80.2...service/s3/v1.81.0) Updates `github.com/aws/smithy-go` from 1.22.3 to 1.22.4 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.22.3...v1.22.4) Updates `github.com/casbin/casbin/v2` from 2.107.0 to 2.108.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.107.0...v2.108.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.36.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.29.17 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.17.70 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.81 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.81.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/smithy-go dependency-version: 1.22.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.108.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 38 +++++++++++++-------------- sda/go.sum | 76 +++++++++++++++++++++++++++--------------------------- 2 files changed, 57 insertions(+), 57 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 4064d9ce55..9f617ab16c 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,13 +3,13 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.36.4 - github.com/aws/aws-sdk-go-v2/config v1.29.16 - github.com/aws/aws-sdk-go-v2/credentials v1.17.69 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79 - github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 - github.com/aws/smithy-go v1.22.3 - github.com/casbin/casbin/v2 v2.107.0 + github.com/aws/aws-sdk-go-v2 v1.36.5 + github.com/aws/aws-sdk-go-v2/config v1.29.17 + github.com/aws/aws-sdk-go-v2/credentials v1.17.70 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.81 + github.com/aws/aws-sdk-go-v2/service/s3 v1.81.0 + github.com/aws/smithy-go v1.22.4 + github.com/casbin/casbin/v2 v2.108.0 github.com/coreos/go-oidc/v3 v3.14.1 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -49,19 +49,19 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index af03d29031..565309f797 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,44 +24,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.36.4 h1:GySzjhVvx0ERP6eyfAbAuAXLtAda5TEy19E5q5W8I9E= -github.com/aws/aws-sdk-go-v2 v1.36.4/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.16 h1:XkruGnXX1nEZ+Nyo9v84TzsX+nj86icbFAeust6uo8A= -github.com/aws/aws-sdk-go-v2/config v1.29.16/go.mod h1:uCW7PNjGwZ5cOGZ5jr8vCWrYkGIhPoTNV23Q/tpHKzg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.69 h1:8B8ZQboRc3uaIKjshve/XlvJ570R7BKNy3gftSbS178= -github.com/aws/aws-sdk-go-v2/credentials v1.17.69/go.mod h1:gPME6I8grR1jCqBFEGthULiolzf/Sexq/Wy42ibKK9c= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31 h1:oQWSGexYasNpYp4epLGZxxjsDo8BMBh6iNWkTXQvkwk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.31/go.mod h1:nc332eGUU+djP3vrMI6blS0woaCfHTe3KiSQUVTMRq0= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79 h1:mGo6WGWry+s5GEf2GLfw3zkHad109FQmtvBV3VYQ8mA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.79/go.mod h1:siwnpWxHYFSSge7Euw9lGMgQBgvRyym352mCuGNHsMQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35 h1:o1v1VFfPcDVlK3ll1L5xHsaQAFdNtZ5GXnNR7SwueC4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.35/go.mod h1:rZUQNYMNG+8uZxz9FOerQJ+FceCiodXvixpeRtdESrU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35 h1:R5b82ubO2NntENm3SAm0ADME+H630HomNJdgv+yZ3xw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.35/go.mod h1:FuA+nmgMRfkzVKYDNEqQadvEMxtxl9+RLT9ribCwEMs= +github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0= +github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY= +github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0= +github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8= +github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0= +github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.81 h1:E5ff1vZlAudg24j5lF6F6/gBpln2LjWxGdQDBSLfVe4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.81/go.mod h1:hHBLCuhHI4Aokvs5vdVoCDBzmFy86yxs5J7LEPQwQEM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35 h1:th/m+Q18CkajTw1iqx2cKkLCij/uz8NMwJFPK91p2ug= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.35/go.mod h1:dkJuf0a1Bc8HAA0Zm2MoTGm/WDC18Td9vSbrQ1+VqE8= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3 h1:VHPZakq2L7w+RLzV54LmQavbvheFaR2u1NomJRSEfcU= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.3/go.mod h1:DX1e/lkbsAt0MkY3NgLYuH4jQvRfw8MYxTe9feR7aXM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16 h1:/ldKrPPXTC421bTNWrUIpq3CxwHwRI/kpc+jPUTJocM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.16/go.mod h1:5vkf/Ws0/wgIMJDQbjI4p2op86hNW6Hie5QtebrDgT8= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16 h1:2HuI7vWKhFWsBhIr2Zq8KfFZT6xqaId2XXnXZjkbEuc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.16/go.mod h1:BrwWnsfbFtFeRjdx0iM1ymvlqDX1Oz68JsQaibX/wG8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2 h1:T6Wu+8E2LeTUqzqQ/Bh1EoFNj1u4jUyveMgmTlu9fDU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.80.2/go.mod h1:chSY8zfqmS0OnhZoO/hpPx/BHfAIL80m77HwhRLYScY= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.4 h1:EU58LP8ozQDVroOEyAfcq0cGc5R/FTZjVoYJ6tvby3w= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.4/go.mod h1:CrtOgCcysxMvrCoHnvNAD7PHWclmoFG78Q2xLK0KKcs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2 h1:XB4z0hbQtpmBnb1FQYvKaCM7UsS6Y/u8jVBwIUGeCTk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.2/go.mod h1:hwRpqkRxnQ58J9blRDrB4IanlXCpcKmsC83EhG77upg= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.21 h1:nyLjs8sYJShFYj6aiyjCBI3EcLn1udWrQTjEF+SOXB0= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.21/go.mod h1:EhdxtZ+g84MSGrSrHzZiUm9PYiZkrADNja15wtRJSJo= -github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= -github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= +github.com/aws/aws-sdk-go-v2/service/s3 v1.81.0 h1:1GmCadhKR3J2sMVKs2bAYq9VnwYeCqfRyZzD4RASGlA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.81.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0= +github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w= +github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= +github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.107.0 h1:Kk1+9S2ou8aTTQd30L+vRvFBNf5YvbN65N5uCzdren8= -github.com/casbin/casbin/v2 v2.107.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.108.0 h1:aMc3I81wfLpQe/uzMdElB1OBhEmPZoWMPb2nfEaKygY= +github.com/casbin/casbin/v2 v2.108.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= From 7a61f6a0e10c4845b4b4e7bc8dea7a8774f79167 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:32:37 +0000 Subject: [PATCH 106/262] Bump github.com/go-viper/mapstructure/v2 in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [github.com/go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure). Updates `github.com/go-viper/mapstructure/v2` from 2.2.1 to 2.3.0 - [Release notes](https://github.com/go-viper/mapstructure/releases) - [Changelog](https://github.com/go-viper/mapstructure/blob/main/CHANGELOG.md) - [Commits](https://github.com/go-viper/mapstructure/compare/v2.2.1...v2.3.0) --- updated-dependencies: - dependency-name: github.com/go-viper/mapstructure/v2 dependency-version: 2.3.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 9f617ab16c..08849b5032 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -88,7 +88,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/go-viper/mapstructure/v2 v2.3.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index 565309f797..fea64a2ef2 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -141,8 +141,8 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= +github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= From c67c671b6d5813ba58aeb7e09d7ca7a9aa322f8d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 27 Jun 2025 16:41:04 +0000 Subject: [PATCH 107/262] Bump github.com/go-viper/mapstructure/v2 Bumps the go_modules group in /sda-download with 1 update: [github.com/go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure). Updates `github.com/go-viper/mapstructure/v2` from 2.2.1 to 2.3.0 - [Release notes](https://github.com/go-viper/mapstructure/releases) - [Changelog](https://github.com/go-viper/mapstructure/blob/main/CHANGELOG.md) - [Commits](https://github.com/go-viper/mapstructure/compare/v2.2.1...v2.3.0) --- updated-dependencies: - dependency-name: github.com/go-viper/mapstructure/v2 dependency-version: 2.3.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 82d0e0feb2..c0e068dd50 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -37,7 +37,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect + github.com/go-viper/mapstructure/v2 v2.3.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 4ef7ddc24d..47055c978c 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -53,8 +53,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= +github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= From c0fa17403db60e4aa114d2984f31d12ec3164223 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 22:58:50 +0000 Subject: [PATCH 108/262] Bump no.elixir:clearinghouse in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:clearinghouse` from 3.0.0 to 3.0.2 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-3.0.0...crypt4gh-3.0.2) --- updated-dependencies: - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 5afd46ff0f..e11ad6fcbf 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -121,7 +121,7 @@ no.elixir clearinghouse - 3.0.0 + 3.0.2 org.slf4j From 5d9482fb0bf4ed1a835ddf38600b1ab42d2f5584 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 23:43:22 +0000 Subject: [PATCH 109/262] Bump the all-modules group in /sda with 2 updates Bumps the all-modules group in /sda with 2 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.81 to 1.17.82 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.81...feature/s3/manager/v1.17.82) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.81.0 to 1.82.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.81.0...service/s3/v1.82.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.82 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.82.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 4 ++-- sda/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 9f617ab16c..967953f2f0 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,8 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.5 github.com/aws/aws-sdk-go-v2/config v1.29.17 github.com/aws/aws-sdk-go-v2/credentials v1.17.70 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.81 - github.com/aws/aws-sdk-go-v2/service/s3 v1.81.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 + github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 github.com/aws/smithy-go v1.22.4 github.com/casbin/casbin/v2 v2.108.0 github.com/coreos/go-oidc/v3 v3.14.1 diff --git a/sda/go.sum b/sda/go.sum index 565309f797..f0b9c40125 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDO github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.81 h1:E5ff1vZlAudg24j5lF6F6/gBpln2LjWxGdQDBSLfVe4= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.81/go.mod h1:hHBLCuhHI4Aokvs5vdVoCDBzmFy86yxs5J7LEPQwQEM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 h1:EO13QJTCD1Ig2IrQnoHTRrn981H9mB7afXsZ89WptI4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82/go.mod h1:AGh1NCg0SH+uyJamiJA5tTQcql4MMRDXGRdMmCxCXzY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzRE github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.81.0 h1:1GmCadhKR3J2sMVKs2bAYq9VnwYeCqfRyZzD4RASGlA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.81.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 h1:JubM8CGDDFaAOmBrd8CRYNr49ZNgEAiLwGwgNMdS0nw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg= github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg= From 38b6bc981cc658a253359681f10f16c2c5a43f58 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 1 Jul 2025 13:08:11 +0200 Subject: [PATCH 110/262] silence linter over reduntant log.fatal, log.panic and os.exit linter suggestions need to be checked first --- .golangci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.golangci.yml b/.golangci.yml index c05076ac27..9edab1a144 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -52,6 +52,8 @@ linters: disabled: true - name: package-comments disabled: true + - name: redundant-test-main-exit # disable for now (non-critical, needs checking) + disabled: true - name: unchecked-type-assertion # disable for now disabled: true - name: unused-parameter # disable for now From 4b038efb5762b233b57401af07ba17e685555991 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 1 Jul 2025 13:11:00 +0200 Subject: [PATCH 111/262] remove unnecessary formatting --- sda-download/api/middleware/middleware_test.go | 2 +- sda-download/internal/database/database_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index 3aec599b3f..ed680004dd 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -260,7 +260,7 @@ func TestTokenMiddleware_Success_FromCache(t *testing.T) { // nolint:bodyclose for _, c := range w.Result().Cookies() { if c.Name == config.Config.Session.Name { - t.Errorf("TestTokenMiddleware_Success_FromCache failed, got a session cookie, when should not have") + t.Error("TestTokenMiddleware_Success_FromCache failed, got a session cookie, when should not have") } } diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index 6f18c2ca18..ca8d608130 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -114,7 +114,7 @@ func TestNewDB(t *testing.T) { err := CatchNewDBPanic() if err == nil { - t.Errorf("NewDB did not report error when it should.") + t.Error("NewDB did not report error when it should.") } db, mock, _ := sqlmock.New(sqlmock.MonitorPingsOption(true)) From 443dd330e80505901109ea43b690d7d69cb38d67 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Jun 2025 21:55:35 +0000 Subject: [PATCH 112/262] Bump org.junit:junit-bom in /sda-sftp-inbox in the all-modules group Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.junit:junit-bom](https://github.com/junit-team/junit-framework). Updates `org.junit:junit-bom` from 5.13.1 to 5.13.2 - [Release notes](https://github.com/junit-team/junit-framework/releases) - [Commits](https://github.com/junit-team/junit-framework/compare/r5.13.1...r5.13.2) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 5.13.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 59121c67a0..b3ae66b7b0 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -29,7 +29,7 @@ org.junit junit-bom - 5.13.1 + 5.13.2 pom import From fc5dcf5837456b7072bedcf98082d0dda7d5b969 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 1 Jul 2025 14:40:33 +0200 Subject: [PATCH 113/262] remove merge_group trigger --- .github/workflows/scan-images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index 465f087701..9d5c393189 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -1,6 +1,5 @@ name: Scan Images on: - merge_group: pull_request: branches: - main From bbbf11ca8649b24aef32cffce2f80ac27ff6c6fe Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 2 Jul 2025 12:09:34 +0000 Subject: [PATCH 114/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index b82fc46e13..f20179bbf3 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.7 -appVersion: v2.0.18 +version: 2.0.8 +appVersion: v2.0.26 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 8b20c32077..a7582ac412 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.7 -appVersion: v2.0.18 +version: 2.0.8 +appVersion: v2.0.26 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 835c13a950..143b7273cd 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.7 -appVersion: v2.0.18 +version: 2.0.8 +appVersion: v2.0.26 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From ea819215d722652e13891e763093b6e450750d20 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 1 Jul 2025 14:46:59 +0200 Subject: [PATCH 115/262] do not scan through all user files to exclude those in a dataset --- sda/internal/database/db_functions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 66be75d5c6..262bdbfa09 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -731,7 +731,7 @@ func (dbs *SDAdb) getUserFiles(userID string, allData bool) ([]*SubmissionFileIn // select all files (that are not part of a dataset) of the user, each one annotated with its latest event const query = "SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f " + "LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " + - "AND f.id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id); " + "AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);" // nolint:rowserrcheck rows, err := db.Query(query, userID) @@ -815,7 +815,7 @@ func (dbs *SDAdb) ListActiveUsers() ([]string, error) { db := dbs.DB var users []string - rows, err := db.Query("SELECT DISTINCT submission_user FROM sda.files WHERE id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id) ORDER BY submission_user ASC;") + rows, err := db.Query("SELECT DISTINCT submission_user FROM sda.files f WHERE NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id) ORDER BY submission_user ASC;") if err != nil { return nil, err } From 3fd77bf210c58aa308f2eaa839a3099e17d2da79 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 8 Jul 2025 10:46:31 +0200 Subject: [PATCH 116/262] [ingest] register configured C4GH keys on startup #minor If the `encryption_keys` table in the database is empty when ingest starts it will add all configured keys. --- sda/cmd/ingest/ingest.go | 41 +++++++++++++++++++---------------- sda/cmd/ingest/ingest_test.go | 27 +++++++++++++++++++++++ 2 files changed, 49 insertions(+), 19 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index c864df6083..e0cebd2d87 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -15,7 +15,6 @@ import ( "os/signal" "strings" "syscall" - "time" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/crypt4gh/model/headers" @@ -80,6 +79,11 @@ func main() { sigc <- syscall.SIGINT panic(errors.New("no C4GH private keys configured")) } + + if err := app.registerC4GHKey(); err != nil { + panic(err) + } + app.Archive, err = storage.NewBackend(app.Conf.Archive) if err != nil { log.Error(err) @@ -112,24 +116,6 @@ func main() { log.Info("starting ingest service") go func() { - start := time.Now() - for i := 1; i > 0; i++ { - h, err := app.DB.ListKeyHashes() - if err != nil { - log.Errorln(err.Error()) - } - if len(h) != 0 { - break - } - - time.Sleep(time.Duration(30 * time.Second)) - if time.Since(start).Seconds() >= float64(300) { - log.Errorln("no crypt4gh key hash registered, restarting") - forever <- false - } - log.Errorln("no crypt4gh key hash registered") - } - messages, err := app.MQ.GetMessages(app.Conf.Broker.Queue) if err != nil { log.Panic(err) @@ -197,6 +183,23 @@ func main() { <-forever } +func (app *Ingest) registerC4GHKey() error { + h, err := app.DB.ListKeyHashes() + if err != nil { + return err + } + if len(h) == 0 { + for num, key := range app.ArchiveKeyList { + publicKey := keys.DerivePublicKey(*key) + if err := app.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), fmt.Sprintf("bootstrapped key: %d", num)); err != nil { + return err + } + } + } + + return nil +} + func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { fileUUID, err := app.DB.GetFileID(correlationID) if err != nil { diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index c2e391e408..02e3f0f018 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -741,3 +741,30 @@ func (ts *TestSuite) TestDetectMisingC4GHKeys() { assert.NoError(ts.T(), err) assert.Equal(ts.T(), 0, len(privateKeys)) } + +func (ts *TestSuite) TestRegisterC4ghKey_newDeployment() { + _, err := ts.ingest.DB.DB.Exec("TRUNCATE sda.encryption_keys CASCADE;") + assert.NoError(ts.T(), err) + + privateKeys, err := config.GetC4GHprivateKeys() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(privateKeys)) + + assert.NoError(ts.T(), ts.ingest.registerC4GHKey()) + + kh, err := ts.ingest.DB.ListKeyHashes() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(kh)) +} + +func (ts *TestSuite) TestRegisterC4ghKey_existingEntry() { + privateKeys, err := config.GetC4GHprivateKeys() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(privateKeys)) + + assert.NoError(ts.T(), ts.ingest.registerC4GHKey()) + + kh, err := ts.ingest.DB.ListKeyHashes() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 1, len(kh)) +} From 9f3276e8118bbcda6cdc274fe32cf97defecb28d Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 9 Jul 2025 07:56:50 +0000 Subject: [PATCH 117/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index f20179bbf3..4a319df19a 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.8 -appVersion: v2.0.26 +version: 2.0.9 +appVersion: v2.1.1 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index a7582ac412..bdfe54c5d5 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.8 -appVersion: v2.0.26 +version: 2.0.9 +appVersion: v2.1.1 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 143b7273cd..ae7d33e464 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.8 -appVersion: v2.0.26 +version: 2.0.9 +appVersion: v2.1.1 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From dcda92d2faff0a345a784c86880dad146f627cfe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 22:37:53 +0000 Subject: [PATCH 118/262] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.25 to 3.0.28 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.25...crypt4gh-3.0.28) Updates `no.elixir:clearinghouse` from 3.0.2 to 3.0.5 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.2...crypt4gh-3.0.5) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.28 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index e11ad6fcbf..b3ab253319 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.25 + 3.0.28 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 3.0.2 + 3.0.5 org.slf4j From c0f903f17fc27e083a93d29d7444cfe57e83dfb2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:01:17 +0000 Subject: [PATCH 119/262] Bump the all-modules group in /sda with 4 updates Bumps the all-modules group in /sda with 4 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2), [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) and [golang.org/x/crypto](https://github.com/golang/crypto). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.82 to 1.17.83 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.82...feature/s3/manager/v1.17.83) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.82.0 to 1.83.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.82.0...service/s3/v1.83.0) Updates `github.com/casbin/casbin/v2` from 2.108.0 to 2.109.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.108.0...v2.109.0) Updates `golang.org/x/crypto` from 0.39.0 to 0.40.0 - [Commits](https://github.com/golang/crypto/compare/v0.39.0...v0.40.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.83 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.83.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.109.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.40.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 14 +++++++------- sda/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index e1c446bcef..55827098fb 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,10 +6,10 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.5 github.com/aws/aws-sdk-go-v2/config v1.29.17 github.com/aws/aws-sdk-go-v2/credentials v1.17.70 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 - github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 + github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 github.com/aws/smithy-go v1.22.4 - github.com/casbin/casbin/v2 v2.108.0 + github.com/casbin/casbin/v2 v2.109.0 github.com/coreos/go-oidc/v3 v3.14.1 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -31,7 +31,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.39.0 + golang.org/x/crypto v0.40.0 golang.org/x/oauth2 v0.30.0 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 @@ -153,9 +153,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.17.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect - golang.org/x/net v0.40.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index d8efb506c4..ac202bd628 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDO github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82 h1:EO13QJTCD1Ig2IrQnoHTRrn981H9mB7afXsZ89WptI4= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.82/go.mod h1:AGh1NCg0SH+uyJamiJA5tTQcql4MMRDXGRdMmCxCXzY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 h1:08otkOELsIi0toRRGMytlJhOctcN8xfKfKFR2NXz3kE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83/go.mod h1:dGsGb2wI8JDWeMAhjVPP+z+dqvYjL6k6o+EujcRNk5c= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= @@ -52,8 +52,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzRE github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0 h1:JubM8CGDDFaAOmBrd8CRYNr49ZNgEAiLwGwgNMdS0nw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.82.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 h1:5Y75q0RPQoAbieyOuGLhjV9P3txvYgXv2lg0UwJOfmE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg= github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg= @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.108.0 h1:aMc3I81wfLpQe/uzMdElB1OBhEmPZoWMPb2nfEaKygY= -github.com/casbin/casbin/v2 v2.108.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.109.0 h1:/Rxcqa8V9t6/mMleX4laRtc/mduA+oYdZr449Rd1lD0= +github.com/casbin/casbin/v2 v2.109.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= @@ -410,8 +410,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -437,8 +437,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -471,8 +471,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -482,8 +482,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -494,8 +494,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 48d4aad2f72722f0e305c905865a20b5e38e4bf2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Jul 2025 23:44:54 +0000 Subject: [PATCH 120/262] Bump golang.org/x/crypto in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [golang.org/x/crypto](https://github.com/golang/crypto). Updates `golang.org/x/crypto` from 0.39.0 to 0.40.0 - [Commits](https://github.com/golang/crypto/compare/v0.39.0...v0.40.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.40.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 10 +++++----- sda-download/go.sum | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index c0e068dd50..c7b669e225 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -15,7 +15,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.39.0 + golang.org/x/crypto v0.40.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.73.0 google.golang.org/protobuf v1.36.6 @@ -67,10 +67,10 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.17.0 // indirect - golang.org/x/net v0.40.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.26.0 // indirect - golang.org/x/tools v0.33.0 // indirect + golang.org/x/net v0.41.0 // indirect + golang.org/x/sys v0.34.0 // indirect + golang.org/x/text v0.27.0 // indirect + golang.org/x/tools v0.34.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 47055c978c..c39269553b 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -175,8 +175,8 @@ golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= -golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= +golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= +golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -188,8 +188,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.40.0 h1:79Xs7wF06Gbdcg4kdCCIQArK11Z1hr5POQ6+fIYHNuY= -golang.org/x/net v0.40.0/go.mod h1:y0hY0exeL2Pku80/zKK7tpntoX23cqL3Oa6njdgRtds= +golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= +golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -204,31 +204,31 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= +golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= +golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.26.0 h1:P42AVeLghgTYr4+xUnTRKDMqpar+PtX7KWuNQL21L8M= -golang.org/x/text v0.26.0/go.mod h1:QK15LZJUUQVJxhz7wXgxSy/CJaTFjd0G+YLonydOVQA= +golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= +golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc= -golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI= +golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= +golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= From 6e6afbd2c299d444171368bf5d49ed339d697591 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 00:24:45 +0000 Subject: [PATCH 121/262] Bump org.junit:junit-bom in /sda-sftp-inbox in the all-modules group Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.junit:junit-bom](https://github.com/junit-team/junit-framework). Updates `org.junit:junit-bom` from 5.13.2 to 5.13.3 - [Release notes](https://github.com/junit-team/junit-framework/releases) - [Commits](https://github.com/junit-team/junit-framework/compare/r5.13.2...r5.13.3) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 5.13.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index b3ae66b7b0..35a4767fb4 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -29,7 +29,7 @@ org.junit junit-bom - 5.13.2 + 5.13.3 pom import From e748e9ea0f01d72faf0f374b4a9b8669a6ae266f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Jul 2025 00:47:35 +0000 Subject: [PATCH 122/262] Bump aquasecurity/trivy-action from 0.31.0 to 0.32.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.31.0 to 0.32.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.31.0...0.32.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.32.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- .github/workflows/scan-images.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 61c90c9344..ac73d0475b 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -105,7 +105,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.31.0 + uses: aquasecurity/trivy-action@0.32.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -123,7 +123,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.31.0 + uses: aquasecurity/trivy-action@0.32.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -172,7 +172,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.31.0 + uses: aquasecurity/trivy-action@0.32.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -218,7 +218,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.31.0 + uses: aquasecurity/trivy-action@0.32.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index 9d5c393189..84ab4a609d 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -27,7 +27,7 @@ jobs: image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.31.0 + uses: aquasecurity/trivy-action@0.32.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -53,7 +53,7 @@ jobs: security-events: write steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.31.0 + uses: aquasecurity/trivy-action@0.32.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From e080b6b890f1f3139f39c03de4a2dd1c76b43ee7 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Thu, 17 Jul 2025 13:04:05 +0000 Subject: [PATCH 123/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 4a319df19a..9acb6d785a 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.9 -appVersion: v2.1.1 +version: 2.0.10 +appVersion: v2.1.5 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index bdfe54c5d5..543aad4bf0 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.9 -appVersion: v2.1.1 +version: 2.0.10 +appVersion: v2.1.5 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index ae7d33e464..19d248d9cf 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 2.0.9 -appVersion: v2.1.1 +version: 2.0.10 +appVersion: v2.1.5 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 3df72b7c2a132efb434b683250211eb2ee40d352 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 11 Jul 2025 09:58:41 +0200 Subject: [PATCH 124/262] Add option to create the crypt4gh secret via the helm chart --- charts/sda-svc/templates/auth-deploy.yaml | 4 ++-- charts/sda-svc/templates/auth-secrets.yaml | 2 +- charts/sda-svc/templates/doa-deploy.yaml | 16 +++++++++++++--- charts/sda-svc/templates/doa-secrets.yaml | 1 + charts/sda-svc/templates/ingest-deploy.yaml | 3 --- .../sda-svc/templates/re-encrypt-deploy.yaml | 3 --- charts/sda-svc/templates/shared-secrets.yaml | 18 ++++++++++++++++++ charts/sda-svc/templates/sync-deploy.yaml | 4 ++-- charts/sda-svc/templates/sync-secrets.yaml | 4 ++-- charts/sda-svc/templates/verify-deploy.yaml | 3 --- charts/sda-svc/values.yaml | 13 ++++++------- 11 files changed, 45 insertions(+), 26 deletions(-) diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index 5df2c4cfea..bb8c937909 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -137,8 +137,8 @@ spec: defaultMode: 0440 secretName: {{ required "A secret for the C4GH public key is needed" .Values.global.c4gh.secretName }} items: - - key: {{ required "The C4GH public key is needed" .Values.global.c4gh.publicFile }} - path: {{ .Values.global.c4gh.publicFile }} + - key: {{ required "The C4GH public key is needed" .Values.global.c4gh.publicKey }} + path: {{ .Values.global.c4gh.publicKey }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls diff --git a/charts/sda-svc/templates/auth-secrets.yaml b/charts/sda-svc/templates/auth-secrets.yaml index 80310d7a8b..abde1f0381 100644 --- a/charts/sda-svc/templates/auth-secrets.yaml +++ b/charts/sda-svc/templates/auth-secrets.yaml @@ -30,7 +30,7 @@ stringData: signaturealg: {{ .Values.global.auth.jwtAlg }} tokenttl: {{ .Values.global.auth.jwtTTL }} {{- end }} - publicfile: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.publicFile }}" + publicfile: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.publicKey }}" resignjwt: {{ .Values.global.auth.resignJwt }} s3Inbox: {{ .Values.global.ingress.hostName.s3Inbox }} db: diff --git a/charts/sda-svc/templates/doa-deploy.yaml b/charts/sda-svc/templates/doa-deploy.yaml index 49fbd9e515..b069f9f7e7 100644 --- a/charts/sda-svc/templates/doa-deploy.yaml +++ b/charts/sda-svc/templates/doa-deploy.yaml @@ -206,7 +206,7 @@ spec: - name: CERT_KEY value: {{ ternary (printf "/etc/ssl/certs/java/tls.key.der" ) (printf "%s/%s" .Values.global.secretsPath .Values.doa.tls.keyFile) (empty .Values.global.pkiService) }} - name: CRYPT4GH_PRIVATE_KEY_PATH - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}" + value: "{{ template "c4ghPath" . }}/{{ (first .Values.global.c4gh.privateKeys).keyName }}" - name: CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH value: "{{ template "c4ghPath" . }}/passphrase" - name: OPENID_CONFIGURATION_URL @@ -315,9 +315,19 @@ spec: path: {{ .Values.global.oidc.pubKey }} {{- end }} - name: c4gh-key - secret: - secretName: {{ .Values.global.c4gh.secretName }} + projected: defaultMode: 0440 + sources: + - secret: + name: {{ .Values.global.c4gh.secretName }} + items: + - key: {{ (first .Values.global.c4gh.privateKeys).keyName }} + path: {{ (first .Values.global.c4gh.privateKeys).keyName }} + - secret: + name: {{ template "sda.fullname" . }}-doa + items: + - key: c4ghPassphrase + path: passphrase {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive diff --git a/charts/sda-svc/templates/doa-secrets.yaml b/charts/sda-svc/templates/doa-secrets.yaml index c7fb633a55..9e33d951a3 100644 --- a/charts/sda-svc/templates/doa-secrets.yaml +++ b/charts/sda-svc/templates/doa-secrets.yaml @@ -8,6 +8,7 @@ metadata: name: {{ template "sda.fullname" . }}-doa type: Opaque data: + c4ghPassphrase: {{ (first .Values.global.c4gh.privateKeys).passphrase }} dbPassword: {{ include "dbPassDoa" . | b64enc }} {{- if and .Values.global.doa.outbox.enabled }} mqPassword: {{ include "mqPassDoa" . | b64enc }} diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index e6be859a6d..90de9d4674 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -105,9 +105,6 @@ spec: secret: defaultMode: 0440 secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls diff --git a/charts/sda-svc/templates/re-encrypt-deploy.yaml b/charts/sda-svc/templates/re-encrypt-deploy.yaml index 9b9a0a56bd..f8b4048de5 100644 --- a/charts/sda-svc/templates/re-encrypt-deploy.yaml +++ b/charts/sda-svc/templates/re-encrypt-deploy.yaml @@ -87,9 +87,6 @@ spec: secret: defaultMode: 0440 secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls diff --git a/charts/sda-svc/templates/shared-secrets.yaml b/charts/sda-svc/templates/shared-secrets.yaml index 49dfb9adb5..14fd283bdc 100644 --- a/charts/sda-svc/templates/shared-secrets.yaml +++ b/charts/sda-svc/templates/shared-secrets.yaml @@ -32,4 +32,22 @@ data: s3InboxAccessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey | quote | trimall "\"" | b64enc }} s3InboxSecretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey | quote | trimall "\"" | b64enc }} {{- end }} +{{- if and .Values.global.c4gh.publicKeyData (first .Values.global.c4gh.privateKeys).keyData }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.global.c4gh.secretName }} +type: Opaque +data: + {{ .Values.global.c4gh.publicKey }}: {{ .Values.global.c4gh.publicKeyData }} +{{- range $pk := .Values.global.c4gh.privateKeys }} + {{- if and $pk.keyName $pk.keyData $pk.passphrase }} + {{ $pk.keyName }}: {{ $pk.keyData }} + {{- end }} +{{- end }} + {{- if and .Values.global.c4gh.syncPubKey .Values.global.c4gh.syncPubKeyData }} + {{ .Values.global.c4gh.syncPubKey }}: {{ .Values.global.c4gh.syncPubKeyData }} + {{- end }} +{{- end }} {{- end }} diff --git a/charts/sda-svc/templates/sync-deploy.yaml b/charts/sda-svc/templates/sync-deploy.yaml index a96e230f45..536f41e6ef 100644 --- a/charts/sda-svc/templates/sync-deploy.yaml +++ b/charts/sda-svc/templates/sync-deploy.yaml @@ -111,8 +111,8 @@ spec: defaultMode: 0440 secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} + - key: {{ (first .Values.global.c4gh.privateKeys).keyName }} + path: {{ (first .Values.global.c4gh.privateKeys).keyName }} - key: {{ .Values.global.c4gh.syncPubKey }} path: {{ .Values.global.c4gh.syncPubKey }} - name: config diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index cc4c50751b..a731e933e0 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -52,8 +52,8 @@ stringData: {{- end }} vhost: {{ include "brokerVhost" . }} c4gh: - filePath: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} - passphrase: {{ .Values.global.c4gh.passphrase }} + filePath: {{ template "c4ghPath" . }}/{{ (first .Values.global.c4gh.privateKeys).keyName }} + passphrase: {{ (first .Values.global.c4gh.privateKeys).passphrase }} syncPubKeyPath: {{ template "c4ghPath" . }}/{{ required "remote sync public c4gh key is missing" .Values.global.c4gh.syncPubKey }} db: {{- if .Values.global.tls.enabled }} diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index e008c76181..73cc038c65 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -96,9 +96,6 @@ spec: secret: defaultMode: 0440 secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index c0db7ebb96..6bf60f21db 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -188,16 +188,15 @@ global: password: "" c4gh: - secretName: "" - # name of the c4gh key file. - keyFile: "" - # name of the public file, required for testing - publicFile: "" - passphrase: "" - backupPubKey: "" + secretName: + publicKey: + publicKeyData: privateKeys: - keyName: passphrase: + keyData: + syncPubKey: + syncPubKeyData: db: host: "" name: "sda" From 313edd1132d4790fa49d60af3851f0e93a93cc9d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 11 Jul 2025 09:59:36 +0200 Subject: [PATCH 125/262] Add job to register c4gh key hash for previously ingested files --- .../templates/job-set-missing-keyhash.yaml | 92 +++++++++++++++++++ charts/sda-svc/values.yaml | 10 ++ 2 files changed, 102 insertions(+) create mode 100644 charts/sda-svc/templates/job-set-missing-keyhash.yaml diff --git a/charts/sda-svc/templates/job-set-missing-keyhash.yaml b/charts/sda-svc/templates/job-set-missing-keyhash.yaml new file mode 100644 index 0000000000..24bb4de82c --- /dev/null +++ b/charts/sda-svc/templates/job-set-missing-keyhash.yaml @@ -0,0 +1,92 @@ +{{- if .Values.jobs.setKeyHash }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sda.fullname" . }}-set-key-hash-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: hook-succeeded +spec: + template: + metadata: + name: {{ template "sda.fullname" . }}-set-key-hash-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + spec: + containers: + - name: {{ template "sda.fullname" . }}-set-key-hash-job + command: ["/usr/local/bin/psql"] + args: [ + "-qf", + "/migratedb/set-key-hash.sql" + ] + env: + - name: PGDATABASE + value: {{ .Values.global.db.name }} + - name: PGHOST + value: {{ .Values.global.db.host }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- if .Values.global.db.port }} + - name: PGPORT + value: {{ .Values.global.db.port | quote }} + {{- end }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + image: {{ .Values.jobs.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumeMounts: + - name: sql + mountPath: /migratedb + restartPolicy: Never + securityContext: + runAsUser: 72 + runAsGroup: 72 + fsGroup: 72 + volumes: + - name: sql + configMap: + name: set-key-hash +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: set-key-hash +data: + set-key-hash.sql: | + DO + $$ + BEGIN + IF (select max(version) from sda.dbschema_version) > 11 then + RAISE NOTICE 'Fixing c4gh key references'; + UPDATE sda.files SET key_hash = (SELECT key_hash FROM sda.encryption_keys WHERE deprecated_at IS NULL ORDER BY created_at ASC LIMIT 1) + WHERE header IS NOT NULL AND key_hash IS NULL; + END IF; + END + $$ +{{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 6bf60f21db..98f9909ef5 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -198,6 +198,12 @@ global: syncPubKey: syncPubKeyData: db: + admin: + # These credentials need at least dbOwner permissions + # in order to perform schema migrations. + secretName: "" + passKey: "" + userKey: "" host: "" name: "sda" user: "" @@ -616,3 +622,7 @@ verify: annotations: {} tls: secretName: "" + +jobs: + image: postgres:15.4-alpine + setKeyHash: false \ No newline at end of file From 1cb57909ec020e54fac74aafbfeb7c7b50f15426 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 14 Jul 2025 09:35:19 +0200 Subject: [PATCH 126/262] Add Job to migrate from version 1 deployment. --- .../templates/job-upgrade-from-v1.yaml | 115 ++++++++++++++++++ charts/sda-svc/values.yaml | 3 +- 2 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 charts/sda-svc/templates/job-upgrade-from-v1.yaml diff --git a/charts/sda-svc/templates/job-upgrade-from-v1.yaml b/charts/sda-svc/templates/job-upgrade-from-v1.yaml new file mode 100644 index 0000000000..d8f6384cdd --- /dev/null +++ b/charts/sda-svc/templates/job-upgrade-from-v1.yaml @@ -0,0 +1,115 @@ +{{- if .Values.jobs.upgradeFomV1 }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sda.fullname" . }}-upgrade-from-v1-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: hook-succeeded +spec: + template: + metadata: + name: {{ template "sda.fullname" . }}-upgrade-from-v1-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + spec: + containers: + - name: {{ template "sda.fullname" . }}-upgrade-from-v1-job + command: ["/usr/local/bin/psql"] + args: [ + "-qf", + "/migratedb/upgrade-from-v1.sql" + ] + env: + - name: PGDATABASE + value: {{ .Values.global.db.name }} + - name: PGHOST + value: {{ .Values.global.db.host }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- if .Values.global.db.port }} + - name: PGPORT + value: {{ .Values.global.db.port | quote }} + {{- end }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + image: {{ .Values.jobs.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumeMounts: + - name: sql + mountPath: /migratedb + restartPolicy: Never + securityContext: + runAsUser: 72 + runAsGroup: 72 + fsGroup: 72 + volumes: + - name: sql + configMap: + name: upgrade-from-v1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: upgrade-from-v1 +data: + upgrade-from-v1.sql: | + DO + $$ + BEGIN + IF (select max(version) from sda.dbschema_version) = 11 then + RAISE NOTICE 'Doing migration from schema version 11 to 17'; + + INSERT INTO sda.dbschema_version VALUES(12, now(), 'Add key hash table'); + CREATE TABLE IF NOT EXISTS sda.encryption_keys ( + key_hash TEXT PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(), + deprecated_at TIMESTAMP WITH TIME ZONE, + description TEXT + ); + ALTER TABLE sda.files ADD COLUMN IF NOT EXISTS key_hash TEXT, + ADD CONSTRAINT fk_files_key_hash FOREIGN KEY (key_hash) REFERENCES sda.encryption_keys(key_hash); + + INSERT INTO sda.dbschema_version VALUES(14, now(), 'Add userinfo table'); + CREATE TABLE IF NOT EXISTS sda.userinfo ( + id TEXT PRIMARY KEY, + name TEXT, + email TEXT, + groups TEXT[] + ); + + INSERT INTO sda.dbschema_version VALUES(17, now(), 'Add submission user to constraint'); + ALTER TABLE sda.files DROP CONSTRAINT unique_ingested; + ALTER TABLE sda.files ADD CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user); + + ELSE + RAISE NOTICE 'Schema migration from 11 does not apply now, skipping'; + END IF; + END + $$ +{{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 98f9909ef5..6cec764aa2 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -625,4 +625,5 @@ verify: jobs: image: postgres:15.4-alpine - setKeyHash: false \ No newline at end of file + setKeyHash: false + upgradeFomV1: false \ No newline at end of file From d66f1960dcb84b80f79efa816a205df2fe85d0b1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 11 Jul 2025 10:00:12 +0200 Subject: [PATCH 127/262] [integration tests] remove obsolete entries from the values file --- .github/integration/scripts/charts/dependencies.sh | 1 - .github/integration/scripts/charts/values.yaml | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index f9d6f47a4e..2dc1871f19 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -109,7 +109,6 @@ yq -i ' .global.backupArchive.s3AccessKey = strenv(MINIO_ACCESS) | .global.backupArchive.s3SecretKey = strenv(MINIO_SECRET) | .global.broker.password = strenv(MQPASSWORD) | -.global.c4gh.passphrase = strenv(C4GHPASSPHRASE) | .global.c4gh.privateKeys[0].passphrase = strenv(C4GHPASSPHRASE) | .global.db.password = strenv(PGPASSWORD) | .global.inbox.s3AccessKey = strenv(MINIO_ACCESS) | diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index 6847412194..747f00d246 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -56,13 +56,11 @@ global: user: PLACEHOLDER_VALUE c4gh: secretName: c4gh - keyFile: c4gh.sec.pem - publicFile: c4gh.pub.pem - passphrase: PLACEHOLDER_VALUE - syncPubKey: c4gh.pub.pem + publicKey: c4gh.pub.pem privateKeys: - keyName: c4gh.sec.pem passphrase: PLACEHOLDER_VALUE + syncPubKey: c4gh.pub.pem db: host: "postgres-sda-db" user: "postgres" From 3b85362f4bc189f5078c7919ca496d1c3697a8d2 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 11 Jul 2025 10:07:19 +0200 Subject: [PATCH 128/262] Update Readme --- charts/sda-svc/README.md | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 5ba3fe87e0..66940ec894 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -10,6 +10,19 @@ Source repositories: Edit the values.yaml file and specify the relevant parts of the `global` section. If no shared credentials for the broker and database are used, the credentials for each service shuld be set in the `credentials` section. +While it is possible to deploy this chart with the crypt4gh keys included in the values file as base64 encoded strings, it is advisable to create the secret containing the crypt4gh keys manually. + +## Upgrading an existing Release to a new version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### To 3.0.0 + +This version adds Jobs that migrates the dtabase schema from a V1 release and sets the first registered crypt4gh key to all ingested files that do not have a c4gh key referenced. +When upgrading from a V1 release *both* `upgradeFomV1` and `setKeyHash` should be run. + +Unless the same queries are being exiecuted manually by a database adminitstrator a `Basic authentication Secret` containg the credentials to perform the upgrade needs to be created. + ### Configuration The following table lists the configurable parameters of the `sda-svc` chart and their default values. @@ -99,9 +112,15 @@ Parameter | Description | Default `global.cega.host` | Full URI to the EGA user authentication service. |`""` `global.cega.user` | Username for the EGA user authentication service. |`""` `global.cega.password` | Password for the EGA user authentication service. |`""` -`global.c4gh.keyFile` | Private C4GH key. |`c4gh.key` -`global.c4gh.passphrase` | Passphrase for the private C4GH key. |`""` -`global.c4gh.publicFile` | Public key corresponding to the private key, provided in /info endpoint. |`""` +`global.c4gh.privateKeys` | List of Private C4GH keys. |`` +`global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. |`""` +`global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. |`""` +`global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. |`""` +`global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. |`""` +`global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. |`""` +`global.db.admin.secretName` | Name of the secret that holds the database admin credentials. |`""` +`global.db.admin.passKey` | Key in the secret that holds the password. |`""` +`global.db.admin.userkey` | Key in the secret that holds the username. |`""` `global.db.host` | Hostname for the database. |`""` `global.db.name` | Database to connect to. |`lega` `global.db.passIngest` | Password used for `data in` services. |`""` @@ -303,3 +322,9 @@ Parameter | Description | Default `releasetest.repository` | inbox container image repository | `neicnordic/sda-helm-test-support` `releasetest.imageTag` | inbox container image version | `latest` `releasetest.imagePullPolicy` | inbox container image pull policy | `Always` + +### Jobs + +`jobs.image` | Container image used for running the DB migration jobs | `postgres:15.4-alpine` +`jobs.setKeyHash` | Populate the key_hash table after migration from V1 | `false` +`jobs.upgradeFomV1` | Upgrade database schema from a version 1 release. | `false` From b9fa14ccf052138739b04e955cf7f88c4520b483 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 11 Jul 2025 10:07:29 +0200 Subject: [PATCH 129/262] Bump Chart version --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 19d248d9cf..e349609656 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 2.0.10 +version: 3.0.0 appVersion: v2.1.5 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From 5a9e040347e4751e9374bc8b56b2998ec981a428 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 21 Jul 2025 12:48:56 +0200 Subject: [PATCH 130/262] Fix typos Co-authored-by: Panos Chatzopoulos --- charts/sda-svc/README.md | 8 ++++---- charts/sda-svc/templates/job-upgrade-from-v1.yaml | 2 +- charts/sda-svc/values.yaml | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 66940ec894..662c7d9168 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -18,10 +18,10 @@ A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an ### To 3.0.0 -This version adds Jobs that migrates the dtabase schema from a V1 release and sets the first registered crypt4gh key to all ingested files that do not have a c4gh key referenced. -When upgrading from a V1 release *both* `upgradeFomV1` and `setKeyHash` should be run. +This version adds Jobs that migrates the database schema from a V1 release and sets the first registered crypt4gh key to all ingested files that do not have a c4gh key referenced. +When upgrading from a V1 release *both* `upgradeFromV1` and `setKeyHash` should be run. -Unless the same queries are being exiecuted manually by a database adminitstrator a `Basic authentication Secret` containg the credentials to perform the upgrade needs to be created. +Unless the same queries are being executed manually by a database administrator a `Basic authentication Secret` containing the credentials to perform the upgrade needs to be created. ### Configuration @@ -327,4 +327,4 @@ Parameter | Description | Default `jobs.image` | Container image used for running the DB migration jobs | `postgres:15.4-alpine` `jobs.setKeyHash` | Populate the key_hash table after migration from V1 | `false` -`jobs.upgradeFomV1` | Upgrade database schema from a version 1 release. | `false` +`jobs.upgradeFromV1` | Upgrade database schema from a version 1 release. | `false` diff --git a/charts/sda-svc/templates/job-upgrade-from-v1.yaml b/charts/sda-svc/templates/job-upgrade-from-v1.yaml index d8f6384cdd..0baab227cb 100644 --- a/charts/sda-svc/templates/job-upgrade-from-v1.yaml +++ b/charts/sda-svc/templates/job-upgrade-from-v1.yaml @@ -1,4 +1,4 @@ -{{- if .Values.jobs.upgradeFomV1 }} +{{- if .Values.jobs.upgradeFromV1 }} apiVersion: batch/v1 kind: Job metadata: diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 6cec764aa2..6a6f091439 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -626,4 +626,4 @@ verify: jobs: image: postgres:15.4-alpine setKeyHash: false - upgradeFomV1: false \ No newline at end of file + upgradeFromV1: false \ No newline at end of file From 0b0b87e4e0d7667ec09c65ec130b48150564bfc6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Jul 2025 08:41:41 +0200 Subject: [PATCH 131/262] [tests] hardcode the crypt4gh version For some reason the lookup of the cryp4gh version started failing at the beginning of the summer. Hardcode the version so we don't get false negatives for the integration tests. --- .github/integration/scripts/charts/dependencies.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index f9d6f47a4e..4794cc9179 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -2,7 +2,7 @@ set -ex YQ_VERSION="v4.20.1" -C4GH_VERSION="$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')" +C4GH_VERSION="v1.14.0" random-string() { head -c 32 /dev/urandom | base64 -w0 | tr -d '/+' | fold -w 32 | head -n 1 From cc34b85b6c70beeeccd7551f534690edee12168e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Jul 2025 07:58:00 +0000 Subject: [PATCH 132/262] Bump the all-modules group in /sda-sftp-inbox with 3 updates --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 5.13.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: commons-io:commons-io dependency-version: 2.20.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-version: 1.12.788 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 35a4767fb4..814dc5c946 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -29,7 +29,7 @@ org.junit junit-bom - 5.13.3 + 5.13.4 pom import @@ -103,12 +103,12 @@ commons-io commons-io - 2.19.0 + 2.20.0 com.amazonaws aws-java-sdk-s3 - 1.12.787 + 1.12.788 com.google.guava From 6a72147c2055e9c7674a3488e9dc040e4d62fd95 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 22 Jul 2025 08:18:54 +0000 Subject: [PATCH 133/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 9acb6d785a..6e44ed8ef2 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.10 -appVersion: v2.1.5 +version: 2.0.11 +appVersion: v2.1.6 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 543aad4bf0..7bdbeacc99 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.10 -appVersion: v2.1.5 +version: 2.0.11 +appVersion: v2.1.6 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index e349609656..53edaa5077 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.0 -appVersion: v2.1.5 +version: 3.0.1 +appVersion: v2.1.6 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 0d00cdef61b3265b2e9ef76af4caad9908dfbcb4 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Jul 2025 09:47:55 +0200 Subject: [PATCH 134/262] [sda][linter] ensure all switches have a default state --- sda/cmd/api/api_test.go | 2 ++ sda/cmd/mapper/mapper.go | 2 ++ sda/cmd/notify/notify.go | 5 ++--- sda/cmd/orchestrate/orchestrate.go | 4 ++++ sda/cmd/s3inbox/proxy.go | 8 +++++++- sda/internal/config/config.go | 18 ++++++++++++++++-- 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 04394b6fab..33da335511 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -697,6 +697,8 @@ func (s *TestSuite) TestAPIGetFiles() { assert.Equal(s.T(), fileInfo.Status, latestStatus) case file2: assert.Equal(s.T(), fileInfo.Status, "registered") + default: + s.Fail("unknown inbox path") } } assert.NoError(s.T(), err) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index a0c11693a6..b38411c86d 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -142,6 +142,8 @@ func main() { continue } + default: + log.Errorf("unknown mappping type, %s", mappings.Type) } if err := delivered.Ack(false); err != nil { diff --git a/sda/cmd/notify/notify.go b/sda/cmd/notify/notify.go index 8e85ca3ac2..93ff7ba8cc 100644 --- a/sda/cmd/notify/notify.go +++ b/sda/cmd/notify/notify.go @@ -4,7 +4,6 @@ package main import ( "encoding/base64" "encoding/json" - "errors" "fmt" "net/smtp" "strconv" @@ -156,7 +155,7 @@ func validator(queue, schemaPath string, delivery amqp091.Delivery) error { } return nil + default: + return fmt.Errorf("unknown queue, %s", queue) } - - return errors.New("Error") } diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index 5c847b6542..457ac838a6 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -200,6 +200,10 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * continue } + default: + log.Errorln("unknown raouting key") + + continue } } } diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index ebf18b8e4b..e8da30b0b9 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -382,18 +382,24 @@ func (p *Proxy) prependBucketToHostPath(r *http.Request) error { case strings.Contains(r.URL.String(), "?location") || strings.Contains(r.URL.String(), "&prefix"): r.URL.Path = "/" + bucket + "/" log.Debug("new Path: ", r.URL.Path) + default: + r.URL.Path = "/" + bucket + "/" } case http.MethodPost: r.URL.Path = "/" + bucket + r.URL.Path log.Debug("new Path: ", r.URL.Path) case http.MethodPut: r.URL.Path = "/" + bucket + r.URL.Path - log.Debug("new Path: ", r.URL.Path) + log.Info("new PUT Path: ", r.URL.Path) case http.MethodDelete: if strings.Contains(r.URL.String(), "?uploadId") { // abort multipart upload r.URL.Path = "/" + bucket + r.URL.Path } + default: + log.Errorf("unknown request: %s", r.Method) + + return errors.New("unknown request method") } log.Infof("User: %v, Request type %v, Path: %v", username, r.Method, r.URL.Path) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 1795ef6db3..e876b7559c 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -319,6 +319,8 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) + default: + return nil, errors.New("archive.type not set") } switch viper.GetString("backup.type") { @@ -326,6 +328,8 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = append(requiredConfVars, []string{"backup.url", "backup.accesskey", "backup.secretkey", "backup.bucket"}...) case POSIX: requiredConfVars = append(requiredConfVars, []string{"backup.location"}...) + default: + return nil, errors.New("backup.type not set") } case "intercept": requiredConfVars = []string{ @@ -355,6 +359,8 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) case POSIX: requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) + default: + return nil, errors.New("inbox.type not set") } case "notify": requiredConfVars = []string{ @@ -671,7 +677,9 @@ func NewConfig(app string) (*Config, error) { } c.configArchive() - c.configSync() + if err := c.configSync(); err != nil { + return nil, err + } c.configSchemas() case "sync-api": if err := c.configBroker(); err != nil { @@ -698,6 +706,8 @@ func NewConfig(app string) (*Config, error) { } c.configSchemas() + default: + return nil, errors.New("unknown app name") } return c, nil @@ -1085,7 +1095,7 @@ func (c *Config) configSMTP() { } // configSync provides configuration for the sync destination storage -func (c *Config) configSync() { +func (c *Config) configSync() error { switch viper.GetString("sync.destination.type") { case S3: c.Sync.Destination.Type = S3 @@ -1096,6 +1106,8 @@ func (c *Config) configSync() { case POSIX: c.Sync.Destination.Type = POSIX c.Sync.Destination.Posix.Location = viper.GetString("sync.destination.location") + default: + return errors.New("sync.destination.type not set") } c.Sync.RemoteHost = viper.GetString("sync.remote.host") @@ -1105,6 +1117,8 @@ func (c *Config) configSync() { c.Sync.RemotePassword = viper.GetString("sync.remote.password") c.Sync.RemoteUser = viper.GetString("sync.remote.user") c.Sync.CenterPrefix = viper.GetString("sync.centerPrefix") + + return nil } // configSync provides configuration for the outgoing sync settings From 8b5b4d4964f7eb08828b088f87904b468f4797b0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Jul 2025 12:29:28 +0200 Subject: [PATCH 135/262] [sda][tests][s3inbox] don't reuse variables from previous step --- sda/cmd/s3inbox/proxy_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 313e389661..0be6752c8c 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -377,7 +377,8 @@ func (s *ProxyTests) TestServeHTTP_allowed() { // Put file works w = httptest.NewRecorder() - r.Method = "PUT" + r, err = http.NewRequest("PUT", "/dummy/file", nil) + assert.NoError(s.T(), err) s.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" proxy.allowedResponse(w, r, s.token) assert.Equal(s.T(), 200, w.Result().StatusCode) From 765920ecc476483ee53ddf86f5a8dedf2d5ae014 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Jul 2025 12:36:06 +0200 Subject: [PATCH 136/262] [sda-download][linter] ensure all switches have a default state --- sda-download/internal/storage/seekable.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sda-download/internal/storage/seekable.go b/sda-download/internal/storage/seekable.go index 845d150a70..5a6a36156c 100644 --- a/sda-download/internal/storage/seekable.go +++ b/sda-download/internal/storage/seekable.go @@ -186,7 +186,6 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { go r.prefetchAt(r.currentOffset) return offset, nil - case io.SeekCurrent: if r.currentOffset+offset < 0 { return r.currentOffset, fmt.Errorf("invalid offset %v from %v would be be before start", offset, r.currentOffset) @@ -199,7 +198,6 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { go r.prefetchAt(r.currentOffset) return r.currentOffset, nil - case io.SeekEnd: if r.objectSize+offset < 0 { return r.currentOffset, fmt.Errorf("invalid offset %v from end in %v bytes object, would be before file start", offset, r.objectSize) @@ -212,9 +210,9 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { go r.prefetchAt(r.currentOffset) return r.currentOffset, nil + default: + return r.currentOffset, errors.New("bad whence") } - - return r.currentOffset, errors.New("bad whence") } // removeFromOutstanding removes a prefetch from the list of outstanding prefetches once it's no longer active From a92b720de14a3e414b96a044f3450be26de3ca17 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 23 Jul 2025 11:03:12 +0000 Subject: [PATCH 137/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 6e44ed8ef2..ae110fb8d6 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.11 -appVersion: v2.1.6 +version: 2.0.12 +appVersion: v2.1.7 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 7bdbeacc99..50a1a50f81 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.11 -appVersion: v2.1.6 +version: 2.0.12 +appVersion: v2.1.7 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 53edaa5077..598334345c 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.1 -appVersion: v2.1.6 +version: 3.0.2 +appVersion: v2.1.7 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From aea345f4b8914dc202ce6b71d099e3aee5fe5cab Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 01:33:55 +0000 Subject: [PATCH 138/262] Bump the all-modules group in /sda-doa with 3 updates Bumps the all-modules group in /sda-doa with 3 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot), [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.3 to 3.5.4 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.3...v3.5.4) Updates `no.elixir:crypt4gh` from 3.0.28 to 3.0.29 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.28...crypt4gh-3.0.29) Updates `no.elixir:clearinghouse` from 3.0.5 to 3.0.6 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.5...crypt4gh-3.0.6) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.29 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index b3ab253319..046da84fb0 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.3 + 3.5.4 no.uio.ifi @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.28 + 3.0.29 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 3.0.5 + 3.0.6 org.slf4j From 4bd41c884fbabf47b35e267de378312513eaa520 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Aug 2025 02:31:38 +0000 Subject: [PATCH 139/262] Bump the all-modules group in /sda with 9 updates Bumps the all-modules group in /sda with 9 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.36.5` | `1.37.2` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.17` | `1.30.3` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.70` | `1.18.3` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.83` | `1.18.3` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.83.0` | `1.86.0` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.22.4` | `1.22.5` | | [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) | `2.109.0` | `2.115.0` | | [github.com/coreos/go-oidc/v3](https://github.com/coreos/go-oidc) | `3.14.1` | `3.15.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.73.0` | `1.74.2` | Updates `github.com/aws/aws-sdk-go-v2` from 1.36.5 to 1.37.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.36.5...v1.37.2) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.17 to 1.30.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.17...v1.30.3) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.70 to 1.18.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.70...config/v1.18.3) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.83 to 1.18.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.3/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.83...config/v1.18.3) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.83.0 to 1.86.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.83.0...service/s3/v1.86.0) Updates `github.com/aws/smithy-go` from 1.22.4 to 1.22.5 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.22.4...v1.22.5) Updates `github.com/casbin/casbin/v2` from 2.109.0 to 2.115.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.109.0...v2.115.0) Updates `github.com/coreos/go-oidc/v3` from 3.14.1 to 3.15.0 - [Release notes](https://github.com/coreos/go-oidc/releases) - [Commits](https://github.com/coreos/go-oidc/compare/v3.14.1...v3.15.0) Updates `google.golang.org/grpc` from 1.73.0 to 1.74.2 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.73.0...v1.74.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.37.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.30.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.18.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.18.3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.86.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/smithy-go dependency-version: 1.22.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.115.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/coreos/go-oidc/v3 dependency-version: 3.15.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.74.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 44 ++++++++++----------- sda/go.sum | 112 ++++++++++++++++++++++++++--------------------------- 2 files changed, 78 insertions(+), 78 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 55827098fb..136e1918c6 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,14 +3,14 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.36.5 - github.com/aws/aws-sdk-go-v2/config v1.29.17 - github.com/aws/aws-sdk-go-v2/credentials v1.17.70 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 - github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 - github.com/aws/smithy-go v1.22.4 - github.com/casbin/casbin/v2 v2.109.0 - github.com/coreos/go-oidc/v3 v3.14.1 + github.com/aws/aws-sdk-go-v2 v1.37.2 + github.com/aws/aws-sdk-go-v2/config v1.30.3 + github.com/aws/aws-sdk-go-v2/credentials v1.18.3 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0 + github.com/aws/smithy-go v1.22.5 + github.com/casbin/casbin/v2 v2.115.0 + github.com/coreos/go-oidc/v3 v3.15.0 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 @@ -33,7 +33,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.40.0 golang.org/x/oauth2 v0.30.0 - google.golang.org/grpc v1.73.0 + google.golang.org/grpc v1.74.2 google.golang.org/protobuf v1.36.6 ) @@ -49,19 +49,19 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect @@ -157,7 +157,7 @@ require ( golang.org/x/sys v0.34.0 // indirect golang.org/x/text v0.27.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index ac202bd628..6115498f94 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,44 +24,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.36.5 h1:0OF9RiEMEdDdZEMqF9MRjevyxAQcf6gY+E7vwBILFj0= -github.com/aws/aws-sdk-go-v2 v1.36.5/go.mod h1:EYrzvCCN9CMUTa5+6lf6MM4tq3Zjp8UhSGR/cBsjai0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11 h1:12SpdwU8Djs+YGklkinSSlcrPyj3H4VifVsKf78KbwA= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.11/go.mod h1:dd+Lkp6YmMryke+qxW/VnKyhMBDTYP41Q2Bb+6gNZgY= -github.com/aws/aws-sdk-go-v2/config v1.29.17 h1:jSuiQ5jEe4SAMH6lLRMY9OVC+TqJLP5655pBGjmnjr0= -github.com/aws/aws-sdk-go-v2/config v1.29.17/go.mod h1:9P4wwACpbeXs9Pm9w1QTh6BwWwJjwYvJ1iCt5QbCXh8= -github.com/aws/aws-sdk-go-v2/credentials v1.17.70 h1:ONnH5CM16RTXRkS8Z1qg7/s2eDOhHhaXVd72mmyv4/0= -github.com/aws/aws-sdk-go-v2/credentials v1.17.70/go.mod h1:M+lWhhmomVGgtuPOhO85u4pEa3SmssPTdcYpP/5J/xc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32 h1:KAXP9JSHO1vKGCr5f4O6WmlVKLFFXgWYAGoJosorxzU= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.32/go.mod h1:h4Sg6FQdexC1yYG9RDnOvLbW1a/P986++/Y/a+GyEM8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83 h1:08otkOELsIi0toRRGMytlJhOctcN8xfKfKFR2NXz3kE= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.83/go.mod h1:dGsGb2wI8JDWeMAhjVPP+z+dqvYjL6k6o+EujcRNk5c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36 h1:SsytQyTMHMDPspp+spo7XwXTP44aJZZAC7fBV2C5+5s= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.36/go.mod h1:Q1lnJArKRXkenyog6+Y+zr7WDpk4e6XlR6gs20bbeNo= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36 h1:i2vNHQiXUvKhs3quBR6aqlgJaiaexz/aNvdCktW/kAM= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.36/go.mod h1:UdyGa7Q91id/sdyHPwth+043HhmP6yP9MBHgbZM0xo8= +github.com/aws/aws-sdk-go-v2 v1.37.2 h1:xkW1iMYawzcmYFYEV0UCMxc8gSsjCGEhBXQkdQywVbo= +github.com/aws/aws-sdk-go-v2 v1.37.2/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= +github.com/aws/aws-sdk-go-v2/config v1.30.3 h1:utupeVnE3bmB221W08P0Moz1lDI3OwYa2fBtUhl7TCc= +github.com/aws/aws-sdk-go-v2/config v1.30.3/go.mod h1:NDGwOEBdpyZwLPlQkpKIO7frf18BW8PaCmAM9iUxQmI= +github.com/aws/aws-sdk-go-v2/credentials v1.18.3 h1:ptfyXmv+ooxzFwyuBth0yqABcjVIkjDL0iTYZBSbum8= +github.com/aws/aws-sdk-go-v2/credentials v1.18.3/go.mod h1:Q43Nci++Wohb0qUh4m54sNln0dbxJw8PvQWkrwOkGOI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 h1:nRniHAvjFJGUCl04F3WaAj7qp/rcz5Gi1OVoj5ErBkc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2/go.mod h1:eJDFKAMHHUvv4a0Zfa7bQb//wFNUXGrbFpYRCHe2kD0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3 h1:Nb2pUE30lySKPGdkiIJ1SZgHsjiebOiRNI7R9NA1WtM= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3/go.mod h1:BO5EKulvhBF1NXwui8lfnuDPBQQU5807yvWASZ/5n6k= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 h1:sPiRHLVUIIQcoVZTNwqQcdtjkqkPopyYmIX0M5ElRf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2/go.mod h1:ik86P3sgV+Bk7c1tBFCwI3VxMoSEwl4YkRB9xn1s340= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 h1:ZdzDAg075H6stMZtbD2o+PyB933M/f20e9WmCBC17wA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2/go.mod h1:eE1IIzXG9sdZCB0pNNpMpsYTLl4YdOQD3njiVN1e/E4= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36 h1:GMYy2EOWfzdP3wfVAGXBNKY5vK4K8vMET4sYOYltmqs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.36/go.mod h1:gDhdAV6wL3PmPqBhiPbnlS447GoWs8HTTOYef9/9Inw= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4 h1:CXV68E2dNqhuynZJPB80bhPQwAKqBWVer887figW6Jc= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.4/go.mod h1:/xFi9KtvBXP97ppCz1TAEvU1Uf66qvid89rbem3wCzQ= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4 h1:nAP2GYbfh8dd2zGZqFRSMlq+/F6cMPBUuCsGAMkN074= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.4/go.mod h1:LT10DsiGjLWh4GbjInf9LQejkYEhBgBCjLG5+lvk4EE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17 h1:t0E6FzREdtCsiLIoLCWsYliNsRBgyGD/MCK571qk4MI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.17/go.mod h1:ygpklyoaypuyDvOM5ujWGrYWpAK3h7ugnmKCU/76Ys4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17 h1:qcLWgdhq45sDM9na4cvXax9dyLitn8EYBRl8Ak4XtG4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.17/go.mod h1:M+jkjBFZ2J6DJrjMv2+vkBbuht6kxJYtJiwoVgX4p4U= -github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0 h1:5Y75q0RPQoAbieyOuGLhjV9P3txvYgXv2lg0UwJOfmE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.83.0/go.mod h1:kUklwasNoCn5YpyAqC/97r6dzTA1SRKJfKq16SXeoDU= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.5 h1:AIRJ3lfb2w/1/8wOOSqYb9fUKGwQbtysJ2H1MofRUPg= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.5/go.mod h1:b7SiVprpU+iGazDUqvRSLf5XmCdn+JtT1on7uNL6Ipc= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3 h1:BpOxT3yhLwSJ77qIY3DoHAQjZsc4HEGfMCE4NGy3uFg= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.3/go.mod h1:vq/GQR1gOFLquZMSrxUK/cpvKCNVYibNyJ1m7JrU88E= -github.com/aws/aws-sdk-go-v2/service/sts v1.34.0 h1:NFOJ/NXEGV4Rq//71Hs1jC/NvPs1ezajK+yQmkwnPV0= -github.com/aws/aws-sdk-go-v2/service/sts v1.34.0/go.mod h1:7ph2tGpfQvwzgistp2+zga9f+bCjlQJPkPUmMgDSD7w= -github.com/aws/smithy-go v1.22.4 h1:uqXzVZNuNexwc/xrh6Tb56u89WDlJY6HS+KC0S4QSjw= -github.com/aws/smithy-go v1.22.4/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 h1:sBpc8Ph6CpfZsEdkz/8bfg8WhKlWMCms5iWj6W/AW2U= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2/go.mod h1:Z2lDojZB+92Wo6EKiZZmJid9pPrDJW2NNIXSlaEfVlU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 h1:blV3dY6WbxIVOFggfYIo2E1Q2lZoy5imS7nKgu5m6Tc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2/go.mod h1:cBWNeLBjHJRSmXAxdS7mwiMUEgx6zup4wQ9J+/PcsRQ= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 h1:oxmDEO14NBZJbK/M8y3brhMFEIGN4j8a6Aq8eY0sqlo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2/go.mod h1:4hH+8QCrk1uRWDPsVfsNDUup3taAjO8Dnx63au7smAU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 h1:0hBNFAPwecERLzkhhBY+lQKUMpXSKVv4Sxovikrioms= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2/go.mod h1:Vcnh4KyR4imrrjGN7A2kP2v9y6EPudqoPKXtnmBliPU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0 h1:utPhv4ECQzJIUbtx7vMN4A8uZxlQ5tSt1H1toPI41h8= +github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0/go.mod h1:1/eZYtTWazDgVl96LmGdGktHFi7prAcGCrJ9JGvBITU= +github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 h1:j7/jTOjWeJDolPwZ/J4yZ7dUsxsWZEsxNwH5O7F8eEA= +github.com/aws/aws-sdk-go-v2/service/sso v1.27.0/go.mod h1:M0xdEPQtgpNT7kdAX4/vOAPkFj60hSQRb7TvW9B0iug= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 h1:ywQF2N4VjqX+Psw+jLjMmUL2g1RDHlvri3NxHA08MGI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0/go.mod h1:Z+qv5Q6b7sWiclvbJyPSOT1BRVU9wfSUPaqQzZ1Xg3E= +github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 h1:bRP/a9llXSSgDPk7Rqn5GD/DQCGo6uk95plBFKoXt2M= +github.com/aws/aws-sdk-go-v2/service/sts v1.36.0/go.mod h1:tgBsFzxwl65BWkuJ/x2EUs59bD4SfYKgikvFDJi1S58= +github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= +github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.109.0 h1:/Rxcqa8V9t6/mMleX4laRtc/mduA+oYdZr449Rd1lD0= -github.com/casbin/casbin/v2 v2.109.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.115.0 h1:56azKfn1tkb/+y6DAasXs5fsz80CpeXJ1yzshuZbzoc= +github.com/casbin/casbin/v2 v2.115.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= @@ -86,8 +86,8 @@ github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJ github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= -github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= -github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= +github.com/coreos/go-oidc/v3 v3.15.0 h1:R6Oz8Z4bqWR7VFQ+sPSvZPQv4x8M+sJkDO5ojgwlyAg= +github.com/coreos/go-oidc/v3 v3.15.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -127,8 +127,8 @@ github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFS github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY= github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -385,16 +385,16 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -513,10 +513,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 81501137feaccd18d7f910e588810385b189d7ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Aug 2025 10:33:43 +0000 Subject: [PATCH 140/262] Bump the all-modules group across 1 directory with 4 updates Bumps the all-modules group with 4 updates in the /sda-download directory: [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go), [golang.org/x/crypto](https://github.com/golang/crypto), [google.golang.org/grpc](https://github.com/grpc/grpc-go) and google.golang.org/protobuf. Updates `github.com/aws/aws-sdk-go` from 1.55.7 to 1.55.8 - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG_PENDING.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.55.7...v1.55.8) Updates `golang.org/x/crypto` from 0.40.0 to 0.41.0 - [Commits](https://github.com/golang/crypto/compare/v0.40.0...v0.41.0) Updates `google.golang.org/grpc` from 1.73.0 to 1.74.2 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.73.0...v1.74.2) Updates `google.golang.org/protobuf` from 1.36.6 to 1.36.7 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-version: 1.55.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.41.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.74.2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-version: 1.36.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 18 ++++++------- sda-download/go.sum | 64 ++++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index c7b669e225..cc3053e7ab 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -4,7 +4,7 @@ go 1.24.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go v1.55.7 + github.com/aws/aws-sdk-go v1.55.8 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -15,10 +15,10 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 - google.golang.org/grpc v1.73.0 - google.golang.org/protobuf v1.36.6 + google.golang.org/grpc v1.74.2 + google.golang.org/protobuf v1.36.7 ) require ( @@ -67,11 +67,11 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.17.0 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 // indirect - golang.org/x/tools v0.34.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect + golang.org/x/tools v0.35.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index c39269553b..95d24c8272 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -3,8 +3,8 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= -github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= +github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -41,8 +41,8 @@ github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -159,24 +159,24 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= -go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= -go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= -go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= -go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= -go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= -go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= -go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= -go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= -go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= +go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= +go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= +go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= +go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= +go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= +go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= +go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= +go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= +go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -188,8 +188,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -204,38 +204,38 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.34.0 h1:qIpSLOxeCYGg9TrcJokLBG4KFA6d795g0xkBkiESGlo= -golang.org/x/tools v0.34.0/go.mod h1:pAP9OwEaY1CAW3HOmg3hLZC5Z0CCmzjAF2UQMSqNARg= +golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= +golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237 h1:cJfm9zPbe1e873mHJzmQ1nwVEeRDU/T1wXDK2kUSU34= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250519155744-55703ea1f237/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.73.0 h1:VIWSmpI2MegBtTuFt5/JWy2oXxtjJ/e89Z70ImfD2ok= -google.golang.org/grpc v1.73.0/go.mod h1:50sbHOUqWoCQGI8V2HQLJM0B+LMlIUjNSZmow7EVBQc= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 8f7f8ef9a9123f85cc06bbcad5537a8843f63984 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 8 Aug 2025 10:06:21 +0200 Subject: [PATCH 141/262] fix(k3d): update the oidc deployment to mount the jwt keys for signing of the mock tokens --- .github/integration/scripts/charts/dependencies.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/integration/scripts/charts/dependencies.yaml b/.github/integration/scripts/charts/dependencies.yaml index c6ed4cb0bf..2166ef7c6c 100644 --- a/.github/integration/scripts/charts/dependencies.yaml +++ b/.github/integration/scripts/charts/dependencies.yaml @@ -91,6 +91,14 @@ spec: requests: cpu: 100m memory: 128Mi + volumeMounts: + - mountPath: /shared/keys + name: jwt + volumes: + - name: jwt + secret: + defaultMode: 288 + secretName: jwk --- apiVersion: v1 kind: Service From 14fdf31efa1aa19e736f212ab23dee394033ae7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 8 Aug 2025 10:13:08 +0200 Subject: [PATCH 142/262] fix(k3d): add requester@demo.org as admin to the rbac to allow the odic mock tokens for admin actions in the sda-api --- .github/integration/sda/rbac.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 2176e704a2..515d7bb456 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -69,6 +69,9 @@ { "role": "testu@lifescience-ri.eu", "rolebinding": "admin" - } + }, { + "role": "requester@demo.org", + "rolebinding": "admin" + } ] } \ No newline at end of file From e3176be1dfef716afc2578feb57cb9bfc190e1a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 8 Aug 2025 11:39:07 +0200 Subject: [PATCH 143/262] fix(helm charts): fix sda-svc helm chart sda-verify mounting ingest secret instead of verify secret --- charts/sda-svc/templates/verify-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index 73cc038c65..23f30efe1a 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -91,7 +91,7 @@ spec: - name: config secret: defaultMode: 0440 - secretName: {{ template "sda.fullname" . }}-ingest + secretName: {{ template "sda.fullname" . }}-verify - name: c4gh secret: defaultMode: 0440 From f92e80bd3b8abcf57d9893adaf8988649bd055e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Fri, 8 Aug 2025 11:50:09 +0200 Subject: [PATCH 144/262] fix(helm charts): update verify-secrets.yaml only set broker.verifyPeer if .Values.global.tls.enabled (similar to other service secrets) --- charts/sda-svc/templates/verify-secrets.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/sda-svc/templates/verify-secrets.yaml b/charts/sda-svc/templates/verify-secrets.yaml index 9b7dd30de1..c6ac6baf19 100644 --- a/charts/sda-svc/templates/verify-secrets.yaml +++ b/charts/sda-svc/templates/verify-secrets.yaml @@ -47,7 +47,9 @@ stringData: {{- end }} ssl: {{ .Values.global.tls.enabled }} user: {{ required "MQ user is required" (include "mqUserIngest" .) }} + {{- if .Values.global.tls.enabled }} verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} vhost: {{ include "brokerVhost" . }} c4gh: privateKeys: From dc34961e3086534100f5effec903f8f480d340ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 11 Aug 2025 14:58:40 +0200 Subject: [PATCH 145/262] fix(helm charts): update default value of broker.Queue for sda-sync from mappings -> mapping_stream as per [sync.md](sda/cmd/sync/sync.md) --- charts/sda-svc/templates/sync-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index a731e933e0..9f8b2d2387 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -41,7 +41,7 @@ stringData: port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} password: {{ required "MQ password is required" (include "mqPassMapper" .) }} - queue: {{ default "mappings" .Values.global.broker.finalizeQueue }} + queue: {{ default "mapping_stream" .Values.global.broker.finalizeQueue }} {{- if ne "" ( default "" .Values.global.broker.serverName ) }} serverName: {{.Values.global.broker.serverName }} {{- end }} From 6a97f0aef8342f89e6ea1eb6df8eaaaf6df98043 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 11 Aug 2025 16:09:43 +0200 Subject: [PATCH 146/262] fix(helm charts): populate broker.queue in [sda sync secret](charts/sda-svc/templates/sync-secrets.yaml) from .Values.global.sync.brokerQueue instead of .Values.global.broker.finalizeQueue --- charts/sda-svc/templates/sync-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index 9f8b2d2387..a0f79a4fe9 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -41,7 +41,7 @@ stringData: port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} password: {{ required "MQ password is required" (include "mqPassMapper" .) }} - queue: {{ default "mapping_stream" .Values.global.broker.finalizeQueue }} + queue: {{ default "mapping_stream" .Values.global.sync.brokerQueue }} {{- if ne "" ( default "" .Values.global.broker.serverName ) }} serverName: {{.Values.global.broker.serverName }} {{- end }} From 1c8c94a54dbd91da04137c511279b42cb4c49983 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 11 Aug 2025 16:11:14 +0200 Subject: [PATCH 147/262] chore(helm charts): bump sda-svc helm chart version -> 3.0.3 --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 598334345c..8ee2736c26 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.0.2 +version: 3.0.3 appVersion: v2.1.7 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From b6dd6245892da11144b9a75d5768969531e02d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 11 Aug 2025 16:22:20 +0200 Subject: [PATCH 148/262] fix(helm charts): update usage of global.sync.brokerQueue in [syncapi secrets](charts/sda-svc/templates/syncapi-secrets.yaml) to also have mappings_stream as default --- charts/sda-svc/templates/syncapi-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/syncapi-secrets.yaml b/charts/sda-svc/templates/syncapi-secrets.yaml index 2c55c2345c..ed27583e5d 100644 --- a/charts/sda-svc/templates/syncapi-secrets.yaml +++ b/charts/sda-svc/templates/syncapi-secrets.yaml @@ -45,7 +45,7 @@ stringData: api: password: {{ required "sync API password is required" .Values.global.sync.api.password }} user: {{ required "sync API user is required" .Values.global.sync.api.user }} - brokerQueue: {{ default "mappings" .Values.global.sync.brokerQueue }} + brokerQueue: {{ default "mapping_stream" .Values.global.sync.brokerQueue }} centerPrefix: {{ .Values.global.sync.centerPrefix }} remote: host: {{ required "remote sync API host is required" .Values.global.sync.remote.host }} From 7df23f4249cdeddc8f7c89529d26a86d5ac28444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Mon, 11 Aug 2025 16:23:55 +0200 Subject: [PATCH 149/262] chore(helm charts): update [README.md](charts/sda-svc/README.md) to specify that mapping_stream is the default value of global.sync.brokerQueue --- charts/sda-svc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 662c7d9168..5d0dc56759 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -172,7 +172,7 @@ Parameter | Description | Default `global.inbox.s3ReadyPath` | Endpoint to verify that the inbox is respondig. |`""` `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` `global.sync.api.user` | User for authenticating to the syncAPI server | `null` -`global.sync.brokerQueue` | Queue to read messages from | `sync` +`global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` `global.sync.destination.storageType` | Storage type for the sync destination, currently only supports S3 | `s3` `global.sync.destination.accesskey` | Access key to S3 sync destination | `null` From 22e0fad354dce91ca8fe6d652d5aae52106346ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Aug 2025 20:33:54 +0000 Subject: [PATCH 150/262] Bump org.springframework.boot:spring-boot-starter-parent Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.3 to 3.5.4 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.3...v3.5.4) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 814dc5c946..c879101125 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.3 + 3.5.4 From 7af1bca6735c691b81c677b7e0eaa832e989a763 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 04:02:32 +0000 Subject: [PATCH 151/262] Bump the all-modules group in /sda with 8 updates Bumps the all-modules group in /sda with 8 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.37.2` | `1.38.0` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.30.3` | `1.31.0` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.18.3` | `1.18.4` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.18.3` | `1.18.4` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.86.0` | `1.87.0` | | [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) | `2.115.0` | `2.118.0` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.40.0` | `0.41.0` | | google.golang.org/protobuf | `1.36.6` | `1.36.7` | Updates `github.com/aws/aws-sdk-go-v2` from 1.37.2 to 1.38.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.37.2...v1.38.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.30.3 to 1.31.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.30.3...v1.31.0) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.18.3 to 1.18.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.4/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.3...config/v1.18.4) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.18.3 to 1.18.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.4/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.3...config/v1.18.4) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.86.0 to 1.87.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.86.0...service/s3/v1.87.0) Updates `github.com/casbin/casbin/v2` from 2.115.0 to 2.118.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.115.0...v2.118.0) Updates `golang.org/x/crypto` from 0.40.0 to 0.41.0 - [Commits](https://github.com/golang/crypto/compare/v0.40.0...v0.41.0) Updates `google.golang.org/protobuf` from 1.36.6 to 1.36.7 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.38.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.31.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.18.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.18.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.87.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.118.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.41.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-version: 1.36.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 42 +++++++++++++------------- sda/go.sum | 88 +++++++++++++++++++++++++++--------------------------- 2 files changed, 65 insertions(+), 65 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 136e1918c6..c83a0a1142 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,13 +3,13 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.37.2 - github.com/aws/aws-sdk-go-v2/config v1.30.3 - github.com/aws/aws-sdk-go-v2/credentials v1.18.3 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3 - github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0 + github.com/aws/aws-sdk-go-v2 v1.38.0 + github.com/aws/aws-sdk-go-v2/config v1.31.0 + github.com/aws/aws-sdk-go-v2/credentials v1.18.4 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.87.0 github.com/aws/smithy-go v1.22.5 - github.com/casbin/casbin/v2 v2.115.0 + github.com/casbin/casbin/v2 v2.118.0 github.com/coreos/go-oidc/v3 v3.15.0 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -31,10 +31,10 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.40.0 + golang.org/x/crypto v0.41.0 golang.org/x/oauth2 v0.30.0 google.golang.org/grpc v1.74.2 - google.golang.org/protobuf v1.36.6 + google.golang.org/protobuf v1.36.7 ) require ( @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.3 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.3 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.28.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.37.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect @@ -153,9 +153,9 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.17.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect - golang.org/x/net v0.41.0 // indirect - golang.org/x/sys v0.34.0 // indirect - golang.org/x/text v0.27.0 // indirect + golang.org/x/net v0.42.0 // indirect + golang.org/x/sys v0.35.0 // indirect + golang.org/x/text v0.28.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 6115498f94..42e9537d47 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.37.2 h1:xkW1iMYawzcmYFYEV0UCMxc8gSsjCGEhBXQkdQywVbo= -github.com/aws/aws-sdk-go-v2 v1.37.2/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= +github.com/aws/aws-sdk-go-v2 v1.38.0 h1:UCRQ5mlqcFk9HJDIqENSLR3wiG1VTWlyUfLDEvY7RxU= +github.com/aws/aws-sdk-go-v2 v1.38.0/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= -github.com/aws/aws-sdk-go-v2/config v1.30.3 h1:utupeVnE3bmB221W08P0Moz1lDI3OwYa2fBtUhl7TCc= -github.com/aws/aws-sdk-go-v2/config v1.30.3/go.mod h1:NDGwOEBdpyZwLPlQkpKIO7frf18BW8PaCmAM9iUxQmI= -github.com/aws/aws-sdk-go-v2/credentials v1.18.3 h1:ptfyXmv+ooxzFwyuBth0yqABcjVIkjDL0iTYZBSbum8= -github.com/aws/aws-sdk-go-v2/credentials v1.18.3/go.mod h1:Q43Nci++Wohb0qUh4m54sNln0dbxJw8PvQWkrwOkGOI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2 h1:nRniHAvjFJGUCl04F3WaAj7qp/rcz5Gi1OVoj5ErBkc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.2/go.mod h1:eJDFKAMHHUvv4a0Zfa7bQb//wFNUXGrbFpYRCHe2kD0= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3 h1:Nb2pUE30lySKPGdkiIJ1SZgHsjiebOiRNI7R9NA1WtM= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.3/go.mod h1:BO5EKulvhBF1NXwui8lfnuDPBQQU5807yvWASZ/5n6k= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2 h1:sPiRHLVUIIQcoVZTNwqQcdtjkqkPopyYmIX0M5ElRf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.2/go.mod h1:ik86P3sgV+Bk7c1tBFCwI3VxMoSEwl4YkRB9xn1s340= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2 h1:ZdzDAg075H6stMZtbD2o+PyB933M/f20e9WmCBC17wA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.2/go.mod h1:eE1IIzXG9sdZCB0pNNpMpsYTLl4YdOQD3njiVN1e/E4= +github.com/aws/aws-sdk-go-v2/config v1.31.0 h1:9yH0xiY5fUnVNLRWO0AtayqwU1ndriZdN78LlhruJR4= +github.com/aws/aws-sdk-go-v2/config v1.31.0/go.mod h1:VeV3K72nXnhbe4EuxxhzsDc/ByrCSlZwUnWH52Nde/I= +github.com/aws/aws-sdk-go-v2/credentials v1.18.4 h1:IPd0Algf1b+Qy9BcDp0sCUcIWdCQPSzDoMK3a8pcbUM= +github.com/aws/aws-sdk-go-v2/credentials v1.18.4/go.mod h1:nwg78FjH2qvsRM1EVZlX9WuGUJOL5od+0qvm0adEzHk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.3 h1:GicIdnekoJsjq9wqnvyi2elW6CGMSYKhdozE7/Svh78= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.3/go.mod h1:R7BIi6WNC5mc1kfRM7XM/VHC3uRWkjc396sfabq4iOo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.3 h1:o9RnO+YZ4X+kt5Z7Nvcishlz0nksIt2PIzDglLMP0vA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.3/go.mod h1:+6aLJzOG1fvMOyzIySYjOFjcguGvVRL68R+uoRencN4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.3 h1:joyyUFhiTQQmVK6ImzNU9TQSNRNeD9kOklqTzyk5v6s= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.3/go.mod h1:+vNIyZQP3b3B1tSLI0lxvrU9cfM7gpdRXMFfm67ZcPc= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2 h1:sBpc8Ph6CpfZsEdkz/8bfg8WhKlWMCms5iWj6W/AW2U= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.2/go.mod h1:Z2lDojZB+92Wo6EKiZZmJid9pPrDJW2NNIXSlaEfVlU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.3 h1:ZV2XK2L3HBq9sCKQiQ/MdhZJppH/rH0vddEAamsHUIs= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.3/go.mod h1:b9F9tk2HdHpbf3xbN7rUZcfmJI26N6NcJu/8OsBFI/0= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2 h1:blV3dY6WbxIVOFggfYIo2E1Q2lZoy5imS7nKgu5m6Tc= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.2/go.mod h1:cBWNeLBjHJRSmXAxdS7mwiMUEgx6zup4wQ9J+/PcsRQ= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2 h1:oxmDEO14NBZJbK/M8y3brhMFEIGN4j8a6Aq8eY0sqlo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.2/go.mod h1:4hH+8QCrk1uRWDPsVfsNDUup3taAjO8Dnx63au7smAU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2 h1:0hBNFAPwecERLzkhhBY+lQKUMpXSKVv4Sxovikrioms= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.2/go.mod h1:Vcnh4KyR4imrrjGN7A2kP2v9y6EPudqoPKXtnmBliPU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0 h1:utPhv4ECQzJIUbtx7vMN4A8uZxlQ5tSt1H1toPI41h8= -github.com/aws/aws-sdk-go-v2/service/s3 v1.86.0/go.mod h1:1/eZYtTWazDgVl96LmGdGktHFi7prAcGCrJ9JGvBITU= -github.com/aws/aws-sdk-go-v2/service/sso v1.27.0 h1:j7/jTOjWeJDolPwZ/J4yZ7dUsxsWZEsxNwH5O7F8eEA= -github.com/aws/aws-sdk-go-v2/service/sso v1.27.0/go.mod h1:M0xdEPQtgpNT7kdAX4/vOAPkFj60hSQRb7TvW9B0iug= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0 h1:ywQF2N4VjqX+Psw+jLjMmUL2g1RDHlvri3NxHA08MGI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.32.0/go.mod h1:Z+qv5Q6b7sWiclvbJyPSOT1BRVU9wfSUPaqQzZ1Xg3E= -github.com/aws/aws-sdk-go-v2/service/sts v1.36.0 h1:bRP/a9llXSSgDPk7Rqn5GD/DQCGo6uk95plBFKoXt2M= -github.com/aws/aws-sdk-go-v2/service/sts v1.36.0/go.mod h1:tgBsFzxwl65BWkuJ/x2EUs59bD4SfYKgikvFDJi1S58= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.3 h1:3ZKmesYBaFX33czDl6mbrcHb6jeheg6LqjJhQdefhsY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.3/go.mod h1:7ryVb78GLCnjq7cw45N6oUb9REl7/vNUwjvIqC5UgdY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.3 h1:ieRzyHXypu5ByllM7Sp4hC5f/1Fy5wqxqY0yB85hC7s= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.3/go.mod h1:O5ROz8jHiOAKAwx179v+7sHMhfobFVi6nZt8DEyiYoM= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.3 h1:SE/e52dq9a05RuxzLcjT+S5ZpQobj3ie3UTaSf2NnZc= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.3/go.mod h1:zkpvBTsR020VVr8TOrwK2TrUW9pOir28sH5ECHpnAfo= +github.com/aws/aws-sdk-go-v2/service/s3 v1.87.0 h1:egoDf+Geuuntmw79Mz6mk9gGmELCPzg5PFEABOHB+6Y= +github.com/aws/aws-sdk-go-v2/service/s3 v1.87.0/go.mod h1:t9MDi29H+HDbkolTSQtbI0HP9DemAWQzUjmWC7LGMnE= +github.com/aws/aws-sdk-go-v2/service/sso v1.28.0 h1:Mc/MKBf2m4VynyJkABoVEN+QzkfLqGj0aiJuEe7cMeM= +github.com/aws/aws-sdk-go-v2/service/sso v1.28.0/go.mod h1:iS5OmxEcN4QIPXARGhavH7S8kETNL11kym6jhoS7IUQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.0 h1:6csaS/aJmqZQbKhi1EyEMM7yBW653Wy/B9hnBofW+sw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.0/go.mod h1:59qHWaY5B+Rs7HGTuVGaC32m0rdpQ68N8QCN3khYiqs= +github.com/aws/aws-sdk-go-v2/service/sts v1.37.0 h1:MG9VFW43M4A8BYeAfaJJZWrroinxeTi2r3+SnmLQfSA= +github.com/aws/aws-sdk-go-v2/service/sts v1.37.0/go.mod h1:JdeBDPgpJfuS6rU/hNglmOigKhyEZtBmbraLE4GK1J8= github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.115.0 h1:56azKfn1tkb/+y6DAasXs5fsz80CpeXJ1yzshuZbzoc= -github.com/casbin/casbin/v2 v2.115.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.118.0 h1:OS0E6J/KkWw3lvmW6f3h7xz4Bvlv8mv91fRk0J4QpFs= +github.com/casbin/casbin/v2 v2.118.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= @@ -410,8 +410,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.40.0 h1:r4x+VvoG5Fm+eJcxMaY8CQM7Lb0l1lsmjGBQ6s8BfKM= -golang.org/x/crypto v0.40.0/go.mod h1:Qr1vMER5WyS2dfPHAlsOj01wgLbsyWtFn/aY+5+ZdxY= +golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= +golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -437,8 +437,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.41.0 h1:vBTly1HeNPEn3wtREYfy4GZ/NECgw2Cnl+nK6Nz3uvw= -golang.org/x/net v0.41.0/go.mod h1:B/K4NNqkfmg07DQYrbwvSluqCJOOXwUjeb/5lOisjbA= +golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= +golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -471,8 +471,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.34.0 h1:H5Y5sJ2L2JRdyv7ROF1he/lPdvFsd0mJHFw2ThKHxLA= -golang.org/x/sys v0.34.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= +golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -482,8 +482,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.33.0 h1:NuFncQrRcaRvVmgRkvM3j/F00gWIAlcmlB8ACEKmGIg= -golang.org/x/term v0.33.0/go.mod h1:s18+ql9tYWp1IfpV9DmCtQDDSRBUjKaw9M1eAv5UeF0= +golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= +golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -494,8 +494,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.27.0 h1:4fGWRpyh641NLlecmyl4LOe6yDdfaYNrGb2zdfo4JV4= -golang.org/x/text v0.27.0/go.mod h1:1D28KMCvyooCX9hBiosv5Tz/+YLxj0j7XhWjpSUF7CU= +golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= +golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -517,8 +517,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= +google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 21bd1bb3ce671c1738c30ca47055e2be1dbbeb92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:09:32 +0000 Subject: [PATCH 152/262] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 18 +++++++++--------- .github/workflows/code-linter.yaml | 6 +++--- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/functionality.yml | 6 +++--- .github/workflows/publish_charts.yml | 2 +- .github/workflows/publish_container.yml | 6 +++--- .github/workflows/release_sda-admin.yaml | 4 ++-- .github/workflows/shellcheck.yml | 2 +- .github/workflows/test.yml | 8 ++++---- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index ac73d0475b..3f5603da60 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -25,7 +25,7 @@ jobs: security-events: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -69,7 +69,7 @@ jobs: security-events: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -149,7 +149,7 @@ jobs: security-events: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -238,7 +238,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Test rabbitmq federation run: docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @@ -250,7 +250,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Test postgres run: docker compose -f .github/integration/postgres.yml run tests @@ -265,7 +265,7 @@ jobs: storage: ["posix", "s3"] steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Test sensitive-data-archive run: docker compose -f .github/integration/sda-${{matrix.storage}}-integration.yml run integration_test @@ -276,7 +276,7 @@ jobs: - build_server_images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dorny/paths-filter@v3 id: changes with: @@ -299,7 +299,7 @@ jobs: storage: [s3, posix] steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: create maven settings.xml uses: s4u/maven-settings-action@v3.1.0 @@ -339,7 +339,7 @@ jobs: storage: "posix" steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Install Helm uses: azure/setup-helm@v4 diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 56be2be57b..d8448c4c08 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -24,7 +24,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run golangci-lint uses: golangci/golangci-lint-action@v8.0.0 @@ -45,7 +45,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run golangci-lint uses: golangci/golangci-lint-action@v8.0.0 @@ -66,7 +66,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run golangci-lint uses: golangci/golangci-lint-action@v8.0.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 57b4c8beba..c5138e6632 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml index e905c30192..f752e0ef42 100644 --- a/.github/workflows/functionality.yml +++ b/.github/workflows/functionality.yml @@ -11,7 +11,7 @@ jobs: sftp-inbox: ${{ steps.changes.outputs.sftp-inbox }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dorny/paths-filter@v3 id: changes with: @@ -40,7 +40,7 @@ jobs: python-version: "3.11" - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run setup scripts run: | @@ -64,7 +64,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Build image run: | diff --git a/.github/workflows/publish_charts.yml b/.github/workflows/publish_charts.yml index e44bd6609e..06c0dffb33 100644 --- a/.github/workflows/publish_charts.yml +++ b/.github/workflows/publish_charts.yml @@ -16,7 +16,7 @@ jobs: continue-on-error: true steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index 839aa0b01b..17726b5385 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -24,7 +24,7 @@ jobs: new_tag: ${{ steps.bump_tag.outputs.new_tag }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: '0' - name: Bump version and push tag @@ -46,7 +46,7 @@ jobs: packages: write steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -101,7 +101,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: '0' diff --git a/.github/workflows/release_sda-admin.yaml b/.github/workflows/release_sda-admin.yaml index e785a4d8db..a28b4d5cef 100644 --- a/.github/workflows/release_sda-admin.yaml +++ b/.github/workflows/release_sda-admin.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get version tag run: | VERSION=$(cat sda-admin/.version) @@ -37,7 +37,7 @@ jobs: - "darwin/arm64" steps: - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up Go uses: actions/setup-go@v5 diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index fd1781d623..caf7e1e375 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: check all scripts uses: ludeeus/action-shellcheck@master diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e51e4db8fe..1226cd1bbe 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} uses: actions/setup-java@v4 with: @@ -44,7 +44,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get dependencies run: | @@ -79,7 +79,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get dependencies run: | @@ -114,7 +114,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get dependencies run: | From f633f7a3a7fd1edd7e478c392a28bd9f8ff6648e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 05:42:18 +0000 Subject: [PATCH 153/262] Bump no.elixir:crypt4gh in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.29 to 3.0.31 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.29...crypt4gh-3.0.31) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.31 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 046da84fb0..41a496a5c7 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.29 + 3.0.31 org.slf4j From 5ad9fe527cf6273c07cac08dfcba88becc65cac5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 12 Aug 2025 08:58:06 +0200 Subject: [PATCH 154/262] fix(helm charts): fix variables populating broker username and password in [sda-verify secret](charts/sda-svc/templates/verify-secrets.yaml) --- charts/sda-svc/templates/verify-secrets.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/sda-svc/templates/verify-secrets.yaml b/charts/sda-svc/templates/verify-secrets.yaml index c6ac6baf19..7fde154e2d 100644 --- a/charts/sda-svc/templates/verify-secrets.yaml +++ b/charts/sda-svc/templates/verify-secrets.yaml @@ -39,14 +39,14 @@ stringData: host: {{ required "A valid MQ host is required" .Values.global.broker.host }} port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} - password: {{ required "MQ password is required" (include "mqPassIngest" .) }} + password: {{ required "MQ password is required" (include "mqPassVerify" .) }} queue: {{ default "archived" .Values.global.broker.verifyQueue }} routingKey: {{ default "verified" .Values.global.broker.routingKey }} {{- if ne "" ( default "" .Values.global.broker.serverName ) }} serverName: {{.Values.global.broker.serverName }} {{- end }} ssl: {{ .Values.global.tls.enabled }} - user: {{ required "MQ user is required" (include "mqUserIngest" .) }} + user: {{ required "MQ user is required" (include "mqUserVerify" .) }} {{- if .Values.global.tls.enabled }} verifyPeer: {{ .Values.global.broker.verifyPeer }} {{- end }} From edfe302057b972207abc95392ff56317f963ce97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl=20Gr=C3=B6nberg?= Date: Tue, 12 Aug 2025 13:01:07 +0200 Subject: [PATCH 155/262] fix(helm charts): remove unsed configs from [syncapi secret](charts/sda-svc/templates/syncapi-secrets.yaml) --- charts/sda-svc/templates/syncapi-secrets.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/charts/sda-svc/templates/syncapi-secrets.yaml b/charts/sda-svc/templates/syncapi-secrets.yaml index ed27583e5d..3cdc05852c 100644 --- a/charts/sda-svc/templates/syncapi-secrets.yaml +++ b/charts/sda-svc/templates/syncapi-secrets.yaml @@ -45,13 +45,6 @@ stringData: api: password: {{ required "sync API password is required" .Values.global.sync.api.password }} user: {{ required "sync API user is required" .Values.global.sync.api.user }} - brokerQueue: {{ default "mapping_stream" .Values.global.sync.brokerQueue }} - centerPrefix: {{ .Values.global.sync.centerPrefix }} - remote: - host: {{ required "remote sync API host is required" .Values.global.sync.remote.host }} - password: {{ required "remote sync API password is required" .Values.global.sync.remote.password }} - port: {{ .Values.global.sync.remote.port }} - user: {{ required "remote sync API user is required" .Values.global.sync.remote.user }} {{- end }} {{- end }} {{- end }} From 1fb14f2535b7a99189f4c323560e27dcfbe7a8a4 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 9 Jul 2025 12:44:53 +0200 Subject: [PATCH 156/262] [sda] remove string concatenations from SQL queries. --- sda/internal/database/db_functions.go | 93 ++++++++++++++------------- 1 file changed, 49 insertions(+), 44 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 262bdbfa09..c9c74cebeb 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -84,11 +84,11 @@ func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFilePath = "SELECT submission_file_path from sda.files where " + - "submission_user= $1 and id = $2 " + - "AND EXISTS (SELECT 1 FROM " + - "(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) " + - "as subquery WHERE event = 'uploaded')" + const getFilePath = `SELECT submission_file_path from sda.files where +submission_user= $1 and id = $2 +AND EXISTS (SELECT 1 FROM +(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) +as subquery WHERE event = 'uploaded');` var filePath string err := db.QueryRow(getFilePath, submissionUser, fileID).Scan(&filePath) @@ -122,12 +122,12 @@ func (dbs *SDAdb) getFileIDByUserPathAndStatus(submissionUser, filePath, status dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFileID = "SELECT id from sda.files " + - "WHERE submission_user=$1 and submission_file_path =$2 and stable_id IS null " + - "AND EXISTS (SELECT 1 FROM " + - "(SELECT event from sda.file_event_log JOIN sda.files ON sda.files.id=sda.file_event_log.file_id " + - "WHERE submission_user=$1 and submission_file_path =$2 order by started_at desc limit 1) " + - "AS subquery WHERE event = $3)" + const getFileID = `SELECT id from sda.files +WHERE submission_user=$1 and submission_file_path =$2 and stable_id IS null +AND EXISTS (SELECT 1 FROM +(SELECT event from sda.file_event_log JOIN sda.files ON sda.files.id=sda.file_event_log.file_id +WHERE submission_user=$1 and submission_file_path =$2 order by started_at desc limit 1) +AS subquery WHERE event = $3);` var fileID string err := db.QueryRow(getFileID, submissionUser, filePath, status).Scan(&fileID) @@ -224,9 +224,10 @@ func (dbs *SDAdb) setArchived(file FileInfo, fileID string) error { } log.Debugf("checksum: %s", file.UploadedChecksum) - const addChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + - "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source)" + - "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + const addChecksum = `INSERT INTO sda.checksums(file_id, checksum, type, source) +VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source) +ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;` + if _, err := db.Exec(addChecksum, fileID, file.UploadedChecksum, "SHA256"); err != nil { return fmt.Errorf("addChecksum error: %s", err.Error()) } @@ -318,16 +319,18 @@ func (dbs *SDAdb) setVerified(file FileInfo, fileID string) error { return fmt.Errorf("setVerified error: %s", err.Error()) } - const addArchiveChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + - "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source)" + - "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + const addArchiveChecksum = `INSERT INTO sda.checksums(file_id, checksum, type, source) +VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source) +ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;` + if _, err := dbs.DB.Exec(addArchiveChecksum, fileID, file.ArchiveChecksum, "SHA256"); err != nil { return fmt.Errorf("addArchiveChecksum error: %s", err.Error()) } - const addUnencryptedChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + - "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source)" + - "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + const addUnencryptedChecksum = `INSERT INTO sda.checksums(file_id, checksum, type, source) +VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source) +ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;` + if _, err := dbs.DB.Exec(addUnencryptedChecksum, fileID, file.DecryptedChecksum, "SHA256"); err != nil { return fmt.Errorf("addUnencryptedChecksum error: %s", err.Error()) } @@ -571,9 +574,10 @@ func (dbs *SDAdb) getFileInfo(id string) (FileInfo, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB const getFileID = "SELECT archive_file_path, archive_file_size from sda.files where id = $1;" - const checkSum = "SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, " + - "MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted, " + - "MAX(checksum) FILTER(where source = 'UPLOADED') as Uploaded from sda.checksums where file_id = $1;" + const checkSum = `SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, +MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted, +MAX(checksum) FILTER(where source = 'UPLOADED') as Uploaded from sda.checksums where file_id = $1;` + var info FileInfo if err := db.QueryRow(getFileID, id).Scan(&info.Path, &info.Size); err != nil { return FileInfo{}, err @@ -729,9 +733,9 @@ func (dbs *SDAdb) getUserFiles(userID string, allData bool) ([]*SubmissionFileIn db := dbs.DB // select all files (that are not part of a dataset) of the user, each one annotated with its latest event - const query = "SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f " + - "LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " + - "AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);" + const query = `SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f +LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 +AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);` // nolint:rowserrcheck rows, err := db.Query(query, userID) @@ -783,9 +787,10 @@ func (dbs *SDAdb) GetCorrID(user, path, accession string) (string, error) { func (dbs *SDAdb) getCorrID(user, path, accession string) (string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "SELECT DISTINCT correlation_id FROM sda.file_event_log e " + - "RIGHT JOIN sda.files f ON e.file_id = f.id " + - "WHERE f.submission_file_path = $1 AND f.submission_user = $2 AND COALESCE(f.stable_id, '') = $3;" + const query = `SELECT DISTINCT correlation_id FROM sda.file_event_log e +RIGHT JOIN sda.files f ON e.file_id = f.id +WHERE f.submission_file_path = $1 AND f.submission_user = $2 AND COALESCE(f.stable_id, '') = $3;` + rows, err := db.Query(query, path, user, accession) if err != nil { return "", err @@ -1004,19 +1009,19 @@ func (dbs *SDAdb) ListUserDatasets(submissionUser string) ([]DatasetInfo, error) db := dbs.DB query := `SELECT dataset_id,event,event_date FROM sda.dataset_event_log WHERE - (dataset_id, event_date) IN ( - SELECT dataset_id,max(event_date) FROM sda.dataset_event_log WHERE - dataset_id IN ( - SELECT stable_id FROM sda.datasets WHERE - id IN ( - SELECT DISTINCT dataset_id FROM sda.file_dataset WHERE - file_id IN ( - SELECT id FROM sda.files WHERE submission_user = $1 AND stable_id IS NOT NULL - ) - ) +(dataset_id, event_date) IN ( + SELECT dataset_id,max(event_date) FROM sda.dataset_event_log WHERE + dataset_id IN ( + SELECT stable_id FROM sda.datasets WHERE + id IN ( + SELECT DISTINCT dataset_id FROM sda.file_dataset WHERE + file_id IN ( + SELECT id FROM sda.files WHERE submission_user = $1 AND stable_id IS NOT NULL ) - GROUP BY dataset_id - );` + ) + ) + GROUP BY dataset_id +);` rows, err := db.Query(query, submissionUser) if err != nil { @@ -1058,9 +1063,9 @@ func (dbs *SDAdb) updateUserInfo(userID, name, email string, groups []string) er dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "INSERT INTO sda.userinfo(id, name, email, groups) VALUES($1, $2, $3, $4)" + - "ON CONFLICT (id)" + - "DO UPDATE SET name = excluded.name, email = excluded.email, groups = excluded.groups;" + const query = `INSERT INTO sda.userinfo(id, name, email, groups) VALUES($1, $2, $3, $4) +ON CONFLICT (id) +DO UPDATE SET name = excluded.name, email = excluded.email, groups = excluded.groups;` result, err := db.Exec(query, userID, name, email, pq.Array(groups)) if err != nil { From 2eff898b6d9890dfa0cdd24844777f93f98673c3 Mon Sep 17 00:00:00 2001 From: zeidlitz Date: Wed, 9 Jul 2025 15:51:14 +0200 Subject: [PATCH 157/262] [api] reduce verbose logging from GIN - Adding a check for log.level and adjust the gin configuration based on it - Adding tests for setup() function, one for loglevel Info and one for Debug --- sda/cmd/api/api.go | 34 ++++++++- sda/cmd/api/api_test.go | 157 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 66af761c55..f91e974816 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -100,7 +100,39 @@ func setup(conf *config.Config) *http.Server { log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) // nolint # FIXME Fatal should only be called from main } - r := gin.Default() + r := gin.New() + r.Use(gin.Recovery()) + + // Enable default Gin logger in debug mode for detailed request logging during development + if log.GetLevel() == log.DebugLevel { + r.Use(gin.LoggerWithConfig(gin.LoggerConfig{ + SkipPaths: []string{"/ready"}, + })) + } + + // Enable structured JSON logging in info mode for clean, parseable logs in production + if log.GetLevel() == log.InfoLevel { + r.Use(gin.LoggerWithConfig( + gin.LoggerConfig{ + Formatter: func(params gin.LogFormatterParams) string { + s, _ := json.Marshal(map[string]any{ + "level": "info", + "method": params.Method, + "path": params.Path, + "remote_addr": params.ClientIP, + "status_code": params.StatusCode, + "time": params.TimeStamp.Format(time.RFC3339), + }) + + return string(s) + "\n" + }, + + Output: gin.DefaultWriter, + SkipPaths: []string{"/ready"}, + }, + )) + } + r.GET("/ready", readinessResponse) r.GET("/files", rbac(e), getFiles) r.GET("/datasets", rbac(e), listDatasets) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 33da335511..a35b2a3df0 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -415,6 +415,9 @@ func (s *TestSuite) SetupSuite() { Conf = c + Conf.API.Host = "0.0.0.0" + Conf.API.Port = 8080 + log.Print("Setup DB for my test") Conf.Database = database.DBConf{ Host: "localhost", @@ -712,6 +715,160 @@ func testEndpoint(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) } +func (s *TestSuite) TestGinLogLevel_Debug() { + // Capture log output from gin in a buffer + var buf bytes.Buffer + gin.DefaultWriter = &buf + gin.DefaultErrorWriter = &buf + + log.SetLevel(log.DebugLevel) + + // A specific port is enforced here so we don't have a conflict when running the LogLevel_Info test + Conf.API.Port = 8081 + srv := setup(Conf) + go func() { + if err := srv.ListenAndServe(); err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failure from API server") + } + }() + + // Allow some time for startup + time.Sleep(500 * time.Millisecond) + + assert.NoError(s.T(), setupJwtAuth()) + + // Send a request towards previously started *http.Server + client := http.Client{Timeout: 5 * time.Second} + r, err := http.NewRequest("GET", fmt.Sprintf("http://%s:%d/files", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance") + } + + r.Header.Add("Authorization", "Bearer "+s.Token) + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput := buf.String() + + // Remove leading and/or trailing newlines and spaces + lines := strings.Split(strings.TrimSpace(logOutput), "\n") + assert.Greater(s.T(), len(lines), 0) + + // Pick the last produced output from GIN + logOutput = lines[len(lines)-1] + assert.Contains(s.T(), logOutput, "[GIN]") + + buf.Reset() + r, err = http.NewRequest("GET", fmt.Sprintf("http://%s:%d/ready", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance for /ready") + } + + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request to /ready") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput = buf.String() + + lines = strings.Split(strings.TrimSpace(logOutput), "\n") + fmt.Println("lines : ", lines) + fmt.Println("len(lines) : ", len(lines)) + if len(lines) > 1 { + assert.NotContains(s.T(), logOutput[len(logOutput)-1], "[GIN]") + } else { + s.T().Log("No log output from /ready, considered PASS on log level debug") + } +} + +func (s *TestSuite) TestGinLogLevel_Info() { + // Capture log output from gin in a buffer + var buf bytes.Buffer + gin.DefaultWriter = &buf + gin.DefaultErrorWriter = &buf + + log.SetLevel(log.InfoLevel) + + // A specific port is enforced here so we don't have a conflict when running the LogLevel_Debug test + Conf.API.Port = 8082 + srv := setup(Conf) + go func() { + if err := srv.ListenAndServe(); err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failure from API server") + } + }() + + // Allow some time for startup + time.Sleep(500 * time.Millisecond) + + assert.NoError(s.T(), setupJwtAuth()) + + // Send a request towards previously started *http.Server + client := http.Client{Timeout: 5 * time.Second} + r, err := http.NewRequest("GET", fmt.Sprintf("http://%s:%d/files", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance") + } + + r.Header.Add("Authorization", "Bearer "+s.Token) + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput := buf.String() + + // Remove leading and/or trailing newlines and spaces + lines := strings.Split(strings.TrimSpace(logOutput), "\n") + assert.Greater(s.T(), len(lines), 0) + + // Pick the last produced output from GIN + logOutput = lines[len(lines)-1] + assert.NotContains(s.T(), logOutput, "[GIN]") + + buf.Reset() + r, err = http.NewRequest("GET", fmt.Sprintf("http://%s:%d/ready", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance for /ready") + } + + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request to /ready") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput = buf.String() + + lines = strings.Split(strings.TrimSpace(logOutput), "\n") + fmt.Println("lines : ", lines) + fmt.Println("len(lines) : ", len(lines)) + if len(lines) > 1 { + assert.NotContains(s.T(), logOutput[len(logOutput)-1], "[GIN]") + } else { + s.T().Log("No log output from /ready, considered PASS on log level info") + } +} + func (s *TestSuite) TestRBAC() { gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) From ad0c18f144ee74ebdc22cad0492c6b6ca05c8928 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 14 Aug 2025 09:39:58 +0200 Subject: [PATCH 158/262] Strategy is only used with matrix --- .github/workflows/code-linter.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index d8448c4c08..cd6f0b1584 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -14,8 +14,6 @@ jobs: lint_download: name: Lint download code runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - name: Set up Go ${{ env.go-version }} uses: actions/setup-go@v5 @@ -35,8 +33,6 @@ jobs: lint_sda: name: Lint sda code runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - name: Set up Go ${{ env.go-version }} uses: actions/setup-go@v5 @@ -56,8 +52,6 @@ jobs: lint_sda_admin: name: Lint sda-admin code runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - name: Set up Go ${{ env.go-version }} uses: actions/setup-go@v5 From 56c53530c1ab33b9627c5cc96c04bd38f4894d04 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 14 Aug 2025 09:40:25 +0200 Subject: [PATCH 159/262] coherent naming schema --- .github/workflows/code-linter.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index cd6f0b1584..52793217f9 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -11,7 +11,7 @@ env: go-version: '1.24' jobs: - lint_download: + lint_sda_download: name: Lint download code runs-on: ubuntu-latest steps: From 4e35e9fb984d3062ad16c3d68e6a3ca439c5adc5 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 14 Aug 2025 09:44:16 +0200 Subject: [PATCH 160/262] Only run the linter if the code has been changed. --- .github/workflows/code-linter.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 52793217f9..495067c419 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -11,7 +11,28 @@ env: go-version: '1.24' jobs: + check_changes: + outputs: + sda-download: ${{ steps.changes.outputs.sda-download }} + sda-admin: ${{ steps.changes.outputs.sda-admin }} + sda: ${{ steps.changes.outputs.sda }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + sda-download: + - 'sda-download/**' + sda-admin: + - 'sda-admin/**' + sda: + - 'sda/**' + lint_sda_download: + needs: check_changes + if: needs.check_changes.outputs.sda-download == 'true' name: Lint download code runs-on: ubuntu-latest steps: @@ -31,6 +52,8 @@ jobs: working-directory: sda-download lint_sda: + needs: check_changes + if: needs.check_changes.outputs.sda == 'true' name: Lint sda code runs-on: ubuntu-latest steps: @@ -50,6 +73,8 @@ jobs: working-directory: sda lint_sda_admin: + needs: check_changes + if: needs.check_changes.outputs.sda-admin == 'true' name: Lint sda-admin code runs-on: ubuntu-latest steps: From 913cc58caf79509eb746fc03cf7922c271042a72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Aug 2025 14:49:28 +0000 Subject: [PATCH 161/262] Bump github.com/docker/docker in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [github.com/docker/docker](https://github.com/docker/docker). Updates `github.com/docker/docker` from 27.2.1+incompatible to 28.0.0+incompatible - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v27.2.1...v28.0.0) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-version: 28.0.0+incompatible dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index c83a0a1142..97b5129c41 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -75,7 +75,7 @@ require ( github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/docker/cli v27.4.1+incompatible // indirect - github.com/docker/docker v27.2.1+incompatible // indirect + github.com/docker/docker v28.0.0+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/structs v1.1.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 42e9537d47..f57d4f911b 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -100,8 +100,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvw github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= -github.com/docker/docker v27.2.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= +github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= From 6075303798b09731dbf7bda74f13cd346e1f17d0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 18 Aug 2025 11:45:39 +0200 Subject: [PATCH 162/262] [integration tests] Update k8s version --- .github/workflows/build_pr_container.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 3f5603da60..39767735a9 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -327,14 +327,14 @@ jobs: strategy: matrix: deployment: ["federated", "isolated"] - version: ["1.30", "1.31"] + version: ["1.32", "1.33"] tls: ["true", "false"] storage: ["posix", "s3"] exclude: - - version: "1.30" + - version: "1.32" tls: "false" storage: "posix" - - version: "1.31" + - version: "1.33" tls: "false" storage: "posix" steps: From 92febc377b2a44da812dd8b4b1ebc0b261ffa319 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 18 Aug 2025 11:49:47 +0200 Subject: [PATCH 163/262] [Makefile] update k3d version --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8958ebb903..ca0f914444 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ k3d-version-check: SHELL:=/bin/bash k3d-version-check: @K3D_VERSION=$$(k3d version | cut -d'v' -f 3 | head -n 1); \ IFS="." read -r -a K3D_VERSION_ARR <<< "$${K3D_VERSION}"; \ - IFS="." read -r -a K3D_VERSION_REQ <<< 5.7.0; \ + IFS="." read -r -a K3D_VERSION_REQ <<< 5.8.0; \ if [[ $${K3D_VERSION_ARR[0]} -lt $${K3D_VERSION_REQ[0]} ||\ ( $${K3D_VERSION_ARR[0]} -eq $${K3D_VERSION_REQ[0]} &&\ ( $${K3D_VERSION_ARR[1]} -lt $${K3D_VERSION_REQ[1]} ||\ From 26b564268f07be4e23ba350c1518fe069058a1ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 05:41:08 +0000 Subject: [PATCH 164/262] Bump the all-modules group in /sda-doa with 5 updates Bumps the all-modules group in /sda-doa with 5 updates: | Package | From | To | | --- | --- | --- | | [io.jsonwebtoken:jjwt-api](https://github.com/jwtk/jjwt) | `0.12.6` | `0.12.7` | | [io.jsonwebtoken:jjwt-impl](https://github.com/jwtk/jjwt) | `0.12.6` | `0.12.7` | | io.jsonwebtoken:jjwt-jackson | `0.12.6` | `0.12.7` | | [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) | `3.0.31` | `3.0.32` | | [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway) | `3.0.6` | `3.0.7` | Updates `io.jsonwebtoken:jjwt-api` from 0.12.6 to 0.12.7 - [Release notes](https://github.com/jwtk/jjwt/releases) - [Changelog](https://github.com/jwtk/jjwt/blob/master/CHANGELOG.md) - [Commits](https://github.com/jwtk/jjwt/compare/0.12.6...0.12.7) Updates `io.jsonwebtoken:jjwt-impl` from 0.12.6 to 0.12.7 - [Release notes](https://github.com/jwtk/jjwt/releases) - [Changelog](https://github.com/jwtk/jjwt/blob/master/CHANGELOG.md) - [Commits](https://github.com/jwtk/jjwt/compare/0.12.6...0.12.7) Updates `io.jsonwebtoken:jjwt-jackson` from 0.12.6 to 0.12.7 Updates `no.elixir:crypt4gh` from 3.0.31 to 3.0.32 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.31...crypt4gh-3.0.32) Updates `no.elixir:clearinghouse` from 3.0.6 to 3.0.7 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.6...crypt4gh-3.0.7) --- updated-dependencies: - dependency-name: io.jsonwebtoken:jjwt-api dependency-version: 0.12.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: io.jsonwebtoken:jjwt-impl dependency-version: 0.12.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: io.jsonwebtoken:jjwt-jackson dependency-version: 0.12.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.32 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 41a496a5c7..bd6d009c33 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -67,18 +67,18 @@ io.jsonwebtoken jjwt-api - 0.12.6 + 0.12.7 io.jsonwebtoken jjwt-impl - 0.12.6 + 0.12.7 runtime io.jsonwebtoken jjwt-jackson - 0.12.6 + 0.12.7 runtime @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.31 + 3.0.32 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 3.0.6 + 3.0.7 org.slf4j From cea2e624f1a6c2b8230291cf2b936f6a396d9466 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Aug 2025 06:28:22 +0000 Subject: [PATCH 165/262] Bump github.com/casbin/casbin/v2 in /sda in the all-modules group Bumps the all-modules group in /sda with 1 update: [github.com/casbin/casbin/v2](https://github.com/casbin/casbin). Updates `github.com/casbin/casbin/v2` from 2.118.0 to 2.120.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.118.0...v2.120.0) --- updated-dependencies: - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.120.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 97b5129c41..6dcd64b9f9 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -9,7 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 github.com/aws/aws-sdk-go-v2/service/s3 v1.87.0 github.com/aws/smithy-go v1.22.5 - github.com/casbin/casbin/v2 v2.118.0 + github.com/casbin/casbin/v2 v2.120.0 github.com/coreos/go-oidc/v3 v3.15.0 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 diff --git a/sda/go.sum b/sda/go.sum index f57d4f911b..4ca3f4fe9f 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.118.0 h1:OS0E6J/KkWw3lvmW6f3h7xz4Bvlv8mv91fRk0J4QpFs= -github.com/casbin/casbin/v2 v2.118.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.120.0 h1:Mo9R/EKZk9aoagFs0OmuCmBYjWJfvbWJiX4aenIJOKY= +github.com/casbin/casbin/v2 v2.120.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= From aeea38fec0ee96431cdc1f5452b32a25f9817545 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 19 Aug 2025 13:45:45 +0200 Subject: [PATCH 166/262] remove Github packages registry configuration --- .github/dependabot.yaml | 8 ------- .github/workflows/build_pr_container.yaml | 29 ----------------------- README.md | 23 ------------------ sda-doa/Dockerfile | 2 -- sda-doa/pom.xml | 7 ------ 5 files changed, 69 deletions(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index cd2f66ea0f..5569d19a39 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -1,10 +1,4 @@ version: 2 -registries: - maven-github-elixir-no: - type: maven-repository - url: 'https://maven.pkg.github.com/ELIXIR-NO' - username: '${{secrets.DOA_MAVEN_GITHUB_USERNAME}}' - password: '${{secrets.DOA_MAVEN_GITHUB_TOKEN}}' updates: - package-ecosystem: "github-actions" directory: "/" @@ -91,7 +85,6 @@ updates: - package-ecosystem: maven directory: "/sda-doa" - registries: "*" groups: all-modules: patterns: @@ -192,7 +185,6 @@ updates: - package-ecosystem: maven target-branch: release_v1 directory: "/sda-doa" - registries: "*" groups: all-modules: patterns: diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 39767735a9..c37d05c195 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -189,21 +189,6 @@ jobs: sarif_file: 'inbox-results.sarif' category: sftp-inbox - - - name: create maven settings.xml - uses: s4u/maven-settings-action@v3.1.0 - with: - servers: | - [{ - "id":"github-fega-norway", - "username": "${{github.actor}}", - "password": "${{ secrets.GITHUB_TOKEN }}" - }] - - - name: Copy settings.xml to sda-doa root - shell: bash - run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml - - name: Build container for sda-doa uses: docker/build-push-action@v6 with: @@ -301,20 +286,6 @@ jobs: - name: Check out code uses: actions/checkout@v5 - - name: create maven settings.xml - uses: s4u/maven-settings-action@v3.1.0 - with: - servers: | - [{ - "id":"github-fega-norway", - "username": "${{github.actor}}", - "password": "${{ secrets.GITHUB_TOKEN }}" - }] - - - name: Copy settings.xml to sda-doa root - shell: bash - run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml - - name: Test sda-doa for ${{ matrix.storage }} storage run: docker compose -f .github/integration/sda-doa-${{ matrix.storage }}-outbox.yml run integration_test diff --git a/README.md b/README.md index 16e33f18d4..8b665a3d63 100644 --- a/README.md +++ b/README.md @@ -46,29 +46,6 @@ To build the `sda-admin` CLI tool: ```sh $ make build-sda-admin ``` - -To build the `sda-doa`: - -SDA-DOA uses two JARs hosted on Github Packages. To build the Docker image successfully , you need to provide authentication credentials to access the GitHub package registry. -#### Required Setup: `settings.xml` - -Create a `settings.xml` file in the root directory (next to the Dockerfile in /sda-doa), and include the following content: - -```xml - - - - github-fega-norway - YOUR_GITHUB_USERNAME - YOUR_GITHUB_PERSONAL_ACCESS_TOKEN - - - -``` -Replace `YOUR_GITHUB_USERNAME` and `YOUR_GITHUB_PERSONAL_ACCESS_TOKEN` with your actual GitHub username and personal access token, respectively. The personal access token should have the `read:packages` scope. - ```sh $ make build-sda-doa ``` diff --git a/sda-doa/Dockerfile b/sda-doa/Dockerfile index 10ae01974d..e8908b716b 100644 --- a/sda-doa/Dockerfile +++ b/sda-doa/Dockerfile @@ -5,8 +5,6 @@ COPY pom.xml . RUN mkdir -p /root/.m2 && \ mkdir /root/.m2/repository -COPY settings.xml /root/.m2 - RUN mvn dependency:go-offline --no-transfer-progress COPY src/ /src/ diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index bd6d009c33..8d4f5ee27c 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -136,13 +136,6 @@ - - - github-fega-norway - https://maven.pkg.github.com/ELIXIR-NO/FEGA-Norway - - - From b0c9090bb6a19a6f79e295a94a7cf4345f731548 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 19 Aug 2025 14:08:35 +0200 Subject: [PATCH 167/262] remove settings.xml mount from doa docker compose files --- .github/integration/sda-doa-posix-outbox.yml | 1 - .github/integration/sda-doa-s3-outbox.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index 9fba0fb7e2..b106439c6b 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -130,7 +130,6 @@ services: volumes: - ../../sda-doa/src:/sda-doa/src - ../../sda-doa/pom.xml:/sda-doa/pom.xml - - ../../sda-doa/settings.xml:/root/.m2/settings.xml - test_file:/sda-doa/outbox - ./tests:/tests - encryption_files:/test diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 8ca9c558db..8ba3dd2912 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -157,7 +157,6 @@ services: volumes: - ../../sda-doa/src:/sda-doa/src - ../../sda-doa/pom.xml:/sda-doa/pom.xml - - ../../sda-doa/settings.xml:/root/.m2/settings.xml - ./tests:/tests - encryption_files:/test - client_certs:/certs From ff73064585b44c4d77800fd97295e713ed30fdfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 08:53:03 +0000 Subject: [PATCH 168/262] Bump google.golang.org/grpc in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `google.golang.org/grpc` from 1.74.2 to 1.75.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.74.2...v1.75.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.75.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 4 ++-- sda-download/go.sum | 30 ++++++++++++++++-------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index cc3053e7ab..9cc0e7028b 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -17,7 +17,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 - google.golang.org/grpc v1.74.2 + google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.7 ) @@ -71,7 +71,7 @@ require ( golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect golang.org/x/tools v0.35.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 95d24c8272..86bee8a80f 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -159,16 +159,16 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= @@ -230,10 +230,12 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From f0dc505bdbf89f9fbaf9f2e9eb8b93822abb1ae4 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 19 Aug 2025 15:00:11 +0200 Subject: [PATCH 169/262] Parameterize all probes This allows for better control when dealing with sluggish network connections for external services. --- charts/sda-svc/Chart.yaml | 2 +- charts/sda-svc/README.md | 22 +++-- charts/sda-svc/templates/api-deploy.yaml | 20 +--- charts/sda-svc/templates/auth-deploy.yaml | 20 +--- charts/sda-svc/templates/doa-deploy.yaml | 10 +- charts/sda-svc/templates/download-deploy.yaml | 30 +----- .../sda-svc/templates/re-encrypt-deploy.yaml | 6 +- charts/sda-svc/templates/s3-inbox-deploy.yaml | 14 +-- .../sda-svc/templates/sftp-inbox-deploy.yaml | 16 +-- charts/sda-svc/templates/syncapi-deploy.yaml | 5 +- charts/sda-svc/values.yaml | 97 +++++++++++++++++++ 11 files changed, 127 insertions(+), 115 deletions(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 8ee2736c26..a8d61244d9 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 3.0.3 +version: 3.0.4 appVersion: v2.1.7 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 5d0dc56759..b8e26a003d 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -239,6 +239,8 @@ Parameter | Description | Default --------- | ----------- | ------- `api.replicaCount` | Desired number of replicas | `2` `api.annotations` | Specific annotation for the auth pod | `{}` +`api.livenessProbe` | Liveness definition for the api pod. |`{}` +`api.readinessProbe` | Rediness definition for the api pod. |`{}` `api.resources.requests.memory` | Memory request for container. |`128Mi` `api.resources.requests.cpu` | CPU request for container. |`100m` `api.resources.limits.memory` | Memory limit for container. |`256Mi` @@ -246,19 +248,17 @@ Parameter | Description | Default `api.tls.secretName` | Secret holding the application TLS certificates |`` `auth.replicaCount` | desired number of replicas | `2` `auth.annotations` | Specific annotation for the auth pod | `{}` +`auth.livenessProbe` | Liveness definition for the auth pod. |`{}` +`auth.readinessProbe` | Rediness definition for the auth pod. |`{}` `auth.resources.requests.memory` | Memory request for container. |`128Mi` `auth.resources.requests.cpu` | CPU request for container. |`100m` `auth.resources.limits.memory` | Memory limit for container. |`256Mi` `auth.resources.limits.cpu` | CPU limit for container. |`250m` -`sync.annotations` | Specific annotation for the sync pod | `{}` -`sync.resources.requests.memory` | Memory request for sync container. |`128Mi` -`sync.resources.requests.cpu` | CPU request for sync container. |`100m` -`sync.resources.limits.memory` | Memory limit for sync container. |`256Mi` -`sync.resources.limits.cpu` | CPU limit for sync container. |`250m` -`sync.deploy` | Set to true if the sync service should be active | `false` `doa.replicaCount` | desired number of replicas | `2` `doa.keystorePass` | keystore password | `changeit` `doa.annotations` | Specific annotation for the doa pod | `{}` +`doa.livenessProbe` | Liveness definition for the doa pod. |`{}` +`doa.readinessProbe` | Rediness definition for the doa pod. |`{}` `doa.resources.requests.memory` | Memory request for dataedge container. |`128Mi` `doa.resources.requests.cpu` | CPU request for dataedge container. |`100m` `doa.resources.limits.memory` | Memory limit for dataedge container. |`1024Mi` @@ -266,6 +266,8 @@ Parameter | Description | Default `download.replicaCount` | desired number of replicas | `2` `download.keystorePass` | keystore password | `changeit` `download.annotations` | Specific annotation for the dataedge pod | `{}` +`download.livenessProbe` | Liveness definition for the download pod. |`{}` +`download.readinessProbe` | Rediness definition for the download pod. |`{}` `download.resources.requests.memory` | Memory request for dataedge container. |`256Mi` `download.resources.requests.cpu` | CPU request for dataedge container. |`100m` `download.resources.limits.memory` | Memory limit for dataedge container. |`512Mi` @@ -289,6 +291,8 @@ Parameter | Description | Default `intercept.resources.limits.cpu` | CPU limit for intercept container. |`2000m` `s3Inbox.replicaCount`| desired number of S3inbox containers | `2` `s3Inbox.annotations` | Specific annotation for the S3inbox pod | `{}` +`s3Inbox.livenessProbe` | Liveness definition for the s3Inbox pod. |`{}` +`s3Inbox.readinessProbe` | Rediness definition for the S3inbox pod. |`{}` `s3Inbox.resources.requests.memory` | Memory request for s3Inbox container. |`128Mi` `s3Inbox.resources.requests.cpu` | CPU request for s3Inbox container. |`100m` `s3Inbox.resources.limits.memory` | Memory limit for s3Inbox container. |`1024Mi` @@ -297,18 +301,22 @@ Parameter | Description | Default `sftpInbox.keystorePass` | sftp inbox keystore password | `changeit` `sftpInbox.nodeHostname` | Node name if the sftp inbox needs to be deployed on a specific node | `""` `sftpInbox.annotations` | Specific annotation for the sftp inbox pod | `{}` +`sftpInbox.livenessProbe` | Liveness definition for the sftpInbox pod. |`{}` +`sftpInbox.readinessProbe` | Rediness definition for the sftpInbox pod. |`{}` `sftpInbox.resources.requests.memory` | Memory request for sftpInbox container. |`128Mi` `sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. |`100m` `sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. |`256Mi` `sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. |`250m` -`sync.replicaCount`| desired number of sync containers | `1` `sync.annotations` | Specific annotation for the sync pod | `{}` +`sync.deploy` | Set to true if the sync service should be active | `false` +`sync.replicaCount`| desired number of sync containers | `1` `sync.resources.requests.memory` | Memory request for sync container. |`128Mi` `sync.resources.requests.cpu` | CPU request for sync container. |`100m` `sync.resources.limits.memory` | Memory limit for sync container. |`512Mi` `sync.resources.limits.cpu` | CPU limit for sync container. |`500m` `syncAPI.replicaCount`| desired number of syncAPI containers | `1` `syncAPI.annotations` | Specific annotation for the syncAPI pod | `{}` +`syncAPI.readinessProbe` | Rediness definition for the syncAPI pod. |`{}` `syncAPI.resources.requests.memory` | Memory request for syncAPI container. |`64Mi` `syncAPI.resources.requests.cpu` | CPU request for syncAPI container. |`100m` `syncAPI.resources.limits.memory` | Memory limit for syncAPI container. |`256Mi` diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index b616940b50..e1867e8dfc 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -78,25 +78,9 @@ spec: containerPort: 8080 protocol: TCP livenessProbe: - httpGet: - port: api - path: /ready - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.api }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.api.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - port: api - path: /ready - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.api }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.api.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.api.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index bb8c937909..e72c6c0eb6 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -79,25 +79,9 @@ spec: containerPort: 8080 protocol: TCP livenessProbe: - httpGet: - port: auth - path: / - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.auth }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.auth.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - port: auth - path: / - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.auth }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.auth.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.auth.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/templates/doa-deploy.yaml b/charts/sda-svc/templates/doa-deploy.yaml index b069f9f7e7..705495196e 100644 --- a/charts/sda-svc/templates/doa-deploy.yaml +++ b/charts/sda-svc/templates/doa-deploy.yaml @@ -257,15 +257,9 @@ spec: containerPort: 8080 protocol: TCP livenessProbe: - tcpSocket: - port: doa - initialDelaySeconds: 120 - periodSeconds: 30 +{{ toYaml .Values.doa.livenessProbe | trim | indent 10 }} readinessProbe: - tcpSocket: - port: doa - initialDelaySeconds: 30 - periodSeconds: 15 +{{ toYaml .Values.doa.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.doa.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index 8bf2e580b0..90650a6f05 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -76,35 +76,9 @@ spec: containerPort: {{ ternary 8443 8080 .Values.global.tls.enabled }} protocol: TCP livenessProbe: - httpGet: - port: download - path: /health - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.download }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.download.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - port: download - path: /health - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.download }} - initialDelaySeconds: 20 - periodSeconds: 10 - startupProbe: - httpGet: - path: /health - port: download - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.download }} - failureThreshold: 30 - periodSeconds: 10 +{{ toYaml .Values.download.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.download.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/templates/re-encrypt-deploy.yaml b/charts/sda-svc/templates/re-encrypt-deploy.yaml index f8b4048de5..7d0bc0e7d6 100644 --- a/charts/sda-svc/templates/re-encrypt-deploy.yaml +++ b/charts/sda-svc/templates/re-encrypt-deploy.yaml @@ -59,11 +59,7 @@ spec: - name: grpchealth containerPort: {{add ( ternary 50443 50051 ( .Values.global.tls.enabled ) ) 1 }} readinessProbe: - initialDelaySeconds: 5 - timeoutSeconds: 2 - grpc: - port: {{add ( ternary 50443 50051 ( .Values.global.tls.enabled ) ) 1 }} - service: "reencrypt.Reencrypt" +{{ toYaml .Values.reencrypt.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.reencrypt.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index 39ae8ea13b..5a34704320 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -97,19 +97,9 @@ spec: containerPort: 8000 protocol: TCP livenessProbe: - httpGet: - path: /health - port: inbox - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - failureThreshold: 1 - periodSeconds: 10 +{{ toYaml .Values.s3Inbox.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - path: /health - port: inbox - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - failureThreshold: 1 - periodSeconds: 5 +{{ toYaml .Values.s3Inbox.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.s3Inbox.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/templates/sftp-inbox-deploy.yaml b/charts/sda-svc/templates/sftp-inbox-deploy.yaml index 5d05514697..8dcd2528e0 100644 --- a/charts/sda-svc/templates/sftp-inbox-deploy.yaml +++ b/charts/sda-svc/templates/sftp-inbox-deploy.yaml @@ -154,21 +154,9 @@ spec: containerPort: 2222 protocol: TCP livenessProbe: - exec: - command: - - /bin/sh - - -c - - netstat -an | grep -q ':2222 ' - initialDelaySeconds: 120 - periodSeconds: 5 +{{ toYaml .Values.sftpInbox.livenessProbe | trim | indent 10 }} readinessProbe: - exec: - command: - - /bin/sh - - -c - - netstat -an | grep -q ':2222 ' - initialDelaySeconds: 30 - periodSeconds: 5 +{{ toYaml .Values.sftpInbox.readinessProbe | trim | indent 10 }} volumeMounts: - name: inbox mountPath: "/ega/inbox" diff --git a/charts/sda-svc/templates/syncapi-deploy.yaml b/charts/sda-svc/templates/syncapi-deploy.yaml index 970f69f9d0..6e0f474a96 100644 --- a/charts/sda-svc/templates/syncapi-deploy.yaml +++ b/charts/sda-svc/templates/syncapi-deploy.yaml @@ -80,10 +80,7 @@ spec: containerPort: 8080 protocol: TCP readinessProbe: - tcpSocket: - port: syncapi - initialDelaySeconds: 10 - periodSeconds: 5 +{{ toYaml .Values.syncapi.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.syncapi.resources | trim | indent 10 }} volumeMounts: diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 6a6f091439..5ec6f3e090 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -373,6 +373,21 @@ credentials: api: name: api replicaCount: 2 + livenessProbe: + # httpGet: + # port: api + # path: /ready + # scheme: "HTTPS" + # initialDelaySeconds: 20 + # periodSeconds: 60 + # timeoutSeconds: 30 + readinessProbe: + httpGet: + port: api + path: /ready + scheme: "HTTPS" + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "128Mi" @@ -390,6 +405,22 @@ api: auth: name: auth replicaCount: 2 + livenessProbe: + # httpGet: + # port: auth + # path: / + # scheme: "HTTPS" + # httpHeaders: + # initialDelaySeconds: 20 + # periodSeconds: 60 + # timeoutSeconds: 30 + readinessProbe: + httpGet: + port: auth + path: / + scheme: "HTTPS" + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "128Mi" @@ -408,6 +439,15 @@ doa: name: doa imagePullPolicy: IfNotPresent replicaCount: 2 + livenessProbe: + # tcpSocket: + # port: doa + # initialDelaySeconds: 120 + # periodSeconds: 30 + readinessProbe: + tcpSocket: + port: doa + periodSeconds: 15 resources: requests: memory: "256Mi" @@ -428,6 +468,21 @@ doa: download: name: download replicaCount: 2 + livenessProbe: + # httpGet: + # path: /health + # port: download + # scheme: "HTTPS" + # initialDelaySeconds: 20 + # periodSeconds: 60 + # timeoutSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: download + scheme: "HTTPS" + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "256Mi" @@ -511,6 +566,12 @@ mapper: reencrypt: replicaCount: 1 + readinessProbe: + periodSeconds: 10 + timeoutSeconds: 2 + grpc: + port: 50443 + service: "reencrypt.Reencrypt" resources: requests: memory: "128Mi" @@ -528,6 +589,22 @@ reencrypt: s3Inbox: name: s3Inbox replicaCount: 2 + livenessProbe: + # httpGet: + # path: /health + # port: inbox + # scheme: "HTTPS" + # failureThreshold: 1 + # periodSeconds: 60 + # timeoutSeconds: 55 + readinessProbe: + httpGet: + path: /health + port: inbox + scheme: "HTTPS" + failureThreshold: 1 + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "128Mi" @@ -545,6 +622,22 @@ s3Inbox: sftpInbox: name: sftpInbox replicaCount: 2 + livenessProbe: + # exec: + # command: + # - /bin/sh + # - -c + # - netstat -an | grep -q ':2222 ' + # initialDelaySeconds: 120 + # periodSeconds: 30 + # timeoutSeconds: 25 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - netstat -an | grep -q ':2222 ' + periodSeconds: 5 resources: requests: memory: "128Mi" @@ -581,6 +674,10 @@ sync: syncapi: name: sync-api replicaCount: 1 + readinessProbe: + tcpSocket: + port: syncapi + periodSeconds: 5 resources: requests: memory: "64Mi" From a748b11053f8e1503f7ef62f9a00359eabccaf26 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 20 Aug 2025 08:57:55 +0200 Subject: [PATCH 170/262] [integration tests] fix chart deploy script --- .github/integration/scripts/charts/deploy_charts.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 1fefa6d0a3..f5f0abab02 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -8,9 +8,13 @@ fi MQ_PORT=5672 PROTOCOL=http +SCHEME=HTTP +GRPC_PORT=50051 if [ "$3" == "true" ]; then MQ_PORT=5671 PROTOCOL=https + SCHEME=HTTPS + GRPC_PORT=50443 fi dir=".github/integration/scripts/charts" @@ -69,6 +73,12 @@ if [ "$1" == "sda-svc" ]; then --set global.sync.api.password="$sync_api_pass" \ --set global.sync.api.user="$sync_api_user" \ --set global.sync.remote.host="$sync_host" \ + --set api.readinessProbe.httpGet.scheme:="$SCHEME" \ + --set auth.readinessProbe.httpGet.scheme:="$SCHEME" \ + --set download.readinessProbe.httpGet.scheme:="$SCHEME" \ + --set s3inbox.readinessProbe.httpGet.scheme:="$SCHEME" \ + --set syncAPI.readinessProbe.httpGet.scheme:="$SCHEME" \ + --set reencrypt.readinessProbe.grpc.port="$GRPC_PORT" \ -f "$dir/values.yaml" \ --wait fi From 105a8d0acdc2474822ce1aa043c9c76e6fb270ef Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Aug 2025 10:21:03 +0000 Subject: [PATCH 171/262] Bump google.golang.org/grpc in /sda in the all-modules group Bumps the all-modules group in /sda with 1 update: [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `google.golang.org/grpc` from 1.74.2 to 1.75.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.74.2...v1.75.0) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-version: 1.75.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 6 +++--- sda/go.sum | 34 ++++++++++++++++++---------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 6dcd64b9f9..eb8da249eb 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -33,7 +33,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.41.0 golang.org/x/oauth2 v0.30.0 - google.golang.org/grpc v1.74.2 + google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.7 ) @@ -84,7 +84,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.9 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.4 // indirect - github.com/go-jose/go-jose/v4 v4.1.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect @@ -157,7 +157,7 @@ require ( golang.org/x/sys v0.35.0 // indirect golang.org/x/text v0.28.0 // indirect golang.org/x/time v0.11.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index 4ca3f4fe9f..e0a3cfbc56 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -125,8 +125,8 @@ github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY= -github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -385,16 +385,16 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= -go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= -go.opentelemetry.io/otel/metric v1.36.0 h1:MoWPKVhQvJ+eeXWHFBOPoBOi20jh6Iq2CcCREuTYufE= -go.opentelemetry.io/otel/metric v1.36.0/go.mod h1:zC7Ks+yeyJt4xig9DEw9kuUFe5C3zLbVjV2PzT6qzbs= -go.opentelemetry.io/otel/sdk v1.36.0 h1:b6SYIuLRs88ztox4EyrvRti80uXIFy+Sqzoh9kFULbs= -go.opentelemetry.io/otel/sdk v1.36.0/go.mod h1:+lC+mTgD+MUWfjJubi2vvXWcVxyr9rmlshZni72pXeY= -go.opentelemetry.io/otel/sdk/metric v1.36.0 h1:r0ntwwGosWGaa0CrSt8cuNuTcccMXERFwHX4dThiPis= -go.opentelemetry.io/otel/sdk/metric v1.36.0/go.mod h1:qTNOhFDfKRwX0yXOqJYegL5WRaW376QbB7P4Pb0qva4= -go.opentelemetry.io/otel/trace v1.36.0 h1:ahxWNuqZjpdiFAyrIoQ4GIiAIhxAunQR6MUoKrsNd4w= -go.opentelemetry.io/otel/trace v1.36.0/go.mod h1:gQ+OnDZzrybY4k4seLzPAWNwVBBVlF2szhehOBB/tGA= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -513,10 +513,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a h1:v2PbRU4K3llS09c7zodFpNePeamkAwG3mPrAery9VeE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250528174236-200df99c418a/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 1dcdfdd53dfd25f94e1b3fd04d1a27af6553272c Mon Sep 17 00:00:00 2001 From: Parisa Date: Wed, 20 Aug 2025 12:38:29 +0200 Subject: [PATCH 172/262] add java analysis to codeql-analysis.yml --- .github/workflows/codeql-analysis.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c5138e6632..1742fb6271 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,11 +2,14 @@ name: "CodeQL" on: push: - branches: [main] + branches: + - main pull_request: - branches: [main] + branches: + - main paths: - '**.go' + - '**.java' schedule: - cron: '0 15 * * 4' @@ -15,6 +18,11 @@ jobs: name: Analyze runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: [ 'java', 'go' ] + steps: - name: Checkout repository uses: actions/checkout@v5 @@ -23,10 +31,12 @@ jobs: - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: - languages: go + languages: ${{ matrix.language }} - name: Autobuild uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" From 6e142ba0604eac3712d7a91e75c9e777dcefb1aa Mon Sep 17 00:00:00 2001 From: Parisa Date: Wed, 20 Aug 2025 15:49:53 +0200 Subject: [PATCH 173/262] skip autobuild for java modules and run the manual build commands --- .github/workflows/codeql-analysis.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 1742fb6271..28dc6e199d 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,7 +33,18 @@ jobs: with: languages: ${{ matrix.language }} + - name: Build sda-doa + if: ${{ matrix.language == 'java' }} + working-directory: ./sda-doa + run: mvn clean install -DskipTests + + - name: Build sda-sftp-inbox + if: ${{ matrix.language == 'java' }} + working-directory: ./sda-sftp-inbox + run: mvn clean install -DskipTests + - name: Autobuild + if: ${{ matrix.language == 'go' }} uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis From f5639d6491c9241ddeecfc37e3b34c6e4acb3b71 Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 21 Aug 2025 08:39:32 +0200 Subject: [PATCH 174/262] set up jdk for building java modules --- .github/workflows/codeql-analysis.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 28dc6e199d..48781a5d4f 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,11 +33,23 @@ jobs: with: languages: ${{ matrix.language }} + - name: Set up JDK 21 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 21 + - name: Build sda-doa if: ${{ matrix.language == 'java' }} working-directory: ./sda-doa run: mvn clean install -DskipTests + - name: Set up JDK 19 + uses: actions/setup-java@v4 + with: + distribution: 'temurin' + java-version: 19 + - name: Build sda-sftp-inbox if: ${{ matrix.language == 'java' }} working-directory: ./sda-sftp-inbox From 65550ea789170037e5f1e58acf62f3b9594cc925 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 21 Aug 2025 13:56:02 +0200 Subject: [PATCH 175/262] [finalize] allow starting without backup enabled --- sda/internal/config/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index e876b7559c..202fef0e2d 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -320,7 +320,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) default: - return nil, errors.New("archive.type not set") + log.Warnln(" archive not configured, backup will not be performed.") } switch viper.GetString("backup.type") { @@ -329,7 +329,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"backup.location"}...) default: - return nil, errors.New("backup.type not set") + log.Warnln(" backup destination not configured, backup will not be performed.") } case "intercept": requiredConfVars = []string{ From b219399ff9f6c4e0e4609463c35cebab3834364e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 21 Aug 2025 14:23:43 +0200 Subject: [PATCH 176/262] [integration test] fix yaml override --- .github/integration/scripts/charts/deploy_charts.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index f5f0abab02..235590de84 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -14,7 +14,7 @@ if [ "$3" == "true" ]; then MQ_PORT=5671 PROTOCOL=https SCHEME=HTTPS - GRPC_PORT=50443 + GRPC_PORT=50444 fi dir=".github/integration/scripts/charts" @@ -73,11 +73,11 @@ if [ "$1" == "sda-svc" ]; then --set global.sync.api.password="$sync_api_pass" \ --set global.sync.api.user="$sync_api_user" \ --set global.sync.remote.host="$sync_host" \ - --set api.readinessProbe.httpGet.scheme:="$SCHEME" \ - --set auth.readinessProbe.httpGet.scheme:="$SCHEME" \ - --set download.readinessProbe.httpGet.scheme:="$SCHEME" \ - --set s3inbox.readinessProbe.httpGet.scheme:="$SCHEME" \ - --set syncAPI.readinessProbe.httpGet.scheme:="$SCHEME" \ + --set api.readinessProbe.httpGet.scheme="$SCHEME" \ + --set auth.readinessProbe.httpGet.scheme="$SCHEME" \ + --set download.readinessProbe.httpGet.scheme="$SCHEME" \ + --set s3Inbox.readinessProbe.httpGet.scheme="$SCHEME" \ + --set syncAPI.readinessProbe.httpGet.scheme="$SCHEME" \ --set reencrypt.readinessProbe.grpc.port="$GRPC_PORT" \ -f "$dir/values.yaml" \ --wait From 7f88a1ac73487b4e74e38e3cf9ddf140ebb6b993 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Aug 2025 15:37:11 +0000 Subject: [PATCH 177/262] Bump github.com/go-viper/mapstructure/v2 Bumps the go_modules group in /sda-download with 1 update: [github.com/go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure). Updates `github.com/go-viper/mapstructure/v2` from 2.3.0 to 2.4.0 - [Release notes](https://github.com/go-viper/mapstructure/releases) - [Changelog](https://github.com/go-viper/mapstructure/blob/main/CHANGELOG.md) - [Commits](https://github.com/go-viper/mapstructure/compare/v2.3.0...v2.4.0) --- updated-dependencies: - dependency-name: github.com/go-viper/mapstructure/v2 dependency-version: 2.4.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 9cc0e7028b..1e2dfcb208 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -37,7 +37,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect - github.com/go-viper/mapstructure/v2 v2.3.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 86bee8a80f..0842b8ef55 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -53,8 +53,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= -github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= From ec1f9f91c8bc3df3534fdfd7b87da090279d6c66 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Aug 2025 07:14:10 +0000 Subject: [PATCH 178/262] Bump github.com/go-viper/mapstructure/v2 in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [github.com/go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure). Updates `github.com/go-viper/mapstructure/v2` from 2.3.0 to 2.4.0 - [Release notes](https://github.com/go-viper/mapstructure/releases) - [Changelog](https://github.com/go-viper/mapstructure/blob/main/CHANGELOG.md) - [Commits](https://github.com/go-viper/mapstructure/compare/v2.3.0...v2.4.0) --- updated-dependencies: - dependency-name: github.com/go-viper/mapstructure/v2 dependency-version: 2.4.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index eb8da249eb..c3e9c56a51 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -88,7 +88,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.26.0 // indirect - github.com/go-viper/mapstructure/v2 v2.3.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index e0a3cfbc56..5bf482f7ce 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -141,8 +141,8 @@ github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-viper/mapstructure/v2 v2.3.0 h1:27XbWsHIqhbdR5TIC911OfYvgSaW93HM+dX7970Q7jk= -github.com/go-viper/mapstructure/v2 v2.3.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= From a4b3fd391b4787125b963a5150b4a690d018c17f Mon Sep 17 00:00:00 2001 From: neicnordic Date: Fri, 22 Aug 2025 07:28:35 +0000 Subject: [PATCH 179/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index ae110fb8d6..1633e8e052 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.12 -appVersion: v2.1.7 +version: 2.0.13 +appVersion: v2.1.24 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 50a1a50f81..c273b3f39a 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.12 -appVersion: v2.1.7 +version: 2.0.13 +appVersion: v2.1.24 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index a8d61244d9..c51702b6b3 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.4 -appVersion: v2.1.7 +version: 3.0.5 +appVersion: v2.1.24 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 37074bc84f6bf7012ec6b699b1809e85a73e8039 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Thu, 21 Aug 2025 08:44:09 +0200 Subject: [PATCH 180/262] use latest release of the mock_lsaai image --- .github/integration/sda-s3-integration.yml | 2 +- .github/integration/sda-sync-integration.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index d47f11c9ad..1d2e7a5cf5 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -334,7 +334,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.3-broker2.1.13-tomcat9.0-jdk21 ports: - "8800:8080" volumes: diff --git a/.github/integration/sda-sync-integration.yml b/.github/integration/sda-sync-integration.yml index 673b617fc5..f09bc3effc 100644 --- a/.github/integration/sda-sync-integration.yml +++ b/.github/integration/sda-sync-integration.yml @@ -268,7 +268,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.3-broker2.1.13-tomcat9.0-jdk21 ports: - "8800:8080" volumes: From 44d0bd01de0b62469dcb91b21281db798cbcfd31 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 25 Aug 2025 12:15:30 +0200 Subject: [PATCH 181/262] [API] Extend swagger file Extend the swagger file with the query parameter for the file listing endpoint. Keeping in mind that this addition should be project agnostic. --- sda/cmd/api/swagger_v1.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index fefbb0e96a..4904de3494 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -300,6 +300,12 @@ paths: schema: type: string required: true + - in: query + name: path_prefix + schema: + type: string + required: false + description: The inbox path prefix to filter entries on. responses: "200": content: From adc6e28a1790aac5e2859390df4aabcbc0b1adbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 10:41:28 +0000 Subject: [PATCH 182/262] Bump actions/setup-java from 4 to 5 Bumps [actions/setup-java](https://github.com/actions/setup-java) from 4 to 5. - [Release notes](https://github.com/actions/setup-java/releases) - [Commits](https://github.com/actions/setup-java/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-java dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 4 ++-- .github/workflows/test.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 48781a5d4f..d2cd99f941 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -34,7 +34,7 @@ jobs: languages: ${{ matrix.language }} - name: Set up JDK 21 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 21 @@ -45,7 +45,7 @@ jobs: run: mvn clean install -DskipTests - name: Set up JDK 19 - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: 'temurin' java-version: 19 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1226cd1bbe..10ce02afa0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: steps: - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: zulu java-version: ${{ matrix.java-version }} From 4bb87122dff60a07c4e22f7e56614a6ea708267b Mon Sep 17 00:00:00 2001 From: Josef Handl Date: Mon, 4 Aug 2025 16:32:15 +0200 Subject: [PATCH 183/262] fix(sda-admin): change sda-cli args order --- tools/sda-admin | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tools/sda-admin b/tools/sda-admin index 6f6ebfdf53..41ae7f356d 100755 --- a/tools/sda-admin +++ b/tools/sda-admin @@ -167,8 +167,7 @@ upload () { # subdirectory of the user's S3 bucket. # if [ "$#" -gt 0 ]; then - "$SDA_CLI" upload \ - -config "$S3_CONFIG" \ + "$SDA_CLI" -config "$S3_CONFIG" upload \ ${target_dir+-targetDir "$target_dir"} \ "$@" fi From 549f502ecf2fa160bb240958e016a017f33181c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 11:38:26 +0000 Subject: [PATCH 184/262] Bump the all-modules group in /sda-sftp-inbox with 2 updates Bumps the all-modules group in /sda-sftp-inbox with 2 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot) and [org.apache.sshd:sshd-sftp](https://github.com/apache/mina-sshd). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.4 to 3.5.5 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.4...v3.5.5) Updates `org.apache.sshd:sshd-sftp` from 2.15.0 to 2.16.0 - [Release notes](https://github.com/apache/mina-sshd/releases) - [Changelog](https://github.com/apache/mina-sshd/blob/master/CHANGES.md) - [Commits](https://github.com/apache/mina-sshd/compare/sshd-2.15.0...sshd-2.16.0) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: org.apache.sshd:sshd-sftp dependency-version: 2.16.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index c879101125..bc3dcae77f 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.4 + 3.5.5 @@ -98,7 +98,7 @@ org.apache.sshd sshd-sftp - 2.15.0 + 2.16.0 commons-io From 567e798422eea4ba089e6fa5ddb9f8b188dda25a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:15:55 +0000 Subject: [PATCH 185/262] Bump the all-modules group in /sda-doa with 4 updates Bumps the all-modules group in /sda-doa with 4 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot), [io.jsonwebtoken:jjwt-api](https://github.com/jwtk/jjwt), [io.jsonwebtoken:jjwt-impl](https://github.com/jwtk/jjwt) and io.jsonwebtoken:jjwt-jackson. Updates `org.springframework.boot:spring-boot-starter-parent` from 3.5.4 to 3.5.5 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.5.4...v3.5.5) Updates `io.jsonwebtoken:jjwt-api` from 0.12.7 to 0.13.0 - [Release notes](https://github.com/jwtk/jjwt/releases) - [Changelog](https://github.com/jwtk/jjwt/blob/master/CHANGELOG.md) - [Commits](https://github.com/jwtk/jjwt/compare/0.12.7...0.13.0) Updates `io.jsonwebtoken:jjwt-impl` from 0.12.7 to 0.13.0 - [Release notes](https://github.com/jwtk/jjwt/releases) - [Changelog](https://github.com/jwtk/jjwt/blob/master/CHANGELOG.md) - [Commits](https://github.com/jwtk/jjwt/compare/0.12.7...0.13.0) Updates `io.jsonwebtoken:jjwt-jackson` from 0.12.7 to 0.13.0 --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.5.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: io.jsonwebtoken:jjwt-api dependency-version: 0.13.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: io.jsonwebtoken:jjwt-impl dependency-version: 0.13.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: io.jsonwebtoken:jjwt-jackson dependency-version: 0.13.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 8d4f5ee27c..6341d1e053 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.5.4 + 3.5.5 no.uio.ifi @@ -67,18 +67,18 @@ io.jsonwebtoken jjwt-api - 0.12.7 + 0.13.0 io.jsonwebtoken jjwt-impl - 0.12.7 + 0.13.0 runtime io.jsonwebtoken jjwt-jackson - 0.12.7 + 0.13.0 runtime From 46ea92874b5b318cfe9ae92707fbeec43647d7b4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:36:35 +0000 Subject: [PATCH 186/262] Bump codecov/codecov-action from 5.4.3 to 5.5.0 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.4.3 to 5.5.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.4.3...v5.5.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 5.5.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1226cd1bbe..e2adbed29e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.3 + uses: codecov/codecov-action@v5.5.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.3 + uses: codecov/codecov-action@v5.5.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.3 + uses: codecov/codecov-action@v5.5.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt From 10f5dcca271daf05126aa1b8ebf4d5581fd20c9c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:37:13 +0000 Subject: [PATCH 187/262] Bump anothrNick/github-tag-action from 1.73.0 to 1.75.0 Bumps [anothrNick/github-tag-action](https://github.com/anothrnick/github-tag-action) from 1.73.0 to 1.75.0. - [Release notes](https://github.com/anothrnick/github-tag-action/releases) - [Commits](https://github.com/anothrnick/github-tag-action/compare/1.73.0...1.75.0) --- updated-dependencies: - dependency-name: anothrNick/github-tag-action dependency-version: 1.75.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index 17726b5385..cd0e7f957a 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -29,7 +29,7 @@ jobs: fetch-depth: '0' - name: Bump version and push tag id: bump_tag - uses: anothrNick/github-tag-action@1.73.0 + uses: anothrNick/github-tag-action@1.75.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} TAG_PREFIX: v From 5a9c48165bfe581c80fe3802a2b5d7c67bebb035 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 13:25:49 +0000 Subject: [PATCH 188/262] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/stretchr/testify](https://github.com/stretchr/testify) and google.golang.org/protobuf. Updates `github.com/stretchr/testify` from 1.10.0 to 1.11.0 - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.10.0...v1.11.0) Updates `google.golang.org/protobuf` from 1.36.7 to 1.36.8 --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-version: 1.11.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-version: 1.36.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 4 ++-- sda-download/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 1e2dfcb208..cd9aeef624 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -14,11 +14,11 @@ require ( github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.0 golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.75.0 - google.golang.org/protobuf v1.36.7 + google.golang.org/protobuf v1.36.8 ) require ( diff --git a/sda-download/go.sum b/sda-download/go.sum index 0842b8ef55..4da07b87ca 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -147,8 +147,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= @@ -236,8 +236,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From ee5beb1139b13e2dd590a04eaa5a8f2cdbb8fb93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Aug 2025 11:23:49 +0000 Subject: [PATCH 189/262] Bump the all-modules group in /sda with 7 updates Bumps the all-modules group in /sda with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.38.0` | `1.38.1` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.0` | `1.31.2` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.18.4` | `1.18.6` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.18.4` | `1.19.0` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.87.0` | `1.87.1` | | [github.com/stretchr/testify](https://github.com/stretchr/testify) | `1.10.0` | `1.11.0` | | google.golang.org/protobuf | `1.36.7` | `1.36.8` | Updates `github.com/aws/aws-sdk-go-v2` from 1.38.0 to 1.38.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.38.0...v1.38.1) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.0 to 1.31.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.31.0...config/v1.31.2) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.18.4 to 1.18.6 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/config/v1.18.6/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.4...config/v1.18.6) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.18.4 to 1.19.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/v1.19.0/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.4...v1.19.0) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.87.0 to 1.87.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.87.0...service/s3/v1.87.1) Updates `github.com/stretchr/testify` from 1.10.0 to 1.11.0 - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.10.0...v1.11.0) Updates `google.golang.org/protobuf` from 1.36.7 to 1.36.8 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.38.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.31.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.18.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.19.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.87.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/stretchr/testify dependency-version: 1.11.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-version: 1.36.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 34 +++++++++++++-------------- sda/go.sum | 68 +++++++++++++++++++++++++++--------------------------- 2 files changed, 51 insertions(+), 51 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index c3e9c56a51..72db488ebe 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.38.0 - github.com/aws/aws-sdk-go-v2/config v1.31.0 - github.com/aws/aws-sdk-go-v2/credentials v1.18.4 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.87.0 + github.com/aws/aws-sdk-go-v2 v1.38.1 + github.com/aws/aws-sdk-go-v2/config v1.31.2 + github.com/aws/aws-sdk-go-v2/credentials v1.18.6 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 github.com/aws/smithy-go v1.22.5 github.com/casbin/casbin/v2 v2.120.0 github.com/coreos/go-oidc/v3 v3.15.0 @@ -30,11 +30,11 @@ require ( github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.10.0 + github.com/stretchr/testify v1.11.0 golang.org/x/crypto v0.41.0 golang.org/x/oauth2 v0.30.0 google.golang.org/grpc v1.75.0 - google.golang.org/protobuf v1.36.7 + google.golang.org/protobuf v1.36.8 ) require ( @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.3 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.3 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.28.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.37.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index 5bf482f7ce..cb70421423 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.38.0 h1:UCRQ5mlqcFk9HJDIqENSLR3wiG1VTWlyUfLDEvY7RxU= -github.com/aws/aws-sdk-go-v2 v1.38.0/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= +github.com/aws/aws-sdk-go-v2 v1.38.1 h1:j7sc33amE74Rz0M/PoCpsZQ6OunLqys/m5antM0J+Z8= +github.com/aws/aws-sdk-go-v2 v1.38.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= -github.com/aws/aws-sdk-go-v2/config v1.31.0 h1:9yH0xiY5fUnVNLRWO0AtayqwU1ndriZdN78LlhruJR4= -github.com/aws/aws-sdk-go-v2/config v1.31.0/go.mod h1:VeV3K72nXnhbe4EuxxhzsDc/ByrCSlZwUnWH52Nde/I= -github.com/aws/aws-sdk-go-v2/credentials v1.18.4 h1:IPd0Algf1b+Qy9BcDp0sCUcIWdCQPSzDoMK3a8pcbUM= -github.com/aws/aws-sdk-go-v2/credentials v1.18.4/go.mod h1:nwg78FjH2qvsRM1EVZlX9WuGUJOL5od+0qvm0adEzHk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.3 h1:GicIdnekoJsjq9wqnvyi2elW6CGMSYKhdozE7/Svh78= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.3/go.mod h1:R7BIi6WNC5mc1kfRM7XM/VHC3uRWkjc396sfabq4iOo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4 h1:0SzCLoPRSK3qSydsaFQWugP+lOBCTPwfcBOm6222+UA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.18.4/go.mod h1:JAet9FsBHjfdI+TnMBX4ModNNaQHAd3dc/Bk+cNsxeM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.3 h1:o9RnO+YZ4X+kt5Z7Nvcishlz0nksIt2PIzDglLMP0vA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.3/go.mod h1:+6aLJzOG1fvMOyzIySYjOFjcguGvVRL68R+uoRencN4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.3 h1:joyyUFhiTQQmVK6ImzNU9TQSNRNeD9kOklqTzyk5v6s= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.3/go.mod h1:+vNIyZQP3b3B1tSLI0lxvrU9cfM7gpdRXMFfm67ZcPc= +github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0= +github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes= +github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU= +github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 h1:2FFgK3oFA8PTNBjprLFfcmkgg7U9YuSimBvR64RUmiA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0/go.mod h1:xdxj6nC1aU/jAO80RIlIj3fU40MOSqutEA9N2XFct04= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 h1:IdCLsiiIj5YJ3AFevsewURCPV+YWUlOW8JiPhoAy8vg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4/go.mod h1:l4bdfCD7XyyZA9BolKBo1eLqgaJxl0/x91PL4Yqe0ao= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 h1:j7vjtr1YIssWQOMeOWRbh3z8g2oY/xPjnZH2gLY4sGw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4/go.mod h1:yDmJgqOiH4EA8Hndnv4KwAo8jCGTSnM5ASG1nBI+toA= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.3 h1:ZV2XK2L3HBq9sCKQiQ/MdhZJppH/rH0vddEAamsHUIs= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.3/go.mod h1:b9F9tk2HdHpbf3xbN7rUZcfmJI26N6NcJu/8OsBFI/0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 h1:BE/MNQ86yzTINrfxPPFS86QCBNQeLKY2A0KhDh47+wI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4/go.mod h1:SPBBhkJxjcrzJBc+qY85e83MQ2q3qdra8fghhkkyrJg= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.3 h1:3ZKmesYBaFX33czDl6mbrcHb6jeheg6LqjJhQdefhsY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.3/go.mod h1:7ryVb78GLCnjq7cw45N6oUb9REl7/vNUwjvIqC5UgdY= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.3 h1:ieRzyHXypu5ByllM7Sp4hC5f/1Fy5wqxqY0yB85hC7s= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.3/go.mod h1:O5ROz8jHiOAKAwx179v+7sHMhfobFVi6nZt8DEyiYoM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.3 h1:SE/e52dq9a05RuxzLcjT+S5ZpQobj3ie3UTaSf2NnZc= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.3/go.mod h1:zkpvBTsR020VVr8TOrwK2TrUW9pOir28sH5ECHpnAfo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.0 h1:egoDf+Geuuntmw79Mz6mk9gGmELCPzg5PFEABOHB+6Y= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.0/go.mod h1:t9MDi29H+HDbkolTSQtbI0HP9DemAWQzUjmWC7LGMnE= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.0 h1:Mc/MKBf2m4VynyJkABoVEN+QzkfLqGj0aiJuEe7cMeM= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.0/go.mod h1:iS5OmxEcN4QIPXARGhavH7S8kETNL11kym6jhoS7IUQ= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.0 h1:6csaS/aJmqZQbKhi1EyEMM7yBW653Wy/B9hnBofW+sw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.0/go.mod h1:59qHWaY5B+Rs7HGTuVGaC32m0rdpQ68N8QCN3khYiqs= -github.com/aws/aws-sdk-go-v2/service/sts v1.37.0 h1:MG9VFW43M4A8BYeAfaJJZWrroinxeTi2r3+SnmLQfSA= -github.com/aws/aws-sdk-go-v2/service/sts v1.37.0/go.mod h1:JdeBDPgpJfuS6rU/hNglmOigKhyEZtBmbraLE4GK1J8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 h1:Beh9oVgtQnBgR4sKKzkUBRQpf1GnL4wt0l4s8h2VCJ0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4/go.mod h1:b17At0o8inygF+c6FOD3rNyYZufPw62o9XJbSfQPgbo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 h1:HVSeukL40rHclNcUqVcBwE1YoZhOkoLeBfhUqR3tjIU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4/go.mod h1:DnbBOv4FlIXHj2/xmrUQYtawRFC9L9ZmQPz+DBc6X5I= +github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1/go.mod h1:w5PC+6GHLkvMJKasYGVloB3TduOtROEMqm15HSuIbw4= +github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0= +github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo= github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -342,8 +342,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= +github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tdewolff/minify/v2 v2.23.6 h1:s0ktOnrlqKfhignkv9M/rh9wCfRlIPiIpjPOq3xUJ84= @@ -519,8 +519,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 25776f60d4b123cae164918f411b17029c2c2e44 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 26 Aug 2025 14:13:49 +0000 Subject: [PATCH 190/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 1633e8e052..90da5bf8d3 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.13 -appVersion: v2.1.24 +version: 2.0.14 +appVersion: v2.1.28 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index c273b3f39a..5b6b1b24a7 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.13 -appVersion: v2.1.24 +version: 2.0.14 +appVersion: v2.1.28 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index c51702b6b3..a635eee840 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.5 -appVersion: v2.1.24 +version: 3.0.6 +appVersion: v2.1.28 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From c4a0aad8c529849897842a763ebc5ab31aa788da Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 14 Aug 2025 23:37:46 +0200 Subject: [PATCH 191/262] verify: add file-id/corr-id to log messages --- sda/cmd/verify/verify.go | 108 +++++++++++++++++++-------------------- 1 file changed, 54 insertions(+), 54 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 53c3623f62..e685db3905 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -74,20 +74,20 @@ func main() { log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-verifiation) failed, reason: (%s)", err.Error()) + log.Errorf("validation of incoming message (ingestion-verification) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Message validation failed", + Error: "Message validation failed, corr-id: " + delivered.CorrelationId, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) + log.Errorf("Failed to Ack message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } // Restart on new message @@ -104,29 +104,29 @@ func main() { // If the file has been canceled by the uploader, don't spend time working on it. status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) + log.Errorf("failed to get file status, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", + Error: "Getheader failed, corr-id: " + delivered.CorrelationId, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } continue } if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with corr-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } continue @@ -134,13 +134,13 @@ func main() { header, err := db.GetHeader(message.FileID) if err != nil { - log.Errorf("GetHeader failed for file with ID: %v, readon: %v", message.FileID, err.Error()) + log.Errorf("GetHeader failed for file with ID: %v, reason: %v", message.FileID, err.Error()) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to nack following getheader error message") + log.Errorf("Failed to nack following getheader error message, file ID: %s", message.FileID) } // store full message info in case we want to fix the db entry and retry infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", + Error: "Getheader failed, file-id: " + message.FileID, Reason: err.Error(), OriginalMessage: message, } @@ -149,7 +149,7 @@ func main() { // Send the message to an error queue so it can be analyzed. if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) } continue @@ -158,27 +158,27 @@ func main() { var file database.FileInfo file.Size, err = archive.GetFileSize(message.ArchivePath, false) if err != nil { //nolint:nestif - log.Errorf("Failed to get archived file size, reson: (%s)", err.Error()) + log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", string(jsonMsg), string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) + log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) } } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) + log.Errorf("Failed to Ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Failed to get archived file size", + Error: "Failed to get archived file size, file-id: " + message.FileID, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) } continue @@ -187,17 +187,17 @@ func main() { archiveFileHash := sha256.New() f, err := archive.NewFileReader(message.ArchivePath) if err != nil { - log.Errorf("Failed to open archived file, reson: %v ", err.Error()) + log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Failed to open archived file", + Error: "Failed to open archived file, file-id: " + message.FileID, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) } // Restart on new message @@ -215,7 +215,7 @@ func main() { } if key == nil { - log.Errorf("no matching key found for file: %s.", message.ArchivePath) + log.Errorf("no matching key found for file, file-id: %s, archive-path: %s", message.FileID, message.ArchivePath) continue } @@ -223,7 +223,7 @@ func main() { mr := io.MultiReader(bytes.NewReader(header), io.TeeReader(f, archiveFileHash)) c4ghr, err := streaming.NewCrypt4GHReader(mr, *key, nil) if err != nil { - log.Errorf("failed to open c4gh decryptor stream, reson: %s", err.Error()) + log.Errorf("failed to open c4gh decryptor stream, file-id: %s, archive-path: %s, reason: %s", message.FileID, message.ArchivePath, err.Error()) continue } @@ -233,22 +233,22 @@ func main() { stream := io.TeeReader(c4ghr, md5hash) if file.DecryptedSize, err = io.Copy(sha256hash, stream); err != nil { - log.Errorf("failed to copy decrypted data, reson: (%s)", err.Error()) + log.Errorf("failed to copy decrypted data, file-id: %s, reason: (%s)", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Failed to verify archived file", + Error: "Failed to verify archived file, file-id: " + message.FileID, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("Failed to publish error message: (%s)", err.Error()) + log.Errorf("Failed to publish error message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message: (%s)", err.Error()) + log.Errorf("Failed to ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } continue @@ -263,37 +263,37 @@ func main() { case message.ReVerify: decrypted, err := db.GetDecryptedChecksum(message.FileID) if err != nil { - log.Errorf("failed to get unencrypted checksum for file: %s, reson: %s", message.FilePath, err.Error()) + log.Errorf("failed to get unencrypted checksum for file, file-id: %s, reason: %s", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } continue } if file.DecryptedChecksum != decrypted { - log.Errorf("encrypted checksum don't match for file: %s", message.FilePath) + log.Errorf("encrypted checksum don't match for file, file-id: %s", message.FileID) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, reason: (%v)", err) + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", message.FileID, err) } continue } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message: (%s)", err.Error()) + log.Errorf("Failed to ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } continue } if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { - log.Errorf("encrypted checksum don't match for file: %s, expected %s, got %s", message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) + log.Errorf("encrypted checksum don't match for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, reason: (%v)", err) + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", message.FileID, err) } continue @@ -301,7 +301,7 @@ func main() { } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message: (%s)", err.Error()) + log.Errorf("Failed to ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } continue @@ -318,37 +318,37 @@ func main() { verifiedMessage, _ := json.Marshal(&c) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession-request.json", conf.Broker.SchemasPath), verifiedMessage) if err != nil { - log.Errorf("Validation of outgoing (ingestion-accession-request) failed, reason: (%s)", err.Error()) + log.Errorf("Validation of outgoing (ingestion-accession-request) failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) // Logging is in ValidateJSON so just restart on new message continue } status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) + log.Errorf("failed to get file status, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Getheader failed", + Error: "Getheader failed, corr-id: " + delivered.CorrelationId, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + log.Errorf("Failed acking canceled work, corr-id:%s, reason: (%s)", delivered.CorrelationId, err.Error()) } continue } if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with corr-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } continue @@ -356,9 +356,9 @@ func main() { fileInfo, err := db.GetFileInfo(message.FileID) if err != nil { - log.Errorf("failed to get info for file: %s", message.FileID) + log.Errorf("failed to get info for file, file-id: %s", message.FileID) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } continue @@ -366,21 +366,21 @@ func main() { if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { if err := db.SetVerified(file, message.FileID); err != nil { - log.Errorf("SetVerified failed, reason: (%s)", err.Error()) + log.Errorf("SetVerified failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } continue } } else { - log.Infof("file with correlation id: %s is already verified", delivered.CorrelationId) + log.Infof("file is already verified, file-id: %s", message.FileID) } if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { - log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) + log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) } continue @@ -389,13 +389,13 @@ func main() { // Send message to verified queue if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) continue } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + log.Errorf("failed to Ack message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) } } } From cc9db8c11260cb4c0fcd3064decee3c31503fd75 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 14 Aug 2025 23:58:46 +0200 Subject: [PATCH 192/262] Add log info on successful verification --- sda/cmd/verify/verify.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index e685db3905..a900af8e12 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -398,6 +398,7 @@ func main() { log.Errorf("failed to Ack message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) } } + log.Infof("Successfully verified the file, file-id: %s, filepath: %s", message.FileID, message.FilePath) } }() From d37ae72024b1c5b8a4a02cd1ab6504d1430b6e1d Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 18 Aug 2025 16:00:53 +0200 Subject: [PATCH 193/262] ingest: add file-id/corr-id to log messages --- sda/cmd/ingest/ingest.go | 114 +++++++++++++++++++-------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index e0cebd2d87..4f48d89b8a 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -126,20 +126,20 @@ func main() { message := schema.IngestionTrigger{} err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.Conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-trigger) failed, reason: (%s)", err.Error()) + log.Errorf("validation of incoming message (ingestion-trigger) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Message validation failed", + Error: "Message validation failed, corr-id: " + delivered.CorrelationId, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } continue @@ -158,23 +158,23 @@ func main() { default: log.Errorln("unexpected ingest message type") if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message for reason: (%s)", err.Error()) + log.Errorf("failed to reject message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } } switch ackNack { case "ack": if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message for reason: (%s)", err.Error()) + log.Errorf("failed to ack message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } case "nack": if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("Failed to Nack message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } default: // will catch `reject`s, failures that should not be requeued. if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message for reason: (%s)", err.Error()) + log.Errorf("failed to reject message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } } } @@ -201,9 +201,9 @@ func (app *Ingest) registerC4GHKey() error { } func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { - fileUUID, err := app.DB.GetFileID(correlationID) + fileID, err := app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file from message (correlationID: %s), reason: %s", correlationID, err.Error()) + log.Errorf("failed to get file-id for file from message (corr-id: %s), reason: %s", correlationID, err.Error()) if strings.Contains(err.Error(), "sql: no rows in result set") { return "reject" } @@ -212,8 +212,8 @@ func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrig } m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileUUID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set event log status for file: %s", correlationID) + if err := app.DB.UpdateFileEventLog(fileID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set event log status for file, corr-id: %s", correlationID) return "nack" } @@ -225,7 +225,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig var fileID string status, err := app.DB.GetFileStatus(correlationID) if err != nil && err.Error() != "sql: no rows in result set" { - log.Errorf("failed to get status for file, reason: (%s)", err.Error()) + log.Errorf("failed to get status for file, corr-id: %s, reason: (%s)", correlationID, err.Error()) return "nack" } @@ -234,14 +234,14 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig case "disabled": fileID, err = app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) + log.Errorf("failed to get file-id for file, corr-id: %s, reason: %s", correlationID, err.Error()) return "nack" } fileInfo, err := app.DB.GetFileInfo(fileID) if err != nil { - log.Errorf("failed to get info for file: %s, reason: %s", fileID, err.Error()) + log.Errorf("failed to get info for file, file-id: %s, reason: %s", fileID, err.Error()) return "nack" } @@ -253,21 +253,21 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig if err != nil { switch { case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): - log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + log.Errorf("Failed to open file to ingest, file-id: %s, reason: (%s)", fileID, err.Error()) jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { - log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) + log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Failed to open file to ingest", + Error: "Failed to open file to ingest, file-id: " + fileID, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, reason: (%s)", fileID, err.Error()) return "reject" } @@ -282,7 +282,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig inboxChecksum := sha256.New() _, err = io.Copy(inboxChecksum, file) if err != nil { - log.Errorf("failed to calculate cheksum for file: %s, reason: %s", fileID, err.Error()) + log.Errorf("failed to calculate the checksum for file, file-id: %s, reason: %s", fileID, err.Error()) return "nack" } @@ -300,20 +300,20 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig archivedMsg, _ := json.Marshal(&msg) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } m, _ := json.Marshal(message) if err = app.DB.UpdateFileEventLog(fileInfo.Path, "enabled", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", correlationID) + log.Errorf("failed to set ingestion status for file from message, file-id: %s", fileID) return "nack" } if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) return "reject" } @@ -322,22 +322,22 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } case "": // Catch all for inboxes that doesn't update the DB - log.Warnln("ingesting unregisterd file") + log.Warnln("ingesting unregistered file, corr-id: " + correlationID) fileID, err = app.DB.RegisterFile(message.FilePath, message.User) if err != nil { - log.Errorf("InsertFile failed, reason: (%s)", err.Error()) + log.Errorf("InsertFile failed, corr-id: %s, reason: (%s)", correlationID, err.Error()) return "nack" } case "uploaded": fileID, err = app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) + log.Errorf("failed to get ID for file, corr-id: %s, reason: %s", correlationID, err.Error()) return "nack" } default: - log.Warnf("unsupported file status: %s", status) + log.Warnf("unsupported file status: %s, corr-id: %s", status, correlationID) return "reject" } @@ -350,24 +350,24 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { - log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) + log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Failed to open file to ingest", + Error: "Failed to open file to ingest, file-id: " + fileID, Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) return "reject" } return "ack" default: - log.Errorf("unexpected error when opening file for reading: %s", err.Error()) + log.Errorf("unexpected error when opening file for reading, file-id: %s, filepath: %s, reason: %s", fileID, message.FilePath, err.Error()) return "nack" } @@ -375,21 +375,21 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig fileSize, err := app.Inbox.GetFileSize(message.FilePath, false) if err != nil { - log.Errorf("Failed to get file size of file to ingest, reason: (%s)", err.Error()) + log.Errorf("Failed to get file size of file to ingest, file-id: %s, filepath: %s, reason: (%s)", fileID, message.FilePath, err.Error()) // Since reading the file worked, this should eventually succeed so it is ok to requeue. return "nack" } dest, err := app.Archive.NewFileWriter(fileID) if err != nil { - log.Errorf("Failed to create archive file, reason: (%s)", err.Error()) + log.Errorf("Failed to create archive file, file-id: %s, reason: (%s)", fileID, err.Error()) // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. return "nack" } m, _ := json.Marshal(message) if err = app.DB.UpdateFileEventLog(fileID, "submitted", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", correlationID) + log.Errorf("failed to set ingestion status for file from message, file-id: %s, corr-id: %s, reason: %s", fileID, correlationID, err.Error()) } // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock @@ -405,7 +405,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig for bytesRead < fileSize { i, _ := io.ReadFull(file, readBuffer) if i == 0 { - log.Errorln("readBuffer returned 0 bytes, this should not happen") + log.Errorln("readBuffer returned 0 bytes, this should not happen, file-id: " + fileID) return "reject" } @@ -418,7 +418,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig h := bytes.NewReader(readBuffer) if _, err = io.Copy(hash, h); err != nil { - log.Errorf("Copy to hash failed while reading file, reason: (%s)", err.Error()) + log.Errorf("Copy to hash failed while reading file, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } @@ -436,26 +436,26 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig break } - log.Warnf("Decryption failed with key, trying next key. Reason: (%s)", err.Error()) + log.Warnf("Decryption failed with key, trying next key. file-id: %s, reason: (%s)", fileID, err.Error()) } // Check if decryption was successful with any key if privateKey == nil { - log.Errorf("All keys failed to decrypt the submitted file") + log.Errorf("All keys failed to decrypt the submitted file, file-id: %s", fileID) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { - log.Errorf("Failed to set ingestion status for file from message: %v", correlationID) + log.Errorf("Failed to set ingestion status for file from message, file-id: %s, corr-id: %s, reason: %s", fileID, correlationID, err.Error()) } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Trying to decrypt the submitted file failed", + Error: "Trying to decrypt the submitted file failed, file-id: " + fileID, Reason: "Decryption failed with the available key(s)", OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) } return "ack" @@ -467,20 +467,20 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig keyhash := hex.EncodeToString(publicKey[:]) err = app.DB.SetKeyHash(keyhash, fileID) if err != nil { - log.Errorf("Key hash %s could not be set for fileID %s: (%s)", keyhash, fileID, err.Error()) + log.Errorf("Key hash %s could not be set for file, file-id: %s, reason: (%s)", keyhash, fileID, err.Error()) return "nack" } log.Debugln("store header") if err := app.DB.StoreHeader(header, fileID); err != nil { - log.Errorf("StoreHeader failed, reason: (%s)", err.Error()) + log.Errorf("StoreHeader failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for header read, reason: %v)", err.Error()) + log.Errorf("Failed to write to read buffer for header read, file-id: %s, reason: %v)", fileID, err.Error()) return "nack" } @@ -488,7 +488,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // Strip header from buffer h := make([]byte, len(header)) if _, err = byteBuf.Read(h); err != nil { - log.Errorf("Failed to strip header from buffer, reason: (%s)", err.Error()) + log.Errorf("Failed to strip header from buffer, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } @@ -497,7 +497,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig readBuffer = readBuffer[:i] } if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for full read, reason: (%s)", err.Error()) + log.Errorf("Failed to write to read buffer for full read, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } @@ -505,12 +505,12 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // Write data to file if _, err = byteBuf.WriteTo(dest); err != nil { - log.Errorf("Failed to write to archive file, reason: (%s)", err.Error()) + log.Errorf("Failed to write to archive file, file-id: %s, reason: (%s)", fileID, err.Error()) _ = file.Close() _ = dest.Close() if err := app.Archive.RemoveFile(fileID); err != nil { - log.Errorf("error when removing uncompleted file: %s", fileID) + log.Errorf("error when removing uncompleted file, file-id: %s, reason: %s", fileID, err.Error()) } return "nack" @@ -527,38 +527,38 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig fileInfo.UploadedChecksum = fmt.Sprintf("%x", hash.Sum(nil)) fileInfo.Size, err = app.Archive.GetFileSize(fileID, true) if err != nil { - log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) + log.Errorf("Couldn't get file size from archive, file-id: %s, reason: %v)", fileID, err.Error()) return "nack" } - log.Debugf("Wrote archived file (corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", correlationID, message.User, message.FilePath, fileID, fileInfo.Size) + log.Debugf("Wrote archived file (file-id: %s, corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", fileID, correlationID, message.User, message.FilePath, fileID, fileInfo.Size) status, err = app.DB.GetFileStatus(correlationID) if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) + log.Errorf("failed to get file status, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) return "nack" } if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping ingestion", correlationID) + log.Infof("file is disabled, stopping ingestion, file-id: %s, corr-id: %s", fileID, correlationID) return "ack" } if err := app.DB.SetArchived(fileInfo, fileID); err != nil { - log.Errorf("SetArchived failed, reason: (%s)", err.Error()) + log.Errorf("SetArchived failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if err := app.DB.UpdateFileEventLog(fileID, "archived", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set event log status for file: %s", correlationID) + log.Errorf("failed to set event log status for file, file-id: %s, corr-id: %s, reason: %s", fileID, correlationID, err.Error()) return "nack" } - log.Debugf("File marked as archived (corr-id: %s, user: %s, filepath: %s, archivepath: %s)", correlationID, message.User, message.FilePath, fileID) + log.Debugf("File marked as archived (file-id: %s, corr-id: %s, user: %s, filepath: %s)", fileID, correlationID, message.User, message.FilePath) // Send message to archived msg := schema.IngestionVerification{ @@ -574,14 +574,14 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) return "reject" } From d2407dc1a225e82d2b8d35766dd52ce2835d4818 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 18 Aug 2025 16:39:06 +0200 Subject: [PATCH 194/262] finalize: add file-id/corr-id to log messages --- sda/cmd/finalize/finalize.go | 67 ++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index df2096cac9..922beb5430 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -4,7 +4,6 @@ package main import ( "encoding/json" - "errors" "fmt" "io" @@ -77,9 +76,9 @@ func main() { log.Debugf("Received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-accession) failed, reason: %v ", err) + log.Errorf("validation of incoming message (ingestion-accession) failed, corr-id: %s, reason: %v ", delivered.CorrelationId, err) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) + log.Errorf("Failed acking canceled work, corr-id: %s, reason: %v", delivered.CorrelationId, err) } continue @@ -90,9 +89,9 @@ func main() { // If the file has been canceled by the uploader, don't spend time working on it. status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, reason: %v", err) + log.Errorf("failed to get file status, corr-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) } continue @@ -100,9 +99,9 @@ func main() { switch status { case "disabled": - log.Infof("file with correlation ID: %s is disabled, stopping work", delivered.CorrelationId) + log.Infof("file with corr-id: %s is disabled, stop the work", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) + log.Errorf("Failed acking canceled work, corr-id: %s, reason: %v", delivered.CorrelationId, err) } continue @@ -110,16 +109,16 @@ func main() { case "verified": case "enabled": case "ready": - log.Infof("File with correlation ID %s is already marked as ready.", delivered.CorrelationId) + log.Infof("File with corr-id: %s is already marked as ready.", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking message, reason: %v", err) + log.Errorf("Failed acking message, corr-id: %s, reason: %v", delivered.CorrelationId, err) } continue default: - log.Warnf("file with correlation ID: %s is not verified yet, stopping work", delivered.CorrelationId) + log.Warnf("file with corr-id: %s is not verified yet, stop the work", delivered.CorrelationId) if err := delivered.Nack(false, true); err != nil { - log.Errorf("Failed acking canceled work, reason: %v", err) + log.Errorf("Failed acking canceled work, corr-id: %s, reason: %v", delivered.CorrelationId, err) } continue @@ -127,9 +126,9 @@ func main() { fileID, err := db.GetFileID(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get ID for file, reason: %v", err) + log.Errorf("failed to get fileID for file, corr-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) } continue @@ -144,16 +143,16 @@ func main() { completeMsg, _ := json.Marshal(&c) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", conf.Broker.SchemasPath), completeMsg) if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%v)", err) + log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%v)", fileID, err) continue } accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID, fileID) if err != nil { - log.Errorf("CheckAccessionIdExists failed, reason: %v ", err) + log.Errorf("CheckAccessionIdExists failed, file-id: %s, reason: %v ", fileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) } continue @@ -161,10 +160,10 @@ func main() { switch accessionIDExists { case "duplicate": - log.Debugf("Seems accession ID already exists (corr-id: %s, accessionid: %s", delivered.CorrelationId, message.AccessionID) + log.Debugf("Seems accession ID already exists (file-id: %s, corr-id: %s, accession-id: %s", fileID, delivered.CorrelationId, message.AccessionID) // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "There is a conflict regarding the file accessionID", + Error: "There is a conflict regarding the file accessionID, file-id: " + fileID, Reason: "The Accession ID already exists in the database, skipping marking it ready.", OriginalMessage: message, } @@ -172,22 +171,22 @@ func main() { // Send the message to an error queue so it can be analyzed. if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { - log.Errorf("failed to publish message, reason: (%v)", err) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%v)", fileID, delivered.CorrelationId, err) } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%v)", err) + log.Errorf("failed to Ack message, file-id: %s, reason: (%v)", fileID, err) } continue case "same": - log.Infoln("file already has a stable ID, marking it as ready") + log.Infoln("file already has a stable ID, marking it as ready, file-id: " + fileID) default: if conf.Backup.Type != "" && conf.Archive.Type != "" { if err = backupFile(delivered); err != nil { - log.Errorf("Failed to backup file with corrID: %v, reason: %v", delivered.CorrelationId, err) + log.Errorf("Failed to backup file, file-id: %s, corr-id: %v, reason: %v", fileID, delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) } continue @@ -195,9 +194,9 @@ func main() { } if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { - log.Errorf("Failed to set accessionID for file with corrID: %v, reason: %v", delivered.CorrelationId, err) + log.Errorf("Failed to set accessionID for file, file-id: %s, corr-id: %v, reason: %v", fileID, delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) } continue @@ -206,25 +205,25 @@ func main() { // Mark file as "ready" if err := db.UpdateFileEventLog(fileID, "ready", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, reason: (%v)", err) + log.Errorf("set status ready failed, file-id: %s, corr-id: %s, reason: (%v)", fileID, delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) } continue } if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, completeMsg); err != nil { - log.Errorf("failed to publish message, reason: (%v)", err) + log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%v)", fileID, delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) } continue } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%v)", err) + log.Errorf("failed to Ack message, file-id: %s, reason: (%v)", fileID, err) } } }() @@ -234,12 +233,12 @@ func main() { func backupFile(delivered amqp.Delivery) error { log.Debug("Backup initiated") - fileUUID, err := db.GetFileID(delivered.CorrelationId) + fileID, err := db.GetFileID(delivered.CorrelationId) if err != nil { return fmt.Errorf("failed to get ID for file, reason: %s", err.Error()) } - filePath, fileSize, err := db.GetArchived(fileUUID) + filePath, fileSize, err := db.GetArchived(fileID) if err != nil { return fmt.Errorf("failed to get file archive information, reason: %v", err) } @@ -251,7 +250,7 @@ func backupFile(delivered amqp.Delivery) error { } if diskFileSize != int64(fileSize) { - return errors.New("file size in archive does not match database for archive file") + return fmt.Errorf("file size in archive does not match database, disk size: %d, db size: %d", diskFileSize, fileSize) } file, err := archive.NewFileReader(filePath) @@ -273,7 +272,7 @@ func backupFile(delivered amqp.Delivery) error { } // Mark file as "backed up" - if err := db.UpdateFileEventLog(fileUUID, "backed up", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { + if err := db.UpdateFileEventLog(fileID, "backed up", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { return fmt.Errorf("UpdateFileEventLog failed, reason: (%v)", err) } From 0b404bb210422e7588b34ddacb0094855c6c648a Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 18 Aug 2025 16:43:00 +0200 Subject: [PATCH 195/262] intercept: add file-id/corr-id to log messages --- sda/cmd/intercept/intercept.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sda/cmd/intercept/intercept.go b/sda/cmd/intercept/intercept.go index 64fd0e4dc4..a36e483b6d 100644 --- a/sda/cmd/intercept/intercept.go +++ b/sda/cmd/intercept/intercept.go @@ -81,10 +81,10 @@ func main() { if routingKey == "" { log.Debugf("msg type: %s", msgType) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "undeliverable", delivered.Body); err != nil { - log.Errorf("failed to publish message, reason: (%v)", err) + log.Errorf("failed to publish message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message for reason: %v", err) + log.Errorf("failed to ack message, corr-id: %s, reason: %v", delivered.CorrelationId, err) } continue @@ -92,10 +92,10 @@ func main() { log.Infof("Routing message (corr-id: %s, routingkey: %s)", delivered.CorrelationId, routingKey) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, routingKey, delivered.Body); err != nil { - log.Errorf("failed to publish message, reason: (%v)", err) + log.Errorf("failed to publish message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message for reason: %v", err) + log.Errorf("failed to ack message, corr-id: %s, reason: %v", delivered.CorrelationId, err) } } }() From b3e440edceb7ebc08342915afadd8507e0b9242e Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 18 Aug 2025 16:51:36 +0200 Subject: [PATCH 196/262] mapper: add corr-id/dataset-id to log messages --- sda/cmd/mapper/mapper.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index b38411c86d..ee09d41c59 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -92,7 +92,7 @@ func main() { case "mapping": log.Debug("Mapping type operation, mapping files to dataset") if err := db.MapFilesToDataset(mappings.DatasetID, mappings.AccessionIDs); err != nil { - log.Errorf("failed to map files to dataset, reason: %v", err) + log.Errorf("failed to map files to dataset, dataset-id: %s, reason: %v", mappings.DatasetID, err) // Nack message so the server gets notified that something is wrong and requeue the message if err := delivered.Nack(false, true); err != nil { @@ -110,7 +110,7 @@ func main() { } err = inbox.RemoveFile(filePath) if err != nil { - log.Errorf("Remove file from inbox failed, reason: %v", err) + log.Errorf("Remove file from inbox failed, corr-id: %s, reason: %v", delivered.CorrelationId, err) } } @@ -143,7 +143,7 @@ func main() { continue } default: - log.Errorf("unknown mappping type, %s", mappings.Type) + log.Errorf("unknown mapping type, %s", mappings.Type) } if err := delivered.Ack(false); err != nil { From 0f525e0b78640c3a98eba8e8ee5066ead98cf838 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 18 Aug 2025 17:07:34 +0200 Subject: [PATCH 197/262] sync: add corr-id to log messages --- sda/cmd/sync/sync.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 8dcf42e3e8..e42968896a 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -96,17 +96,17 @@ func main() { err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (dataset-mapping) failed, reason: (%s)", err.Error()) + log.Errorf("validation of incoming message (dataset-mapping) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Message validation failed in sync service", + Error: "Message validation failed in sync service, corr-id: " + delivered.CorrelationId, Reason: err.Error(), OriginalMessage: string(delivered.Body), } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) } if err := delivered.Ack(false); err != nil { log.Errorf("failed to Ack message, reason: (%s)", err.Error()) @@ -129,7 +129,7 @@ func main() { for _, aID := range message.AccessionIDs { if err := syncFiles(aID); err != nil { - log.Errorf("failed to sync archived file %s, reason: (%s)", aID, err.Error()) + log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) if err := delivered.Nack(false, false); err != nil { log.Errorf("failed to nack following GetFileSize error message") } @@ -165,17 +165,17 @@ func syncFiles(stableID string) error { log.Debugf("syncing file %s", stableID) inboxPath, err := db.GetInboxPath(stableID) if err != nil { - return fmt.Errorf("failed to get inbox path for file with stable ID: %s", stableID) + return fmt.Errorf("failed to get inbox path for file with stable ID: %s, reason: %v", stableID, err) } archivePath, err := db.GetArchivePath(stableID) if err != nil { - return fmt.Errorf("failed to get archive path for file with stable ID: %s", stableID) + return fmt.Errorf("failed to get archive path for file with stable ID: %s, reason: %v", stableID, err) } fileSize, err := archive.GetFileSize(archivePath, false) if err != nil { - return err + return fmt.Errorf("failed to get filesize for file with archivepath: %s, reason: %v", archivePath, err) } file, err := archive.NewFileReader(archivePath) From 20c2504390a092007e3915d52f421c4d1469cf99 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 19 Aug 2025 15:31:25 +0200 Subject: [PATCH 198/262] feat: remove corr-id/file-id from MQ related errors --- sda/cmd/finalize/finalize.go | 32 ++++++++--------- sda/cmd/ingest/ingest.go | 30 ++++++++-------- sda/cmd/intercept/intercept.go | 8 ++--- sda/cmd/notify/notify.go | 2 +- sda/cmd/sync/sync.go | 4 +-- sda/cmd/verify/verify.go | 64 +++++++++++++++++----------------- 6 files changed, 70 insertions(+), 70 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 922beb5430..ed160ea436 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -78,7 +78,7 @@ func main() { if err != nil { log.Errorf("validation of incoming message (ingestion-accession) failed, corr-id: %s, reason: %v ", delivered.CorrelationId, err) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("Failed acking canceled work, reason: %v", err) } continue @@ -91,7 +91,7 @@ func main() { if err != nil { log.Errorf("failed to get file status, corr-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -101,7 +101,7 @@ func main() { case "disabled": log.Infof("file with corr-id: %s is disabled, stop the work", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("Failed acking canceled work, reason: %v", err) } continue @@ -111,14 +111,14 @@ func main() { case "ready": log.Infof("File with corr-id: %s is already marked as ready.", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking message, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("Failed acking message, reason: %v", err) } continue default: log.Warnf("file with corr-id: %s is not verified yet, stop the work", delivered.CorrelationId) if err := delivered.Nack(false, true); err != nil { - log.Errorf("Failed acking canceled work, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("Failed acking canceled work, reason: %v", err) } continue @@ -128,7 +128,7 @@ func main() { if err != nil { log.Errorf("failed to get fileID for file, corr-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -152,7 +152,7 @@ func main() { if err != nil { log.Errorf("CheckAccessionIdExists failed, file-id: %s, reason: %v ", fileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -163,7 +163,7 @@ func main() { log.Debugf("Seems accession ID already exists (file-id: %s, corr-id: %s, accession-id: %s", fileID, delivered.CorrelationId, message.AccessionID) // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "There is a conflict regarding the file accessionID, file-id: " + fileID, + Error: "There is a conflict regarding the file accessionID", Reason: "The Accession ID already exists in the database, skipping marking it ready.", OriginalMessage: message, } @@ -171,11 +171,11 @@ func main() { // Send the message to an error queue so it can be analyzed. if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%v)", fileID, delivered.CorrelationId, err) + log.Errorf("failed to publish message, reason: %v", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("failed to Ack message, reason: %v", err) } continue @@ -186,7 +186,7 @@ func main() { if err = backupFile(delivered); err != nil { log.Errorf("Failed to backup file, file-id: %s, corr-id: %v, reason: %v", fileID, delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -196,7 +196,7 @@ func main() { if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { log.Errorf("Failed to set accessionID for file, file-id: %s, corr-id: %v, reason: %v", fileID, delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -207,23 +207,23 @@ func main() { if err := db.UpdateFileEventLog(fileID, "ready", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { log.Errorf("set status ready failed, file-id: %s, corr-id: %s, reason: (%v)", fileID, delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("failed to Nack message, reason: %v", err) } continue } if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, completeMsg); err != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%v)", fileID, delivered.CorrelationId, err) + log.Errorf("failed to publish message, reason: %v", err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("failed to Nack message, reason: %v", err) } continue } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("failed to Ack message, reason: %v", err) } } }() diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 4f48d89b8a..9aac940f2c 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -129,17 +129,17 @@ func main() { log.Errorf("validation of incoming message (ingestion-trigger) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Message validation failed, corr-id: " + delivered.CorrelationId, + Error: "Message validation failed", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: %v", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("Failed acking canceled work, reason: %v", err) } continue @@ -158,23 +158,23 @@ func main() { default: log.Errorln("unexpected ingest message type") if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to reject message, reason: %v", err) } } switch ackNack { case "ack": if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to ack message, reason: %v", err) } case "nack": if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to Nack message, reason: %v", err) } default: // will catch `reject`s, failures that should not be requeued. if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to reject message, reason: %v", err) } } } @@ -261,13 +261,13 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Failed to open file to ingest, file-id: " + fileID, + Error: "Failed to open file to ingest", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, file-id: %s, reason: (%s)", fileID, err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } @@ -313,7 +313,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } @@ -354,13 +354,13 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Failed to open file to ingest, file-id: " + fileID, + Error: "Failed to open file to ingest", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } @@ -449,13 +449,13 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Trying to decrypt the submitted file failed, file-id: " + fileID, + Error: "Trying to decrypt the submitted file failed", Reason: "Decryption failed with the available key(s)", OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) + log.Errorf("failed to publish message, reason: %v", err) } return "ack" @@ -581,7 +581,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { // TODO fix resend mechanism - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } diff --git a/sda/cmd/intercept/intercept.go b/sda/cmd/intercept/intercept.go index a36e483b6d..64fd0e4dc4 100644 --- a/sda/cmd/intercept/intercept.go +++ b/sda/cmd/intercept/intercept.go @@ -81,10 +81,10 @@ func main() { if routingKey == "" { log.Debugf("msg type: %s", msgType) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "undeliverable", delivered.Body); err != nil { - log.Errorf("failed to publish message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) + log.Errorf("failed to publish message, reason: (%v)", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to ack message for reason: %v", err) } continue @@ -92,10 +92,10 @@ func main() { log.Infof("Routing message (corr-id: %s, routingkey: %s)", delivered.CorrelationId, routingKey) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, routingKey, delivered.Body); err != nil { - log.Errorf("failed to publish message, corr-id: %s, reason: (%v)", delivered.CorrelationId, err) + log.Errorf("failed to publish message, reason: (%v)", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to ack message for reason: %v", err) } } }() diff --git a/sda/cmd/notify/notify.go b/sda/cmd/notify/notify.go index 93ff7ba8cc..32a3153464 100644 --- a/sda/cmd/notify/notify.go +++ b/sda/cmd/notify/notify.go @@ -71,7 +71,7 @@ func main() { log.Errorf("Failed to send email, error %v", err) if e := d.Nack(false, false); e != nil { - log.Errorf("Failed to Nack message (corr-id: %s, errror: %v) ", d.CorrelationId, e) + log.Errorf("Failed to Nack message, error: %v) ", e) } continue diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index e42968896a..723b494b69 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -99,14 +99,14 @@ func main() { log.Errorf("validation of incoming message (dataset-mapping) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Message validation failed in sync service, corr-id: " + delivered.CorrelationId, + Error: "Message validation failed in sync service", Reason: err.Error(), OriginalMessage: string(delivered.Body), } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: (%v)", err) } if err := delivered.Ack(false); err != nil { log.Errorf("failed to Ack message, reason: (%s)", err.Error()) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index a900af8e12..473a6bc9e9 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -77,17 +77,17 @@ func main() { log.Errorf("validation of incoming message (ingestion-verification) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Message validation failed, corr-id: " + delivered.CorrelationId, + Error: "Message validation failed", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: %v", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("Failed to Ack message, reason: %v", err) } // Restart on new message @@ -107,18 +107,18 @@ func main() { log.Errorf("failed to get file status, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Getheader failed, corr-id: " + delivered.CorrelationId, + Error: "Getheader failed", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } continue @@ -126,7 +126,7 @@ func main() { if status == "disabled" { log.Infof("file with corr-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } continue @@ -136,11 +136,11 @@ func main() { if err != nil { log.Errorf("GetHeader failed for file with ID: %v, reason: %v", message.FileID, err.Error()) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to nack following getheader error message, file ID: %s", message.FileID) + log.Errorf("Failed to nack following getheader error message") } // store full message info in case we want to fix the db entry and retry infoErrorMessage := broker.InfoError{ - Error: "Getheader failed, file-id: " + message.FileID, + Error: "Getheader failed", Reason: err.Error(), OriginalMessage: message, } @@ -149,7 +149,7 @@ func main() { // Send the message to an error queue so it can be analyzed. if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } continue @@ -167,18 +167,18 @@ func main() { } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ - Error: "Failed to get archived file size, file-id: " + message.FileID, + Error: "Failed to get archived file size", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(fileError) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } continue @@ -190,14 +190,14 @@ func main() { log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Failed to open archived file, file-id: " + message.FileID, + Error: "Failed to open archived file", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } // Restart on new message @@ -237,18 +237,18 @@ func main() { // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Failed to verify archived file, file-id: " + message.FileID, + Error: "Failed to verify archived file", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("Failed to publish error message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) + log.Errorf("Failed to publish error message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } continue @@ -265,7 +265,7 @@ func main() { if err != nil { log.Errorf("failed to get unencrypted checksum for file, file-id: %s, reason: %s", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue @@ -276,13 +276,13 @@ func main() { if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", message.FileID, err) + log.Errorf("failed to Nack message, reason: (%v)", err) } continue } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } continue @@ -293,7 +293,7 @@ func main() { if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%v)", message.FileID, err) + log.Errorf("failed to Nack message, reason: (%v)", err) } continue @@ -301,7 +301,7 @@ func main() { } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } continue @@ -328,18 +328,18 @@ func main() { log.Errorf("failed to get file status, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ - Error: "Getheader failed, corr-id: " + delivered.CorrelationId, + Error: "Getheader failed", Reason: err.Error(), OriginalMessage: message, } body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, corr-id:%s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } continue @@ -348,7 +348,7 @@ func main() { if status == "disabled" { log.Infof("file with corr-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } continue @@ -358,7 +358,7 @@ func main() { if err != nil { log.Errorf("failed to get info for file, file-id: %s", message.FileID) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue @@ -368,7 +368,7 @@ func main() { if err := db.SetVerified(file, message.FileID); err != nil { log.Errorf("SetVerified failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue @@ -380,7 +380,7 @@ func main() { if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, file-id: %s, reason: (%s)", message.FileID, err.Error()) + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue @@ -389,13 +389,13 @@ func main() { // Send message to verified queue if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { // TODO fix resend mechanism - log.Errorf("failed to publish message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) + log.Errorf("failed to publish message, reason: (%s)", err.Error()) continue } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, file-id: %s, corr-id: %s, reason: (%s)", message.FileID, delivered.CorrelationId, err.Error()) + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } } log.Infof("Successfully verified the file, file-id: %s, filepath: %s", message.FileID, message.FilePath) From 1292774d4d7f4cb35f9627758296c0a291c3de9e Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 19 Aug 2025 15:57:21 +0200 Subject: [PATCH 199/262] feat: don't print both file-id and corr-id --- sda/cmd/finalize/finalize.go | 8 ++++---- sda/cmd/ingest/ingest.go | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index ed160ea436..302a63ab5f 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -160,7 +160,7 @@ func main() { switch accessionIDExists { case "duplicate": - log.Debugf("Seems accession ID already exists (file-id: %s, corr-id: %s, accession-id: %s", fileID, delivered.CorrelationId, message.AccessionID) + log.Debugf("Seems accession ID already exists (file-id: %s, accession-id: %s", fileID, message.AccessionID) // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ Error: "There is a conflict regarding the file accessionID", @@ -184,7 +184,7 @@ func main() { default: if conf.Backup.Type != "" && conf.Archive.Type != "" { if err = backupFile(delivered); err != nil { - log.Errorf("Failed to backup file, file-id: %s, corr-id: %v, reason: %v", fileID, delivered.CorrelationId, err) + log.Errorf("failed to backup file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } @@ -194,7 +194,7 @@ func main() { } if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { - log.Errorf("Failed to set accessionID for file, file-id: %s, corr-id: %v, reason: %v", fileID, delivered.CorrelationId, err) + log.Errorf("failed to set accessionID for file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } @@ -205,7 +205,7 @@ func main() { // Mark file as "ready" if err := db.UpdateFileEventLog(fileID, "ready", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, file-id: %s, corr-id: %s, reason: (%v)", fileID, delivered.CorrelationId, err) + log.Errorf("set status ready failed, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 9aac940f2c..23437ae66f 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -389,7 +389,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig m, _ := json.Marshal(message) if err = app.DB.UpdateFileEventLog(fileID, "submitted", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set ingestion status for file from message, file-id: %s, corr-id: %s, reason: %s", fileID, correlationID, err.Error()) + log.Errorf("failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock @@ -444,7 +444,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig log.Errorf("All keys failed to decrypt the submitted file, file-id: %s", fileID) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { - log.Errorf("Failed to set ingestion status for file from message, file-id: %s, corr-id: %s, reason: %s", fileID, correlationID, err.Error()) + log.Errorf("Failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. @@ -532,17 +532,17 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "nack" } - log.Debugf("Wrote archived file (file-id: %s, corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", fileID, correlationID, message.User, message.FilePath, fileID, fileInfo.Size) + log.Debugf("Wrote archived file (file-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", fileID, message.User, message.FilePath, fileID, fileInfo.Size) status, err = app.DB.GetFileStatus(correlationID) if err != nil { - log.Errorf("failed to get file status, file-id: %s, corr-id: %s, reason: (%s)", fileID, correlationID, err.Error()) + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if status == "disabled" { - log.Infof("file is disabled, stopping ingestion, file-id: %s, corr-id: %s", fileID, correlationID) + log.Infof("file is disabled, stopping ingestion, file-id: %s", fileID) return "ack" } @@ -554,11 +554,11 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } if err := app.DB.UpdateFileEventLog(fileID, "archived", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set event log status for file, file-id: %s, corr-id: %s, reason: %s", fileID, correlationID, err.Error()) + log.Errorf("failed to set event log status for file, file-id: %s, reason: %s", fileID, err.Error()) return "nack" } - log.Debugf("File marked as archived (file-id: %s, corr-id: %s, user: %s, filepath: %s)", fileID, correlationID, message.User, message.FilePath) + log.Debugf("File marked as archived (file-id: %s, user: %s, filepath: %s)", fileID, message.User, message.FilePath) // Send message to archived msg := schema.IngestionVerification{ From b9a45f2b71c9cd29cbc8b04e543deaae3f11e69d Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 19 Aug 2025 16:06:36 +0200 Subject: [PATCH 200/262] refactor: use string formatting instead of concatenation --- sda/cmd/finalize/finalize.go | 2 +- sda/cmd/ingest/ingest.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 302a63ab5f..06f091e417 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -180,7 +180,7 @@ func main() { continue case "same": - log.Infoln("file already has a stable ID, marking it as ready, file-id: " + fileID) + log.Infof("file already has a stable ID, marking it as ready, file-id: %s", fileID) default: if conf.Backup.Type != "" && conf.Archive.Type != "" { if err = backupFile(delivered); err != nil { diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 23437ae66f..4ec54f23c4 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -322,7 +322,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } case "": // Catch all for inboxes that doesn't update the DB - log.Warnln("ingesting unregistered file, corr-id: " + correlationID) + log.Warnf("ingesting unregistered file, corr-id: %s", correlationID) fileID, err = app.DB.RegisterFile(message.FilePath, message.User) if err != nil { log.Errorf("InsertFile failed, corr-id: %s, reason: (%s)", correlationID, err.Error()) @@ -405,7 +405,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig for bytesRead < fileSize { i, _ := io.ReadFull(file, readBuffer) if i == 0 { - log.Errorln("readBuffer returned 0 bytes, this should not happen, file-id: " + fileID) + log.Errorf("readBuffer returned 0 bytes, this should not happen, file-id: %s", fileID) return "reject" } From 42f3cc30b156c05dcad5f304c7847a7636200854 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 21 Aug 2025 09:20:00 +0200 Subject: [PATCH 201/262] Apply reviewers comments Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/verify/verify.go Co-authored-by: Joakim Bygdell Update sda/cmd/verify/verify.go Co-authored-by: Joakim Bygdell Update sda/cmd/verify/verify.go Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/verify/verify.go Co-authored-by: Joakim Bygdell Update sda/cmd/verify/verify.go Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell Update sda/cmd/mapper/mapper.go Co-authored-by: Joakim Bygdell Update sda/cmd/sync/sync.go Co-authored-by: Joakim Bygdell --- sda/cmd/finalize/finalize.go | 18 +++++++++--------- sda/cmd/ingest/ingest.go | 20 ++++++++++---------- sda/cmd/mapper/mapper.go | 2 +- sda/cmd/sync/sync.go | 2 +- sda/cmd/verify/verify.go | 10 +++++----- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 06f091e417..c6d439e50e 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -76,7 +76,7 @@ func main() { log.Debugf("Received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-accession) failed, corr-id: %s, reason: %v ", delivered.CorrelationId, err) + log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -89,7 +89,7 @@ func main() { // If the file has been canceled by the uploader, don't spend time working on it. status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to get file status, correlation-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } @@ -99,7 +99,7 @@ func main() { switch status { case "disabled": - log.Infof("file with corr-id: %s is disabled, stop the work", delivered.CorrelationId) + log.Infof("file with correlation-id: %s is disabled, aborting work", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -109,14 +109,14 @@ func main() { case "verified": case "enabled": case "ready": - log.Infof("File with corr-id: %s is already marked as ready.", delivered.CorrelationId) + log.Infof("File with correlation-id: %s is already marked as ready.", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking message, reason: %v", err) } continue default: - log.Warnf("file with corr-id: %s is not verified yet, stop the work", delivered.CorrelationId) + log.Warnf("file with correlation-id: %s is not verified yet, aborting work", delivered.CorrelationId) if err := delivered.Nack(false, true); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -126,7 +126,7 @@ func main() { fileID, err := db.GetFileID(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get fileID for file, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to get fileID for file with correlation-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } @@ -143,7 +143,7 @@ func main() { completeMsg, _ := json.Marshal(&c) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", conf.Broker.SchemasPath), completeMsg) if err != nil { - log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%v)", fileID, err) + log.Errorf("Validation of outgoing message ingestion-completion failed, reason: (%v). Message body: %s\n", err, string(completeMsg)) continue } @@ -160,7 +160,7 @@ func main() { switch accessionIDExists { case "duplicate": - log.Debugf("Seems accession ID already exists (file-id: %s, accession-id: %s", fileID, message.AccessionID) + log.Errorf("Accession ID already exists in the system. (accession-id: %s)\n", message.AccessionID) // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ Error: "There is a conflict regarding the file accessionID", @@ -250,7 +250,7 @@ func backupFile(delivered amqp.Delivery) error { } if diskFileSize != int64(fileSize) { - return fmt.Errorf("file size in archive does not match database, disk size: %d, db size: %d", diskFileSize, fileSize) + return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)\n", diskFileSize, fileSize) } file, err := archive.NewFileReader(filePath) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 4ec54f23c4..820f96ed42 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -126,7 +126,7 @@ func main() { message := schema.IngestionTrigger{} err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.Conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-trigger) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Message validation failed", @@ -203,7 +203,7 @@ func (app *Ingest) registerC4GHKey() error { func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { fileID, err := app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get file-id for file from message (corr-id: %s), reason: %s", correlationID, err.Error()) + log.Errorf("failed to get file-id for file from message (correlation-id: %s), reason: %s", correlationID, err.Error()) if strings.Contains(err.Error(), "sql: no rows in result set") { return "reject" } @@ -213,7 +213,7 @@ func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrig m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set event log status for file, corr-id: %s", correlationID) + log.Errorf("failed to set event log status for file, correlation-id: %s", correlationID) return "nack" } @@ -225,7 +225,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig var fileID string status, err := app.DB.GetFileStatus(correlationID) if err != nil && err.Error() != "sql: no rows in result set" { - log.Errorf("failed to get status for file, corr-id: %s, reason: (%s)", correlationID, err.Error()) + log.Errorf("failed to get status for file, correlation-id: %s, reason: (%s)", correlationID, err.Error()) return "nack" } @@ -234,7 +234,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig case "disabled": fileID, err = app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get file-id for file, corr-id: %s, reason: %s", correlationID, err.Error()) + log.Errorf("failed to get file-id for file, correlation-id: %s, reason: %s", correlationID, err.Error()) return "nack" } @@ -253,7 +253,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig if err != nil { switch { case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): - log.Errorf("Failed to open file to ingest, file-id: %s, reason: (%s)", fileID, err.Error()) + log.Errorf("Failed to open file to ingest, file-id: %s, inbox path: %s, reason: (%s)", fileID, message.FilePath, err.Error()) jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { @@ -322,22 +322,22 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } case "": // Catch all for inboxes that doesn't update the DB - log.Warnf("ingesting unregistered file, corr-id: %s", correlationID) + log.Infof("ingesting unregistered file, correlation-id: %s", correlationID) fileID, err = app.DB.RegisterFile(message.FilePath, message.User) if err != nil { - log.Errorf("InsertFile failed, corr-id: %s, reason: (%s)", correlationID, err.Error()) + log.Errorf("InsertFile failed, correlation-id: %s, reason: (%s)", correlationID, err.Error()) return "nack" } case "uploaded": fileID, err = app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file, corr-id: %s, reason: %s", correlationID, err.Error()) + log.Errorf("failed to get ID for file, correlation-id: %s, reason: %s", correlationID, err.Error()) return "nack" } default: - log.Warnf("unsupported file status: %s, corr-id: %s", status, correlationID) + log.Warnf("unsupported file status: %s, correlation-id: %s", status, correlationID) return "reject" } diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index ee09d41c59..af89b37386 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -110,7 +110,7 @@ func main() { } err = inbox.RemoveFile(filePath) if err != nil { - log.Errorf("Remove file from inbox failed, corr-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("Remove file from inbox failed, reason: %v", err) } } diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 723b494b69..7d71051a70 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -96,7 +96,7 @@ func main() { err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (dataset-mapping) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("validation of incoming message (dataset-mapping) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Message validation failed in sync service", diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 473a6bc9e9..868a17fd6a 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -74,7 +74,7 @@ func main() { log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-verification) failed, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("validation of incoming message (ingestion-verification) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Message validation failed", @@ -104,7 +104,7 @@ func main() { // If the file has been canceled by the uploader, don't spend time working on it. status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to get file status, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Getheader failed", @@ -124,7 +124,7 @@ func main() { continue } if status == "disabled" { - log.Infof("file with corr-id: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with correlation-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } @@ -325,7 +325,7 @@ func main() { } status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, corr-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) + log.Errorf("failed to get file status, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Getheader failed", @@ -346,7 +346,7 @@ func main() { } if status == "disabled" { - log.Infof("file with corr-id: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with correlation-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } From 9eb058ec0a8ac84a657b95b0446de496333fcfae Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 21 Aug 2025 11:25:27 +0200 Subject: [PATCH 202/262] fix: linter --- sda/cmd/finalize/finalize.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index c6d439e50e..385cfb3231 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -250,7 +250,7 @@ func backupFile(delivered amqp.Delivery) error { } if diskFileSize != int64(fileSize) { - return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)\n", diskFileSize, fileSize) + return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)", diskFileSize, fileSize) } file, err := archive.NewFileReader(filePath) From f8a2d46d8a3ed928ec4568e800eef153e3dcc2a9 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 25 Aug 2025 09:08:11 +0200 Subject: [PATCH 203/262] apply reviewers comment on styling Co-authored-by: Joakim Bygdell Update sda/cmd/finalize/finalize.go Co-authored-by: Joakim Bygdell Update sda/cmd/ingest/ingest.go Co-authored-by: Joakim Bygdell --- sda/cmd/finalize/finalize.go | 4 ++-- sda/cmd/ingest/ingest.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 385cfb3231..092dc3fb44 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -126,7 +126,7 @@ func main() { fileID, err := db.GetFileID(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get fileID for file with correlation-id: %s, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to get file-id for file with correlation-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: %v", err) } @@ -160,7 +160,7 @@ func main() { switch accessionIDExists { case "duplicate": - log.Errorf("Accession ID already exists in the system. (accession-id: %s)\n", message.AccessionID) + log.Errorf("accession ID already exists in the system. (accession-id: %s)\n", message.AccessionID) // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ Error: "There is a conflict regarding the file accessionID", diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 820f96ed42..4ba5a9f901 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -213,7 +213,7 @@ func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrig m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set event log status for file, correlation-id: %s", correlationID) + log.Errorf("failed to update event log for file with id : %s", fileID) return "nack" } From f179430b747b52541e407f8adfbccc06e450b498 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 26 Aug 2025 11:04:24 +0200 Subject: [PATCH 204/262] apply reviewers comments: grammar, add one fileID --- sda/cmd/finalize/finalize.go | 2 +- sda/cmd/verify/verify.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 092dc3fb44..8f7ca8acfb 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -160,7 +160,7 @@ func main() { switch accessionIDExists { case "duplicate": - log.Errorf("accession ID already exists in the system. (accession-id: %s)\n", message.AccessionID) + log.Errorf("accession ID already exists in the system, file-id: %s, accession-id: %s\n", fileID, message.AccessionID) // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ Error: "There is a conflict regarding the file accessionID", diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 868a17fd6a..572ac6c5ce 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -289,7 +289,7 @@ func main() { } if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { - log.Errorf("encrypted checksum don't match for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) + log.Errorf("encrypted checksum mismatch for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { From f4502ca1caa5a7c649fe334dd7964cb1e755330b Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 27 Aug 2025 09:11:06 +0200 Subject: [PATCH 205/262] feat: better log info for when inboxes not updating the DB --- sda/cmd/ingest/ingest.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 4ba5a9f901..63da694cb5 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -321,11 +321,11 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "ack" } case "": - // Catch all for inboxes that doesn't update the DB - log.Infof("ingesting unregistered file, correlation-id: %s", correlationID) + // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox + log.Infof("registering file, correlation-id: %s", correlationID) fileID, err = app.DB.RegisterFile(message.FilePath, message.User) if err != nil { - log.Errorf("InsertFile failed, correlation-id: %s, reason: (%s)", correlationID, err.Error()) + log.Errorf("failed to register file, correlation-id: %s, reason: (%s)", correlationID, err.Error()) return "nack" } From fab452c3a9078249401c0d532c2cd011b5d2aba4 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 27 Aug 2025 12:53:00 +0000 Subject: [PATCH 206/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 90da5bf8d3..c3eba44180 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.14 -appVersion: v2.1.28 +version: 2.0.15 +appVersion: v2.1.29 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 5b6b1b24a7..5dbd2436ba 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.14 -appVersion: v2.1.28 +version: 2.0.15 +appVersion: v2.1.29 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index a635eee840..95a7622bba 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.6 -appVersion: v2.1.28 +version: 3.0.7 +appVersion: v2.1.29 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 01ea4cfd9a102a3ad8ef1b40caf92423ed677319 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 28 Aug 2025 08:32:42 +0200 Subject: [PATCH 207/262] [internal/config] Set up reencrypt to be able to load multiple C4gh keys --- sda/internal/config/config.go | 12 ++++++----- sda/internal/config/config_test.go | 32 ++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 202fef0e2d..7287c83987 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -66,8 +66,8 @@ type Grpc struct { type ReEncConfig struct { Grpc - Crypt4GHKey *[32]byte - Timeout int + C4ghPrivateKeyList []*[32]byte + Timeout int } type Sync struct { @@ -386,8 +386,7 @@ func NewConfig(app string) (*Config, error) { } case "reencrypt": requiredConfVars = []string{ - "c4gh.filepath", - "c4gh.passphrase", + "c4gh.privateKeys", } case "s3inbox": requiredConfVars = []string{ @@ -1007,10 +1006,13 @@ func (c *Config) configReEncryptServer() (err error) { c.ReEncrypt.Port = 50443 } - c.ReEncrypt.Crypt4GHKey, err = GetC4GHKey() + c.ReEncrypt.C4ghPrivateKeyList, err = GetC4GHprivateKeys() if err != nil { return err } + if len(c.ReEncrypt.C4ghPrivateKeyList) == 0 { + return errors.New("no C4GH private keys configured") + } return nil } diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 2752f8ff70..4cde11881e 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -468,11 +468,13 @@ func (ts *ConfigTestSuite) TestConfigReEncryptServer() { ts.T().FailNow() } - viper.Set("c4gh.filepath", keyPath+"/c4gh.key") - viper.Set("c4gh.passphrase", "test") + viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ + {FilePath: keyPath + "/c4gh.key", Passphrase: "test"}, + }) config, err := NewConfig("reencrypt") assert.NoError(ts.T(), err) assert.Equal(ts.T(), 50051, config.ReEncrypt.Port) + assert.Equal(ts.T(), 1, len(config.ReEncrypt.C4ghPrivateKeyList)) viper.Set("grpc.CACert", certPath+"/ca.crt") viper.Set("grpc.serverCert", certPath+"/tls.crt") @@ -483,6 +485,32 @@ func (ts *ConfigTestSuite) TestConfigReEncryptServer() { assert.Equal(ts.T(), certPath+"/tls.crt", config.ReEncrypt.ServerCert) } +func (ts *ConfigTestSuite) TestConfigReEncryptServer_multipleKeys() { + keyPath, _ := os.MkdirTemp("", "key") + defer os.RemoveAll(keyPath) + keyFile1 := keyPath + "/c4gh1.key" + keyFile2 := keyPath + "/c4gh2.key" + + _, err := helper.CreatePrivateKeyFile(keyFile1, "test") + assert.NoError(ts.T(), err) + _, err = helper.CreatePrivateKeyFile(keyFile2, "test") + assert.NoError(ts.T(), err) + + viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ + {FilePath: keyFile1, Passphrase: "test"}, + {FilePath: keyFile2, Passphrase: "test"}, + }) + + config, err := NewConfig("reencrypt") + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(config.ReEncrypt.C4ghPrivateKeyList)) +} +func (ts *ConfigTestSuite) TestConfigReEncryptServer_noKeys() { + viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{}) + _, err := NewConfig("reencrypt") + assert.Error(ts.T(), err) +} + func (ts *ConfigTestSuite) TestConfigReEncryptClient() { ts.SetupTest() conf, err := configReEncryptClient() From 319cc5a8e0daba3408d93e836f9b64bdf207afba Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 28 Aug 2025 08:37:18 +0200 Subject: [PATCH 208/262] [reencrypt] Loop through all keys until the header can be reencrypted Since the service doesn't have access to the database there is no possibility to know which exact key to use. And giving the service access to the database changes the entire code design. --- sda/cmd/reencrypt/reencrypt.go | 16 ++++++++-------- sda/cmd/reencrypt/reencrypt_test.go | 19 ++++++++++--------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/sda/cmd/reencrypt/reencrypt.go b/sda/cmd/reencrypt/reencrypt.go index 5c25e721a9..0f954c3fb5 100644 --- a/sda/cmd/reencrypt/reencrypt.go +++ b/sda/cmd/reencrypt/reencrypt.go @@ -33,7 +33,7 @@ import ( // server struct is used to implement reencrypt.ReEncryptServer. type server struct { re.UnimplementedReencryptServer - c4ghPrivateKey *[32]byte + c4ghPrivateKeyList []*[32]byte } // hServer struct is used to implement the proxy grpc health.HealthServer. @@ -90,14 +90,14 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r newReaderPublicKeyList := [][chacha20poly1305.KeySize]byte{} newReaderPublicKeyList = append(newReaderPublicKeyList, newReaderPublicKey) - log.Debugf("crypt4ghkey: %v", *s.c4ghPrivateKey) - - newheader, err := headers.ReEncryptHeader(in.GetOldheader(), *s.c4ghPrivateKey, newReaderPublicKeyList, extraHeaderPackets...) - if err != nil { - return nil, status.Error(400, err.Error()) + for _, key := range s.c4ghPrivateKeyList { + newheader, err := headers.ReEncryptHeader(in.GetOldheader(), *key, newReaderPublicKeyList, extraHeaderPackets...) + if err == nil { + return &re.ReencryptResponse{Header: newheader}, nil + } } - return &re.ReencryptResponse{Header: newheader}, nil + return nil, status.Error(400, err.Error()) } // Check implements the healthgrpc.HealthServer Check method for the proxy grpc Health server. @@ -218,7 +218,7 @@ func main() { } s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: conf.ReEncrypt.Crypt4GHKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: conf.ReEncrypt.C4ghPrivateKeyList}) reflection.Register(s) // Add health service for reencrypt server diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 71da4b588f..7792174540 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -33,7 +33,7 @@ type ReEncryptTests struct { FileData []byte KeyPath string FileHeader []byte - PrivateKey *[32]byte + PrivateKeyList []*[32]byte UserPrivateKey [32]byte UserPublicKey [32]byte UserPubKeyString string @@ -72,10 +72,11 @@ func (ts *ReEncryptTests) SetupTest() { } ts.UserPubKeyString = base64.StdEncoding.EncodeToString(buf.Bytes()) - viper.Set("c4gh.filepath", ts.KeyPath+"/c4gh.key") - viper.Set("c4gh.passphrase", "test") + viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ + {FilePath: ts.KeyPath + "/c4gh.key", Passphrase: "test"}, + }) - ts.PrivateKey, err = config.GetC4GHKey() + ts.PrivateKeyList, err = config.GetC4GHprivateKeys() if err != nil { ts.T().FailNow() } @@ -97,7 +98,7 @@ func (ts *ReEncryptTests) TestReencryptHeader() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) if err := s.Serve(lis); err != nil { ts.T().Fail() } @@ -139,7 +140,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_DataEditList() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) if err := s.Serve(lis); err != nil { ts.T().Fail() } @@ -213,7 +214,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_BadPubKey() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) _ = s.Serve(lis) }() @@ -243,7 +244,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_NoHeader() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) _ = s.Serve(lis) }() @@ -298,7 +299,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_TLS() { ) opts := []grpc.ServerOption{grpc.Creds(serverCreds)} s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) if err := s.Serve(lis); err != nil { ts.T().Fail() } From 747f61a2d41e4cd85224c585a933341aa7a757ee Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 28 Aug 2025 08:38:11 +0200 Subject: [PATCH 209/262] Fix complementary test The test shouldn't be done like this but refactoring need to wait until all tests can be done at the same time. --- sda/cmd/api/api_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index a35b2a3df0..23ba26c055 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -49,7 +49,7 @@ var BrokerAPI string type server struct { reencrypt.UnimplementedReencryptServer - c4ghPrivateKey *[32]byte + c4ghPrivateKeyList []*[32]byte } func (s *server) ReencryptHeader(ctx context.Context, req *reencrypt.ReencryptRequest) (*reencrypt.ReencryptResponse, error) { @@ -532,7 +532,7 @@ func (s *TestSuite) SetupSuite() { go func() { var opts []grpc.ServerOption s.GrpcListener.gs = grpc.NewServer(opts...) - reencrypt.RegisterReencryptServer(s.GrpcListener.gs, &server{c4ghPrivateKey: Conf.ReEncrypt.Crypt4GHKey}) + reencrypt.RegisterReencryptServer(s.GrpcListener.gs, &server{c4ghPrivateKeyList: Conf.ReEncrypt.C4ghPrivateKeyList}) if err := s.GrpcListener.gs.Serve(s.GrpcListener.Listener); err != nil { s.T().Fail() } From 35cdd5b5fd5a26c77e9e707540feb032cd8023b1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 28 Aug 2025 08:52:29 +0200 Subject: [PATCH 210/262] [charts] give reencrypt all keys --- charts/sda-svc/templates/re-encrypt-deploy.yaml | 2 +- charts/sda-svc/templates/reencrypt-secrets.yaml | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/charts/sda-svc/templates/re-encrypt-deploy.yaml b/charts/sda-svc/templates/re-encrypt-deploy.yaml index 7d0bc0e7d6..ee7eb02134 100644 --- a/charts/sda-svc/templates/re-encrypt-deploy.yaml +++ b/charts/sda-svc/templates/re-encrypt-deploy.yaml @@ -82,7 +82,7 @@ spec: - name: c4gh secret: defaultMode: 0440 - secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} + secretName: {{ required "A secret for the c4gh keys is required" .Values.global.c4gh.secretName }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls diff --git a/charts/sda-svc/templates/reencrypt-secrets.yaml b/charts/sda-svc/templates/reencrypt-secrets.yaml index de613fc92e..fbf718c186 100644 --- a/charts/sda-svc/templates/reencrypt-secrets.yaml +++ b/charts/sda-svc/templates/reencrypt-secrets.yaml @@ -9,10 +9,11 @@ type: Opaque stringData: config.yaml: |- c4gh: - {{- with (first .Values.global.c4gh.privateKeys) }} - filePath: {{ template "c4ghPath" $ }}/{{ .keyName }} - passphrase: {{ required "a passphrase for the c4gh key is required" .passphrase }} - {{- end }} + privateKeys: + {{- range $k := .Values.global.c4gh.privateKeys }} + - filePath: {{ template "c4ghPath" $ }}/{{ $k.keyName }} + passphrase: {{ $k.passphrase }} + {{- end }} {{- if .Values.global.tls.enabled }} grpc: caCert: {{ template "tlsPath" . }}/ca.crt From d9e22799338de7c85a2c12ee55e7afc33612e64f Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 29 Aug 2025 11:12:22 +0200 Subject: [PATCH 211/262] allow ingest to register configured keys --- postgresql/initdb.d/04_grants.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index c9778aa4ed..a04fe93f3e 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -44,6 +44,7 @@ GRANT INSERT ON sda.file_event_log TO ingest; GRANT SELECT ON sda.file_event_log TO ingest; GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO ingest; GRANT SELECT ON sda.encryption_keys TO ingest; +GRANT INSERT ON sda.encryption_keys TO ingest; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO ingest; From 204bdc48708c48fabf27d2d4763e5a80d9f7ccb2 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Fri, 29 Aug 2025 10:39:29 +0000 Subject: [PATCH 212/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index c3eba44180..497ef4d6b8 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.15 -appVersion: v2.1.29 +version: 2.0.16 +appVersion: v2.1.31 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 5dbd2436ba..2d1e5859e2 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.15 -appVersion: v2.1.29 +version: 2.0.16 +appVersion: v2.1.31 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 95a7622bba..a8b8cf9c62 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.7 -appVersion: v2.1.29 +version: 3.0.8 +appVersion: v2.1.31 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From dc8cf3bbe4ab808f42f6c6a376f05eb7477851b2 Mon Sep 17 00:00:00 2001 From: zeidlitz Date: Mon, 18 Aug 2025 11:36:47 +0200 Subject: [PATCH 213/262] [api] audit trail information in the logs - Add a new logger called auditLogging that can be used to log "audit" level events. - Add a new configuration field for audit logging --- sda/cmd/api/api.go | 40 +++++++++++++++++++++++++++++++---- sda/internal/config/config.go | 31 ++++++++++++++++----------- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index f91e974816..174b5c61cc 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -45,9 +45,10 @@ type dataset struct { } var ( - Conf *config.Config - err error - auth *userauth.ValidateFromToken + Conf *config.Config + err error + auth *userauth.ValidateFromToken + auditLogger *log.Logger ) func main() { @@ -234,11 +235,21 @@ func checkDB(db *database.SDAdb, timeout time.Duration) error { return db.DB.PingContext(ctx) } +func auditLog(auditFields log.Fields) { + auditFields["audit"] = true + Conf.API.AuditLogger.WithFields(auditFields).Info("Incoming audit event") +} + func rbac(e *casbin.Enforcer) gin.HandlerFunc { return func(c *gin.Context) { token, err := auth.Authenticate(c.Request) if err != nil { - log.Debugln("bad token") + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "authentication error": err.Error(), + "path": c.Request.URL.Path, + }) + } c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err.Error()}) return @@ -246,16 +257,37 @@ func rbac(e *casbin.Enforcer) gin.HandlerFunc { ok, err := e.Enforce(token.Subject(), c.Request.URL.String(), c.Request.Method) if err != nil { + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "authorization": "error (err.Error())", + "user": token.Subject(), + "path": c.Request.URL.Path, + }) + } log.Debugf("rbac enforcement failed, reason: %s\n", err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } if !ok { + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "authorization": "failed", + "user": token.Subject(), + "path": c.Request.URL.Path, + }) + } c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "not authorized"}) return } + + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "user": token.Subject(), + "path": c.Request.URL.Path, + }) + } log.Debugln("authorized") } } diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 7287c83987..941c2b2658 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -88,17 +88,18 @@ type SyncAPIConf struct { } type APIConf struct { - RBACpolicy []byte - CACert string - ServerCert string - ServerKey string - Host string - Port int - Session SessionConfig - DB *database.SDAdb - MQ *broker.AMQPBroker - INBOX storage.Backend - Grpc Grpc + RBACpolicy []byte + CACert string + ServerCert string + ServerKey string + Host string + Port int + Session SessionConfig + DB *database.SDAdb + MQ *broker.AMQPBroker + INBOX storage.Backend + Grpc Grpc + AuditLogger *log.Logger } type SessionConfig struct { @@ -712,7 +713,6 @@ func NewConfig(app string) (*Config, error) { return c, nil } -// configDatabase provides configuration for the database func (c *Config) configAPI() error { c.apiDefaults() api := APIConf{} @@ -728,6 +728,12 @@ func (c *Config) configAPI() error { api.ServerKey = viper.GetString("api.serverKey") api.ServerCert = viper.GetString("api.serverCert") api.CACert = viper.GetString("api.CACert") + if viper.GetBool("api.audit") { + api.AuditLogger = log.New() + api.AuditLogger.SetFormatter(&log.JSONFormatter{}) + api.AuditLogger.SetOutput(os.Stdout) + api.AuditLogger.SetLevel(log.InfoLevel) + } c.API = api @@ -742,6 +748,7 @@ func (c *Config) apiDefaults() { viper.SetDefault("api.session.secure", true) viper.SetDefault("api.session.httponly", true) viper.SetDefault("api.session.name", "api_session_key") + viper.SetDefault("api.audit", true) } // configArchive provides configuration for the archive storage From f3043d8186451c489eaad81d430eb5718c577df6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 2 Sep 2025 10:42:36 +0200 Subject: [PATCH 214/262] Add codeowners file Now we will be notified again when dependabot creates new PRs. --- .github/CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..0bb2e47ee3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @neicnordic/sensitive-data-development-collaboration From f550ea44b6fb0ab6f1fdbe5abfedcf1d8d847d35 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 2 Sep 2025 10:46:30 +0200 Subject: [PATCH 215/262] [dependabot] remove obsolete reviewers field --- .github/dependabot.yaml | 65 +++++++++++++---------------------------- 1 file changed, 21 insertions(+), 44 deletions(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 5569d19a39..4c843fe89e 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -5,8 +5,6 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" # Each subfolder needs to be checked separately - package-ecosystem: docker @@ -14,36 +12,31 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + - package-ecosystem: docker directory: "/sda-download" schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + - package-ecosystem: docker directory: "/rabbitmq" schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + - package-ecosystem: docker directory: "/sda" schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + - package-ecosystem: docker directory: "/sda-sftp-inbox" schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + # Each subfolder needs to be checked separately - package-ecosystem: gomod @@ -53,8 +46,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly @@ -65,8 +57,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly @@ -78,8 +69,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly @@ -90,8 +80,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly @@ -100,8 +89,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + ## release v1 branch - package-ecosystem: "github-actions" @@ -110,8 +98,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + commit-message: prefix: "[release_v1]" ### Docker @@ -121,8 +108,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + commit-message: prefix: "[release_v1]" @@ -132,8 +118,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + commit-message: prefix: "[release_v1]" @@ -143,8 +128,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + commit-message: prefix: "[release_v1]" @@ -154,8 +138,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + commit-message: prefix: "[release_v1]" @@ -165,8 +148,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + commit-message: prefix: "[release_v1]" @@ -176,8 +158,7 @@ updates: schedule: interval: weekly open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + commit-message: prefix: "[release_v1]" @@ -190,8 +171,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly commit-message: @@ -205,8 +185,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly commit-message: @@ -220,8 +199,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly commit-message: @@ -235,8 +213,7 @@ updates: patterns: - "*" open-pull-requests-limit: 10 - reviewers: - - "neicnordic/sensitive-data-development-collaboration" + schedule: interval: weekly commit-message: From ab65e3f62d0fbf91bc9eaeb152a4b778811e5cbb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 10:04:13 +0000 Subject: [PATCH 216/262] Bump aquasecurity/trivy-action from 0.32.0 to 0.33.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.32.0 to 0.33.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.32.0...0.33.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.33.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- .github/workflows/scan-images.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index c37d05c195..acfe1e4cca 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -105,7 +105,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@0.33.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -123,7 +123,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@0.33.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -172,7 +172,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@0.33.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -203,7 +203,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@0.33.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index 84ab4a609d..28dd648260 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -27,7 +27,7 @@ jobs: image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@0.33.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -53,7 +53,7 @@ jobs: security-events: write steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.32.0 + uses: aquasecurity/trivy-action@0.33.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From f83de11db083e6ad69c96fcc6636e7efacc8ec1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 10:48:44 +0000 Subject: [PATCH 217/262] Bump github.com/stretchr/testify Bumps the all-modules group in /sda-download with 1 update: [github.com/stretchr/testify](https://github.com/stretchr/testify). Updates `github.com/stretchr/testify` from 1.11.0 to 1.11.1 - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.11.0...v1.11.1) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-version: 1.11.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index cd9aeef624..26729d0c66 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -14,7 +14,7 @@ require ( github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.11.0 + github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.41.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.75.0 diff --git a/sda-download/go.sum b/sda-download/go.sum index 4da07b87ca..07eca7c911 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -147,8 +147,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= -github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= From d5cc32c7eaaa4022077cdb86af25dd407334d71c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 11:24:00 +0000 Subject: [PATCH 218/262] Bump org.yaml:snakeyaml in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [org.yaml:snakeyaml](https://bitbucket.org/snakeyaml/snakeyaml). Updates `org.yaml:snakeyaml` from 2.4 to 2.5 - [Commits](https://bitbucket.org/snakeyaml/snakeyaml/branches/compare/snakeyaml-2.5..snakeyaml-2.4) --- updated-dependencies: - dependency-name: org.yaml:snakeyaml dependency-version: '2.5' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 6341d1e053..4d1f896852 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -132,7 +132,7 @@ org.yaml snakeyaml - 2.4 + 2.5 From 2f36b641fbec8b6c7092b3f60a892e58b72c68ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 12:33:23 +0000 Subject: [PATCH 219/262] Bump the all-modules group in /sda with 8 updates Bumps the all-modules group in /sda with 8 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.38.1` | `1.38.3` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.2` | `1.31.6` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.18.6` | `1.18.10` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.19.0` | `1.19.4` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.87.1` | `1.87.3` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.22.5` | `1.23.0` | | [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) | `2.120.0` | `2.121.0` | | [github.com/stretchr/testify](https://github.com/stretchr/testify) | `1.11.0` | `1.11.1` | Updates `github.com/aws/aws-sdk-go-v2` from 1.38.1 to 1.38.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.38.1...v1.38.3) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.2 to 1.31.6 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.2...config/v1.31.6) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.18.6 to 1.18.10 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.6...config/v1.18.10) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.19.0 to 1.19.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.19.0...service/m2/v1.19.4) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.87.1 to 1.87.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.87.1...service/s3/v1.87.3) Updates `github.com/aws/smithy-go` from 1.22.5 to 1.23.0 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.22.5...v1.23.0) Updates `github.com/casbin/casbin/v2` from 2.120.0 to 2.121.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.120.0...v2.121.0) Updates `github.com/stretchr/testify` from 1.11.0 to 1.11.1 - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.11.0...v1.11.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.38.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.31.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.18.10 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.19.4 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.87.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/smithy-go dependency-version: 1.23.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.121.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/stretchr/testify dependency-version: 1.11.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 40 +++++++++++++-------------- sda/go.sum | 80 +++++++++++++++++++++++++++--------------------------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 72db488ebe..395ad67f53 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,13 +3,13 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.38.1 - github.com/aws/aws-sdk-go-v2/config v1.31.2 - github.com/aws/aws-sdk-go-v2/credentials v1.18.6 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 - github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 - github.com/aws/smithy-go v1.22.5 - github.com/casbin/casbin/v2 v2.120.0 + github.com/aws/aws-sdk-go-v2 v1.38.3 + github.com/aws/aws-sdk-go-v2/config v1.31.6 + github.com/aws/aws-sdk-go-v2/credentials v1.18.10 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4 + github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3 + github.com/aws/smithy-go v1.23.0 + github.com/casbin/casbin/v2 v2.121.0 github.com/coreos/go-oidc/v3 v3.15.0 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -30,7 +30,7 @@ require ( github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.11.0 + github.com/stretchr/testify v1.11.1 golang.org/x/crypto v0.41.0 golang.org/x/oauth2 v0.30.0 google.golang.org/grpc v1.75.0 @@ -49,19 +49,19 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index cb70421423..8a763a93ba 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,44 +24,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.38.1 h1:j7sc33amE74Rz0M/PoCpsZQ6OunLqys/m5antM0J+Z8= -github.com/aws/aws-sdk-go-v2 v1.38.1/go.mod h1:9Q0OoGQoboYIAJyslFyF1f5K1Ryddop8gqMhWx/n4Wg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0 h1:6GMWV6CNpA/6fbFHnoAjrv4+LGfyTqZz2LtCHnspgDg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.0/go.mod h1:/mXlTIVG9jbxkqDnr5UQNQxW1HRYxeGklkM9vAFeabg= -github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0= -github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 h1:2FFgK3oFA8PTNBjprLFfcmkgg7U9YuSimBvR64RUmiA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0/go.mod h1:xdxj6nC1aU/jAO80RIlIj3fU40MOSqutEA9N2XFct04= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4 h1:IdCLsiiIj5YJ3AFevsewURCPV+YWUlOW8JiPhoAy8vg= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.4/go.mod h1:l4bdfCD7XyyZA9BolKBo1eLqgaJxl0/x91PL4Yqe0ao= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4 h1:j7vjtr1YIssWQOMeOWRbh3z8g2oY/xPjnZH2gLY4sGw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.4/go.mod h1:yDmJgqOiH4EA8Hndnv4KwAo8jCGTSnM5ASG1nBI+toA= +github.com/aws/aws-sdk-go-v2 v1.38.3 h1:B6cV4oxnMs45fql4yRH+/Po/YU+597zgWqvDpYMturk= +github.com/aws/aws-sdk-go-v2 v1.38.3/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= +github.com/aws/aws-sdk-go-v2/config v1.31.6 h1:a1t8fXY4GT4xjyJExz4knbuoxSCacB5hT/WgtfPyLjo= +github.com/aws/aws-sdk-go-v2/config v1.31.6/go.mod h1:5ByscNi7R+ztvOGzeUaIu49vkMk2soq5NaH5PYe33MQ= +github.com/aws/aws-sdk-go-v2/credentials v1.18.10 h1:xdJnXCouCx8Y0NncgoptztUocIYLKeQxrCgN6x9sdhg= +github.com/aws/aws-sdk-go-v2/credentials v1.18.10/go.mod h1:7tQk08ntj914F/5i9jC4+2HQTAuJirq7m1vZVIhEkWs= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 h1:wbjnrrMnKew78/juW7I2BtKQwa1qlf6EjQgS69uYY14= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6/go.mod h1:AtiqqNrDioJXuUgz3+3T0mBWN7Hro2n9wll2zRUc0ww= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4 h1:BTl+TXrpnrpPWb/J3527GsJ/lMkn7z3GO12j6OlsbRg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4/go.mod h1:cG2tenc/fscpChiZE29a2crG9uo2t6nQGflFllFL8M8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 h1:uF68eJA6+S9iVr9WgX1NaRGyQ/6MdIyc4JNUo6TN1FA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6/go.mod h1:qlPeVZCGPiobx8wb1ft0GHT5l+dc6ldnwInDFaMvC7Y= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 h1:pa1DEC6JoI0zduhZePp3zmhWvk/xxm4NB8Hy/Tlsgos= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6/go.mod h1:gxEjPebnhWGJoaDdtDkA0JX46VRg1wcTHYe63OfX5pE= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4 h1:BE/MNQ86yzTINrfxPPFS86QCBNQeLKY2A0KhDh47+wI= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.4/go.mod h1:SPBBhkJxjcrzJBc+qY85e83MQ2q3qdra8fghhkkyrJg= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0 h1:6+lZi2JeGKtCraAj1rpoZfKqnQ9SptseRZioejfUOLM= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.0/go.mod h1:eb3gfbVIxIoGgJsi9pGne19dhCBpK6opTYpQqAmdy44= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4 h1:Beh9oVgtQnBgR4sKKzkUBRQpf1GnL4wt0l4s8h2VCJ0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.4/go.mod h1:b17At0o8inygF+c6FOD3rNyYZufPw62o9XJbSfQPgbo= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4 h1:ueB2Te0NacDMnaC+68za9jLwkjzxGWm0KB5HTUHjLTI= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.4/go.mod h1:nLEfLnVMmLvyIG58/6gsSA03F1voKGaCfHV7+lR8S7s= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4 h1:HVSeukL40rHclNcUqVcBwE1YoZhOkoLeBfhUqR3tjIU= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.4/go.mod h1:DnbBOv4FlIXHj2/xmrUQYtawRFC9L9ZmQPz+DBc6X5I= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1 h1:2n6Pd67eJwAb/5KCX62/8RTU0aFAAW7V5XIGSghiHrw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.1/go.mod h1:w5PC+6GHLkvMJKasYGVloB3TduOtROEMqm15HSuIbw4= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo= -github.com/aws/smithy-go v1.22.5 h1:P9ATCXPMb2mPjYBgueqJNCA5S9UfktsW0tTxi+a7eqw= -github.com/aws/smithy-go v1.22.5/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.6 h1:R0tNFJqfjHL3900cqhXuwQ+1K4G0xc9Yf8EDbFXCKEw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.6/go.mod h1:y/7sDdu+aJvPtGXr4xYosdpq9a6T9Z0jkXfugmti0rI= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.6 h1:hncKj/4gR+TPauZgTAsxOxNcvBayhUlYZ6LO/BYiQ30= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.6/go.mod h1:OiIh45tp6HdJDDJGnja0mw8ihQGz3VGrUflLqSL0SmM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6 h1:LHS1YAIJXJ4K9zS+1d/xa9JAA9sL2QyXIQCQFQW/X08= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6/go.mod h1:c9PCiTEuh0wQID5/KqA32J+HAgZxN9tOGXKCiYJjTZI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.6 h1:nEXUSAwyUfLTgnc9cxlDWy637qsq4UWwp3sNAfl0Z3Y= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.6/go.mod h1:HGzIULx4Ge3Do2V0FaiYKcyKzOqwrhUZgCI77NisswQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3 h1:ETkfWcXP2KNPLecaDa++5bsQhCRa5M5sLUJa5DWYIIg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3/go.mod h1:+/3ZTqoYb3Ur7DObD00tarKMLMuKg8iqz5CHEanqTnw= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 h1:8OLZnVJPvjnrxEwHFg9hVUof/P4sibH+Ea4KKuqAGSg= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.1/go.mod h1:27M3BpVi0C02UiQh1w9nsBEit6pLhlaH3NHna6WUbDE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 h1:gKWSTnqudpo8dAxqBqZnDoDWCiEh/40FziUjr/mo6uA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2/go.mod h1:x7+rkNmRoEN1U13A6JE2fXne9EWyJy54o3n6d4mGaXQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 h1:YZPjhyaGzhDQEvsffDEcpycq49nl7fiGcfJTIo8BszI= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.2/go.mod h1:2dIN8qhQfv37BdUYGgEC8Q3tteM3zFxTI1MLO2O3J3c= +github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= +github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.120.0 h1:Mo9R/EKZk9aoagFs0OmuCmBYjWJfvbWJiX4aenIJOKY= -github.com/casbin/casbin/v2 v2.120.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.121.0 h1:lrgTnLJTsdpe8Kdgi+NedM9+K7ftYBaK19OE+IZUwdk= +github.com/casbin/casbin/v2 v2.121.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= @@ -342,8 +342,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8= -github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/tdewolff/minify/v2 v2.23.6 h1:s0ktOnrlqKfhignkv9M/rh9wCfRlIPiIpjPOq3xUJ84= From 913baed98e1ac1c0cf44ca2be9db3a3408331600 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Sep 2025 12:33:30 +0000 Subject: [PATCH 220/262] Bump org.yaml:snakeyaml in /sda-sftp-inbox in the all-modules group Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.yaml:snakeyaml](https://bitbucket.org/snakeyaml/snakeyaml). Updates `org.yaml:snakeyaml` from 2.4 to 2.5 - [Commits](https://bitbucket.org/snakeyaml/snakeyaml/branches/compare/snakeyaml-2.5..snakeyaml-2.4) --- updated-dependencies: - dependency-name: org.yaml:snakeyaml dependency-version: '2.5' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index bc3dcae77f..0a3f9e46d9 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -161,7 +161,7 @@ org.yaml snakeyaml - 2.4 + 2.5 junit From 41219f8183213798b9548c3cba3ff3ea5b630a34 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 26 Aug 2025 13:52:17 +0200 Subject: [PATCH 221/262] [internal database] enable filtration in GetUserFiles Make it possible to use path prefix when listing files for a user. --- sda/internal/database/db_functions.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index c9c74cebeb..c7d0dad074 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -708,14 +708,14 @@ func (dbs *SDAdb) getArchivePath(stableID string) (string, error) { } // GetUserFiles retrieves all the files a user submitted -func (dbs *SDAdb) GetUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) GetUserFiles(userID, pathPrefix string, allData bool) ([]*SubmissionFileInfo, error) { var err error files := []*SubmissionFileInfo{} // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { - files, err = dbs.getUserFiles(userID, allData) + files, err = dbs.getUserFiles(userID, pathPrefix, allData) if err == nil { break } @@ -726,7 +726,7 @@ func (dbs *SDAdb) GetUserFiles(userID string, allData bool) ([]*SubmissionFileIn } // getUserFiles is the actual function performing work for GetUserFiles -func (dbs *SDAdb) getUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) getUserFiles(userID, pathPrefix string, allData bool) ([]*SubmissionFileInfo, error) { dbs.checkAndReconnectIfNeeded() files := []*SubmissionFileInfo{} @@ -734,11 +734,12 @@ func (dbs *SDAdb) getUserFiles(userID string, allData bool) ([]*SubmissionFileIn // select all files (that are not part of a dataset) of the user, each one annotated with its latest event const query = `SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f -LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 +LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id +WHERE f.submission_user = $1 and f.submission_file_path LIKE $2 AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);` // nolint:rowserrcheck - rows, err := db.Query(query, userID) + rows, err := db.Query(query, userID, fmt.Sprintf("%s%%", pathPrefix)) if err != nil { return nil, err } From 54ebf62cc1c909db3f87a891f74f96908ae7be67 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 26 Aug 2025 13:52:58 +0200 Subject: [PATCH 222/262] [internal database] update tests with path prefix case --- sda/internal/database/db_functions_test.go | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 369220aea3..bbdbc73572 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -458,9 +458,14 @@ func (suite *DatabaseTests) TestGetUserFiles() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) testCases := 5 testUser := "GetFilesUser" + sub := "submission_a" - for i := 0; i < testCases; i++ { - fileID, err := db.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", testUser, i), testUser) + for i := range testCases { + if i == 2 { + sub = "submission_b" + } + + fileID, err := db.RegisterFile(fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", testUser, sub, i), testUser) assert.NoError(suite.T(), err, "failed to register file in database") err = db.UpdateFileEventLog(fileID, "uploaded", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") @@ -469,11 +474,11 @@ func (suite *DatabaseTests) TestGetUserFiles() { err = db.UpdateFileEventLog(fileID, "ready", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") } - filelist, err := db.GetUserFiles("unknownuser", true) + filelist, err := db.GetUserFiles("unknownuser", "", true) assert.NoError(suite.T(), err, "failed to get (empty) file list of unknown user") assert.Empty(suite.T(), filelist, "file list of unknown user is not empty") - filelist, err = db.GetUserFiles(testUser, true) + filelist, err = db.GetUserFiles(testUser, "", true) assert.NoError(suite.T(), err, "failed to get file list") assert.Equal(suite.T(), testCases, len(filelist), "file list is of incorrect length") @@ -481,6 +486,10 @@ func (suite *DatabaseTests) TestGetUserFiles() { assert.Equal(suite.T(), "ready", fileInfo.Status, "incorrect file status") assert.Contains(suite.T(), fileInfo.AccessionID, "stableID-00", "incorrect file accession ID") } + + filteredFilelist, err := db.GetUserFiles(testUser, fmt.Sprintf("%s/submission_b", testUser), true) + assert.NoError(suite.T(), err, "failed to get file list") + assert.Equal(suite.T(), 3, len(filteredFilelist), "file list is of incorrect length") } func (suite *DatabaseTests) TestGetCorrID() { From d1205bff70312e3da7b2f1ce80a5e83fabffdc5f Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 26 Aug 2025 09:58:34 +0200 Subject: [PATCH 223/262] [api] add test for filtering user files --- sda/cmd/api/api_test.go | 109 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 23ba26c055..bb1f83a311 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -707,6 +707,60 @@ func (s *TestSuite) TestAPIGetFiles() { assert.NoError(s.T(), err) } +func (s *TestSuite) TestAPIGetFiles_filteredSelection() { + testUsers := []string{"dummy", "User-B", "User-C"} + for _, user := range testUsers { + sub := "submission_a" + + for i := range 5 { + if i == 2 { + sub = "submission_b" + } + + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", user, sub, i), strings.ReplaceAll(user, "_", "@")) + if err != nil { + s.FailNow("failed to register file in database") + } + + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + if err != nil { + s.FailNow("failed to update satus of file in database") + } + } + } + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") + } + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") + } + + // Mock request and response holders + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/files?path_prefix=submission_b", http.NoBody) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.GET("/files", rbac(e), getFiles) + + router.ServeHTTP(w, r) + okResponse := w.Result() + defer okResponse.Body.Close() + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + + files := []database.SubmissionFileInfo{} + err = json.NewDecoder(okResponse.Body).Decode(&files) + assert.NoError(s.T(), err, "failed to list users from DB") + assert.Equal(s.T(), 3, len(files)) +} + func TestApiTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } @@ -1867,6 +1921,61 @@ func (s *TestSuite) TestListUserFiles() { assert.Contains(s.T(), files[0].AccessionID, "accession_user_example.org_0") } +func (s *TestSuite) TestListUserFiles_filteredSelection() { + testUsers := []string{"user_example.org", "User-B", "User-C"} + for _, user := range testUsers { + sub := "submission_a" + + for i := range 5 { + if i == 2 { + sub = "submission_b" + } + + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", user, sub, i), strings.ReplaceAll(user, "_", "@")) + if err != nil { + s.FailNow("failed to register file in database") + } + + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + if err != nil { + s.FailNow("failed to update satus of file in database") + } + } + } + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") + } + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") + } + + // Mock request and response holders + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/users/user@example.org/files?path_prefix=submission_b", http.NoBody) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.GET("/users/:username/files", rbac(e), listUserFiles) + + router.ServeHTTP(w, r) + okResponse := w.Result() + defer okResponse.Body.Close() + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + + files := []database.SubmissionFileInfo{} + err = json.NewDecoder(okResponse.Body).Decode(&files) + assert.NoError(s.T(), err, "failed to list users from DB") + assert.Equal(s.T(), 3, len(files)) + assert.Contains(s.T(), files[0].Status, "uploaded") +} + func (s *TestSuite) TestAddC4ghHash() { gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) From 9bed0e8f21f30b56f073aaf4c4cc8d9975494b93 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 26 Aug 2025 14:43:35 +0200 Subject: [PATCH 224/262] [api] enable path prefix filtering when listing a users files --- sda/cmd/api/api.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 174b5c61cc..28a7250343 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -304,7 +304,12 @@ func getFiles(c *gin.Context) { return } - files, err := Conf.API.DB.GetUserFiles(token.Subject(), false) + prefix := c.Query("path_prefix") + if prefix != "" { + prefix = fmt.Sprintf("%s/%s", strings.ReplaceAll(token.Subject(), "@", "_"), prefix) + } + + files, err := Conf.API.DB.GetUserFiles(token.Subject(), prefix, false) if err != nil { // something went wrong with querying or parsing rows c.JSON(502, err.Error()) @@ -726,7 +731,11 @@ func listUserFiles(c *gin.Context) { username = strings.TrimPrefix(username, "/") username = strings.TrimSuffix(username, "/files") log.Debugln(username) - files, err := Conf.API.DB.GetUserFiles(username, true) + prefix := c.Query("path_prefix") + if prefix != "" { + prefix = fmt.Sprintf("%s/%s", strings.ReplaceAll(username, "@", "_"), prefix) + } + files, err := Conf.API.DB.GetUserFiles(username, prefix, true) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) From 9f3cc1e3b5906320f06ca0272fd26f497b69a455 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 26 Aug 2025 09:59:12 +0200 Subject: [PATCH 225/262] [api] fix rbac enforcement path --- sda/cmd/api/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 28a7250343..6d9c64d501 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -255,7 +255,7 @@ func rbac(e *casbin.Enforcer) gin.HandlerFunc { return } - ok, err := e.Enforce(token.Subject(), c.Request.URL.String(), c.Request.Method) + ok, err := e.Enforce(token.Subject(), c.Request.URL.Path, c.Request.Method) if err != nil { if Conf.API.AuditLogger != nil { auditLog(log.Fields{ From 04158ce3907ec24c8805429abc97c974c54c1051 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 26 Aug 2025 15:46:14 +0200 Subject: [PATCH 226/262] [api] update readme --- sda/cmd/api/api.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 59a410df1c..1a124c8134 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -19,6 +19,12 @@ Endpoints: [{"inboxPath":"requester_demo.org/data/file1.c4gh","fileStatus":"uploaded","createAt":"2023-11-13T10:12:43.144242Z"}] ``` + It is possible to limit the returned results by supplying a base path prefix in the query. In this case only files that have a path that starts with `submission-1` will be returned. + + ```bash + curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/files?path_prefix=submission-1 + ``` + If the `token` is invalid, 401 is returned. - `/datasets` @@ -215,6 +221,12 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/users/submitter@example.org/files ``` + It is possible to limit the returned results by supplying a base path prefix in the query. In this case only files that have a path that starts with `submission-1` will be returned. + + ```bash + curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/users/submitter@example.org/files?path_prefix=submission-1 + ``` + - Error codes - `200` Query execute ok. - `401` Token user is not in the list of admins. From d05b2a918f416cf54c18af15c2af59268c0aa319 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 2 Sep 2025 09:08:38 +0200 Subject: [PATCH 227/262] Apply suggestions from code review Co-authored-by: Nanjiang Shu --- sda/cmd/api/api_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index bb1f83a311..71a1f44766 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -757,7 +757,7 @@ func (s *TestSuite) TestAPIGetFiles_filteredSelection() { files := []database.SubmissionFileInfo{} err = json.NewDecoder(okResponse.Body).Decode(&files) - assert.NoError(s.T(), err, "failed to list users from DB") + assert.NoError(s.T(), err, "failed to list files from DB") assert.Equal(s.T(), 3, len(files)) } @@ -1971,7 +1971,7 @@ func (s *TestSuite) TestListUserFiles_filteredSelection() { files := []database.SubmissionFileInfo{} err = json.NewDecoder(okResponse.Body).Decode(&files) - assert.NoError(s.T(), err, "failed to list users from DB") + assert.NoError(s.T(), err, "failed to list user files from DB") assert.Equal(s.T(), 3, len(files)) assert.Contains(s.T(), files[0].Status, "uploaded") } From 8f28de1362e867af7626ad4ca9864ba27c8a3b89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 3 Sep 2025 11:44:31 +0000 Subject: [PATCH 228/262] Bump no.elixir:clearinghouse in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:clearinghouse` from 3.0.7 to 3.0.8 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.7...crypt4gh-3.0.8) --- updated-dependencies: - dependency-name: no.elixir:clearinghouse dependency-version: 3.0.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 4d1f896852..50b9f937f9 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -121,7 +121,7 @@ no.elixir clearinghouse - 3.0.7 + 3.0.8 org.slf4j From e6993a0d916af2d901bee349399e5f8d04345d3d Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 3 Sep 2025 12:27:39 +0000 Subject: [PATCH 229/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 497ef4d6b8..f569323a20 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.16 -appVersion: v2.1.31 +version: 2.0.17 +appVersion: v2.1.38 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 2d1e5859e2..d6a63f6647 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.16 -appVersion: v2.1.31 +version: 2.0.17 +appVersion: v2.1.38 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index a8b8cf9c62..306e29522b 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.8 -appVersion: v2.1.31 +version: 3.0.9 +appVersion: v2.1.38 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 0b3c8fc0aeab49f534025d2235ab4672d519de12 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 26 Nov 2024 13:33:46 +0100 Subject: [PATCH 230/262] [test] integration test no longer refers to user id in file paths --- .github/integration/tests/sda/20_ingest-verify_test.sh | 2 +- .github/integration/tests/sda/21_cancel_test.sh | 4 ++-- .github/integration/tests/sda/22_error_test.sh | 6 +++--- .../integration/tests/sda/30_backup-finalize_test.sh | 2 +- .github/integration/tests/sda/31_cancel_test2.sh | 8 ++++---- .../integration/tests/sda/32_test_race_condition.sh | 6 +++--- .github/integration/tests/sda/92_handle_file_errors.sh | 10 ++++++---- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/integration/tests/sda/20_ingest-verify_test.sh b/.github/integration/tests/sda/20_ingest-verify_test.sh index f9443cfb31..0c750f0cf7 100644 --- a/.github/integration/tests/sda/20_ingest-verify_test.sh +++ b/.github/integration/tests/sda/20_ingest-verify_test.sh @@ -36,7 +36,7 @@ for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/"$file.c4gh" \ + --arg filepath "$file.c4gh" \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/.github/integration/tests/sda/21_cancel_test.sh b/.github/integration/tests/sda/21_cancel_test.sh index ab6fd64b23..c7482bdbad 100644 --- a/.github/integration/tests/sda/21_cancel_test.sh +++ b/.github/integration/tests/sda/21_cancel_test.sh @@ -34,7 +34,7 @@ cancel_payload=$( jq -r -c -n \ --arg type cancel \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878_20k_b37.bam.c4gh \ + --arg filepath NA12878_20k_b37.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -65,7 +65,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878_20k_b37.bam.c4gh \ + --arg filepath NA12878_20k_b37.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/.github/integration/tests/sda/22_error_test.sh b/.github/integration/tests/sda/22_error_test.sh index 461b0f5dca..ea79144e87 100644 --- a/.github/integration/tests/sda/22_error_test.sh +++ b/.github/integration/tests/sda/22_error_test.sh @@ -29,7 +29,7 @@ if [ -n "$PGSSLCERT" ]; then fi ## get correlation id from message -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NB12878.bam.c4gh';") +CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NB12878.bam.c4gh';") properties=$( jq -c -n \ @@ -50,7 +50,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NB12878.bam.c4gh \ + --arg filepath NB12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -96,7 +96,7 @@ verify_payload=$( --arg user test@dummy.com \ --arg archive_path "$CORRID" \ --arg file_id "$CORRID" \ - --arg filepath test_dummy.org/NB12878.bam.c4gh \ + --arg filepath NB12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ --argjson re_verify false \ '$ARGS.named|@base64' diff --git a/.github/integration/tests/sda/30_backup-finalize_test.sh b/.github/integration/tests/sda/30_backup-finalize_test.sh index e4a6d4f81e..6ad7828261 100644 --- a/.github/integration/tests/sda/30_backup-finalize_test.sh +++ b/.github/integration/tests/sda/30_backup-finalize_test.sh @@ -94,7 +94,7 @@ EOD # check DB for archive file names for file in NA12878.bam.c4gh NA12878.bai.c4gh NA12878_20k_b37.bam.c4gh NA12878_20k_b37.bai.c4gh; do - archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = 'test_dummy.org/$file';") + archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = '$file';") size=$(s3cmd -c direct ls s3://backup/"$archiveName" | tr -s ' ' | cut -d ' ' -f 3) if [ "$size" -eq 0 ]; then echo "Failed to get size of $file from backup site" diff --git a/.github/integration/tests/sda/31_cancel_test2.sh b/.github/integration/tests/sda/31_cancel_test2.sh index 74cce3f025..0095a06870 100644 --- a/.github/integration/tests/sda/31_cancel_test2.sh +++ b/.github/integration/tests/sda/31_cancel_test2.sh @@ -7,7 +7,7 @@ ENC_SHA=$(sha256sum NA12878.bam.c4gh | cut -d' ' -f 1) ENC_MD5=$(md5sum NA12878.bam.c4gh | cut -d' ' -f 1) ## get correlation id from message -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NA12878.bam.c4gh';") +CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NA12878.bam.c4gh';") properties=$( @@ -30,7 +30,7 @@ cancel_payload=$( jq -r -c -n \ --arg type cancel \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878.bam.c4gh \ + --arg filepath NA12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -67,7 +67,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878.bam.c4gh \ + --arg filepath NA12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -110,7 +110,7 @@ accession_payload=$( jq -r -c -n \ --arg type accession \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878.bam.c4gh \ + --arg filepath NA12878.bam.c4gh \ --arg accession_id EGAF74900000001 \ --argjson decrypted_checksums "$decrypted_checksums" \ '$ARGS.named|@base64' diff --git a/.github/integration/tests/sda/32_test_race_condition.sh b/.github/integration/tests/sda/32_test_race_condition.sh index 58260e828c..eb04ba8e6f 100644 --- a/.github/integration/tests/sda/32_test_race_condition.sh +++ b/.github/integration/tests/sda/32_test_race_condition.sh @@ -14,7 +14,7 @@ ENC_SHA=$(sha256sum race_file.c4gh | cut -d' ' -f 1) ENC_MD5=$(md5sum race_file.c4gh | cut -d' ' -f 1) ## get correlation id from message -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/race_file.c4gh';") +CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'race_file.c4gh';") properties=$( jq -c -n \ @@ -36,7 +36,7 @@ accession_payload=$( jq -r -c -n \ --arg type accession \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/race_file.c4gh \ + --arg filepath race_file.c4gh \ --arg accession_id EGAF74900000099 \ --argjson decrypted_checksums "$decrypted_checksums" \ '$ARGS.named|@base64' @@ -74,7 +74,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/race_file.c4gh \ + --arg filepath race_file.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/.github/integration/tests/sda/92_handle_file_errors.sh b/.github/integration/tests/sda/92_handle_file_errors.sh index bc28ef3f35..7316d1bace 100644 --- a/.github/integration/tests/sda/92_handle_file_errors.sh +++ b/.github/integration/tests/sda/92_handle_file_errors.sh @@ -45,7 +45,7 @@ bad_file_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/bad.file.c4gh \ + --arg filepath bad.file.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -76,12 +76,14 @@ missing_file_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/missing.file.c4gh \ + --arg filepath missing.file.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) -CORRID=${CORRID//a/b} +FILEID=$(psql -U postgres -h postgres -d sda -At -c "SELECT DISTINCT(file_id) FROM sda.file_event_log WHERE correlation_id = '$CORRID';") +psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, message) VALUES('$FILEID', 'uploaded', '$CORRID', 'test@dummy.org', '{\"uploaded\": \"message\"}');" + properties=$( jq -c -n \ --argjson delivery_mode 2 \ @@ -133,7 +135,7 @@ truncated_file_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/truncated.c4gh \ + --arg filepath truncated.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) From 64c94494e6fc354cbcb2a192373a5aebf7921167 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 26 Nov 2024 13:36:39 +0100 Subject: [PATCH 231/262] [inbox] dont put user id in filepaths --- sda/cmd/s3inbox/proxy.go | 13 +++++++++---- sda/cmd/s3inbox/proxy_test.go | 11 ++++++----- sda/internal/helper/helper.go | 9 +++++++++ 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index e8da30b0b9..a3f61d5659 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -22,6 +22,7 @@ import ( "github.com/minio/minio-go/v6/pkg/signer" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" @@ -150,8 +151,9 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw username := token.Subject() rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) + anonymizedFilepath := helper.AnonymizeFilepath(rawFilepath, username) - filepath, err := formatUploadFilePath(rawFilepath) + filepath, err := formatUploadFilePath(anonymizedFilepath) if err != nil { reportError(http.StatusNotAcceptable, err.Error(), w) @@ -192,7 +194,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") - message, err := p.CreateMessageFromRequest(r, token) + message, err := p.CreateMessageFromRequest(r, token, anonymizedFilepath) if err != nil { p.internalServerError(w, r, err.Error()) @@ -295,6 +297,7 @@ func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { } } + log.Debugf("Sending message with id %s", p.fileIDs[r.URL.Path]) if err := p.messenger.SendMessage(p.fileIDs[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { return fmt.Errorf("error when sending message to broker: %v", err) } @@ -488,7 +491,7 @@ func (p *Proxy) detectRequestType(r *http.Request) S3RequestType { // CreateMessageFromRequest is a function that can take a http request and // figure out the correct rabbitmq message to send from it. -func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token) (Event, error) { +func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, user string) (Event, error) { event := Event{} checksum := Checksum{} var err error @@ -500,7 +503,9 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token) (Eve // Case for simple upload event.Operation = "upload" - event.Filepath = strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) + rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) + event.Filepath = helper.AnonymizeFilepath(rawFilepath, user) + event.Username = claims.Subject() checksum.Type = "sha256" event.Checksum = []any{checksum} diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 0be6752c8c..d09545284c 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -468,12 +468,13 @@ func (s *ProxyTests) TestMessageFormatting() { r.Header.Set("x-amz-content-sha256", "checksum") claims := jwt.New() - assert.NoError(s.T(), claims.Set("sub", "user@host.domain")) + user := "user@host.domain" + assert.NoError(s.T(), claims.Set("sub", user)) // start proxy that denies everything proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" - msg, err := proxy.CreateMessageFromRequest(r, claims) + msg, err := proxy.CreateMessageFromRequest(r, claims, user) assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) @@ -489,7 +490,7 @@ func (s *ProxyTests) TestMessageFormatting() { // Test single shot upload r.Method = "PUT" - msg, err = proxy.CreateMessageFromRequest(r, jwt.New()) + msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), msg.Username) assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), "upload", msg.Operation) @@ -507,7 +508,7 @@ func (s *ProxyTests) TestDatabaseConnection() { // PUT a file into the system filename := "/dummy/db-test-file" - + anonymFilename := "db-test-file" stringReader := strings.NewReader("a brand new string") r, _ := http.NewRequest("PUT", filename, stringReader) w := httptest.NewRecorder() @@ -526,7 +527,7 @@ func (s *ProxyTests) TestDatabaseConnection() { // Check that the file is in the database var fileID string query := "SELECT id FROM sda.files WHERE submission_file_path = $1;" - err = db.DB.QueryRow(query, filename[1:]).Scan(&fileID) + err = db.DB.QueryRow(query, anonymFilename).Scan(&fileID) assert.Nil(s.T(), err, "Failed to query database") assert.NotNil(s.T(), fileID, "File not found in database") diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index e337768d83..23a54e24d9 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "time" "github.com/lestrrat-go/jwx/v2/jwa" @@ -446,3 +447,11 @@ func CreatePrivateKeyFile(keyFile string, passphrase string) ([32]byte, error) { return publicKey, nil } + +func AnonymizeFilepath(fp string, username string) string { + return strings.ReplaceAll(fp, strings.Replace(username, "@", "_", 1)+"/", "") +} + +func UnanonymizeFilepath(fp string, username string) string { + return strings.Replace(username, "@", "_", 1) + "/" + fp +} From 8d9a17deac9264124b0f7f4d3541b9f9ca51b604 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 26 Nov 2024 13:38:56 +0100 Subject: [PATCH 232/262] [ingest] user id not part of filepath --- sda/cmd/ingest/ingest.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 63da694cb5..5c64923567 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -22,6 +22,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" @@ -249,7 +250,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // What if the file in the inbox is different this time? // Check uploaded checksum in the DB against the checksum of the file. // Would be easy if the inbox message had the checksum or that the s3inbox added the checksum to the DB. - file, err := app.Inbox.NewFileReader(message.FilePath) + file, err := app.Inbox.NewFileReader(helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { switch { case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): @@ -342,7 +343,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "reject" } - file, err := app.Inbox.NewFileReader(message.FilePath) + file, err := app.Inbox.NewFileReader(helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { switch { case (strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:")): @@ -373,7 +374,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } } - fileSize, err := app.Inbox.GetFileSize(message.FilePath, false) + fileSize, err := app.Inbox.GetFileSize(helper.UnanonymizeFilepath(message.FilePath, message.User), false) if err != nil { log.Errorf("Failed to get file size of file to ingest, file-id: %s, filepath: %s, reason: (%s)", fileID, message.FilePath, err.Error()) // Since reading the file worked, this should eventually succeed so it is ok to requeue. From d50c535d1f77dfb69a2cd5c3e524109d0dc81737 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 26 Nov 2024 13:39:12 +0100 Subject: [PATCH 233/262] [mapper] user id not part of filepath --- sda/cmd/mapper/mapper.go | 11 +++++---- sda/internal/database/db_functions.go | 33 +++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index af89b37386..51dba4abb8 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -10,6 +10,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" @@ -103,14 +104,14 @@ func main() { } for _, aID := range mappings.AccessionIDs { - log.Debugf("Mapped file to dataset (corr-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) - filePath, err := db.GetInboxPath(aID) + fileInfo, err := db.GetFileInfoFromAccessionID(aID) if err != nil { - log.Errorf("failed to get inbox path for file with stable ID: %s", aID) + log.Errorf("failed to get file info for file with stable ID: %s", aID) } - err = inbox.RemoveFile(filePath) + + err = inbox.RemoveFile(helper.UnanonymizeFilepath(fileInfo.FilePath, fileInfo.User)) if err != nil { - log.Errorf("Remove file from inbox failed, reason: %v", err) + log.Errorf("Remove file from inbox %s failed, reason: %v", fileInfo.FilePath, err) } } diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index c7d0dad074..40372c8d98 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -611,6 +611,39 @@ func (dbs *SDAdb) GetHeaderForStableID(stableID string) ([]byte, error) { return header, nil } +// GetFileInfoFromAccessionID retrieves the file information needed for mapping +func (dbs *SDAdb) GetFileInfoFromAccessionID(accessionID string) (SyncData, error) { + var ( + s SyncData + err error + ) + + for count := 1; count <= RetryTimes; count++ { + s, err = dbs.getFileInfoFromAccessionID(accessionID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(3, float64(count))) * time.Second) + } + + return s, err +} + +// getFileInfoFromAccessionID is the actual function performing work for GetFileInfoFromAccessionID +func (dbs *SDAdb) getFileInfoFromAccessionID(accessionID string) (SyncData, error) { + dbs.checkAndReconnectIfNeeded() + + const query = "SELECT submission_user, submission_file_path from sda.files WHERE stable_id = $1;" + var data SyncData + if err := dbs.DB.QueryRow(query, accessionID).Scan(&data.User, &data.FilePath); err != nil { + log.Warnf("Error while searching for id %s: %v", accessionID, err) + + return SyncData{}, err + } + + return data, nil +} + // GetSyncData retrieves the file information needed to sync a dataset func (dbs *SDAdb) GetSyncData(accessionID string) (SyncData, error) { var ( From fe8cdfa0924bae2ccc6848c44f2f10aa56b655e3 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 3 Dec 2024 13:55:23 +0100 Subject: [PATCH 234/262] [tests] refactor: keep all installations together --- .github/integration/tests/sda/01_install_dependencies.sh | 2 ++ .github/integration/tests/sda/10_upload_test.sh | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/tests/sda/01_install_dependencies.sh b/.github/integration/tests/sda/01_install_dependencies.sh index 9020696c1a..174d96c6dd 100644 --- a/.github/integration/tests/sda/01_install_dependencies.sh +++ b/.github/integration/tests/sda/01_install_dependencies.sh @@ -13,3 +13,5 @@ for t in curl expect jq openssh-client postgresql-client xxd; do apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null fi done + +pip -q install s3cmd diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh index 8c045f66ea..e21f474bea 100644 --- a/.github/integration/tests/sda/10_upload_test.sh +++ b/.github/integration/tests/sda/10_upload_test.sh @@ -20,8 +20,6 @@ done ## truncate database psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files, sda.encryption_keys CASCADE;" -pip -q install s3cmd - for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" if [ ! -f "$file.c4gh" ]; then From 0c10f51caea8f7e3a393658e44f5e5893c892c7e Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 13 Dec 2024 10:20:46 +0100 Subject: [PATCH 235/262] [tests] refactor, make tests more robust --- .../tests/sda/60_api_admin_test.sh | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index f68ef3ce9d..588f011d7b 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e cd shared || true @@ -41,6 +41,7 @@ fi # Reupload a file under a different name, test to delete it s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh +stream_size=$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/error_stream | jq '.messages_ready') echo "waiting for upload to complete" URI=http://rabbitmq:15672 @@ -48,15 +49,14 @@ if [ -n "$PGSSLCERT" ]; then URI=https://rabbitmq:15671 fi RETRY_TIMES=0 -until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 4 ]; do + +sleep 10 + +if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready')" ]; then echo "waiting for upload to complete" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - echo "::error::Time out while waiting for upload to complete" - exit 1 - fi - sleep 2 -done + echo "but too bad will eexit" + exit 1 +fi # get the fileId of the new file fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" @@ -109,17 +109,13 @@ URI=http://rabbitmq:15672 if [ -n "$PGSSLCERT" ]; then URI=https://rabbitmq:15671 fi -RETRY_TIMES=0 -until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do +sleep 10 + +if [ $((stream_size++)) -eq "$(curl -s -u guest:guest $URI/api/queues/sda/inbox | jq '.messages_ready')" ]; then echo "waiting for upload to complete" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 3 ]; then - echo "::error::Time out while waiting for upload to complete" - #exit 1 - break - fi - sleep 2 -done + echo "sad" + exit 1 +fi # Ingest it new_payload=$( From eb83befd0e1b69ab2d2c8ec78562a3631082ca69 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 13 Dec 2024 10:24:41 +0100 Subject: [PATCH 236/262] [tests] remove user id from filepaths --- .github/integration/tests/sda/60_api_admin_test.sh | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 588f011d7b..60a73d6300 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -59,7 +59,7 @@ if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/ fi # get the fileId of the new file -fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NC12878.bam.c4gh") | .fileID')" output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) if [ -z "$output" ] ; then @@ -120,7 +120,7 @@ fi # Ingest it new_payload=$( jq -c -n \ - --arg filepath "test_dummy.org/NE12878.bam.c4gh" \ + --arg filepath "NE12878.bam.c4gh" \ --arg user "test@dummy.org" \ '$ARGS.named' ) @@ -131,7 +131,7 @@ if [ "$resp" != "200" ]; then exit 1 fi -fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NE12878.bam.c4gh") | .fileID')" # wait for the fail to get the correct status RETRY_TIMES=0 @@ -146,7 +146,7 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_e done # Try to delete file not in inbox -fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NE12878.bam.c4gh") | .fileID')" resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "404" ]; then echo "Error when deleting the file, expected 404 got: $resp" @@ -156,8 +156,7 @@ fi # Test downloading the file from the inbox # Reupload a file under a different name, test to download it s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh -fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" - +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NC12878.bam.c4gh") | .fileID')" output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) if [ -z "$output" ] ; then echo "Uploaded file NC12878.bam.c4gh not found in inbox" From ecf4a6610aac4f10ba4e6894ba41279a5aece9b1 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 13 Dec 2024 10:25:34 +0100 Subject: [PATCH 237/262] [api] correct filepath for s3 --- sda/cmd/api/api.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 6d9c64d501..379cc51d1d 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -28,6 +28,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" @@ -396,8 +397,8 @@ func deleteFile(c *gin.Context) { return } - var retryTimes = 5 - for count := 1; count <= retryTimes; count++ { + filePath = helper.UnanonymizeFilepath(filePath, submissionUser) + for count := 1; count <= 5; count++ { err = inbox.RemoveFile(filePath) if err == nil { break From 617c4ccab6d3a32e04f4fe6840287903d0046a9e Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 16 Dec 2024 09:11:15 +0100 Subject: [PATCH 238/262] [tests] improve log messages --- .../integration/tests/sda/60_api_admin_test.sh | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 60a73d6300..5ee07ab527 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -48,13 +48,10 @@ URI=http://rabbitmq:15672 if [ -n "$PGSSLCERT" ]; then URI=https://rabbitmq:15671 fi -RETRY_TIMES=0 - sleep 10 if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready')" ]; then - echo "waiting for upload to complete" - echo "but too bad will eexit" + echo "Upload did not complete successfully" exit 1 fi @@ -104,16 +101,11 @@ fi # Re-upload the file and use the api to ingest it, then try to delete it s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NE12878.bam.c4gh - -URI=http://rabbitmq:15672 -if [ -n "$PGSSLCERT" ]; then - URI=https://rabbitmq:15671 -fi +echo "waiting for upload to complete" sleep 10 if [ $((stream_size++)) -eq "$(curl -s -u guest:guest $URI/api/queues/sda/inbox | jq '.messages_ready')" ]; then - echo "waiting for upload to complete" - echo "sad" + echo "Upload did not complete successfully" exit 1 fi @@ -213,3 +205,5 @@ if [ "$resp" != "404" ]; then echo "Trying to download a non existing file, expected 404 got: $resp" exit 1 fi + +echo "API admin tests completed successfully" \ No newline at end of file From 43e40e360741c728b716df801d43cd2f863584bd Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 17 Dec 2024 08:59:04 +0100 Subject: [PATCH 239/262] [tests] clean up --- .../tests/sda/60_api_admin_test.sh | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 5ee07ab527..2094287a13 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -41,19 +41,24 @@ fi # Reupload a file under a different name, test to delete it s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh -stream_size=$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/error_stream | jq '.messages_ready') +stream_size=$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready') -echo "waiting for upload to complete" URI=http://rabbitmq:15672 if [ -n "$PGSSLCERT" ]; then URI=https://rabbitmq:15671 fi -sleep 10 +stream_size=$((stream_size + 1)) +RETRY_TIMES=0 -if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready')" ]; then - echo "Upload did not complete successfully" - exit 1 -fi +until [ $((stream_size)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready')" ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "Upload did not complete successfully" + exit 1 + fi + sleep 2 +done # get the fileId of the new file fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NC12878.bam.c4gh") | .fileID')" @@ -101,13 +106,18 @@ fi # Re-upload the file and use the api to ingest it, then try to delete it s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NE12878.bam.c4gh -echo "waiting for upload to complete" -sleep 10 +stream_size=$((stream_size + 1)) +RETRY_TIMES=0 -if [ $((stream_size++)) -eq "$(curl -s -u guest:guest $URI/api/queues/sda/inbox | jq '.messages_ready')" ]; then - echo "Upload did not complete successfully" - exit 1 -fi +until [ $((stream_size)) -eq "$(curl -s -u guest:guest $URI/api/queues/sda/inbox | jq '.messages_ready')" ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "Upload did not complete successfully" + exit 1 + fi + sleep 2 +done # Ingest it new_payload=$( From 6c75cd1e51329bb67e60a6628b140f3c700907dc Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 17 Dec 2024 09:54:36 +0100 Subject: [PATCH 240/262] [mapper] readd log message --- sda/cmd/mapper/mapper.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 51dba4abb8..189264de3f 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -104,6 +104,7 @@ func main() { } for _, aID := range mappings.AccessionIDs { + log.Debugf("Mapped file to dataset (corr-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) fileInfo, err := db.GetFileInfoFromAccessionID(aID) if err != nil { log.Errorf("failed to get file info for file with stable ID: %s", aID) From e68f94e4eb6b8c87d16c2db7b50ebb78456e83e0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 14 Apr 2025 09:56:09 +0200 Subject: [PATCH 241/262] [ingest test] fix tests for username file path --- sda/cmd/ingest/ingest_test.go | 70 +++++++++++++++++------------------ 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 02e3f0f018..ead55be340 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -170,6 +170,7 @@ type TestSuite struct { pubKeyList [][32]byte ingest Ingest tempDir string + UserName string } func (ts *TestSuite) SetupSuite() { @@ -237,6 +238,8 @@ func (ts *TestSuite) SetupSuite() { if err := ts.ingest.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), "the test key"); err != nil { ts.FailNow("failed to register the public key") } + + ts.UserName = "test-ingest" } func (ts *TestSuite) TearDownSuite() { @@ -257,7 +260,13 @@ func (ts *TestSuite) SetupTest() { ts.FailNow("failed to create temp folder") } - f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") + // Ensure a folder with the user name exists + err = os.Mkdir(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName), 0750) + if err != nil { + ts.FailNow("failed to create temp folder") + } + + f, err := os.CreateTemp(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName), "") if err != nil { ts.FailNow("failed to create test file") } @@ -416,38 +425,36 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // messages of type `ingest` func (ts *TestSuite) TestIngestFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } func (ts *TestSuite) TestIngestFile_secondTime() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -456,30 +463,28 @@ func (ts *TestSuite) TestIngestFile_secondTime() { assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) } func (ts *TestSuite) TestIngestFile_unknownInboxType() { - userName := "test-ingest-unknown" message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(uuid.New().String(), message)) } func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -492,19 +497,18 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -530,7 +534,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName, ts.filePath)) if err != nil { ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } @@ -564,19 +568,18 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -596,19 +599,18 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -632,19 +634,18 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -687,7 +688,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName, ts.filePath)) if err != nil { ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } @@ -723,14 +724,13 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_missingFile() { // prepare the DB entries - userName := "test-ingest" corrID := uuid.New().String() basepath := filepath.Dir(ts.filePath) message := schema.IngestionTrigger{ Type: "ingest", FilePath: fmt.Sprintf("%s/missing.file.c4gh", basepath), - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) From 1d9b2604d815a07617468a27ecbcd3da6956e6d4 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Apr 2025 13:14:07 +0200 Subject: [PATCH 242/262] [helper_test] add test for anonymize/unanonymize file paths --- sda/internal/helper/helper_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/sda/internal/helper/helper_test.go b/sda/internal/helper/helper_test.go index 42f538817c..c6821cfae0 100644 --- a/sda/internal/helper/helper_test.go +++ b/sda/internal/helper/helper_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/lestrrat-go/jwx/v2/jwk" @@ -151,3 +152,24 @@ func (ts *HelperTest) TestCreatePrivateKeyFile() { // Verify the public key is not empty assert.NotEqual(ts.T(), [32]byte{}, publicKey, "Public key should not be empty") } + +func (ts *HelperTest) TestAnonymizeFilepath() { + filePath := "main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := AnonymizeFilepath(filepath.Join(strings.Replace(userName, "@", "_", 1), filePath), userName) + assert.Equal(ts.T(), filePath, newPath) +} + +func (ts *HelperTest) TestAnonymizeFilepath_noUser() { + filePath := "main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := AnonymizeFilepath(filePath, userName) + assert.Equal(ts.T(), filePath, newPath) +} + +func (ts *HelperTest) TestUnanonymizeFilepath() { + filePath := "main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := UnanonymizeFilepath(filePath, userName) + assert.Equal(ts.T(), filepath.Join(strings.Replace(userName, "@", "_", 1), filePath), newPath) +} From 3d3d65d1e53fc7bfd26ebf0c1302d94d1cfae664 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 22 Apr 2025 13:14:40 +0200 Subject: [PATCH 243/262] [helper] use filepath.Join when concatenating file paths --- sda/internal/helper/helper.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index 23a54e24d9..d912ccb389 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -453,5 +453,5 @@ func AnonymizeFilepath(fp string, username string) string { } func UnanonymizeFilepath(fp string, username string) string { - return strings.Replace(username, "@", "_", 1) + "/" + fp + return filepath.Join(strings.Replace(username, "@", "_", 1), fp) } From cacb2875ffe30143c8c0e4408c8d380f308a78af Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 27 May 2025 11:21:49 +0200 Subject: [PATCH 244/262] [api] unanonymize filepath for download #major --- sda/cmd/api/api.go | 9 +++++++-- sda/cmd/api/api_test.go | 6 ++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 379cc51d1d..77463a5963 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -492,8 +492,13 @@ func downloadFile(c *gin.Context) { return } - // Get inbox file handle - file, err := inbox.NewFileReader(filePath) + // Get inbox file handle #noqa + file, err := inbox.NewFileReader( + helper.UnanonymizeFilepath( + filePath, + strings.TrimPrefix(c.Param("username"), "/"), + ), + ) if err != nil { log.Errorf("inbox file %s not found or failed to read, %s", filePath, err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to read inbox file") diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 71a1f44766..ddbf40d4cc 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -2588,8 +2588,7 @@ func (s *TestSuite) TestDownloadFile() { defer ts.Close() // Register the file in the database - filePath := fmt.Sprintf("/%s/%s", s.User, filepath.Base(s.GoodC4ghFile)) - fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) + fileID, err := Conf.API.DB.RegisterFile(filepath.Base(s.GoodC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") @@ -2708,8 +2707,7 @@ func (s *TestSuite) TestDownloadFile_badC4ghFile() { defer ts.Close() // Register a file in the database (but don't create the actual file) - filePath := fmt.Sprintf("/%s/%s", s.User, filepath.Base(s.BadC4ghFile)) - fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) + fileID, err := Conf.API.DB.RegisterFile(filepath.Base(s.BadC4ghFile), s.User) assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") assert.NoError(s.T(), err, "failed to update satus of file in database") From 7a5cafad34e39127cc3581b69805cb8efde340cf Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 13 Aug 2025 15:58:35 +0200 Subject: [PATCH 245/262] [helper] support old style file paths as well. --- sda/internal/helper/helper.go | 4 ++++ sda/internal/helper/helper_test.go | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index d912ccb389..08ad03e002 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -453,5 +453,9 @@ func AnonymizeFilepath(fp string, username string) string { } func UnanonymizeFilepath(fp string, username string) string { + if strings.HasPrefix(fp, strings.Replace(username, "@", "_", 1)) { + return fp + } + return filepath.Join(strings.Replace(username, "@", "_", 1), fp) } diff --git a/sda/internal/helper/helper_test.go b/sda/internal/helper/helper_test.go index c6821cfae0..01c9227cea 100644 --- a/sda/internal/helper/helper_test.go +++ b/sda/internal/helper/helper_test.go @@ -173,3 +173,9 @@ func (ts *HelperTest) TestUnanonymizeFilepath() { newPath := UnanonymizeFilepath(filePath, userName) assert.Equal(ts.T(), filepath.Join(strings.Replace(userName, "@", "_", 1), filePath), newPath) } +func (ts *HelperTest) TestUnanonymizeFilepath_oldMessage() { + filePath := "test.user_demo.org/main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := UnanonymizeFilepath(filePath, userName) + assert.Equal(ts.T(), filePath, newPath) +} From 59c2a60553aef4159514968d5bdcdcb7313cfec0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 13 Aug 2025 15:59:02 +0200 Subject: [PATCH 246/262] [integration tests] old style file paths should be ok. --- .github/integration/tests/sda/20_ingest-verify_test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/integration/tests/sda/20_ingest-verify_test.sh b/.github/integration/tests/sda/20_ingest-verify_test.sh index 0c750f0cf7..f500d68781 100644 --- a/.github/integration/tests/sda/20_ingest-verify_test.sh +++ b/.github/integration/tests/sda/20_ingest-verify_test.sh @@ -32,6 +32,10 @@ for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))' ) + if [ "$file" = "NA12878.bam" ]; then + file="test_dummy.org/NA12878.bam" + fi + ingest_payload=$( jq -r -c -n \ --arg type ingest \ From 0a71b413e80f5c51a2e2001ce4df2a6b24989745 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 18 Aug 2025 11:09:44 +0200 Subject: [PATCH 247/262] [charts] Create job to clean up inbox paths This job removes the username if it is the first part of the inbox path. --- charts/sda-svc/README.md | 11 ++- .../templates/job-remove-user-name.yaml | 68 +++++++++++++++++++ charts/sda-svc/values.yaml | 1 + 3 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 charts/sda-svc/templates/job-remove-user-name.yaml diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index b8e26a003d..148ed4acfa 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -18,8 +18,13 @@ A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an ### To 3.0.0 -This version adds Jobs that migrates the database schema from a V1 release and sets the first registered crypt4gh key to all ingested files that do not have a c4gh key referenced. -When upgrading from a V1 release *both* `upgradeFromV1` and `setKeyHash` should be run. +This version adds Jobs that performes the following actions: + +- Migrates the database schema from a V1 release. +- Sets the first registered crypt4gh key to all ingested files that do not have a c4gh key referenced. +- Removes the submitters username from the submission file paths in the database. + +When upgrading from a V1 release *All these jobs* should be run. Unless the same queries are being executed manually by a database administrator a `Basic authentication Secret` containing the credentials to perform the upgrade needs to be created. @@ -30,7 +35,7 @@ The following table lists the configurable parameters of the `sda-svc` chart and Parameter | Description | Default --------- | ----------- | ------- `image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` -`image.tag` | Tag version to deploy | `` +`image.tag` | Tag version to deploy | `chart appVersion` `image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` diff --git a/charts/sda-svc/templates/job-remove-user-name.yaml b/charts/sda-svc/templates/job-remove-user-name.yaml new file mode 100644 index 0000000000..650b959e45 --- /dev/null +++ b/charts/sda-svc/templates/job-remove-user-name.yaml @@ -0,0 +1,68 @@ +{{- if .Values.jobs.removeUserName }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sda.fullname" . }}-remove-user-name-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: hook-succeeded +spec: + template: + metadata: + name: {{ template "sda.fullname" . }}-remove-user-name-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + spec: + containers: + - name: {{ template "sda.fullname" . }}-remove-user-name-job + command: ["/usr/local/bin/psql"] + args: [ + "-qc", + "UPDATE sda.files SET submission_file_path = REPLACE(submission_file_path, CONCAT(REPLACE(submission_user, '@', '_'),'/'), '');" + ] + env: + - name: PGDATABASE + value: {{ .Values.global.db.name }} + - name: PGHOST + value: {{ .Values.global.db.host }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- if .Values.global.db.port }} + - name: PGPORT + value: {{ .Values.global.db.port | quote }} + {{- end }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + image: {{ .Values.jobs.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + restartPolicy: Never + securityContext: + runAsUser: 72 + runAsGroup: 72 + fsGroup: 72 +{{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 5ec6f3e090..5e171e857a 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -722,5 +722,6 @@ verify: jobs: image: postgres:15.4-alpine + removeUserName: false setKeyHash: false upgradeFromV1: false \ No newline at end of file From 3af7770f634722af55d57e3906c428e3fc204283 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 1 Sep 2025 11:34:45 +0200 Subject: [PATCH 248/262] Update charts/sda-svc/README.md Co-authored-by: Nanjiang Shu --- charts/sda-svc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 148ed4acfa..18506b5800 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -18,7 +18,7 @@ A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an ### To 3.0.0 -This version adds Jobs that performes the following actions: +This version adds jobs that perform the following actions: - Migrates the database schema from a V1 release. - Sets the first registered crypt4gh key to all ingested files that do not have a c4gh key referenced. From e79b1801a774cbf40f8689bbcc98812fd1c732f6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 3 Sep 2025 09:54:01 +0200 Subject: [PATCH 249/262] [api] No need to concatenate the user name to the file path anymore #major --- sda/cmd/api/api.go | 14 +++----------- sda/cmd/api/api_test.go | 4 ++-- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 77463a5963..b0bfe22554 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -305,12 +305,7 @@ func getFiles(c *gin.Context) { return } - prefix := c.Query("path_prefix") - if prefix != "" { - prefix = fmt.Sprintf("%s/%s", strings.ReplaceAll(token.Subject(), "@", "_"), prefix) - } - - files, err := Conf.API.DB.GetUserFiles(token.Subject(), prefix, false) + files, err := Conf.API.DB.GetUserFiles(token.Subject(), c.Query("path_prefix"), false) if err != nil { // something went wrong with querying or parsing rows c.JSON(502, err.Error()) @@ -737,11 +732,8 @@ func listUserFiles(c *gin.Context) { username = strings.TrimPrefix(username, "/") username = strings.TrimSuffix(username, "/files") log.Debugln(username) - prefix := c.Query("path_prefix") - if prefix != "" { - prefix = fmt.Sprintf("%s/%s", strings.ReplaceAll(username, "@", "_"), prefix) - } - files, err := Conf.API.DB.GetUserFiles(username, prefix, true) + + files, err := Conf.API.DB.GetUserFiles(username, c.Query("path_prefix"), true) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index ddbf40d4cc..85f57fb799 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -717,7 +717,7 @@ func (s *TestSuite) TestAPIGetFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", user, sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } @@ -1931,7 +1931,7 @@ func (s *TestSuite) TestListUserFiles_filteredSelection() { sub = "submission_b" } - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", user, sub, i), strings.ReplaceAll(user, "_", "@")) + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) if err != nil { s.FailNow("failed to register file in database") } From d9ec94ca08acb15f4f6082228dd98cb269a97cbf Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 4 Sep 2025 10:54:49 +0200 Subject: [PATCH 250/262] [s3inbox] feed correct file path to msg function --- sda/cmd/s3inbox/proxy.go | 5 ++--- sda/cmd/s3inbox/proxy_test.go | 7 ++++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index a3f61d5659..0604ec6883 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -491,7 +491,7 @@ func (p *Proxy) detectRequestType(r *http.Request) S3RequestType { // CreateMessageFromRequest is a function that can take a http request and // figure out the correct rabbitmq message to send from it. -func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, user string) (Event, error) { +func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anonymizedFilepath string) (Event, error) { event := Event{} checksum := Checksum{} var err error @@ -503,8 +503,7 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, user // Case for simple upload event.Operation = "upload" - rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) - event.Filepath = helper.AnonymizeFilepath(rawFilepath, user) + event.Filepath = anonymizedFilepath event.Username = claims.Subject() checksum.Type = "sha256" diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index d09545284c..13e901b58a 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -474,12 +474,12 @@ func (s *ProxyTests) TestMessageFormatting() { // start proxy that denies everything proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" - msg, err := proxy.CreateMessageFromRequest(r, claims, user) + msg, err := proxy.CreateMessageFromRequest(r, claims, "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), int64(1234), msg.Filesize) - assert.Equal(s.T(), "user/new_file.txt", msg.Filepath) + assert.Equal(s.T(), "new_file.txt", msg.Filepath) assert.Equal(s.T(), "user@host.domain", msg.Username) c, _ := json.Marshal(msg.Checksum[0]) @@ -490,10 +490,11 @@ func (s *ProxyTests) TestMessageFormatting() { // Test single shot upload r.Method = "PUT" - msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), msg.Username) + msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), "upload", msg.Operation) + assert.Equal(s.T(), "new_file.txt", msg.Filepath) } func (s *ProxyTests) TestDatabaseConnection() { From 4c94ee2ce8905dad65ca25aa8d0c37eb8186970e Mon Sep 17 00:00:00 2001 From: neicnordic Date: Thu, 4 Sep 2025 13:11:35 +0000 Subject: [PATCH 251/262] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index f569323a20..e18fd38071 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 2.0.17 -appVersion: v2.1.38 +version: 2.0.18 +appVersion: v3.0.0 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index d6a63f6647..0ecc70fde0 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 2.0.17 -appVersion: v2.1.38 +version: 2.0.18 +appVersion: v3.0.0 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 306e29522b..1640bf8b46 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 3.0.9 -appVersion: v2.1.38 +version: 3.0.10 +appVersion: v3.0.0 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 22932b9b445d9a0d6f4f859fb7ab03065fb585b2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:31:08 +0000 Subject: [PATCH 252/262] Bump com.amazonaws:aws-java-sdk-s3 Bumps the all-modules group in /sda-sftp-inbox with 1 update: [com.amazonaws:aws-java-sdk-s3](https://github.com/aws/aws-sdk-java). Updates `com.amazonaws:aws-java-sdk-s3` from 1.12.788 to 1.12.791 - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.788...1.12.791) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-version: 1.12.791 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 0a3f9e46d9..37324fd37d 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -108,7 +108,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.788 + 1.12.791 com.google.guava From 55540d29343a014e4902420621c5bb6b173f3f6f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:42:05 +0000 Subject: [PATCH 253/262] Bump no.elixir:crypt4gh in /sda-doa in the all-modules group Bumps the all-modules group in /sda-doa with 1 update: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.32 to 3.0.33 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.32...crypt4gh-3.0.33) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.33 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 50b9f937f9..a7d98e50cc 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.32 + 3.0.33 org.slf4j From b92c7d095394b9467241984818a3ec039ad29c72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:44:25 +0000 Subject: [PATCH 254/262] Bump codecov/codecov-action from 5.5.0 to 5.5.1 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.5.0 to 5.5.1. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.5.0...v5.5.1) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 5.5.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 307fdbe382..75a764deb0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.0 + uses: codecov/codecov-action@v5.5.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.0 + uses: codecov/codecov-action@v5.5.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.5.0 + uses: codecov/codecov-action@v5.5.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt From 5fd7087e9e645888afda3daa8e4cbae93db92fd1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:45:39 +0000 Subject: [PATCH 255/262] Bump actions/setup-python from 5 to 6 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 5 to 6. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-python dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/functionality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml index f752e0ef42..5e6cbca4fb 100644 --- a/.github/workflows/functionality.yml +++ b/.github/workflows/functionality.yml @@ -35,7 +35,7 @@ jobs: fail-fast: false steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" From 15e592c6aa8a0849be2e549a939b279a9e65f198 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:46:29 +0000 Subject: [PATCH 256/262] Bump aquasecurity/trivy-action from 0.33.0 to 0.33.1 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.33.0 to 0.33.1. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.33.0...0.33.1) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-version: 0.33.1 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- .github/workflows/scan-images.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index acfe1e4cca..813b943d93 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -105,7 +105,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.33.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -123,7 +123,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.33.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -172,7 +172,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.33.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -203,7 +203,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.33.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml index 28dd648260..c1ea49fdcc 100644 --- a/.github/workflows/scan-images.yml +++ b/.github/workflows/scan-images.yml @@ -27,7 +27,7 @@ jobs: image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.33.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -53,7 +53,7 @@ jobs: security-events: write steps: - name: Run Trivy vulnerability scanner - uses: aquasecurity/trivy-action@0.33.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From e55c16e3dbb779245ca3f4ee565d7570d59d8de6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:49:08 +0000 Subject: [PATCH 257/262] Bump actions/setup-go from 5 to 6 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 5 to 6. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v5...v6) --- updated-dependencies: - dependency-name: actions/setup-go dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- .github/workflows/release_sda-admin.yaml | 2 +- .github/workflows/test.yml | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 495067c419..03d87c40b1 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -37,7 +37,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go @@ -58,7 +58,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go @@ -79,7 +79,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go diff --git a/.github/workflows/release_sda-admin.yaml b/.github/workflows/release_sda-admin.yaml index a28b4d5cef..43c7f1d387 100644 --- a/.github/workflows/release_sda-admin.yaml +++ b/.github/workflows/release_sda-admin.yaml @@ -40,7 +40,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: '>=1.22' id: go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 307fdbe382..7649917713 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go @@ -73,7 +73,7 @@ jobs: steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go @@ -108,7 +108,7 @@ jobs: steps: - name: Set up Go ${{ matrix.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ matrix.go-version }} id: go From 8389ffe7e2630d621173611a7feaa78dff8e86f8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 19:56:55 +0000 Subject: [PATCH 258/262] Bump the all-modules group in /sda with 9 updates Bumps the all-modules group in /sda with 9 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.38.3` | `1.39.0` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.31.6` | `1.31.7` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.18.10` | `1.18.11` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.19.4` | `1.19.5` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.87.3` | `1.88.0` | | [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) | `2.121.0` | `2.122.0` | | [github.com/spf13/viper](https://github.com/spf13/viper) | `1.20.1` | `1.21.0` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.41.0` | `0.42.0` | | [golang.org/x/oauth2](https://github.com/golang/oauth2) | `0.30.0` | `0.31.0` | Updates `github.com/aws/aws-sdk-go-v2` from 1.38.3 to 1.39.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.38.3...v1.39.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.31.6 to 1.31.7 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.31.6...config/v1.31.7) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.18.10 to 1.18.11 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.18.10...config/v1.18.11) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.19.4 to 1.19.5 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/m2/v1.19.4...service/pi/v1.19.5) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.87.3 to 1.88.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.87.3...service/s3/v1.88.0) Updates `github.com/casbin/casbin/v2` from 2.121.0 to 2.122.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.121.0...v2.122.0) Updates `github.com/spf13/viper` from 1.20.1 to 1.21.0 - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.20.1...v1.21.0) Updates `golang.org/x/crypto` from 0.41.0 to 0.42.0 - [Commits](https://github.com/golang/crypto/compare/v0.41.0...v0.42.0) Updates `golang.org/x/oauth2` from 0.30.0 to 0.31.0 - [Commits](https://github.com/golang/oauth2/compare/v0.30.0...v0.31.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-version: 1.39.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.31.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.18.11 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.19.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.88.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.122.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/spf13/viper dependency-version: 1.21.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.42.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-version: 0.31.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 56 +++++++++++++------------- sda/go.sum | 116 ++++++++++++++++++++++++++--------------------------- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 395ad67f53..27e056de21 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,13 +3,13 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.38.3 - github.com/aws/aws-sdk-go-v2/config v1.31.6 - github.com/aws/aws-sdk-go-v2/credentials v1.18.10 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4 - github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3 + github.com/aws/aws-sdk-go-v2 v1.39.0 + github.com/aws/aws-sdk-go-v2/config v1.31.7 + github.com/aws/aws-sdk-go-v2/credentials v1.18.11 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5 + github.com/aws/aws-sdk-go-v2/service/s3 v1.88.0 github.com/aws/smithy-go v1.23.0 - github.com/casbin/casbin/v2 v2.121.0 + github.com/casbin/casbin/v2 v2.122.0 github.com/coreos/go-oidc/v3 v3.15.0 github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 @@ -29,10 +29,10 @@ require ( github.com/rabbitmq/amqp091-go v1.10.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.1 + github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.41.0 - golang.org/x/oauth2 v0.30.0 + golang.org/x/crypto v0.42.0 + golang.org/x/oauth2 v0.31.0 google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.8 ) @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect github.com/andybalholm/brotli v1.1.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.6 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.6 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.29.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.38.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect github.com/bytedance/sonic v1.13.2 // indirect @@ -131,13 +131,13 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.9.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.14.0 // indirect - github.com/spf13/cast v1.8.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tdewolff/minify/v2 v2.23.6 // indirect github.com/tdewolff/parse/v2 v2.8.1 // indirect @@ -150,12 +150,12 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yosssi/ace v0.0.5 // indirect - go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.17.0 // indirect golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect golang.org/x/time v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 8a763a93ba..6e98f3a961 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= -github.com/aws/aws-sdk-go-v2 v1.38.3 h1:B6cV4oxnMs45fql4yRH+/Po/YU+597zgWqvDpYMturk= -github.com/aws/aws-sdk-go-v2 v1.38.3/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= +github.com/aws/aws-sdk-go-v2 v1.39.0 h1:xm5WV/2L4emMRmMjHFykqiA4M/ra0DJVSWUkDyBjbg4= +github.com/aws/aws-sdk-go-v2 v1.39.0/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= -github.com/aws/aws-sdk-go-v2/config v1.31.6 h1:a1t8fXY4GT4xjyJExz4knbuoxSCacB5hT/WgtfPyLjo= -github.com/aws/aws-sdk-go-v2/config v1.31.6/go.mod h1:5ByscNi7R+ztvOGzeUaIu49vkMk2soq5NaH5PYe33MQ= -github.com/aws/aws-sdk-go-v2/credentials v1.18.10 h1:xdJnXCouCx8Y0NncgoptztUocIYLKeQxrCgN6x9sdhg= -github.com/aws/aws-sdk-go-v2/credentials v1.18.10/go.mod h1:7tQk08ntj914F/5i9jC4+2HQTAuJirq7m1vZVIhEkWs= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6 h1:wbjnrrMnKew78/juW7I2BtKQwa1qlf6EjQgS69uYY14= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.6/go.mod h1:AtiqqNrDioJXuUgz3+3T0mBWN7Hro2n9wll2zRUc0ww= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4 h1:BTl+TXrpnrpPWb/J3527GsJ/lMkn7z3GO12j6OlsbRg= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.4/go.mod h1:cG2tenc/fscpChiZE29a2crG9uo2t6nQGflFllFL8M8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6 h1:uF68eJA6+S9iVr9WgX1NaRGyQ/6MdIyc4JNUo6TN1FA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.6/go.mod h1:qlPeVZCGPiobx8wb1ft0GHT5l+dc6ldnwInDFaMvC7Y= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6 h1:pa1DEC6JoI0zduhZePp3zmhWvk/xxm4NB8Hy/Tlsgos= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.6/go.mod h1:gxEjPebnhWGJoaDdtDkA0JX46VRg1wcTHYe63OfX5pE= +github.com/aws/aws-sdk-go-v2/config v1.31.7 h1:zS1O6hr6t0nZdBCMFc/c9OyZFyLhXhf/B2IZ9Y0lRQE= +github.com/aws/aws-sdk-go-v2/config v1.31.7/go.mod h1:GpHmi1PQDdL5pP4JaB00pU0ek4EXVcYH7IkjkUadQmM= +github.com/aws/aws-sdk-go-v2/credentials v1.18.11 h1:1Fnb+7Dk96/VYx/uYfzk5sU2V0b0y2RWZROiMZCN/Io= +github.com/aws/aws-sdk-go-v2/credentials v1.18.11/go.mod h1:iuvn9v10dkxU4sDgtTXGWY0MrtkEcmkUmjv4clxhuTc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 h1:Is2tPmieqGS2edBnmOJIbdvOA6Op+rRpaYR60iBAwXM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7/go.mod h1:F1i5V5421EGci570yABvpIXgRIBPb5JM+lSkHF6Dq5w= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5 h1:fSuJX/VBJKufwJG/szWgUdRJVyRiEQDDXNh/6NPrTBg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5/go.mod h1:LvN0noQuST+3Su55Wl++BkITpptnfN9g6Ohkv4zs9To= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 h1:UCxq0X9O3xrlENdKf1r9eRJoKz/b0AfGkpp3a7FPlhg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7/go.mod h1:rHRoJUNUASj5Z/0eqI4w32vKvC7atoWR0jC+IkmVH8k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 h1:Y6DTZUn7ZUC4th9FMBbo8LVE+1fyq3ofw+tRwkUd3PY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7/go.mod h1:x3XE6vMnU9QvHN/Wrx2s44kwzV2o2g5x/siw4ZUJ9g8= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.6 h1:R0tNFJqfjHL3900cqhXuwQ+1K4G0xc9Yf8EDbFXCKEw= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.6/go.mod h1:y/7sDdu+aJvPtGXr4xYosdpq9a6T9Z0jkXfugmti0rI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 h1:BszAktdUo2xlzmYHjWMq70DqJ7cROM8iBd3f6hrpuMQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7/go.mod h1:XJ1yHki/P7ZPuG4fd3f0Pg/dSGA2cTQBCLw82MH2H48= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.6 h1:hncKj/4gR+TPauZgTAsxOxNcvBayhUlYZ6LO/BYiQ30= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.6/go.mod h1:OiIh45tp6HdJDDJGnja0mw8ihQGz3VGrUflLqSL0SmM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6 h1:LHS1YAIJXJ4K9zS+1d/xa9JAA9sL2QyXIQCQFQW/X08= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.6/go.mod h1:c9PCiTEuh0wQID5/KqA32J+HAgZxN9tOGXKCiYJjTZI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.6 h1:nEXUSAwyUfLTgnc9cxlDWy637qsq4UWwp3sNAfl0Z3Y= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.6/go.mod h1:HGzIULx4Ge3Do2V0FaiYKcyKzOqwrhUZgCI77NisswQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3 h1:ETkfWcXP2KNPLecaDa++5bsQhCRa5M5sLUJa5DWYIIg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.87.3/go.mod h1:+/3ZTqoYb3Ur7DObD00tarKMLMuKg8iqz5CHEanqTnw= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.1 h1:8OLZnVJPvjnrxEwHFg9hVUof/P4sibH+Ea4KKuqAGSg= -github.com/aws/aws-sdk-go-v2/service/sso v1.29.1/go.mod h1:27M3BpVi0C02UiQh1w9nsBEit6pLhlaH3NHna6WUbDE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2 h1:gKWSTnqudpo8dAxqBqZnDoDWCiEh/40FziUjr/mo6uA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.2/go.mod h1:x7+rkNmRoEN1U13A6JE2fXne9EWyJy54o3n6d4mGaXQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.2 h1:YZPjhyaGzhDQEvsffDEcpycq49nl7fiGcfJTIo8BszI= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.2/go.mod h1:2dIN8qhQfv37BdUYGgEC8Q3tteM3zFxTI1MLO2O3J3c= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 h1:zmZ8qvtE9chfhBPuKB2aQFxW5F/rpwXUgmcVCgQzqRw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7/go.mod h1:vVYfbpd2l+pKqlSIDIOgouxNsGu5il9uDp0ooWb0jys= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 h1:mLgc5QIgOy26qyh5bvW+nDoAppxgn3J2WV3m9ewq7+8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7/go.mod h1:wXb/eQnqt8mDQIQTTmcw58B5mYGxzLGZGK8PWNFZ0BA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 h1:u3VbDKUCWarWiU+aIUK4gjTr/wQFXV17y3hgNno9fcA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7/go.mod h1:/OuMQwhSyRapYxq6ZNpPer8juGNrB4P5Oz8bZ2cgjQE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.88.0 h1:k5JXPr+2SrPDwM3PdygZUenn0lVPLa3KOs7cCYqinFs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.88.0/go.mod h1:xajPTguLoeQMAOE44AAP2RQoUhF8ey1g5IFHARv71po= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.2 h1:rcoTaYOhGE/zfxE1uR6X5fvj+uKkqeCNRE0rBbiQM34= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.2/go.mod h1:Ql6jE9kyyWI5JHn+61UT/Y5Z0oyVJGmgmJbZD5g4unY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3 h1:BSIfeFtU9tlSt8vEYS7KzurMoAuYzYPWhcZiMtxVf2M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3/go.mod h1:XclEty74bsGBCr1s0VSaA11hQ4ZidK4viWK7rRfO88I= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.3 h1:yEiZ0ztgji2GsCb/6uQSITXcGdtmWMfLRys0jJFiUkc= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.3/go.mod h1:Z+Gd23v97pX9zK97+tX4ppAgqCt3Z2dIXB02CtBncK8= github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -72,8 +72,8 @@ github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/casbin/casbin/v2 v2.121.0 h1:lrgTnLJTsdpe8Kdgi+NedM9+K7ftYBaK19OE+IZUwdk= -github.com/casbin/casbin/v2 v2.121.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.122.0 h1:T960HruD6W3JZmoYQjKAu0YBLUL++5l7LormNJIcmkc= +github.com/casbin/casbin/v2 v2.122.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= @@ -302,8 +302,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= -github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -320,16 +320,16 @@ github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= -github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= -github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= -github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= -github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -397,8 +397,8 @@ go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mx go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -410,8 +410,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -437,10 +437,10 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo= +golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -471,8 +471,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -482,8 +482,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -494,8 +494,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 5edf9d1937379c291a25dc57b2745342d3ead1b6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 10 Sep 2025 11:55:52 +0200 Subject: [PATCH 259/262] [tests] update reencrypt config in the download tests --- sda-download/dev_utils/config-notls.yaml | 5 ++++- sda-download/dev_utils/config.yaml | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index 13b90a638a..f67d8d7f5a 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -19,8 +19,11 @@ grpc: port: 50051 c4gh: - passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" + privateKeys: + - filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" db: host: "localhost" diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index 062299df62..a75f9bfec3 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -41,8 +41,11 @@ session: name: "sda_session_key" c4gh: - passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" + privateKeys: + - filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" transientPassphrase: "" transientKeyPath: "" From 4595c4e272cdae0608fe0e55f229e09a89db7036 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 8 Sep 2025 20:20:43 +0000 Subject: [PATCH 260/262] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/spf13/viper](https://github.com/spf13/viper) and [golang.org/x/crypto](https://github.com/golang/crypto). Updates `github.com/spf13/viper` from 1.20.1 to 1.21.0 - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.20.1...v1.21.0) Updates `golang.org/x/crypto` from 0.41.0 to 0.42.0 - [Commits](https://github.com/golang/crypto/compare/v0.41.0...v0.42.0) --- updated-dependencies: - dependency-name: github.com/spf13/viper dependency-version: 1.21.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.42.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 24 ++++++++++----------- sda-download/go.sum | 52 ++++++++++++++++++++++----------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 26729d0c66..be1582d707 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -13,9 +13,9 @@ require ( github.com/lib/pq v1.10.9 github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.1 + github.com/spf13/viper v1.21.0 github.com/stretchr/testify v1.11.1 - golang.org/x/crypto v0.41.0 + golang.org/x/crypto v0.42.0 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 google.golang.org/grpc v1.75.0 google.golang.org/protobuf v1.36.8 @@ -55,22 +55,22 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/sagikazarmark/locafero v0.9.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.14.0 // indirect - github.com/spf13/cast v1.8.0 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - go.uber.org/multierr v1.11.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/arch v0.17.0 // indirect - golang.org/x/net v0.42.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - golang.org/x/tools v0.35.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/tools v0.36.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 07eca7c911..ad2264e99f 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -117,8 +117,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sagikazarmark/locafero v0.9.0 h1:GbgQGNtTrEmddYDSAH9QLRyfAHY12md+8YFTqyMTC9k= -github.com/sagikazarmark/locafero v0.9.0/go.mod h1:UBUyz37V+EdMS3hDF3QWIiVr/2dPrx49OMO0Bn0hJqk= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= @@ -126,17 +126,17 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIG github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.14.0 h1:9tH6MapGnn/j0eb0yIXiLjERO8RB6xIVZRDCX7PtqWA= -github.com/spf13/afero v1.14.0/go.mod h1:acJQ8t0ohCGuMN3O+Pv0V0hgMxNYDlvdk+VTfyZmbYo= -github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk= -github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= -github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -169,14 +169,14 @@ go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFh go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -188,8 +188,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.42.0 h1:jzkYrhi3YQWD6MLBJcsklgQsoAcw89EcZbJw8Z614hs= -golang.org/x/net v0.42.0/go.mod h1:FF1RA5d3u7nAYA4z2TkclSCKh68eSXtiFwcWQpPXdt8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -204,31 +204,31 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.34.0 h1:O/2T7POpk0ZZ7MAzMeWFSg6S5IpWd/RXDlM9hgM3DR4= -golang.org/x/term v0.34.0/go.mod h1:5jC53AEywhIVebHgPVeg0mj8OD3VO9OzclacVrqpaAw= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.35.0 h1:mBffYraMEf7aa0sB+NuKnuCy8qI/9Bughn8dC2Gu5r0= -golang.org/x/tools v0.35.0/go.mod h1:NKdj5HkL/73byiZSJjqJgKn3ep7KjFkBOkR/Hps3VPw= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= From 5589e23ab68e87d993a66ab25874055ffdcd7f08 Mon Sep 17 00:00:00 2001 From: Krzysztof Kochel Date: Mon, 15 Sep 2025 23:12:05 +0200 Subject: [PATCH 261/262] Fix tests --- .github/integration/tests/sda/40_mapper_test.sh | 2 +- .github/integration/tests/sda/45_sync_test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/tests/sda/40_mapper_test.sh b/.github/integration/tests/sda/40_mapper_test.sh index 49cb486d71..314ffa4bf2 100644 --- a/.github/integration/tests/sda/40_mapper_test.sh +++ b/.github/integration/tests/sda/40_mapper_test.sh @@ -115,7 +115,7 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dat echo "::error::Time out while waiting for dataset to be released" exit 1 fi - sleep 2 + sleep 4 done echo "dataset released successfully" diff --git a/.github/integration/tests/sda/45_sync_test.sh b/.github/integration/tests/sda/45_sync_test.sh index 4d44aef99e..5dc6d12215 100644 --- a/.github/integration/tests/sda/45_sync_test.sh +++ b/.github/integration/tests/sda/45_sync_test.sh @@ -17,7 +17,7 @@ for file in NA12878.bai NA12878_20k_b37.bai; do echo "::error::Time out while waiting for files to be synced" exit 1 fi - sleep 2 + sleep 4 done done From f4d3673ae1fb67e8399fe60406bca3f28c50caef Mon Sep 17 00:00:00 2001 From: Krzysztof Kochel Date: Mon, 15 Sep 2025 23:54:51 +0200 Subject: [PATCH 262/262] Fix tests --- .github/integration/sda-doa-posix-outbox.yml | 6 ++--- .github/integration/sda-doa-s3-outbox.yml | 6 ++--- .github/integration/sda-posix-integration.yml | 2 +- .github/integration/sda-sync-integration.yml | 22 +++++++++---------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index b106439c6b..729bf0283a 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -15,7 +15,7 @@ services: build: context: ../../postgresql container_name: postgres - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres depends_on: certfixer: condition: service_completed_successfully @@ -53,7 +53,7 @@ services: rabbitmq: build: context: ../../rabbitmq - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq container_name: rabbitmq depends_on: certfixer: @@ -86,7 +86,7 @@ services: container_name: doa build: context: ../../sda-doa - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-doa depends_on: postgres: condition: service_healthy diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 8ba3dd2912..dcd24b5042 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -15,7 +15,7 @@ services: build: context: ../../postgresql container_name: postgres - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres depends_on: certfixer: condition: service_completed_successfully @@ -53,7 +53,7 @@ services: rabbitmq: build: context: ../../rabbitmq - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq container_name: rabbitmq depends_on: certfixer: @@ -113,7 +113,7 @@ services: container_name: doa build: context: ../../sda-doa - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-doa depends_on: postgres: condition: service_healthy diff --git a/.github/integration/sda-posix-integration.yml b/.github/integration/sda-posix-integration.yml index 26cb4c3a68..23fec2afae 100644 --- a/.github/integration/sda-posix-integration.yml +++ b/.github/integration/sda-posix-integration.yml @@ -37,7 +37,7 @@ services: - shared:/shared reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-reencrypt ] container_name: reencrypt depends_on: diff --git a/.github/integration/sda-sync-integration.yml b/.github/integration/sda-sync-integration.yml index f09bc3effc..d9192a36a6 100644 --- a/.github/integration/sda-sync-integration.yml +++ b/.github/integration/sda-sync-integration.yml @@ -27,7 +27,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres ports: - "15432:5432" restart: always @@ -49,7 +49,7 @@ services: interval: 10s timeout: 5s retries: 6 - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" restart: always @@ -81,7 +81,7 @@ services: GOLANG_VERSION: ${GOLANG_VERSION:-1.20} context: ../../sda target: debug - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-s3inbox ] container_name: s3inbox depends_on: @@ -112,7 +112,7 @@ services: - "18001:8001" ingest: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-ingest ] container_name: ingest depends_on: @@ -137,7 +137,7 @@ services: - shared:/shared verify: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-verify ] container_name: verify depends_on: @@ -162,7 +162,7 @@ services: - shared:/shared finalize: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-finalize ] container_name: finalize depends_on: @@ -187,7 +187,7 @@ services: - shared:/shared mapper: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-mapper ] container_name: mapper depends_on: @@ -211,7 +211,7 @@ services: - shared:/shared sync: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-sync ] container_name: sync depends_on: @@ -235,7 +235,7 @@ services: - shared:/shared sync-api: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-syncapi ] container_name: sync-api depends_on: @@ -309,7 +309,7 @@ services: - DB_USER=api extra_hosts: - "localhost:host-gateway" - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} ports: - "8090:8080" restart: always @@ -319,7 +319,7 @@ services: - shared:/shared reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-reencrypt ] container_name: reencrypt depends_on: