From 8c16ee9f47fed45da5b20a1f96bc57164696eb2c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:06:01 +0000 Subject: [PATCH 001/250] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) and google.golang.org/protobuf. Updates `github.com/aws/aws-sdk-go` from 1.55.5 to 1.55.6 - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG_PENDING.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.55.5...v1.55.6) Updates `google.golang.org/protobuf` from 1.36.2 to 1.36.3 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 4 ++-- sda-download/go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 042125200d..67e647a9f1 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -4,7 +4,7 @@ go 1.23.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go v1.55.5 + github.com/aws/aws-sdk-go v1.55.6 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 @@ -18,7 +18,7 @@ require ( golang.org/x/crypto v0.32.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.36.2 + google.golang.org/protobuf v1.36.3 ) require ( diff --git a/sda-download/go.sum b/sda-download/go.sum index ccffa3e18c..5dc1695a04 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -3,8 +3,8 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= -github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= +github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -239,8 +239,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= -google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From c0f93801ab83f4df79129b6d25fb5292a92f654c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:12:26 +0000 Subject: [PATCH 002/250] Bump golangci/golangci-lint-action from 6.1.1 to 6.2.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.1.1 to 6.2.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v6.1.1...v6.2.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 515c0b8512..9620790c14 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.1.1 + uses: golangci/golangci-lint-action@v6.2.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-download @@ -48,7 +48,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.1.1 + uses: golangci/golangci-lint-action@v6.2.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.1.1 + uses: golangci/golangci-lint-action@v6.2.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-admin From eff9302dbd723f531646d72c9bbad708c3c61e94 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:12:28 +0000 Subject: [PATCH 003/250] Bump helm/chart-releaser-action from 1.6.0 to 1.7.0 Bumps [helm/chart-releaser-action](https://github.com/helm/chart-releaser-action) from 1.6.0 to 1.7.0. - [Release notes](https://github.com/helm/chart-releaser-action/releases) - [Commits](https://github.com/helm/chart-releaser-action/compare/v1.6.0...v1.7.0) --- updated-dependencies: - dependency-name: helm/chart-releaser-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish_charts.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_charts.yml b/.github/workflows/publish_charts.yml index ba1793e375..e44bd6609e 100644 --- a/.github/workflows/publish_charts.yml +++ b/.github/workflows/publish_charts.yml @@ -29,6 +29,6 @@ jobs: uses: azure/setup-helm@v4 - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.6.0 + uses: helm/chart-releaser-action@v1.7.0 env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" From e86da75d008c75b0284629ab88b909ce5131e470 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 19:54:59 +0000 Subject: [PATCH 004/250] Bump the all-modules group in /sda with 6 updates Bumps the all-modules group in /sda with 6 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.32.8` | `1.33.0` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.28.10` | `1.29.1` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.51` | `1.17.54` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.48` | `1.17.52` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.72.2` | `1.73.2` | | google.golang.org/protobuf | `1.36.2` | `1.36.3` | Updates `github.com/aws/aws-sdk-go-v2` from 1.32.8 to 1.33.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.32.8...v1.33.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.28.10 to 1.29.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.28.10...config/v1.29.1) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.51 to 1.17.54 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.51...credentials/v1.17.54) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.48 to 1.17.52 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.48...credentials/v1.17.52) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.72.2 to 1.73.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.72.2...service/s3/v1.73.2) Updates `google.golang.org/protobuf` from 1.36.2 to 1.36.3 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 32 +++++++++++++-------------- sda/go.sum | 64 +++++++++++++++++++++++++++--------------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 9f3b06f12c..b4fbb8c1ab 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.23.1 require ( - github.com/aws/aws-sdk-go-v2 v1.32.8 - github.com/aws/aws-sdk-go-v2/config v1.28.10 - github.com/aws/aws-sdk-go-v2/credentials v1.17.51 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 - github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 + github.com/aws/aws-sdk-go-v2 v1.33.0 + github.com/aws/aws-sdk-go-v2/config v1.29.1 + github.com/aws/aws-sdk-go-v2/credentials v1.17.54 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.52 + github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2 github.com/aws/smithy-go v1.22.1 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 @@ -34,7 +34,7 @@ require ( golang.org/x/crypto v0.32.0 golang.org/x/oauth2 v0.25.0 google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.36.2 + google.golang.org/protobuf v1.36.3 ) require ( @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index 3d5f5d7244..9cc681e1ef 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.32.8 h1:cZV+NUS/eGxKXMtmyhtYPJ7Z4YLoI/V8bkTdRZfYhGo= -github.com/aws/aws-sdk-go-v2 v1.32.8/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= +github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs= +github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= -github.com/aws/aws-sdk-go-v2/config v1.28.10 h1:fKODZHfqQu06pCzR69KJ3GuttraRJkhlC8g80RZ0Dfg= -github.com/aws/aws-sdk-go-v2/config v1.28.10/go.mod h1:PvdxRYZ5Um9QMq9PQ0zHHNdtKK+he2NHtFCUFMXWXeg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.51 h1:F/9Sm6Y6k4LqDesZDPJCLxQGXNNHd/ZtJiWd0lCZKRk= -github.com/aws/aws-sdk-go-v2/credentials v1.17.51/go.mod h1:TKbzCHm43AoPyA+iLGGcruXd4AFhF8tOmLex2R9jWNQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 h1:IBAoD/1d8A8/1aA8g4MBVtTRHhXRiNAgwdbo/xRM2DI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23/go.mod h1:vfENuCM7dofkgKpYzuzf1VT1UKkA/YL3qanfBn7HCaA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 h1:XnXVe2zRyPf0+fAW5L05esmngvBpC6DQZK7oZB/z/Co= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48/go.mod h1:S3wey90OrS4f7kYxH6PT175YyEcHTORY07++HurMaRM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 h1:jSJjSBzw8VDIbWv+mmvBSP8ezsztMYJGH+eKqi9AmNs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27/go.mod h1:/DAhLbFRgwhmvJdOfSm+WwikZrCuUJiA4WgJG0fTNSw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 h1:l+X4K77Dui85pIj5foXDhPlnqcNRG2QUyvca300lXh8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27/go.mod h1:KvZXSFEXm6x84yE8qffKvT3x8J5clWnVFXphpohhzJ8= +github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= +github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.52 h1:6kI83R98XOnnyzHv9g9KTYXFawMyeQq8NeEERWMAwJk= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.52/go.mod h1:Juj7unpf3CIrWpEyJZhRJ6rJl9IYX7Hd8HOlwaZq/LE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 h1:AmB5QxnD+fBFrg9LcqzkgF/CaYvMyU/BTlejG4t1S7Q= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27/go.mod h1:Sai7P3xTiyv9ZUYO3IFxMnmiIP759/67iQbU4kdmkyU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28 h1:7kpeALOUeThs2kEjlAxlADAVfxKmkYAedlpZ3kdoSJ4= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28/go.mod h1:pyaOYEdp1MJWgtXLy6q80r3DhsVdOIOZNB9hdTcJIvI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 h1:iwYS40JnrBeA9e9aI5S6KKN4EB2zR4iUVYN0nwVivz4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8/go.mod h1:Fm9Mi+ApqmFiknZtGpohVcBGvpTu542VC4XO9YudRi0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 h1:cWno7lefSH6Pp+mSznagKCgfDGeZRin66UvYUqAkyeA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8/go.mod h1:tPD+VjU3ABTBoEJ3nctu5Nyg4P4yjqSH5bJGGkY4+XE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 h1:/Mn7gTedG86nbpjT4QEKsN1D/fThiYe1qvq7WsBGNHg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8/go.mod h1:Ae3va9LPmvjj231ukHB6UeT8nS7wTPfC3tMZSZMwNYg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 h1:a7aQ3RW+ug4IbhoQp29NZdc7vqrzKZZfWZSaQAXOZvQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 h1:YqtxripbjWb2QLyzRK9pByfEDvgg95gpC2AyDq4hFE8= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.9/go.mod h1:lV8iQpg6OLOfBnqbGMBKYjilBlf633qwHnBEiMSPoHY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 h1:6dBT1Lz8fK11m22R+AqfRsFn8320K0T5DTGxxOQBSMw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8/go.mod h1:/kiBvRQXBc6xeJTYzhSdGvJ5vm1tjaDEjH+MSeRJnlY= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 h1:VwhTrsTuVn52an4mXx29PqRzs2Dvu921NpGk7y43tAM= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.6/go.mod h1:+8h7PZb3yY5ftmVLD7ocEoE98hdc8PoKS0H3wfx1dlc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2 h1:e6um6+DWYQP1XCa+E9YVtG/9v1qk5lyAOelMOVwSyO8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2/go.mod h1:dIW8puxSbYLSPv/ju0d9A3CpwXdtqvJtYKDMVmPLOWE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9 h1:2aInXbh02XsbO0KobPGMNXyv2QP73VDKsWPNJARj/+4= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9/go.mod h1:dgXS1i+HgWnYkPXqNoPIPKeUsUUYHaUbThC90aDnNiE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2 h1:F3h8VYq9ZLBXYurmwrT8W0SPhgCcU0q+0WZJfT1dFt0= +github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2/go.mod h1:jGJ/v7FIi7Ys9t54tmEFnrxuaWeJLpwNgKp2DXAVhOU= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -497,8 +497,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= -google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= +google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 2f91167e29b3ba96a7c87d014f02df926d84cfdf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Jan 2025 20:01:22 +0000 Subject: [PATCH 005/250] Bump org.bouncycastle:bcprov-jdk18on Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.bouncycastle:bcprov-jdk18on](https://github.com/bcgit/bc-java). Updates `org.bouncycastle:bcprov-jdk18on` from 1.79 to 1.80 - [Changelog](https://github.com/bcgit/bc-java/blob/main/docs/releasenotes.html) - [Commits](https://github.com/bcgit/bc-java/commits) --- updated-dependencies: - dependency-name: org.bouncycastle:bcprov-jdk18on dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index b428127b05..b9d8e11481 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -111,7 +111,7 @@ org.bouncycastle bcprov-jdk18on - 1.79 + 1.80 From 07a56c767fb66a7e63ac9affc5f25a45bb164082 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:52:58 +0000 Subject: [PATCH 006/250] Bump golang.org/x/net in /sda-download in the go_modules group Bumps the go_modules group in /sda-download with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.30.0 to 0.33.0 - [Commits](https://github.com/golang/net/compare/v0.30.0...v0.33.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 042125200d..2888a44d81 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -71,7 +71,7 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect - golang.org/x/net v0.30.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.25.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index ccffa3e18c..1d7c9d284a 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -193,8 +193,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From dfaa639f7e3c5df7a0ce876e29a1a1efb546ee38 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 22 Jan 2025 12:52:59 +0000 Subject: [PATCH 007/250] Bump golang.org/x/net in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.30.0 to 0.33.0 - [Commits](https://github.com/golang/net/compare/v0.30.0...v0.33.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index b4fbb8c1ab..a52a441abc 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -157,7 +157,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.30.0 // indirect + golang.org/x/net v0.33.0 // indirect golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 9cc681e1ef..f727bf1cc9 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -431,8 +431,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 883a2f3b34f2f3d17f46109bb65f99383f6fdf0b Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 22 Jan 2025 13:31:37 +0000 Subject: [PATCH 008/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index f190be213e..525e82dad2 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.71 -appVersion: v0.3.170 +version: 0.8.72 +appVersion: v0.3.184 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 12e02ca42c..3d409e594c 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.72 -appVersion: v0.3.170 +version: 0.7.73 +appVersion: v0.3.184 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 1630e27757..379aeba89c 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.0 -appVersion: v0.3.170 +version: 0.30.1 +appVersion: v0.3.184 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From d1e3beb7ef0a218d8457a757e24296b37180f98b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 19:11:42 +0000 Subject: [PATCH 009/250] Bump codecov/codecov-action from 5.1.2 to 5.3.1 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.1.2 to 5.3.1. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.1.2...v5.3.1) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ffb2dec24d..1a08ef7156 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.1.2 + uses: codecov/codecov-action@v5.3.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.1.2 + uses: codecov/codecov-action@v5.3.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.1.2 + uses: codecov/codecov-action@v5.3.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt From 1b6c323a46bb2a6154a17ff1ccb81f72ba15572d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 19:27:18 +0000 Subject: [PATCH 010/250] Bump the all-modules group across 1 directory with 2 updates Bumps the all-modules group with 2 updates in the /sda-doa directory: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot) and [io.minio:minio](https://github.com/minio/minio-java). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.1 to 3.4.2 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.1...v3.4.2) Updates `io.minio:minio` from 8.5.15 to 8.5.17 - [Release notes](https://github.com/minio/minio-java/releases) - [Commits](https://github.com/minio/minio-java/compare/8.5.15...8.5.17) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: io.minio:minio dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 3df3a678c2..a25299e1a0 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.1 + 3.4.2 no.uio.ifi @@ -84,7 +84,7 @@ io.minio minio - 8.5.15 + 8.5.17 net.logstash.logback From e8e7d3c08bc2547f0418ad23ade5544b013668e8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 19:37:00 +0000 Subject: [PATCH 011/250] Bump the all-modules group in /sda with 8 updates Bumps the all-modules group in /sda with 8 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.33.0` | `1.34.0` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.1` | `1.29.2` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.54` | `1.17.55` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.52` | `1.17.54` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.73.2` | `1.74.1` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.22.1` | `1.22.2` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.69.4` | `1.70.0` | | google.golang.org/protobuf | `1.36.3` | `1.36.4` | Updates `github.com/aws/aws-sdk-go-v2` from 1.33.0 to 1.34.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.33.0...v1.34.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.1 to 1.29.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.1...config/v1.29.2) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.54 to 1.17.55 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.54...credentials/v1.17.55) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.52 to 1.17.54 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.52...credentials/v1.17.54) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.73.2 to 1.74.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.73.2...service/s3/v1.74.1) Updates `github.com/aws/smithy-go` from 1.22.1 to 1.22.2 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.22.1...v1.22.2) Updates `google.golang.org/grpc` from 1.69.4 to 1.70.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.69.4...v1.70.0) Updates `google.golang.org/protobuf` from 1.36.3 to 1.36.4 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/smithy-go dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 44 +++++++++++----------- sda/go.sum | 108 ++++++++++++++++++++++++++--------------------------- 2 files changed, 76 insertions(+), 76 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index a52a441abc..2aea17d8e2 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,12 +3,12 @@ module github.com/neicnordic/sensitive-data-archive go 1.23.1 require ( - github.com/aws/aws-sdk-go-v2 v1.33.0 - github.com/aws/aws-sdk-go-v2/config v1.29.1 - github.com/aws/aws-sdk-go-v2/credentials v1.17.54 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.52 - github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2 - github.com/aws/smithy-go v1.22.1 + github.com/aws/aws-sdk-go-v2 v1.34.0 + github.com/aws/aws-sdk-go-v2/config v1.29.2 + github.com/aws/aws-sdk-go-v2/credentials v1.17.55 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54 + github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 + github.com/aws/smithy-go v1.22.2 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 github.com/gin-gonic/gin v1.10.0 @@ -33,8 +33,8 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.32.0 golang.org/x/oauth2 v0.25.0 - google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.36.3 + google.golang.org/grpc v1.70.0 + google.golang.org/protobuf v1.36.4 ) require ( @@ -49,19 +49,19 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect @@ -161,7 +161,7 @@ require ( golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.6.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index f727bf1cc9..d301513a50 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,44 +24,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.33.0 h1:Evgm4DI9imD81V0WwD+TN4DCwjUMdc94TrduMLbgZJs= -github.com/aws/aws-sdk-go-v2 v1.33.0/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= -github.com/aws/aws-sdk-go-v2/config v1.29.1 h1:JZhGawAyZ/EuJeBtbQYnaoftczcb2drR2Iq36Wgz4sQ= -github.com/aws/aws-sdk-go-v2/config v1.29.1/go.mod h1:7bR2YD5euaxBhzt2y/oDkt3uNRb6tjFp98GlTFueRwk= -github.com/aws/aws-sdk-go-v2/credentials v1.17.54 h1:4UmqeOqJPvdvASZWrKlhzpRahAulBfyTJQUaYy4+hEI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.54/go.mod h1:RTdfo0P0hbbTxIhmQrOsC/PquBZGabEPnCaxxKRPSnI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24 h1:5grmdTdMsovn9kPZPI23Hhvp0ZyNm5cRO+IZFIYiAfw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.24/go.mod h1:zqi7TVKTswH3Ozq28PkmBmgzG1tona7mo9G2IJg4Cis= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.52 h1:6kI83R98XOnnyzHv9g9KTYXFawMyeQq8NeEERWMAwJk= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.52/go.mod h1:Juj7unpf3CIrWpEyJZhRJ6rJl9IYX7Hd8HOlwaZq/LE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28 h1:igORFSiH3bfq4lxKFkTSYDhJEUCYo6C8VKiWJjYwQuQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.28/go.mod h1:3So8EA/aAYm36L7XIvCVwLa0s5N0P7o2b1oqnx/2R4g= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28 h1:1mOW9zAUMhTSrMDssEHS/ajx8JcAj/IcftzcmNlmVLI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.28/go.mod h1:kGlXVIWDfvt2Ox5zEaNglmq0hXPHgQFNMix33Tw22jA= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28 h1:7kpeALOUeThs2kEjlAxlADAVfxKmkYAedlpZ3kdoSJ4= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.28/go.mod h1:pyaOYEdp1MJWgtXLy6q80r3DhsVdOIOZNB9hdTcJIvI= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2 h1:e6um6+DWYQP1XCa+E9YVtG/9v1qk5lyAOelMOVwSyO8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.2/go.mod h1:dIW8puxSbYLSPv/ju0d9A3CpwXdtqvJtYKDMVmPLOWE= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9 h1:TQmKDyETFGiXVhZfQ/I0cCFziqqX58pi4tKJGYGFSz0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.9/go.mod h1:HVLPK2iHQBUx7HfZeOQSEu3v2ubZaAY2YPbAm5/WUyY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9 h1:2aInXbh02XsbO0KobPGMNXyv2QP73VDKsWPNJARj/+4= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.9/go.mod h1:dgXS1i+HgWnYkPXqNoPIPKeUsUUYHaUbThC90aDnNiE= -github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2 h1:F3h8VYq9ZLBXYurmwrT8W0SPhgCcU0q+0WZJfT1dFt0= -github.com/aws/aws-sdk-go-v2/service/s3 v1.73.2/go.mod h1:jGJ/v7FIi7Ys9t54tmEFnrxuaWeJLpwNgKp2DXAVhOU= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.11 h1:kuIyu4fTT38Kj7YCC7ouNbVZSSpqkZ+LzIfhCr6Dg+I= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.11/go.mod h1:Ro744S4fKiCCuZECXgOi760TiYylUM8ZBf6OGiZzJtY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10 h1:l+dgv/64iVlQ3WsBbnn+JSbkj01jIi+SM0wYsj3y/hY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.10/go.mod h1:Fzsj6lZEb8AkTE5S68OhcbBqeWPsR8RnGuKPr8Todl8= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.9 h1:BRVDbewN6VZcwr+FBOszDKvYeXY1kJ+GGMCcpghlw0U= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.9/go.mod h1:f6vjfZER1M17Fokn0IzssOTMT2N8ZSq+7jnNF0tArvw= -github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= -github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU= +github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= +github.com/aws/aws-sdk-go-v2/config v1.29.2 h1:JuIxOEPcSKpMB0J+khMjznG9LIhIBdmqNiEcPclnwqc= +github.com/aws/aws-sdk-go-v2/config v1.29.2/go.mod h1:HktTHregOZwNSM/e7WTfVSu9RCX+3eOv+6ij27PtaYs= +github.com/aws/aws-sdk-go-v2/credentials v1.17.55 h1:CDhKnDEaGkLA5ZszV/qw5uwN5M8rbv9Cl0JRN+PRsaM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.55/go.mod h1:kPD/vj+RB5MREDUky376+zdnjZpR+WgdBBvwrmnlmKE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 h1:kU7tmXNaJ07LsyN3BUgGqAmVmQtq0w6duVIHAKfp0/w= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25/go.mod h1:OiC8+OiqrURb1wrwmr/UbOVLFSWEGxjinj5C299VQdo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54 h1:6BWOAho3Cgdy4cmNJ4HWY8VZgqODEU7Gw78XXireNZI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54/go.mod h1:n+t/oyYErOV3jf/GxNTVlizSM9RMV1yH7jvcIvld3Do= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 h1:kznaW4f81mNMlREkU9w3jUuJvU5g/KsqDV43ab7Rp6s= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.12/go.mod h1:bZy9r8e0/s0P7BSDHgMLXK2KvdyRRBIQ2blKlvLt0IU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 h1:mUwIpAvILeKFnRx4h1dEgGEFGuV8KJ3pEScZWVFYuZA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11/go.mod h1:JDJtD+b8HNVv71axz8+S5492KM8wTzHRFpMKQbPlYxw= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 h1:g9d+TOsu3ac7SgmY2dUf1qMgu/uJVTlQ4VCbH6hRxSw= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.10/go.mod h1:WZfNmntu92HO44MVZAubQaz3qCuIdeOdog2sADfU6hU= +github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= +github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= @@ -387,16 +387,16 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -493,12 +493,12 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= -google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 17aae49dc1f0e67d276abf7492f9c498eb03a17f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 19:49:05 +0000 Subject: [PATCH 012/250] Bump org.springframework.boot:spring-boot-starter-parent Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.1 to 3.4.2 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.1...v3.4.2) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index b9d8e11481..405c9ff8f7 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.1 + 3.4.2 From 79319f79c5a37ece1f516cc21a08e113791bb823 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 27 Jan 2025 19:57:53 +0000 Subject: [PATCH 013/250] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [google.golang.org/grpc](https://github.com/grpc/grpc-go) and google.golang.org/protobuf. Updates `google.golang.org/grpc` from 1.69.4 to 1.70.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.69.4...v1.70.0) Updates `google.golang.org/protobuf` from 1.36.3 to 1.36.4 --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 6 +++--- sda-download/go.sum | 32 ++++++++++++++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index facbf71654..693cb3d24d 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -17,8 +17,8 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.32.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.36.3 + google.golang.org/grpc v1.70.0 + google.golang.org/protobuf v1.36.4 ) require ( @@ -75,7 +75,7 @@ require ( golang.org/x/sys v0.29.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.25.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 54fe4f1ab5..8c29acb7ab 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -164,16 +164,16 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= +go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= +go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= +go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= +go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= +go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= +go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= +go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= +go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= +go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= @@ -235,12 +235,12 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/protobuf v1.36.3 h1:82DV7MYdb8anAVi3qge1wSnMDrnKK7ebr+I0hHRN1BU= -google.golang.org/protobuf v1.36.3/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o= +google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= +google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= +google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= +google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From e3ccaa1e7ca9c4fdba7aa41ffd48924a05dfa2e3 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 28 Jan 2025 08:58:33 +0000 Subject: [PATCH 014/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 525e82dad2..7842ccb2fe 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.72 -appVersion: v0.3.184 +version: 0.8.73 +appVersion: v0.3.188 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 3d409e594c..05f8896db9 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.73 -appVersion: v0.3.184 +version: 0.7.74 +appVersion: v0.3.188 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 379aeba89c..2b25ee27b7 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.1 -appVersion: v0.3.184 +version: 0.30.2 +appVersion: v0.3.188 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 67f6f6074ae0eb8c83f314b7f0254d0a8484cf91 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2025 19:49:43 +0000 Subject: [PATCH 015/250] Bump the all-modules group in /sda with 5 updates Bumps the all-modules group in /sda with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.34.0` | `1.36.0` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.2` | `1.29.4` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.55` | `1.17.57` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.54` | `1.17.57` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.74.1` | `1.75.2` | Updates `github.com/aws/aws-sdk-go-v2` from 1.34.0 to 1.36.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.34.0...v1.36.0) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.2 to 1.29.4 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.2...config/v1.29.4) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.55 to 1.17.57 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.55...credentials/v1.17.57) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.54 to 1.17.57 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.54...credentials/v1.17.57) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.74.1 to 1.75.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.74.1...service/s3/v1.75.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 30 +++++++++++++-------------- sda/go.sum | 60 +++++++++++++++++++++++++++--------------------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 2aea17d8e2..304f9b037e 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.23.1 require ( - github.com/aws/aws-sdk-go-v2 v1.34.0 - github.com/aws/aws-sdk-go-v2/config v1.29.2 - github.com/aws/aws-sdk-go-v2/credentials v1.17.55 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54 - github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 + github.com/aws/aws-sdk-go-v2 v1.36.0 + github.com/aws/aws-sdk-go-v2/config v1.29.4 + github.com/aws/aws-sdk-go-v2/credentials v1.17.57 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57 + github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2 github.com/aws/smithy-go v1.22.2 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index d301513a50..e9671a051b 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.34.0 h1:9iyL+cjifckRGEVpRKZP3eIxVlL06Qk1Tk13vreaVQU= -github.com/aws/aws-sdk-go-v2 v1.34.0/go.mod h1:JgstGg0JjWU1KpVJjD5H0y0yyAIpSdKEq556EI6yOOM= +github.com/aws/aws-sdk-go-v2 v1.36.0 h1:b1wM5CcE65Ujwn565qcwgtOTT1aT4ADOHHgglKjG7fk= +github.com/aws/aws-sdk-go-v2 v1.36.0/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= -github.com/aws/aws-sdk-go-v2/config v1.29.2 h1:JuIxOEPcSKpMB0J+khMjznG9LIhIBdmqNiEcPclnwqc= -github.com/aws/aws-sdk-go-v2/config v1.29.2/go.mod h1:HktTHregOZwNSM/e7WTfVSu9RCX+3eOv+6ij27PtaYs= -github.com/aws/aws-sdk-go-v2/credentials v1.17.55 h1:CDhKnDEaGkLA5ZszV/qw5uwN5M8rbv9Cl0JRN+PRsaM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.55/go.mod h1:kPD/vj+RB5MREDUky376+zdnjZpR+WgdBBvwrmnlmKE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25 h1:kU7tmXNaJ07LsyN3BUgGqAmVmQtq0w6duVIHAKfp0/w= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.25/go.mod h1:OiC8+OiqrURb1wrwmr/UbOVLFSWEGxjinj5C299VQdo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54 h1:6BWOAho3Cgdy4cmNJ4HWY8VZgqODEU7Gw78XXireNZI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.54/go.mod h1:n+t/oyYErOV3jf/GxNTVlizSM9RMV1yH7jvcIvld3Do= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29 h1:Ej0Rf3GMv50Qh4G4852j2djtoDb7AzQ7MuQeFHa3D70= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.29/go.mod h1:oeNTC7PwJNoM5AznVr23wxhLnuJv0ZDe5v7w0wqIs9M= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29 h1:6e8a71X+9GfghragVevC5bZqvATtc3mAMgxpSNbgzF0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.29/go.mod h1:c4jkZiQ+BWpNqq7VtrxjwISrLrt/VvPq3XiopkUIolI= +github.com/aws/aws-sdk-go-v2/config v1.29.4 h1:ObNqKsDYFGr2WxnoXKOhCvTlf3HhwtoGgc+KmZ4H5yg= +github.com/aws/aws-sdk-go-v2/config v1.29.4/go.mod h1:j2/AF7j/qxVmsNIChw1tWfsVKOayJoGRDjg1Tgq7NPk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.57 h1:kFQDsbdBAR3GZsB8xA+51ptEnq9TIj3tS4MuP5b+TcQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.57/go.mod h1:2kerxPUUbTagAr/kkaHiqvj/bcYHzi2qiJS/ZinllU0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 h1:7lOW8NUwE9UZekS1DYoiPdVAqZ6A+LheHWb+mHbNOq8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27/go.mod h1:w1BASFIPOPUae7AgaH4SbjNbfdkxuggLyGfNFTn8ITY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57 h1:4hFrvTb32jty/LpKdIwWhMgqITPxNo9l1X1hjUyVCZ4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57/go.mod h1:n6n8rfggAVPgDVldL1zk9QUzIWImRb6OWI8t9CfDImM= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 h1:lWm9ucLSRFiI4dQQafLrEOmEDGry3Swrz0BIRdiHJqQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31/go.mod h1:Huu6GG0YTfbPphQkDSo4dEGmQRTKb9k9G7RdtyQWxuI= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 h1:ACxDklUKKXb48+eg5ROZXi1vDgfMyfIA/WyvqHcHI0o= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31/go.mod h1:yadnfsDwqXeVaohbGc/RaD287PuyRw2wugkh5ZL2J6k= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29 h1:g9OUETuxA8i/Www5Cby0R3WSTe7ppFTZXHVLNskNS4w= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.29/go.mod h1:CQk+koLR1QeY1+vm7lqNfFii07DEderKq6T3F1L2pyc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 h1:8IwBjuLdqIO1dGB+dZ9zJEl8wzY3bVYxcs0Xyu/Lsc0= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31/go.mod h1:8tMBcuVjL4kP/ECEIWTCWtwV2kj6+ouEKl4cqR4iWLw= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3 h1:EP1ITDgYVPM2dL1bBBntJ7AW5yTjuWGz9XO+CZwpALU= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.3/go.mod h1:5lWNWeAgWenJ/BZ/CP9k9DjLbC0pjnM045WjXRPPi14= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10 h1:hN4yJBGswmFTOVYqmbz1GBs9ZMtQe8SrYxPwrkrlRv8= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.10/go.mod h1:TsxON4fEZXyrKY+D+3d2gSTyJkGORexIYab9PTf56DA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10 h1:fXoWC2gi7tdJYNTPnnlSGzEVwewUchOi8xVq/dkg8Qs= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.10/go.mod h1:cvzBApD5dVazHU8C2rbBQzzzsKc8m5+wNJ9mCRZLKPc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1 h1:9LawY3cDJ3HE+v2GMd5SOkNLDwgN4K7TsCjyVBYu/L4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.74.1/go.mod h1:hHnELVnIHltd8EOF3YzahVX6F6y2C6dNqpRj1IMkS5I= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.12 h1:kznaW4f81mNMlREkU9w3jUuJvU5g/KsqDV43ab7Rp6s= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.12/go.mod h1:bZy9r8e0/s0P7BSDHgMLXK2KvdyRRBIQ2blKlvLt0IU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11 h1:mUwIpAvILeKFnRx4h1dEgGEFGuV8KJ3pEScZWVFYuZA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.11/go.mod h1:JDJtD+b8HNVv71axz8+S5492KM8wTzHRFpMKQbPlYxw= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.10 h1:g9d+TOsu3ac7SgmY2dUf1qMgu/uJVTlQ4VCbH6hRxSw= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.10/go.mod h1:WZfNmntu92HO44MVZAubQaz3qCuIdeOdog2sADfU6hU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 h1:siiQ+jummya9OLPDEyHVb2dLW4aOMe22FGDd0sAfuSw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5/go.mod h1:iHVx2J9pWzITdP5MJY6qWfG34TfD9EA+Qi3eV6qQCXw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 h1:O+8vD2rGjfihBewr5bT+QUfYUHIxCVgG61LHoT59shM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12/go.mod h1:usVdWJaosa66NMvmCrr08NcWDBRv4E6+YFG2pUdw1Lk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 h1:tkVNm99nkJnFo1H9IIQb5QkCiPcvCDn3Pos+IeTbGRA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12/go.mod h1:dIVlquSPUMqEJtx2/W17SM2SuESRaVEhEV9alcMqxjw= +github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2 h1:dyC+iA2+Yc7iDMDh0R4eT6fi8TgBduc+BOWCy6Br0/o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2/go.mod h1:FHSHmyEUkzRbaFFqqm6bkLAOQHgqhsLmfCahvCBMiyA= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 h1:c5WJ3iHz7rLIgArznb3JCSQT3uUMiz9DLZhIX+1G8ok= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.14/go.mod h1:+JJQTxB6N4niArC14YNtxcQtwEqzS3o9Z32n7q33Rfs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 h1:f1L/JtUkVODD+k1+IiSJUUv8A++2qVr+Xvb3xWXETMU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13/go.mod h1:tvqlFoja8/s0o+UruA1Nrezo/df0PzdunMDDurUfg6U= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 h1:fqg6c1KVrc3SYWma/egWue5rKI4G2+M4wMQN2JosNAA= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.12/go.mod h1:7Yn+p66q/jt38qMoVfNvjbm3D89mGBnkwDcijgtih8w= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= From fcfc7d76c8729f96a30881a5c5513f8529eddbae Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 14 Jan 2025 13:41:47 +0100 Subject: [PATCH 016/250] [download] improve gin logging --- sda-download/api/api.go | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/sda-download/api/api.go b/sda-download/api/api.go index 4cbcb67714..4ccdb2a1f4 100644 --- a/sda-download/api/api.go +++ b/sda-download/api/api.go @@ -2,6 +2,7 @@ package api import ( "crypto/tls" + "encoding/json" "fmt" "net/http" "time" @@ -31,12 +32,31 @@ func healthResponse(c *gin.Context) { func Setup() *http.Server { // Set up routing log.Info("(2/5) Registering endpoint handlers") - + if log.GetLevel() != log.DebugLevel { + gin.SetMode(gin.ReleaseMode) + } router := gin.New() - router.Use( - gin.LoggerWithWriter(gin.DefaultWriter, "/health"), - gin.Recovery(), - ) + if log.GetLevel() == log.DebugLevel { + router.Use(gin.LoggerWithConfig( + gin.LoggerConfig{ + Formatter: func(params gin.LogFormatterParams) string { + //.Format("2006-01-02 - 15:04:05") + s, _ := json.Marshal(map[string]any{ + "level": "debug", + "method": params.Method, + "path": params.Path, + "remote_addr": params.ClientIP, + "status_code": params.StatusCode, + "time": params.TimeStamp.Format(time.RFC3339), + }) + + return string(s) + "\n" + }, + Output: gin.DefaultWriter, + SkipPaths: []string{"/health"}, + }, + )) + } router.HandleMethodNotAllowed = true From c7df49b9bc0b3392381faf402e59bcab8dffd345 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 14 Jan 2025 13:42:03 +0100 Subject: [PATCH 017/250] [download] clean up log messages --- sda-download/api/sda/sda.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index e531606c0d..76a25340a1 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -103,7 +103,6 @@ func reencryptHeader(oldHeader []byte, reencKey string) ([]byte, error) { return nil, err } - log.Debugf("Response from the reencrypt service: %v", res) return res.Header, nil } @@ -383,7 +382,6 @@ func Download(c *gin.Context) { } log.Debugf("Public key from the request header = %v", reencKey) - log.Debugf("old c4gh file header = %v\n", fileDetails.Header) newHeader, err := reencryptHeader(fileDetails.Header, reencKey) if err != nil { log.Errorf("Failed to reencrypt the file header, reason: %v", err) @@ -391,7 +389,6 @@ func Download(c *gin.Context) { return } - log.Debugf("Reencrypted c4gh file header = %v", newHeader) newHr := bytes.NewReader(newHeader) From c9223f8af756e278db3a97cb11ffa4c9bd9e573b Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 14 Jan 2025 13:41:47 +0100 Subject: [PATCH 018/250] [download] improve gin logging --- sda-download/api/api.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/sda-download/api/api.go b/sda-download/api/api.go index 4ccdb2a1f4..ae1a4c8dde 100644 --- a/sda-download/api/api.go +++ b/sda-download/api/api.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "net/http" + "strings" "time" "github.com/gin-gonic/gin" @@ -40,7 +41,6 @@ func Setup() *http.Server { router.Use(gin.LoggerWithConfig( gin.LoggerConfig{ Formatter: func(params gin.LogFormatterParams) string { - //.Format("2006-01-02 - 15:04:05") s, _ := json.Marshal(map[string]any{ "level": "debug", "method": params.Method, @@ -52,8 +52,14 @@ func Setup() *http.Server { return string(s) + "\n" }, - Output: gin.DefaultWriter, - SkipPaths: []string{"/health"}, + + Skip: func(c *gin.Context) bool { + // skip logging HEAD requests to / and all requests to /health + // (HEAD request to health are redirected to path "") + return (c.Request.Method == "HEAD" && strings.Trim(c.FullPath(), "/") == "") || + c.FullPath() == "/health" + }, + Output: gin.DefaultWriter, }, )) } From a1703044192d5174b4810c9db2f303300dac0ebe Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 5 Feb 2025 07:41:32 +0000 Subject: [PATCH 019/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 7842ccb2fe..4ff6d3b1a2 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.73 -appVersion: v0.3.188 +version: 0.8.74 +appVersion: v0.3.190 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 05f8896db9..d36da9a99a 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.74 -appVersion: v0.3.188 +version: 0.7.75 +appVersion: v0.3.190 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 2b25ee27b7..99ffcd5cee 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.2 -appVersion: v0.3.188 +version: 0.30.3 +appVersion: v0.3.190 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From eed68d27b76bef34f76dd5fbae0fd9cab8f4b502 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 16 Dec 2024 11:01:25 +0100 Subject: [PATCH 020/250] [Makefile] install `ingess-nginx` wheh creating a k3d cluster. --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0e0c16104a..ee4c5a7841 100644 --- a/Makefile +++ b/Makefile @@ -170,7 +170,8 @@ k3d-version-check: echo "kubectl is missing";\ fi k3d-create-cluster: - @k3d cluster create + @k3d cluster create --k3s-arg "--disable=traefik@server:0" --port "80:80@loadbalancer" --port "443:443@loadbalancer"; \ + helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace k3d-delete-cluster: @k3d cluster delete k3d-deploy-dependencies: From a82259cff0565cb3ec126c7b4d9e7dab6e24dcca Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 30 Jan 2025 09:01:59 +0100 Subject: [PATCH 021/250] Ensure all dependencies exist when running locally --- .../integration/scripts/charts/dependencies.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index f01365c514..1b885de96b 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -9,7 +9,7 @@ random-string() { } if [ "$1" == "local" ]; then - if [ ! "$(command crypt4gh)" ]; then + if [ ! "$(command crypt4gh --version)" ]; then echo "crypt4gh not installed, get it from here: https://github.com/neicnordic/crypt4gh/releases/latest" exit 1 elif [ "$(crypt4gh --version | cut -d ' ' -f1)" == "GA4GH" ]; then @@ -18,10 +18,20 @@ if [ "$1" == "local" ]; then exit 1 fi - if [ ! "$(command yq)" ]; then + if [ ! "$(command yq --version)" ]; then echo "yq not installed, get it from here: https://github.com/mikefarah/yq/releases/latest" exit 1 fi + + if [ ! "$(command jq --version)" ]; then + echo "jq not installed" + exit 1 + fi + + if [ ! "$(command xxd --version 2>&1)" ]; then + echo "xxd not installed" + exit 1 + fi else sudo curl --retry 100 -sL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -o /usr/bin/yq && sudo chmod +x /usr/bin/yq @@ -104,4 +114,4 @@ yq -i ' .releasetest.secrets.accessToken = strenv(TEST_TOKEN) ' "$values_file" -kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json" \ No newline at end of file +kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json" From 9cddf9bb127643997df09f3afe1f622e3e831a49 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 16 Dec 2024 11:02:11 +0100 Subject: [PATCH 022/250] Update Readme.md --- README.md | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index ac5a977d50..9ddb0021c3 100644 --- a/README.md +++ b/README.md @@ -176,13 +176,9 @@ wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash ``` -Once installed a cluster named `test-cluster` can be created as such: +#### Create a cluster -```sh -k3d cluster create test-cluster -``` - -Or by using the `make k3d-create-cluster` command, you can create a cluster named `k3s-default`. +Once installed a cluster can be created using the `make k3d-create-cluster` command, you can create a cluster named `k3s-default`. The new cluster's connection details will automatically be merged into your default kubeconfig and activated. The command below should show the created node. @@ -190,6 +186,16 @@ The new cluster's connection details will automatically be merged into your defa kubectl get nodes ``` +The Nginx ingress controller is deployed and will bind to ports 80 and 443 of the host system. A deployed service with an ingress definition can then be targeted by setting the `Host: HOSTNAME` header for that service. + +```sh +curl -H "Host: test" http://localhost/ +``` + +For testing ingress endpoints with other applications like a web browser, the hostname in the ingress definition should have the `.127.0.0.1.nip.io` ending in order to not have to modify the `/etc/hosts` file, ex. `app.127.0.0.1.nip.io`. + +#### Remove the cluster + Removing the cluster can be done using the `make k3d-delete-cluster` command or as shown below if a specific name is used during creation. ```sh @@ -220,7 +226,7 @@ Deployment of the charts can be done as describe below in more detail, or by usi #### Bootstrap the dependencies -This script requires [yq](https://github.com/mikefarah/yq/releases/latest) and the GO version of [crypt4gh](https://github.com/neicnordic/crypt4gh/releases/latest) +This script requires [yq](https://github.com/mikefarah/yq/releases/latest), the GO version of [crypt4gh](https://github.com/neicnordic/crypt4gh/releases/latest) as well as [xxd](https://manpages.org/xxd) and [jq](https://manpages.org/jq) to be installed. ```sh bash .github/integration/scripts/charts/dependencies.sh local From d3ec22a5724420cea145b66ea592863552e8f6ae Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 3 Feb 2025 19:23:13 +0100 Subject: [PATCH 023/250] Update chart deployment to enable ingress --- .github/integration/scripts/charts/deploy_charts.sh | 1 + .github/integration/scripts/charts/values.yaml | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 774ccc911c..204f096fba 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -36,6 +36,7 @@ if [ "$1" == "sda-mq" ]; then --set image.pullPolicy=IfNotPresent \ --set global.adminPassword="$ADMINPASS" \ --set global.adminUser=admin \ + --set global.ingress.hostName=broker.127.0.0.1.nip.io \ --set global.tls.enabled="$3" \ --set global.tls.clusterIssuer=cert-issuer \ --set persistence.enabled=false \ diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index 885b0f5c9b..7c119d3cde 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -1,12 +1,13 @@ global: schemaType: "isolated" ingress: - deploy: false + deploy: true hostName: - api: pipeline-sda-svc-api - auth: pipeline-sda-svc-auth - download: pipeline-sda-svc-download - s3Inbox: pipeline-sda-svc-inbox + api: api.127.0.0.1.nip.io + auth: auth.127.0.0.1.nip.io + download: download.127.0.0.1.nip.io + s3Inbox: inbox.127.0.0.1.nip.io + syncapi: sync-api.127.0.0.1.nip.io log: level: "debug" tls: From 28cbb7ab22f54540e89c759a82734bb63883f192 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 3 Feb 2025 19:33:52 +0100 Subject: [PATCH 024/250] [Readme] add section about interacting through ingress --- README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README.md b/README.md index 9ddb0021c3..7235160039 100644 --- a/README.md +++ b/README.md @@ -253,6 +253,16 @@ Once the DB and MQ are installed the SDA stack can be installed, here the desire bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$(date +%F)" false s3 ``` +#### Testing with ingress + +Once everything is deployed it is posible to interact with the services using the following hostnames: + +- api.127.0.0.1.nip.io +- auth.127.0.0.1.nip.io +- broker.127.0.0.1.nip.io +- download.127.0.0.1.nip.io +- inbox.127.0.0.1.nip.io + #### Cleanup all deployed components Once the testing is concluded all deployed components can be removed. From c4ac1b4156fc63173d7063f1f49bc96f96cb04e2 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 3 Feb 2025 19:37:24 +0100 Subject: [PATCH 025/250] [Readme] only refer to the Makefile commands --- README.md | 34 +++++----------------------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 7235160039..e9ccd0e971 100644 --- a/README.md +++ b/README.md @@ -179,7 +179,6 @@ curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash #### Create a cluster Once installed a cluster can be created using the `make k3d-create-cluster` command, you can create a cluster named `k3s-default`. - The new cluster's connection details will automatically be merged into your default kubeconfig and activated. The command below should show the created node. ```sh @@ -198,10 +197,6 @@ For testing ingress endpoints with other applications like a web browser, the ho Removing the cluster can be done using the `make k3d-delete-cluster` command or as shown below if a specific name is used during creation. -```sh -k3d cluster delete test-cluster -``` - #### Install kubectl If `kubectl` is not installed, run the following command to download the latest stable version. (substitue `linux/amd64` with `darwin/arm64` if you are using a Mac). @@ -228,30 +223,15 @@ Deployment of the charts can be done as describe below in more detail, or by usi This script requires [yq](https://github.com/mikefarah/yq/releases/latest), the GO version of [crypt4gh](https://github.com/neicnordic/crypt4gh/releases/latest) as well as [xxd](https://manpages.org/xxd) and [jq](https://manpages.org/jq) to be installed. -```sh -bash .github/integration/scripts/charts/dependencies.sh local -``` +Bootstrap the dependencies with the command: `make k3d-deploy-dependencies`. #### Deploy the Sensitive Data Archive components -Start by building the required containers using the `make build-all` command, once that has completed the images can be imported to the cluster. - -```sh -bash .github/integration/scripts/charts/import_local_images.sh -``` - -The Postgres and RabbitMQ Needs to be deployed first, the bool at the end specifies if TLS should be enabled or not for the deployes services. -Replace `sda-db` in the example below with the helmc hart that shuld be installed. (`sda-db` or `sda-mq`) +Start by building and importing the required containers using the `make k3d-import-images`. -```sh -bash .github/integration/scripts/charts/deploy_charts.sh sda-db "$(date +%F)" false -``` +The Postgres and RabbitMQ Needs to be deployed first using the following commands: `make k3d-deplploy-postgres` and `make k3d-deploy-rabbitmq`. -Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`) - -```sh -bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$(date +%F)" false s3 -``` +Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`), `make k3d-deplpoy-sda-posix` or `make k3d-deplpoy-sda-s3`. #### Testing with ingress @@ -265,8 +245,4 @@ Once everything is deployed it is posible to interact with the services using th #### Cleanup all deployed components -Once the testing is concluded all deployed components can be removed. - -```sh -bash .github/integration/scripts/charts/cleanup.sh -``` +Once the testing is concluded all deployed components can be removed with the command `make k3d-cleanup-all-deployments` From 81229e80e30e062c60692ae744c79b65af3eb344 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 3 Feb 2025 19:38:13 +0100 Subject: [PATCH 026/250] [Readme] Restructure regarding installation --- README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e9ccd0e971..287875c91f 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,14 @@ wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash ``` +#### Install kubectl + +If `kubectl` is not installed, run the following command to download the latest stable version. (substitue `linux/amd64` with `darwin/arm64` if you are using a Mac). + +```sh +curl -sLO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" +``` + #### Create a cluster Once installed a cluster can be created using the `make k3d-create-cluster` command, you can create a cluster named `k3s-default`. @@ -197,14 +205,6 @@ For testing ingress endpoints with other applications like a web browser, the ho Removing the cluster can be done using the `make k3d-delete-cluster` command or as shown below if a specific name is used during creation. -#### Install kubectl - -If `kubectl` is not installed, run the following command to download the latest stable version. (substitue `linux/amd64` with `darwin/arm64` if you are using a Mac). - -```sh -curl -sLO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -``` - ### Deploy the components Deployment of the charts can be done as describe below in more detail, or by using the corresponding command in the [Makefile](./Makefile) From 8932b4b445823a33054fce08d084a47f5fc7e66c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 19:22:21 +0000 Subject: [PATCH 027/250] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [golang.org/x/crypto](https://github.com/golang/crypto) and google.golang.org/protobuf. Updates `golang.org/x/crypto` from 0.32.0 to 0.33.0 - [Commits](https://github.com/golang/crypto/compare/v0.32.0...v0.33.0) Updates `google.golang.org/protobuf` from 1.36.4 to 1.36.5 --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 8 ++++---- sda-download/go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 693cb3d24d..c4453bad24 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -15,10 +15,10 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.32.0 + golang.org/x/crypto v0.33.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 google.golang.org/grpc v1.70.0 - google.golang.org/protobuf v1.36.4 + google.golang.org/protobuf v1.36.5 ) require ( @@ -72,8 +72,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/tools v0.25.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 8c29acb7ab..0cecfdfbe3 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -180,8 +180,8 @@ golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -209,23 +209,23 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -239,8 +239,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= -google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 489f1effc3f388b8545ff242e547328ef57bd506 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Feb 2025 19:35:53 +0000 Subject: [PATCH 028/250] Bump the all-modules group in /sda with 8 updates Bumps the all-modules group in /sda with 8 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.36.0` | `1.36.1` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.4` | `1.29.6` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.57` | `1.17.59` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.57` | `1.17.60` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.75.2` | `1.76.0` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.32.0` | `0.33.0` | | [golang.org/x/oauth2](https://github.com/golang/oauth2) | `0.25.0` | `0.26.0` | | google.golang.org/protobuf | `1.36.4` | `1.36.5` | Updates `github.com/aws/aws-sdk-go-v2` from 1.36.0 to 1.36.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.36.0...v1.36.1) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.4 to 1.29.6 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.4...config/v1.29.6) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.57 to 1.17.59 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.57...credentials/v1.17.59) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.57 to 1.17.60 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.57...feature/s3/manager/v1.17.60) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.75.2 to 1.76.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.75.2...service/s3/v1.76.0) Updates `golang.org/x/crypto` from 0.32.0 to 0.33.0 - [Commits](https://github.com/golang/crypto/compare/v0.32.0...v0.33.0) Updates `golang.org/x/oauth2` from 0.25.0 to 0.26.0 - [Commits](https://github.com/golang/oauth2/compare/v0.25.0...v0.26.0) Updates `google.golang.org/protobuf` from 1.36.4 to 1.36.5 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 40 +++++++++++++------------- sda/go.sum | 84 +++++++++++++++++++++++++++--------------------------- 2 files changed, 62 insertions(+), 62 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 304f9b037e..d686bc3c55 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.23.1 require ( - github.com/aws/aws-sdk-go-v2 v1.36.0 - github.com/aws/aws-sdk-go-v2/config v1.29.4 - github.com/aws/aws-sdk-go-v2/credentials v1.17.57 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57 - github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2 + github.com/aws/aws-sdk-go-v2 v1.36.1 + github.com/aws/aws-sdk-go-v2/config v1.29.6 + github.com/aws/aws-sdk-go-v2/credentials v1.17.59 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.60 + github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0 github.com/aws/smithy-go v1.22.2 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 @@ -31,10 +31,10 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.32.0 - golang.org/x/oauth2 v0.25.0 + golang.org/x/crypto v0.33.0 + golang.org/x/oauth2 v0.26.0 google.golang.org/grpc v1.70.0 - google.golang.org/protobuf v1.36.4 + google.golang.org/protobuf v1.36.5 ) require ( @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect @@ -158,8 +158,8 @@ require ( golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/sys v0.30.0 // indirect + golang.org/x/text v0.22.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/sda/go.sum b/sda/go.sum index e9671a051b..7c40e58ab6 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.36.0 h1:b1wM5CcE65Ujwn565qcwgtOTT1aT4ADOHHgglKjG7fk= -github.com/aws/aws-sdk-go-v2 v1.36.0/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= +github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E= +github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= -github.com/aws/aws-sdk-go-v2/config v1.29.4 h1:ObNqKsDYFGr2WxnoXKOhCvTlf3HhwtoGgc+KmZ4H5yg= -github.com/aws/aws-sdk-go-v2/config v1.29.4/go.mod h1:j2/AF7j/qxVmsNIChw1tWfsVKOayJoGRDjg1Tgq7NPk= -github.com/aws/aws-sdk-go-v2/credentials v1.17.57 h1:kFQDsbdBAR3GZsB8xA+51ptEnq9TIj3tS4MuP5b+TcQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.57/go.mod h1:2kerxPUUbTagAr/kkaHiqvj/bcYHzi2qiJS/ZinllU0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27 h1:7lOW8NUwE9UZekS1DYoiPdVAqZ6A+LheHWb+mHbNOq8= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.27/go.mod h1:w1BASFIPOPUae7AgaH4SbjNbfdkxuggLyGfNFTn8ITY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57 h1:4hFrvTb32jty/LpKdIwWhMgqITPxNo9l1X1hjUyVCZ4= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.57/go.mod h1:n6n8rfggAVPgDVldL1zk9QUzIWImRb6OWI8t9CfDImM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31 h1:lWm9ucLSRFiI4dQQafLrEOmEDGry3Swrz0BIRdiHJqQ= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.31/go.mod h1:Huu6GG0YTfbPphQkDSo4dEGmQRTKb9k9G7RdtyQWxuI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31 h1:ACxDklUKKXb48+eg5ROZXi1vDgfMyfIA/WyvqHcHI0o= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.31/go.mod h1:yadnfsDwqXeVaohbGc/RaD287PuyRw2wugkh5ZL2J6k= +github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg= +github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.60 h1:ssZzp6JAGAbOYUTppPfKLa3Cbmx0PtnPsjh4RSy06Ao= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.60/go.mod h1:0fi8BNjII7rWunx2Cvezfnu1iZDCw7EWEiSQyC+Kgww= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31 h1:8IwBjuLdqIO1dGB+dZ9zJEl8wzY3bVYxcs0Xyu/Lsc0= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.31/go.mod h1:8tMBcuVjL4kP/ECEIWTCWtwV2kj6+ouEKl4cqR4iWLw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvro3THr6NlqErJc3wY= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5 h1:siiQ+jummya9OLPDEyHVb2dLW4aOMe22FGDd0sAfuSw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.5/go.mod h1:iHVx2J9pWzITdP5MJY6qWfG34TfD9EA+Qi3eV6qQCXw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12 h1:O+8vD2rGjfihBewr5bT+QUfYUHIxCVgG61LHoT59shM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.12/go.mod h1:usVdWJaosa66NMvmCrr08NcWDBRv4E6+YFG2pUdw1Lk= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12 h1:tkVNm99nkJnFo1H9IIQb5QkCiPcvCDn3Pos+IeTbGRA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.12/go.mod h1:dIVlquSPUMqEJtx2/W17SM2SuESRaVEhEV9alcMqxjw= -github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2 h1:dyC+iA2+Yc7iDMDh0R4eT6fi8TgBduc+BOWCy6Br0/o= -github.com/aws/aws-sdk-go-v2/service/s3 v1.75.2/go.mod h1:FHSHmyEUkzRbaFFqqm6bkLAOQHgqhsLmfCahvCBMiyA= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.14 h1:c5WJ3iHz7rLIgArznb3JCSQT3uUMiz9DLZhIX+1G8ok= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.14/go.mod h1:+JJQTxB6N4niArC14YNtxcQtwEqzS3o9Z32n7q33Rfs= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13 h1:f1L/JtUkVODD+k1+IiSJUUv8A++2qVr+Xvb3xWXETMU= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.13/go.mod h1:tvqlFoja8/s0o+UruA1Nrezo/df0PzdunMDDurUfg6U= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.12 h1:fqg6c1KVrc3SYWma/egWue5rKI4G2+M4wMQN2JosNAA= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.12/go.mod h1:7Yn+p66q/jt38qMoVfNvjbm3D89mGBnkwDcijgtih8w= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 h1:cCBJaT7EeEojpJ4s7wTDbhZlHVJOgNHN7iw6qVurGaw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q= +github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0 h1:ehvUZNVrGA1Usa6yYo8A8pUqrigRelWXSbcCqYpRLeI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0/go.mod h1:KuLNrwYJFaC2AVZ+CVVc12k9NyqwgWsoNNHjwqF6QNk= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -410,8 +410,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= +golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -433,8 +433,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= -golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE= +golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -459,16 +459,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -476,8 +476,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= +golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -497,8 +497,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/protobuf v1.36.4 h1:6A3ZDJHn/eNqc1i+IdefRzy/9PokBTPvcqMySR7NNIM= -google.golang.org/protobuf v1.36.4/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= +google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 971938c6ce80ed4ea3a24b41cc2bfac4f9cf9514 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 11 Feb 2025 09:25:14 +0000 Subject: [PATCH 029/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 4ff6d3b1a2..00f2e2afdb 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.74 -appVersion: v0.3.190 +version: 0.8.75 +appVersion: v0.3.192 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index d36da9a99a..55509c11a6 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.75 -appVersion: v0.3.190 +version: 0.7.76 +appVersion: v0.3.192 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 99ffcd5cee..a4de649150 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.3 -appVersion: v0.3.190 +version: 0.30.4 +appVersion: v0.3.192 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From cbcdcd7c9cee94e83f51550136abd9654c80bd7c Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 4 Feb 2025 22:43:41 +0100 Subject: [PATCH 030/250] persist fileID over changes of the service instance --- sda/cmd/s3inbox/proxy.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 7bc63cf11b..46880265dc 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -199,6 +199,27 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } + // The following block is for treating the case when the client loses connection to the server and then it reconnects to a + // different instance of s3inbox. For more details see https://github.com/NBISweden/BigPicture-Deployment/issues/283. + if p.fileIds[r.URL.Path] == "" { + + corrID, err := p.database.GetCorrID(username, filepath, "") + if err != nil { + p.internalServerError(w, r, fmt.Sprintf("failed to retrieve corrID from database: %v", err)) + + return + } + + p.fileIds[r.URL.Path], err = p.database.GetFileID(corrID) + if err != nil { + p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) + + return + } + + log.Debugf("resuming work on file with fileId: %v", p.fileIds[r.URL.Path]) + } + log.Debugf("marking file %v as 'uploaded' in database", p.fileIds[r.URL.Path]) err = p.database.UpdateFileEventLog(p.fileIds[r.URL.Path], "uploaded", p.fileIds[r.URL.Path], "inbox", "{}", string(jsonMessage)) if err != nil { From beae0cc99b2e8f92085685da95c8752c9c5b7ffa Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 4 Feb 2025 23:19:54 +0100 Subject: [PATCH 031/250] silence the linter over nestif --- sda/cmd/s3inbox/proxy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 46880265dc..1c1a36167f 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -177,6 +177,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw } // Send message to upstream and set file as uploaded in the database + // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") message, err := p.CreateMessageFromRequest(r, token) From d0b2d9edbe5f6a435f82a99f331cbb4e02a34d8a Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 5 Feb 2025 11:27:43 +0100 Subject: [PATCH 032/250] link to issue in this repo --- sda/cmd/s3inbox/proxy.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 1c1a36167f..7d55955b3a 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -201,7 +201,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw } // The following block is for treating the case when the client loses connection to the server and then it reconnects to a - // different instance of s3inbox. For more details see https://github.com/NBISweden/BigPicture-Deployment/issues/283. + // different instance of s3inbox. For more details see #1358. if p.fileIds[r.URL.Path] == "" { corrID, err := p.database.GetCorrID(username, filepath, "") From 76460ce6ff46f1eacd35e1b71f0e4bb5c9194627 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 10 Feb 2025 14:48:21 +0100 Subject: [PATCH 033/250] add fun getFileIDByUserPathAndStatus --- sda/internal/database/db_functions.go | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index adb2a505f9..8a69754250 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -100,6 +100,45 @@ func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, return filePath, nil } +// GetFileIDByUserPathAndStatus checks if a file exists in the database for a given user and submission filepath +// and returns its fileID for the latest specified status +func (dbs *SDAdb) GetFileIDByUserPathAndStatus(submissionUser, filePath, status string) (string, error) { + var ( + err error + fileID string + ) + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + fileID, err = dbs.getFileIDByUserPathAndStatus(submissionUser, filePath, status) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return fileID, err +} + +func (dbs *SDAdb) getFileIDByUserPathAndStatus(submissionUser, filePath, status string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const getFileID = "SELECT id from sda.files " + + "WHERE submission_user=$1 and submission_file_path =$2 and stable_id IS null " + + "AND EXISTS (SELECT 1 FROM " + + "(SELECT event from sda.file_event_log JOIN sda.files ON sda.files.id=sda.file_event_log.file_id " + + "WHERE submission_user=$1 and submission_file_path =$2 order by started_at desc limit 1) " + + "AS subquery WHERE event = $3)" + + var fileID string + err := db.QueryRow(getFileID, submissionUser, filePath, status).Scan(&fileID) + if err != nil { + return "", err + } + + return fileID, nil +} + // UpdateFileEventLog updates the status in of the file in the database. // The message parameter is the rabbitmq message sent on file upload. func (dbs *SDAdb) UpdateFileEventLog(fileUUID, event, corrID, user, details, message string) error { From fcd61ee0efc6a51091e8e35bf1a8156471411e4e Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 10 Feb 2025 18:32:42 +0100 Subject: [PATCH 034/250] add tests for getFileIDByUserPathAndStatus --- sda/internal/database/db_functions_test.go | 33 ++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index b169831cd9..34aafb6b24 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1228,3 +1228,36 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { _, err = db.getInboxFilePathFromID(user, fileID) assert.Error(suite.T(), err) } + +func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + user := "UserX" + filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) + fileID, err := db.RegisterFile(filePath, user) + if err != nil { + suite.FailNow("Failed to register file") + } + + // check happy path + fileID2, err := db.getFileIDByUserPathAndStatus(user, filePath, "registered") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), fileID, fileID2) + + // update the status of the file + err = db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}") + if err != nil { + suite.FailNow("Failed to update file event log") + } + + // check that an error and an empty fileID are returned when the most recent status is not registered + fileID2, err = db.getFileIDByUserPathAndStatus(user, filePath, "registered") + assert.Error(suite.T(), err) + assert.Equal(suite.T(), "", fileID2) + + // check that the function works for other statuses as well + fileID2, err = db.getFileIDByUserPathAndStatus(user, filePath, "archived") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), fileID, fileID2) +} From e4e911f2f31e68a2f3a56b5b2b2e0d47a035c7ed Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 10 Feb 2025 14:49:57 +0100 Subject: [PATCH 035/250] replace corrID lookup with user,filepath and latest status --- sda/cmd/s3inbox/proxy.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 7d55955b3a..96074cdebc 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -204,14 +204,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // different instance of s3inbox. For more details see #1358. if p.fileIds[r.URL.Path] == "" { - corrID, err := p.database.GetCorrID(username, filepath, "") - if err != nil { - p.internalServerError(w, r, fmt.Sprintf("failed to retrieve corrID from database: %v", err)) - - return - } - - p.fileIds[r.URL.Path], err = p.database.GetFileID(corrID) + p.fileIds[r.URL.Path], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) From 8d0998f20b55035cd166543a6707a77288cf54fc Mon Sep 17 00:00:00 2001 From: Alex Aperis <76202622+aaperis@users.noreply.github.com> Date: Tue, 11 Feb 2025 10:01:25 +0100 Subject: [PATCH 036/250] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- sda/cmd/s3inbox/proxy.go | 1 - sda/internal/database/db_functions.go | 2 +- sda/internal/database/db_functions_test.go | 3 +++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 96074cdebc..50940e6c9e 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -203,7 +203,6 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // The following block is for treating the case when the client loses connection to the server and then it reconnects to a // different instance of s3inbox. For more details see #1358. if p.fileIds[r.URL.Path] == "" { - p.fileIds[r.URL.Path], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 8a69754250..9e643b1a1c 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -110,7 +110,7 @@ func (dbs *SDAdb) GetFileIDByUserPathAndStatus(submissionUser, filePath, status // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { fileID, err = dbs.getFileIDByUserPathAndStatus(submissionUser, filePath, status) - if err == nil { + if err == nil || strings.Contains(err.Error(), "sql: no rows in result set") { break } time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 34aafb6b24..2c744d724e 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -1239,6 +1239,9 @@ func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { if err != nil { suite.FailNow("Failed to register file") } + // sanity check - should fail + _, err = db.GetFileIDByUserPathAndStatus("wrong-user", filePath, "registered") + assert.EqualError(suite.T(), err, "sql: no rows in result set") // check happy path fileID2, err := db.getFileIDByUserPathAndStatus(user, filePath, "registered") From 2722d65e9099181f921054e9ccd2a5080eb404c1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 29 Nov 2024 09:47:30 +0100 Subject: [PATCH 037/250] Mock AAI for manual testing --- .github/integration/sda-s3-integration.yml | 97 ++++++--- .github/integration/sda/aai-mock/aai-mock.sql | 206 ++++++++++++++++++ .../sda/aai-mock/application.properties | 12 + .../sda/aai-mock/clients/aai-auth.yaml | 8 + .../integration/sda/aai-mock/ga4gh-broker.sql | 11 + .../aai-mock/ga4gh-broker/application.yaml | 63 ++++++ .../ga4gh-broker/users/test-user-visa.yaml | 34 +++ .github/integration/sda/aai-mock/keys.jwks | 18 ++ .../sda/aai-mock/userinfos/dummy-user.yaml | 33 +++ .../sda/aai-mock/userinfos/test-user.yaml | 33 +++ .github/integration/sda/config.yaml | 14 +- 11 files changed, 489 insertions(+), 40 deletions(-) create mode 100644 .github/integration/sda/aai-mock/aai-mock.sql create mode 100644 .github/integration/sda/aai-mock/application.properties create mode 100644 .github/integration/sda/aai-mock/clients/aai-auth.yaml create mode 100644 .github/integration/sda/aai-mock/ga4gh-broker.sql create mode 100644 .github/integration/sda/aai-mock/ga4gh-broker/application.yaml create mode 100644 .github/integration/sda/aai-mock/ga4gh-broker/users/test-user-visa.yaml create mode 100644 .github/integration/sda/aai-mock/keys.jwks create mode 100644 .github/integration/sda/aai-mock/userinfos/dummy-user.yaml create mode 100644 .github/integration/sda/aai-mock/userinfos/test-user.yaml diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 59ecdff77e..8b2050de0b 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -83,7 +83,7 @@ services: context: ../../sda target: debug image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-s3inbox ] + command: [sda-s3inbox] container_name: s3inbox depends_on: credentials: @@ -102,6 +102,8 @@ services: - BROKER_ROUTINGKEY=inbox - DB_PASSWORD=inbox - DB_USER=inbox + extra_hosts: + - "localhost:host-gateway" restart: always volumes: - ./sda/config.yaml:/config.yaml @@ -112,7 +114,7 @@ services: ingest: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-ingest ] + command: [sda-ingest] container_name: ingest depends_on: credentials: @@ -137,7 +139,7 @@ services: verify: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-verify ] + command: [sda-verify] container_name: verify depends_on: credentials: @@ -162,7 +164,7 @@ services: finalize: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-finalize ] + command: [sda-finalize] container_name: finalize depends_on: credentials: @@ -187,7 +189,7 @@ services: mapper: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-mapper ] + command: [sda-mapper] container_name: mapper depends_on: credentials: @@ -224,7 +226,13 @@ services: extra_hosts: - "localhost:host-gateway" healthcheck: - test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] + test: + [ + "CMD", + "python3", + "-c", + 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)', + ] interval: 10s timeout: 2s retries: 6 @@ -237,7 +245,7 @@ services: - shared:/shared api: - command: [ sda-api ] + command: [sda-api] container_name: api depends_on: credentials: @@ -253,6 +261,8 @@ services: - BROKER_USER=api - DB_PASSWORD=api - DB_USER=api + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8090:8080" @@ -264,7 +274,7 @@ services: reencrypt: image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-reencrypt ] + command: [sda-reencrypt] container_name: reencrypt depends_on: credentials: @@ -282,13 +292,7 @@ services: credentials: condition: service_completed_successfully command: - [ - "python", - "/cega/users.py", - "0.0.0.0", - "8443", - "/shared/users.json" - ] + ["python", "/cega/users.py", "0.0.0.0", "8443", "/shared/users.json"] environment: - CEGA_USERS_PASSWORD=test - CEGA_USERS_USER=test @@ -300,7 +304,7 @@ services: - shared:/shared auth-cega: - command: [ sda-auth ] + command: [sda-auth] container_name: auth-cega depends_on: cega-nss: @@ -313,6 +317,8 @@ services: - AUTH_CEGA_SECRET=test - DB_PASSWORD=auth - DB_USER=auth + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8888:8080" @@ -321,32 +327,57 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared - auth-oidc: - command: [ sda-auth ] - container_name: auth-oidc + auth-aai: + command: [sda-auth] + container_name: auth-aai depends_on: - postgres: - condition: service_healthy - oidc: - condition: service_healthy + credentials: + condition: service_completed_successfully + mock-aai: + condition: service_started environment: - AUTH_RESIGNJWT=false - - OIDC_ID=XC56EL11xx - - OIDC_SECRET=wHPVQaYXmdDHg - - OIDC_PROVIDER=http://localhost:8080 - - OIDC_REDIRECTURL=http://localhost:8889/oidc/login - DB_PASSWORD=auth - DB_USER=auth extra_hosts: - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - - "8889:8080" + - "8801:8080" restart: always volumes: - ./sda/config.yaml:/config.yaml - shared:/shared - + mock-aai: + container_name: ls-aai-mock + depends_on: + aai-db: + condition: service_healthy + environment: + - DOCKERHOST=localhost + extra_hosts: + - "localhost:host-gateway" + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 + ports: + - "8800:8080" + volumes: + - "./sda/aai-mock:/etc/lsaai-mock" + aai-db: + container_name: ls-aai-db + environment: + MYSQL_ROOT_PASSWORD: "aaiPass" + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: "aai" + MYSQL_USER: "aai" + MYSQL_PASSWORD: "aaiPass" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 2s + retries: 6 + image: mysql/mysql-server:latest + volumes: + - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql integration_test: container_name: tester command: @@ -356,7 +387,7 @@ services: depends_on: auth-cega: condition: service_started - auth-oidc: + auth-aai: condition: service_started cega-nss: condition: service_started @@ -371,11 +402,11 @@ services: s3inbox: condition: service_started verify: - condition: service_started + condition: service_started api: - condition: service_started + condition: service_started reencrypt: - condition: service_started + condition: service_started extra_hosts: - "localhost:host-gateway" environment: diff --git a/.github/integration/sda/aai-mock/aai-mock.sql b/.github/integration/sda/aai-mock/aai-mock.sql new file mode 100644 index 0000000000..ecc0ad5c24 --- /dev/null +++ b/.github/integration/sda/aai-mock/aai-mock.sql @@ -0,0 +1,206 @@ +CREATE TABLE IF NOT EXISTS access_token ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + token_value VARCHAR(4096), + expiration TIMESTAMP NULL, + token_type VARCHAR(256), + refresh_token_id BIGINT, + client_id VARCHAR(256) NOT NULL, + auth_holder_id BIGINT, + approved_site_id BIGINT +); + +CREATE TABLE IF NOT EXISTS authorization_code ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + code VARCHAR(256), + auth_holder_id BIGINT, + expiration TIMESTAMP NULL +); + +CREATE TABLE IF NOT EXISTS approved_site ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + user_id VARCHAR(256), + client_id VARCHAR(256), + creation_date TIMESTAMP NULL, + access_date TIMESTAMP NULL, + timeout_date TIMESTAMP NULL, + whitelisted_site_id BIGINT +); + +CREATE TABLE IF NOT EXISTS approved_site_scope ( + owner_id BIGINT, + scope VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS authentication_holder ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + user_auth_id BIGINT, + approved BOOLEAN, + redirect_uri VARCHAR(2048), + client_id VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_authority ( + owner_id BIGINT, + authority VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_resource_id ( + owner_id BIGINT, + resource_id VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_response_type ( + owner_id BIGINT, + response_type VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_extension ( + owner_id BIGINT, + extension VARCHAR(2048), + val VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_scope ( + owner_id BIGINT, + scope VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_request_parameter ( + owner_id BIGINT, + param VARCHAR(2048), + val TEXT +); + +CREATE TABLE IF NOT EXISTS saved_user_auth ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + acr VARCHAR(1024), + auth_time BIGINT DEFAULT NULL, + name VARCHAR(1024), + authenticated BOOLEAN, + authentication_attributes TEXT +); + +CREATE TABLE IF NOT EXISTS saved_user_auth_authority ( + owner_id BIGINT, + authority VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS refresh_token ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + token_value VARCHAR(4096), + expiration TIMESTAMP NULL, + auth_holder_id BIGINT, + client_id VARCHAR(256) NOT NULL +); + +CREATE TABLE IF NOT EXISTS token_scope ( + owner_id BIGINT, + scope VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS device_code ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + device_code VARCHAR(1024), + user_code VARCHAR(1024), + expiration TIMESTAMP NULL, + client_id VARCHAR(256), + approved BOOLEAN, + auth_holder_id BIGINT, + recorded_error TEXT DEFAULT NULL +); + +CREATE TABLE IF NOT EXISTS device_code_scope ( + owner_id BIGINT NOT NULL, + scope VARCHAR(256) NOT NULL +); + +CREATE TABLE IF NOT EXISTS device_code_request_parameter ( + owner_id BIGINT, + param VARCHAR(2048), + val VARCHAR(2048) +); + +alter table access_token + add constraint access_token_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete set null; + +alter table access_token + add constraint access_token_refresh_token_id_fk + foreign key (refresh_token_id) references refresh_token (id) + on update cascade on delete set null; + +alter table approved_site_scope + add constraint approved_site_scope_approved_site_id_fk + foreign key (owner_id) references approved_site (id) + on update cascade on delete cascade; + +alter table authentication_holder_authority + add constraint authentication_holder_authority_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_extension + add constraint authentication_holder_extension_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_request_parameter + add constraint auth_holder_request_parameter_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_resource_id + add constraint authentication_holder_resource_id_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_response_type + add constraint authentication_holder_response_type_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder + add constraint authentication_holder_saved_user_auth_id_fk + foreign key (user_auth_id) references saved_user_auth (id) + on update cascade on delete cascade; + +alter table authentication_holder_scope + add constraint authentication_holder_scope_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authorization_code + add constraint authorization_code_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table device_code + add constraint device_code_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete set null; + +alter table device_code_request_parameter + add constraint device_code_request_parameter_device_code_id_fk + foreign key (owner_id) references device_code (id) + on update cascade on delete cascade; + +alter table device_code_scope + add constraint device_code_scope_device_code_id_fk + foreign key (owner_id) references device_code (id) + on update cascade on delete cascade; + +alter table refresh_token + add constraint refresh_token_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete set null; + +alter table saved_user_auth_authority + add constraint saved_user_auth_authority_saved_user_auth_id_fk + foreign key (owner_id) references saved_user_auth (id) + on update cascade on delete cascade; + +alter table token_scope + add constraint token_scope_refresh_token_id_fk + foreign key (owner_id) references access_token (id) + on update cascade on delete cascade; \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/application.properties b/.github/integration/sda/aai-mock/application.properties new file mode 100644 index 0000000000..c5c30f8d1d --- /dev/null +++ b/.github/integration/sda/aai-mock/application.properties @@ -0,0 +1,12 @@ +main.oidc.issuer.url=http://${DOCKERHOST}:8800/oidc/ +web.baseURL=https://${DOCKERHOST}:8800/oidc + +# GA4GH broker +ga4gh.broker.url=http://${DOCKERHOST}:8800/ga4gh-broker/ +ga4gh.broker.username=broker-aai-user +ga4gh.broker.password=broker-aai-pass + +# DATABASE +jdbc.url=jdbc:mysql://ls-aai-db:3306/aai +jdbc.user=aai +jdbc.password=aaiPass \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/clients/aai-auth.yaml b/.github/integration/sda/aai-mock/clients/aai-auth.yaml new file mode 100644 index 0000000000..e785e83cf4 --- /dev/null +++ b/.github/integration/sda/aai-mock/clients/aai-auth.yaml @@ -0,0 +1,8 @@ +client-name: "auth" +client-id: "XC56EL11xx" +client-secret: "wHPVQaYXmdDHg" +redirect-uris: ["http://localhost:8801/oidc/login"] +token-endpoint-auth-method: "client_secret_basic" +scope: ["openid", "profile", "email", "ga4gh_passport_v1", "eduperson_entitlement"] +grant-types: ["authorization_code"] +post-logout-redirect-uris: ["http://localhost:8801/oidc/login"] \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/ga4gh-broker.sql b/.github/integration/sda/aai-mock/ga4gh-broker.sql new file mode 100644 index 0000000000..424907c502 --- /dev/null +++ b/.github/integration/sda/aai-mock/ga4gh-broker.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS visas( + id BIGINT AUTO_INCREMENT NOT NULL, + user_id VARCHAR(255) NOT NULL, + source VARCHAR(1024) NOT NULL, + linked_identity VARCHAR(1024) NOT NULL, + exp datetime NOT NULL, + jwt LONGTEXT NOT NULL, + CONSTRAINT pk_visas PRIMARY KEY (id) +); + +CREATE INDEX idx_visaentity_user_id_source ON visas (user_id, source); \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/ga4gh-broker/application.yaml b/.github/integration/sda/aai-mock/ga4gh-broker/application.yaml new file mode 100644 index 0000000000..47fa94f03c --- /dev/null +++ b/.github/integration/sda/aai-mock/ga4gh-broker/application.yaml @@ -0,0 +1,63 @@ +--- + +spring: + main: + allow-bean-definition-overriding: true + mvc: + pathmatch: + matching-strategy: ant_path_matcher + +logging: + level: + root: info + cz.muni.ics: debug + +mock-broker: + enabled: True + issuer: http://${DOCKERHOST}:8080/oidc/ + jku: http://${DOCKERHOST}:8800/ga4gh-broker/jwk + path-to-jwk-file: /etc/lsaai-mock/keys.jwks + visa-configs-path: /etc/lsaai-mock/ga4gh-broker/users/ + passport-repositories: [] +# Sample configuration for repository +# passport-repositories: +# - name: REMS-LOCAL-API +# url: http://rems-api.local/api/permissions/{user_id} +# jwks: http://rems-api.local/api/jwk +# headers: +# - header: x-rems-api-key +# value: api-key +# - header: x-rems-user-id +# value: user-id + +visa-writers: + enabled: False +# Uncomment to enable writing from visa sources +# enabled: True +# db: +# url: jdbc:mysql://localhost:3307/broker +# username: brokerUser +# password: brokerPass +# driver: "com.mysql.cj.jdbc.Driver" +# repositories: +# - name: Local REMS +# jwks: http://rems.local/api/jwk +# auth: +# api-key: +# api-key-header: X-visa-writer +# api-key-value: X-vis-writer-key + +passport-v1-0: + enabled: True + auth: + - username: "broker-aai-user" + password: "broker-aai-pass" + +passport-v1-2: + enabled: False +# Uncomment to enable token-exchange integration with AAI mock +# enabled: True +# auth: +# introspection-client-id: broker +# introspection-client-secret: broker-secret +# introspection-uri: http://localhost:8080/oidc/introspect \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/ga4gh-broker/users/test-user-visa.yaml b/.github/integration/sda/aai-mock/ga4gh-broker/users/test-user-visa.yaml new file mode 100644 index 0000000000..dbdcc3515e --- /dev/null +++ b/.github/integration/sda/aai-mock/ga4gh-broker/users/test-user-visa.yaml @@ -0,0 +1,34 @@ +--- +# Identifier of the user - has to match the value of 'sub' claim in AAI +userId: "testu@lifescience-ri.eu" +visas: + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "affiliate@dummy.uni" + type: "AffiliationAndRole" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "faculty@dummy.uni" + type: "AffiliationAndRole" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "https://lifescience-ri.eu/ls-login/ls-aai-aup.html" + type: "AcceptedTermsAndPolicies" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "https://doi.org/10.1038/s41431-018-0219-y" + type: "ResearcherStatus" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "urn:sampledataset:gdi:testcase" + type: "ControlledAccessGrants" + by: "system" \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/keys.jwks b/.github/integration/sda/aai-mock/keys.jwks new file mode 100644 index 0000000000..e95becd288 --- /dev/null +++ b/.github/integration/sda/aai-mock/keys.jwks @@ -0,0 +1,18 @@ +{ + "keys": [ + { + "n": "4bmY8T5jhlCFtS4ccyn8ZDftWk27yOcPgKbak991_m1rs3474RKHYjc9GwX0ICSaUN02fCdSGaTNhU586MfhRsSj3uU_TIJ4M3eDengcllT16DTJOyHZgVzHT2QRUsozFuG2tTBCwqYildOXXuCeaKUQ8Jh5sjKmCcQZO_Ht4omGIsVZPo9SrvshZeHdkdvctvjvT5WXoCZaHvBRpyiBmQKUv_LfYsgpF4PFm40j84GAk0CsZ_xOJXBnU6wLfzHCq1pgLqaXK_q6T_gSJ5t5eWbv0IkUmXDG-TuOtKIz_Xip2lJQ6Rmb_QmOY-MeJavL41NpCu-2FwTvCNqMbHuWuQ", + "e": "AQAB", + "d": "BbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKPRushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF-eqipSQXKghdKZCVAsY8io0MmpXB1omDSygp7h3j52yHdayE2muav-VTAPOYn5QwG0_gGgVqYrR9x7CMiTuyOIuGNO4Wlly4_5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq_77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsnmccF9plVTzem7N_vo-T-hFRPUO9TZUao91mMV8iV5Q", + "p": "9Z2W0FtzR3aJdH1wPJ8OSUZEy2cdc3GeZd-llT5MAoyTsl0Nr1h5uJc-xfNUa_HUJzCi1jAyo6wU_Q237MWCtm2MRDUtVv5_tIziGl_-_dg2PbX1-77x1PchknaCROaLOgxgh2CJzg9MZ0Zp5d70u4Cr_Jog8A8xbG47Q4Oln0s", + "q": "60S5Jpn2QjHJiwNNmEqxKwJVLHrsmne5WBxCQKVyn59utohxC5UGtszgWaSRlyHB7DX9VxqTHlOaR_iCyhOGOam3JavgMxQnGZwRVHw3msJlnVeRSajQ3dS_DG_GhPuqtGOazW90S-wNsqR9mXmCkhqR89wu0Z-Slr6FgF0Vq4s", + "dp": "h-G8St8dIr8YgoLXMiKIw88FRkZwy6pjCFc3gx6o7IwbNgU3Cff3t0CtcNQSs3lhdqD4pWiay2PrZKhPRcVrcEOg4ydMQyGfQyLh_9d1-bfBIljlknXr0nc6co3vufCVu_KXAK5O7toV2djhw1Oar5UbES-v4z90h1lgnhRZ74s", + "dq": "ai8zTavRjKDB8R8Fh4W2ns3HuXKllTYSDiV69Yh2ynp_0pyUQQD67chcPblcCJUw3Dal6AlTey2KhY8xIJldTGtAAR2o7ATdfN6Gp9YElrHEgRZDubYaxvQ9YhSUyfJ4YkBOXwg9ugKJUCaTlZBVETIyy5iF_RCDbcwBnyTkIe8", + "qi": "mbnk1BlUoAQMZ3hrbWpMNxF120_DAvag_eprSBsaThgVFQKtf32Eh6M5O9BbXF_q8bFN3sTVCLO8Wx7jJHNlErJchzsMJx3M6ekytJ3X5pfBm1EctLDQQNLo9Z4ll2MlgfGXZikCYx1RF1wZOsdvpv_q-FT3T7EvXgJfsDCh-n8", + "use": "sig", + "alg": "RS256", + "kty": "RSA", + "kid": "rsa1" + } + ] +} \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/userinfos/dummy-user.yaml b/.github/integration/sda/aai-mock/userinfos/dummy-user.yaml new file mode 100644 index 0000000000..ae0ef3b970 --- /dev/null +++ b/.github/integration/sda/aai-mock/userinfos/dummy-user.yaml @@ -0,0 +1,33 @@ +web-display-name: "Dummy User - submitter" + +sub: "test@dummy.org" + +name: "Dummy User" +preferred-username: "dummyu" +given-name: "Dummy" +family-name: "User" +middle-name: "" +nickname: "duus" +locale: "EN" + +email: "test@dummy.org" +email-verified: True + +country: "eu" + +eduperson-entitlement: ["urn:geant:lifescience-ri.eu:group:researchers#lifescience-ri.eu"] + +voperson-external-affiliation: ["unknown@orcid.org", "affiliate@dummy.uni", "faculty@dummy.uni"] + +eduperson-orcid: "https://orcid.org/0000-0000-XXXX-XXXX" + +schac-home-organization: ["dummy.uni", "orcid.org"] + +eduperson-scoped-affiliation: ["affiliate@lifescience-ri.eu", "member@lifescience-ri.eu"] + +voperson-current-external-affiliation: ["affiliate@dummy.uni", "faculty@dummy.uni"] + +authenticating-entity: "https://idp.dummy.uni/mock" + +acr: "https://refeds.org/profile/mfa" +authn-instant: "2023-01-01T10:15:30Z" \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/userinfos/test-user.yaml b/.github/integration/sda/aai-mock/userinfos/test-user.yaml new file mode 100644 index 0000000000..54c2bd7454 --- /dev/null +++ b/.github/integration/sda/aai-mock/userinfos/test-user.yaml @@ -0,0 +1,33 @@ +web-display-name: "Test User - admin" + +sub: "testu@lifescience-ri.eu" + +name: "Test User" +preferred-username: "testu" +given-name: "Test" +family-name: "User" +middle-name: "dummy" +nickname: "tedu" +locale: "EN" + +email: "testu@dummy.uni" +email-verified: True + +country: "eu" + +eduperson-entitlement: ["urn:geant:lifescience-ri.eu:group:researchers#lifescience-ri.eu"] + +voperson-external-affiliation: ["unknown@orcid.org", "affiliate@dummy.uni", "faculty@dummy.uni"] + +eduperson-orcid: "https://orcid.org/0000-0000-XXXX-XXXX" + +schac-home-organization: ["dummy.uni", "orcid.org"] + +eduperson-scoped-affiliation: ["affiliate@lifescience-ri.eu", "member@lifescience-ri.eu"] + +voperson-current-external-affiliation: ["affiliate@dummy.uni", "faculty@dummy.uni"] + +authenticating-entity: "https://idp.dummy.uni/mock" + +acr: "https://refeds.org/profile/mfa" +authn-instant: "2023-05-01T10:15:30Z" \ No newline at end of file diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 50dfe73a2b..e11150a377 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -77,17 +77,17 @@ c4gh: passphrase: "c4ghpass" oidc: - id: - jwkPath: /jwk - provider: http://oidc:8080 - redirectUrl: http://auth_oidc:8080/oidc/login - secret: + id: XC56EL11xx + jwkPath: jwk + provider: http://localhost:8800/oidc/ + redirectUrl: http://localhost:8801/oidc/login + secret: wHPVQaYXmdDHg server: cert: "" key: "" - jwtpubkeypath: "/shared/keys/pub/" - jwtpubkeyurl: "http://oidc:8080/jwk" + # jwtpubkeypath: "/shared/keys/pub/" + jwtpubkeyurl: "http://localhost:8800/oidc/jwk" sync: api: From 5b1a423cb8b6a9a14d5c85053628aaed564ae7c2 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 3 Feb 2025 14:13:13 +0100 Subject: [PATCH 038/250] Update tests to work with the mock AAI --- .../scripts/make_sda_credentials.sh | 51 ++++++++++++++++--- .github/integration/scripts/sign_jwt.py | 13 ++--- .github/integration/sda-s3-integration.yml | 44 ++++------------ .github/integration/sda/config.yaml | 2 +- .github/integration/sda/rbac.json | 2 +- .../integration/tests/sda/10_upload_test.sh | 34 ------------- .github/integration/tests/sda/11_api_test.sh | 4 +- .../integration/tests/sda/40_mapper_test.sh | 2 +- .../tests/sda/60_api_admin_test.sh | 2 +- 9 files changed, 67 insertions(+), 87 deletions(-) diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 9b305650e8..83bc347d0e 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -29,19 +29,58 @@ done mkdir -p /shared/keys/pub if [ ! -f "/shared/keys/jwt.key" ]; then echo "creating jwt key" - openssl ecparam -genkey -name prime256v1 -noout -out /shared/keys/jwt.key - openssl ec -in /shared/keys/jwt.key -outform PEM -pubout >/shared/keys/pub/jwt.pub + cat << 'EOF' > /shared/keys/jwt.key +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDhuZjxPmOGUIW1 +LhxzKfxkN+1aTbvI5w+AptqT33X+bWuzfjvhEodiNz0bBfQgJJpQ3TZ8J1IZpM2F +Tnzox+FGxKPe5T9Mgngzd4N6eByWVPXoNMk7IdmBXMdPZBFSyjMW4ba1MELCpiKV +05de4J5opRDwmHmyMqYJxBk78e3iiYYixVk+j1Ku+yFl4d2R29y2+O9PlZegJloe +8FGnKIGZApS/8t9iyCkXg8WbjSPzgYCTQKxn/E4lcGdTrAt/McKrWmAuppcr+rpP ++BInm3l5Zu/QiRSZcMb5O460ojP9eKnaUlDpGZv9CY5j4x4lq8vjU2kK77YXBO8I +2oxse5a5AgMBAAECggEABbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKP +RushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF+eqipSQXKghdKZCV +AsY8io0MmpXB1omDSygp7h3j52yHdayE2muav+VTAPOYn5QwG0/gGgVqYrR9x7CM +iTuyOIuGNO4Wlly4/5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI +3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq/77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsn +mccF9plVTzem7N/vo+T+hFRPUO9TZUao91mMV8iV5QKBgQD1nZbQW3NHdol0fXA8 +nw5JRkTLZx1zcZ5l36WVPkwCjJOyXQ2vWHm4lz7F81Rr8dQnMKLWMDKjrBT9Dbfs +xYK2bYxENS1W/n+0jOIaX/792DY9tfX7vvHU9yGSdoJE5os6DGCHYInOD0xnRmnl +3vS7gKv8miDwDzFsbjtDg6WfSwKBgQDrRLkmmfZCMcmLA02YSrErAlUseuyad7lY +HEJApXKfn262iHELlQa2zOBZpJGXIcHsNf1XGpMeU5pH+ILKE4Y5qbclq+AzFCcZ +nBFUfDeawmWdV5FJqNDd1L8Mb8aE+6q0Y5rNb3RL7A2ypH2ZeYKSGpHz3C7Rn5KW +voWAXRWriwKBgQCH4bxK3x0ivxiCgtcyIojDzwVGRnDLqmMIVzeDHqjsjBs2BTcJ +9/e3QK1w1BKzeWF2oPilaJrLY+tkqE9FxWtwQ6DjJ0xDIZ9DIuH/13X5t8EiWOWS +devSdzpyje+58JW78pcArk7u2hXZ2OHDU5qvlRsRL6/jP3SHWWCeFFnviwKBgGov +M02r0YygwfEfBYeFtp7Nx7lypZU2Eg4levWIdsp6f9KclEEA+u3IXD25XAiVMNw2 +pegJU3stioWPMSCZXUxrQAEdqOwE3XzehqfWBJaxxIEWQ7m2Gsb0PWIUlMnyeGJA +Tl8IPboCiVAmk5WQVREyMsuYhf0Qg23MAZ8k5CHvAoGBAJm55NQZVKAEDGd4a21q +TDcRddtPwwL2oP3qa0gbGk4YFRUCrX99hIejOTvQW1xf6vGxTd7E1QizvFse4yRz +ZRKyXIc7DCcdzOnpMrSd1+aXwZtRHLSw0EDS6PWeJZdjJYHxl2YpAmMdURdcGTrH +b6b/6vhU90+xL14CX7Awofp/ +-----END PRIVATE KEY----- +EOF + cat << 'EOF' > /shared/keys/pub/jwt.pub +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4bmY8T5jhlCFtS4ccyn8 +ZDftWk27yOcPgKbak991/m1rs3474RKHYjc9GwX0ICSaUN02fCdSGaTNhU586Mfh +RsSj3uU/TIJ4M3eDengcllT16DTJOyHZgVzHT2QRUsozFuG2tTBCwqYildOXXuCe +aKUQ8Jh5sjKmCcQZO/Ht4omGIsVZPo9SrvshZeHdkdvctvjvT5WXoCZaHvBRpyiB +mQKUv/LfYsgpF4PFm40j84GAk0CsZ/xOJXBnU6wLfzHCq1pgLqaXK/q6T/gSJ5t5 +eWbv0IkUmXDG+TuOtKIz/Xip2lJQ6Rmb/QmOY+MeJavL41NpCu+2FwTvCNqMbHuW +uQIDAQAB +-----END PUBLIC KEY----- +EOF chmod 644 /shared/keys/pub/jwt.pub /shared/keys/jwt.key fi echo "creating token" -token="$(python /scripts/sign_jwt.py)" +python /scripts/sign_jwt.py testu@lifescience-ri.eu > "/shared/token" cat >/shared/s3cfg < Date: Wed, 12 Feb 2025 07:25:32 +0100 Subject: [PATCH 039/250] [tests] make sure the sync test doesn't fail with the mock aai --- .github/integration/sda-sync-integration.yml | 54 +++++++++++++------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/.github/integration/sda-sync-integration.yml b/.github/integration/sda-sync-integration.yml index 5160ab9623..2af84742e1 100644 --- a/.github/integration/sda-sync-integration.yml +++ b/.github/integration/sda-sync-integration.yml @@ -89,7 +89,7 @@ services: condition: service_completed_successfully minio: condition: service_healthy - oidc: + mock-aai: condition: service_healthy postgres: condition: service_healthy @@ -101,6 +101,8 @@ services: - BROKER_ROUTINGKEY=inbox - DB_PASSWORD=inbox - DB_USER=inbox + extra_hosts: + - "localhost:host-gateway" restart: always volumes: - ./sda/config.yaml:/config.yaml @@ -251,30 +253,42 @@ services: volumes: - ./sda/config.yaml:/config.yaml - oidc: - container_name: oidc - command: - - /bin/sh - - -c - - | - pip install --upgrade pip - pip install aiohttp Authlib joserfc requests - python -u /oidc.py + mock-aai: + container_name: ls-aai-mock depends_on: - credentials: - condition: service_completed_successfully + aai-db: + condition: service_healthy + environment: + - DOCKERHOST=localhost + extra_hosts: + - "localhost:host-gateway" healthcheck: - test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] + test: + [ "CMD", "/bin/true" ] interval: 10s timeout: 2s retries: 6 - image: python:3.11-slim + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 ports: - - "8080:8080" - restart: always + - "8800:8080" volumes: - - ./sda/oidc.py:/oidc.py - - shared:/shared + - "./sda/aai-mock:/etc/lsaai-mock" + aai-db: + container_name: ls-aai-db + environment: + MYSQL_ROOT_PASSWORD: "aaiPass" + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: "aai" + MYSQL_USER: "aai" + MYSQL_PASSWORD: "aaiPass" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 2s + retries: 6 + image: mysql/mysql-server:latest + volumes: + - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql api: command: [ sda-api ] @@ -284,7 +298,7 @@ services: condition: service_completed_successfully postgres: condition: service_healthy - oidc: + mock-aai: condition: service_healthy rabbitmq: condition: service_healthy @@ -293,6 +307,8 @@ services: - BROKER_USER=api - DB_PASSWORD=api - DB_USER=api + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8090:8080" From 491c8999e08f3b53c646ce2fdd07f2b4a57eb0cf Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 12 Feb 2025 13:50:23 +0000 Subject: [PATCH 040/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 4ff6d3b1a2..6cf63a3456 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.74 -appVersion: v0.3.190 +version: 0.8.75 +appVersion: v0.3.193 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index d36da9a99a..eb5c0d3765 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.75 -appVersion: v0.3.190 +version: 0.7.76 +appVersion: v0.3.193 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 99ffcd5cee..3a391dd55c 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.3 -appVersion: v0.3.190 +version: 0.30.4 +appVersion: v0.3.193 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 6c192c4ccf056fdc9f6e53a8388b0e9afeaa894a Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 11 Feb 2025 15:21:46 +0100 Subject: [PATCH 041/250] add release_v1 prefix to commit message of dependabot prs to release_v1, for better differentiation --- .github/dependabot.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 0bfb3f1f50..bb05ba941b 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -113,6 +113,8 @@ updates: open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" + commit-message: + prefix: "[release_v1]" - package-ecosystem: docker target-branch: release_v1 @@ -122,6 +124,8 @@ updates: open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" + commit-message: + prefix: "[release_v1]" - package-ecosystem: docker target-branch: release_v1 @@ -131,6 +135,8 @@ updates: open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" + commit-message: + prefix: "[release_v1]" - package-ecosystem: docker target-branch: release_v1 @@ -140,6 +146,8 @@ updates: open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" + commit-message: + prefix: "[release_v1]" - package-ecosystem: docker target-branch: release_v1 @@ -149,6 +157,8 @@ updates: open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" + commit-message: + prefix: "[release_v1]" - package-ecosystem: docker target-branch: release_v1 @@ -158,6 +168,8 @@ updates: open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" + commit-message: + prefix: "[release_v1]" ### JAVA - package-ecosystem: maven @@ -172,6 +184,8 @@ updates: - "neicnordic/sensitive-data-development-collaboration" schedule: interval: daily + commit-message: + prefix: "[release_v1]" - package-ecosystem: maven target-branch: release_v1 @@ -185,6 +199,8 @@ updates: - "neicnordic/sensitive-data-development-collaboration" schedule: interval: daily + commit-message: + prefix: "[release_v1]" ### GO - package-ecosystem: gomod target-branch: release_v1 @@ -198,6 +214,8 @@ updates: - "neicnordic/sensitive-data-development-collaboration" schedule: interval: daily + commit-message: + prefix: "[release_v1]" - package-ecosystem: gomod target-branch: release_v1 @@ -211,3 +229,5 @@ updates: - "neicnordic/sensitive-data-development-collaboration" schedule: interval: daily + commit-message: + prefix: "[release_v1]" From 150803305414e79e78422aa7aa489546d8b29217 Mon Sep 17 00:00:00 2001 From: Parisa Date: Mon, 17 Feb 2025 12:51:41 +0100 Subject: [PATCH 042/250] set all intervals to weekly in dependabot.yaml --- .github/dependabot.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index bb05ba941b..11cc977139 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -3,7 +3,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: weekly open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" @@ -56,7 +56,7 @@ updates: reviewers: - "neicnordic/sensitive-data-development-collaboration" schedule: - interval: "weekly" + interval: weekly - package-ecosystem: gomod directory: "/sda-download" @@ -109,7 +109,7 @@ updates: target-branch: release_v1 directory: "/postgresql" schedule: - interval: daily + interval: weekly open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" @@ -120,7 +120,7 @@ updates: target-branch: release_v1 directory: "/rabbitmq" schedule: - interval: daily + interval: weekly open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" @@ -131,7 +131,7 @@ updates: target-branch: release_v1 directory: "/sda" schedule: - interval: daily + interval: weekly open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" @@ -142,7 +142,7 @@ updates: target-branch: release_v1 directory: "/sda-doa" schedule: - interval: daily + interval: weekly open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" @@ -153,7 +153,7 @@ updates: target-branch: release_v1 directory: "/sda-download" schedule: - interval: daily + interval: weekly open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" @@ -164,7 +164,7 @@ updates: target-branch: release_v1 directory: "/sda-sftp-inbox" schedule: - interval: daily + interval: weekly open-pull-requests-limit: 10 reviewers: - "neicnordic/sensitive-data-development-collaboration" @@ -183,7 +183,7 @@ updates: reviewers: - "neicnordic/sensitive-data-development-collaboration" schedule: - interval: daily + interval: weekly commit-message: prefix: "[release_v1]" @@ -198,7 +198,7 @@ updates: reviewers: - "neicnordic/sensitive-data-development-collaboration" schedule: - interval: daily + interval: weekly commit-message: prefix: "[release_v1]" ### GO @@ -213,7 +213,7 @@ updates: reviewers: - "neicnordic/sensitive-data-development-collaboration" schedule: - interval: daily + interval: weekly commit-message: prefix: "[release_v1]" @@ -228,6 +228,6 @@ updates: reviewers: - "neicnordic/sensitive-data-development-collaboration" schedule: - interval: daily + interval: weekly commit-message: prefix: "[release_v1]" From 60cc576f35f5f0f52a77c9263967c8d7a3909b2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 13:09:08 +0000 Subject: [PATCH 043/250] Bump golangci/golangci-lint-action from 6.2.0 to 6.5.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.2.0 to 6.5.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v6.2.0...v6.5.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 9620790c14..f10a243b76 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.2.0 + uses: golangci/golangci-lint-action@v6.5.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-download @@ -48,7 +48,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.2.0 + uses: golangci/golangci-lint-action@v6.5.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.2.0 + uses: golangci/golangci-lint-action@v6.5.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-admin From 06a420f7a69e4b3200edd08fb941f32915e2dc24 Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 23 Jan 2025 16:06:30 +0100 Subject: [PATCH 044/250] add integrationtest-sda-doa and build-sda-doa to Makefile --- .github/integration/scripts/make_certs.sh | 3 + .github/integration/sda-doa-posix-outbox.yml | 132 ++++++++++++++ .github/integration/sda-doa-s3-outbox.yml | 163 ++++++++++++++++++ Makefile | 4 + sda-doa/Dockerfile | 2 +- sda-doa/docker-compose-posix-outbox.yml | 79 --------- sda-doa/docker-compose-s3-outbox.yml | 93 ---------- .../doa/LocalEGADOAApplicationTests.java | 95 +++------- sda-doa/test/mock_auth.py | 16 +- sda-doa/test/run/run_sda_doa_tests.sh | 49 ++++++ sda-doa/test/setup/make_crypt4gh_files.sh | 41 +++++ .../test/setup/make_crypt4gh_private_key.sh | 19 ++ sda-doa/test/setup/run_db_command.sh | 65 +++++++ 13 files changed, 510 insertions(+), 251 deletions(-) create mode 100644 .github/integration/sda-doa-posix-outbox.yml create mode 100644 .github/integration/sda-doa-s3-outbox.yml delete mode 100644 sda-doa/docker-compose-posix-outbox.yml delete mode 100644 sda-doa/docker-compose-s3-outbox.yml create mode 100644 sda-doa/test/run/run_sda_doa_tests.sh create mode 100644 sda-doa/test/setup/make_crypt4gh_files.sh create mode 100644 sda-doa/test/setup/make_crypt4gh_private_key.sh create mode 100644 sda-doa/test/setup/run_db_command.sh diff --git a/.github/integration/scripts/make_certs.sh b/.github/integration/scripts/make_certs.sh index bde3bd1a05..7b0d66692c 100755 --- a/.github/integration/scripts/make_certs.sh +++ b/.github/integration/scripts/make_certs.sh @@ -34,6 +34,7 @@ openssl x509 -req -in "$out_dir/mq.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey # Create client certificate openssl req -config "$script_dir/ssl.cnf" -new -nodes -newkey rsa:4096 -keyout "$out_dir/client.key" -out "$out_dir/client.csr" -extensions client_cert -subj "/CN=admin" +openssl pkcs8 -topk8 -inform PEM -outform DER -in "$out_dir/client.key" -out "$out_dir/client.der" -nocrypt openssl x509 -req -in "$out_dir/client.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey "$out_dir/ca-key.pem" -set_serial 01 -out "$out_dir/client.crt" -extensions client_cert -extfile "$script_dir/ssl.cnf" if [ -n "$KEYSTORE_PASSWORD" ]; then @@ -68,7 +69,9 @@ chmod 600 /certs/*.key cp -p "$out_dir/ca.crt" /client_certs/ca.crt cp -p "$out_dir/client.crt" /client_certs/ cp -p "$out_dir/client.key" /client_certs/ +cp -p "$out_dir/client.der" /client_certs/ chmod 600 /client_certs/*.key +chmod 644 /client_certs/*.der # needed if testing locally mkdir -p /temp/certs diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml new file mode 100644 index 0000000000..fada2ffd63 --- /dev/null +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -0,0 +1,132 @@ +services: + certfixer: + command: + - /bin/sh + - /scripts/make_certs.sh + container_name: certfixer + image: alpine:latest + volumes: + - ./scripts:/scripts + - certs:/certs + - client_certs:/client_certs + - /tmp:/temp + + postgres: + build: + context: ../../postgresql + container_name: postgres + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + depends_on: + certfixer: + condition: service_completed_successfully + ports: + - 5432:5432 + environment: + - LEGA_IN_PASSWORD=password + - LEGA_OUT_PASSWORD=password + - POSTGRES_PASSWORD=rootpasswd + - POSTGRES_SERVER_CACERT=/certs/ca.crt + - POSTGRES_SERVER_CERT=/certs/db.crt + - POSTGRES_SERVER_KEY=/certs/db.key + + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "lega_out" ] + interval: 5s + timeout: 20s + retries: 3 + volumes: + - certs:/certs + + mockauth: + container_name: mockauth + image: python:3.11-slim + ports: + - 8000:8000 + volumes: + - ../../sda-doa/test/mock_auth.py:/mock_auth.py + - client_certs:/client_certs + command: > + sh -c "pip install --upgrade pip && pip install aiohttp Authlib && python -u /mock_auth.py 0.0.0.0 8000" + + + + rabbitmq: + build: + context: ../../rabbitmq + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + container_name: rabbitmq + depends_on: + certfixer: + condition: service_completed_successfully + environment: + - RABBITMQ_SERVER_CACERT=/etc/rabbitmq/ssl/ca.crt + - RABBITMQ_SERVER_CERT=/etc/rabbitmq/ssl/mq.crt + - RABBITMQ_SERVER_KEY=/etc/rabbitmq/ssl/mq.key + - RABBITMQ_SERVER_VERIFY=verify_none + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms", + ] + interval: 5s + timeout: 20s + retries: 3 + restart: always + volumes: + - certs:/etc/rabbitmq/ssl/ + - rabbitmq_data:/var/lib/rabbitmq + ports: + - 5671:5671 + - 25671:15671 + + doa: + container_name: doa + build: + context: ../../sda-doa + dockerfile: ../../sda-doa/Dockerfile + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-sda-doa + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + certfixer: + condition: service_completed_successfully + links: + - rabbitmq + - postgres + ports: + - "8080:8080" + environment: + - SSL_MODE=verify-ca + - POSTGRES_USER=postgres + - POSTGRES_DB=sda + - POSTGRES_PASSWORD=rootpasswd + - DB_INSTANCE=postgres + - SSL_ENABLED=false + - BROKER_HOST=rabbitmq + - BROKER_VALIDATE=false + - OPENID_CONFIGURATION_URL=http://mockauth:8000/openid-configuration + - USERINFO_ENDPOINT_URL=http://mockauth:8000/userinfo + - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem + - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass + - OUTBOX_TYPE=POSIX + - OUTBOX_LOCATION=/outbox/%s/files/ + - SSL_ENABLED=false + - ROOT_CERT_PATH=/certs/ca.crt + - CERT_PATH=/certs/client.crt + - CERT_KEY=/certs/client.der + - BROKER_USERNAME=guest + + volumes: + - client_certs:/certs + - ../../sda-doa/test/body.enc:/test/body.enc + - ../../sda-doa/outbox:/outbox + +volumes: + certs: + client_certs: + rabbitmq_data: diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml new file mode 100644 index 0000000000..437ae05f01 --- /dev/null +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -0,0 +1,163 @@ +services: + certfixer: + command: + - /bin/sh + - /scripts/make_certs.sh + container_name: certfixer + image: alpine:latest + volumes: + - ./scripts:/scripts + - certs:/certs + - client_certs:/client_certs + - /tmp:/temp + + postgres: + build: + context: ../../postgresql + container_name: postgres + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + depends_on: + certfixer: + condition: service_completed_successfully + ports: + - 5432:5432 + environment: + - LEGA_IN_PASSWORD=password + - LEGA_OUT_PASSWORD=password + - POSTGRES_PASSWORD=rootpasswd + - POSTGRES_SERVER_CACERT=/certs/ca.crt + - POSTGRES_SERVER_CERT=/certs/db.crt + - POSTGRES_SERVER_KEY=/certs/db.key + + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "lega_out" ] + interval: 5s + timeout: 20s + retries: 3 + volumes: + - certs:/certs + + mockauth: + container_name: mockauth + image: python:3.11-slim + ports: + - 8000:8000 + volumes: + - ../../sda-doa/test/mock_auth.py:/mock_auth.py + - client_certs:/client_certs + command: > + sh -c "pip install --upgrade pip && pip install aiohttp Authlib && python -u /mock_auth.py 0.0.0.0 8000" + + rabbitmq: + build: + context: ../../rabbitmq + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + container_name: rabbitmq + depends_on: + certfixer: + condition: service_completed_successfully + environment: + - RABBITMQ_SERVER_CACERT=/etc/rabbitmq/ssl/ca.crt + - RABBITMQ_SERVER_CERT=/etc/rabbitmq/ssl/mq.crt + - RABBITMQ_SERVER_KEY=/etc/rabbitmq/ssl/mq.key + - RABBITMQ_SERVER_VERIFY=verify_none + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms", + ] + interval: 5s + timeout: 20s + retries: 3 + restart: always + volumes: + - certs:/etc/rabbitmq/ssl/ + - rabbitmq_data:/var/lib/rabbitmq + ports: + - 5671:5671 + - 25671:15671 + + outbox: + container_name: outbox + image: minio/minio + ports: + - 9000:9000 + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=miniostorage + command: server /data + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 10s + timeout: 5s + retries: 5 + + init-bucket: + container_name: init-bucket + image: minio/mc + depends_on: + outbox: + condition: service_healthy + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage; + /usr/bin/mc mb s3/lega; + exit 0; + " + + doa: + container_name: doa + build: + context: ../../sda-doa + dockerfile: ../../sda-doa/Dockerfile + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-sda-doa + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + certfixer: + condition: service_completed_successfully + links: + - rabbitmq + - postgres + ports: + - "8080:8080" + environment: + - SSL_MODE=verify-ca + - POSTGRES_USER=postgres + - POSTGRES_DB=sda + - POSTGRES_PASSWORD=rootpasswd + - DB_INSTANCE=postgres + - SSL_ENABLED=false + - BROKER_HOST=rabbitmq + - BROKER_VALIDATE=false + - OPENID_CONFIGURATION_URL=http://mockauth:8000/openid-configuration + - USERINFO_ENDPOINT_URL=http://mockauth:8000/userinfo + - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem + - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass + - OUTBOX_TYPE=S3 + - SSL_ENABLED=false + - ROOT_CERT_PATH=/certs/ca.crt + - CERT_PATH=/certs/client.crt + - CERT_KEY=/certs/client.der + - BROKER_USERNAME=guest + - S3_OUT_ENDPOINT=outbox + - S3_OUT_PORT=9000 + - S3_OUT_SECURE=false + + volumes: + - client_certs:/certs + - ../../sda-doa/test/body.enc:/test/body.enc + + +volumes: + postgres: + certs: + client_certs: + postgres_data: + rabbitmq_data: + shared: \ No newline at end of file diff --git a/Makefile b/Makefile index ee4c5a7841..618337f598 100644 --- a/Makefile +++ b/Makefile @@ -36,6 +36,8 @@ build-sda-sftp-inbox: @cd sda-sftp-inbox && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-sftp-inbox . build-sda-admin: @cd sda-admin && go build +build-sda-doa: + @cd sda-doa && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-sda-doa . go-version-check: SHELL:=/bin/bash @@ -127,6 +129,8 @@ integrationtest-sda-sync-run: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml run integration_test integrationtest-sda-sync-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans +integrationtest-sda-doa: + @cd sda-doa/test/run && sh run_sda_doa_tests.sh # lint go code lint-all: lint-sda lint-sda-download lint-sda-admin diff --git a/sda-doa/Dockerfile b/sda-doa/Dockerfile index b534cd27b9..10ae01974d 100644 --- a/sda-doa/Dockerfile +++ b/sda-doa/Dockerfile @@ -1,4 +1,4 @@ -FROM maven:3-eclipse-temurin-21-alpine as builder +FROM maven:3-eclipse-temurin-21-alpine AS builder COPY pom.xml . diff --git a/sda-doa/docker-compose-posix-outbox.yml b/sda-doa/docker-compose-posix-outbox.yml deleted file mode 100644 index 3a593186bf..0000000000 --- a/sda-doa/docker-compose-posix-outbox.yml +++ /dev/null @@ -1,79 +0,0 @@ -version: '3.3' - -services: - - db: - build: - context: ../../postgresql - container_name: postgres - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres - ports: - - 5432:5432 - environment: - - LEGA_IN_PASSWORD=password - - LEGA_OUT_PASSWORD=password - - POSTGRES_SERVER_CERT=/etc/ega/pg.cert - - POSTGRES_SERVER_KEY=/etc/ega/pg.key - - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert - - PG_VERIFY_PEER=1 - - POSTGRES_PASSWORD=rootpasswd - secrets: - - source: server.pem - target: /etc/ega/pg.cert - uid: '70' - gid: '70' - mode: 0600 - - source: server-key.pem - target: /etc/ega/pg.key - uid: '70' - gid: '70' - mode: 0600 - - source: rootCA.pem - target: /etc/ega/CA.cert - uid: '70' - gid: '70' - mode: 0600 - volumes: - - db:/ega - - mockauth: - image: cscfi/beacon-python - ports: - - 8000:8000 - volumes: - - ./test/mock_auth.py:/mock_auth.py - entrypoint: ["python", "/mock_auth.py", "0.0.0.0", "8000"] - - private-mq: - build: - context: ../../rabbitmq - container_name: rabbitmq - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq - ports: - - 5671:5671 - - 15671:15671 - environment: - - SSL_VERIFY=verify_peer - - SSL_FAIL_IF_NO_PEER_CERT=false - - SSL_DEPTH=2 - - USER_NAME=admin - - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - - VIRTUAL_HOST=sda - secrets: - - source: server.pem - target: /etc/rabbitmq/ssl.cert - - source: server-key.pem - target: /etc/rabbitmq/ssl.key - - source: rootCA.pem - target: /etc/rabbitmq/CA.cert - -volumes: - db: - -secrets: - rootCA.pem: - file: test/rootCA.pem - server.pem: - file: test/localhost.pem - server-key.pem: - file: test/localhost-key.pem diff --git a/sda-doa/docker-compose-s3-outbox.yml b/sda-doa/docker-compose-s3-outbox.yml deleted file mode 100644 index a659a4c910..0000000000 --- a/sda-doa/docker-compose-s3-outbox.yml +++ /dev/null @@ -1,93 +0,0 @@ -version: '3.3' - -services: - - db: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres - ports: - - 5432:5432 - environment: - - LEGA_IN_PASSWORD=password - - LEGA_OUT_PASSWORD=password - - POSTGRES_SERVER_CERT=/etc/ega/pg.cert - - POSTGRES_SERVER_KEY=/etc/ega/pg.key - - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert - - PG_VERIFY_PEER=1 - - POSTGRES_PASSWORD=rootpasswd - secrets: - - source: server.pem - target: /etc/ega/pg.cert - uid: '70' - gid: '70' - mode: 0600 - - source: server-key.pem - target: /etc/ega/pg.key - uid: '70' - gid: '70' - mode: 0600 - - source: rootCA.pem - target: /etc/ega/CA.cert - uid: '70' - gid: '70' - mode: 0600 - volumes: - - db:/ega - - mockauth: - image: cscfi/beacon-python - ports: - - 8000:8000 - volumes: - - ./test/mock_auth.py:/mock_auth.py - entrypoint: ["python", "/mock_auth.py", "0.0.0.0", "8000"] - - private-mq: - image: uiobmi/localega-broker-private:latest - ports: - - 5671:5671 - - 15671:15671 - environment: - - SSL_VERIFY=verify_peer - - SSL_FAIL_IF_NO_PEER_CERT=false - - SSL_DEPTH=2 - - USER_NAME=admin - - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - - VIRTUAL_HOST=sda - secrets: - - source: server.pem - target: /etc/rabbitmq/ssl.cert - - source: server-key.pem - target: /etc/rabbitmq/ssl.key - - source: rootCA.pem - target: /etc/rabbitmq/CA.cert - - outbox: - image: minio/minio - ports: - - 9000:9000 - environment: - - MINIO_ACCESS_KEY=minio - - MINIO_SECRET_KEY=miniostorage - command: server /data - - createbucket: - image: minio/mc - depends_on: - - s3 - entrypoint: > - /bin/sh -c " - /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage; - /usr/bin/mc mb s3/lega; - exit 0; - " - -volumes: - db: - -secrets: - rootCA.pem: - file: test/rootCA.pem - server.pem: - file: test/localhost.pem - server-key.pem: - file: test/localhost-key.pem diff --git a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java index 2ece458ab2..a84a4ef545 100644 --- a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java +++ b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java @@ -31,10 +31,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.security.PrivateKey; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.util.Properties; import java.util.UUID; @Slf4j @@ -47,43 +43,6 @@ class LocalEGADOAApplicationTests { @SneakyThrows @BeforeAll public static void setup() { - String url = String.format("jdbc:postgresql://%s:%s/%s", "localhost", "5432", "sda"); - Properties props = new Properties(); -// props.setProperty("user", "lega_in"); //will be used when lega_in user is set in db again - props.setProperty("user", "postgres"); - props.setProperty("password", "rootpasswd"); - props.setProperty("ssl", "true"); - props.setProperty("application_name", "LocalEGA"); - props.setProperty("sslmode", "verify-full"); - props.setProperty("sslrootcert", new File("test/rootCA.pem").getAbsolutePath()); - props.setProperty("sslcert", new File("test/localhost-client.pem").getAbsolutePath()); - props.setProperty("sslkey", new File("test/localhost-client-key.der").getAbsolutePath()); - Connection connection = DriverManager.getConnection(url, props); - PreparedStatement file = connection.prepareStatement("SELECT local_ega.insert_file('body.enc','requester@elixir-europe.org');"); - file.executeQuery(); - PreparedStatement header = connection.prepareStatement("UPDATE local_ega.files SET header = '637279707434676801000000010000006c00000000000000aa7ad1bb4f93bf5e4fb3bc28a95bc4d80bf2fd8075e69eb2ee15e0a4f08f1d78ab98c8fd9b50e675f71311936e8d0c6f73538962b836355d5d4371a12eae46addb43518b5236fb9554249710a473026f34b264a61d2ba52ed11abc1efa1d3478fa40a710' WHERE id = 1;"); - header.executeUpdate(); - PreparedStatement finalize = connection.prepareStatement("UPDATE local_ega.files SET archive_path = 'test/body.enc', status = 'READY', stable_id = 'EGAF00000000014' WHERE id = 1;"); - finalize.executeUpdate(); - connection.close(); - -// props.setProperty("user", "lega_out"); //will be used when lega_out user is set in db again - connection = DriverManager.getConnection(url, props); - PreparedStatement dataset = connection.prepareStatement("INSERT INTO local_ega_ebi.filedataset(file_id, dataset_stable_id) values(1, 'EGAD00010000919');"); - dataset.executeUpdate(); - - PreparedStatement dataset_event_registered = connection.prepareStatement(prepareInsertQueryDatasetEvent("EGAD00010000919", "registered", "mapping")); - dataset_event_registered.executeUpdate(); - - Thread.sleep(1000 * 3); - - PreparedStatement dataset_event_released = connection.prepareStatement(prepareInsertQueryDatasetEvent("EGAD00010000919", "released", "release")); - dataset_event_released.executeUpdate(); - - PreparedStatement datasetReferenceInsert = connection.prepareStatement("INSERT INTO sda.dataset_references(dataset_id, reference_id, reference_scheme) values('1', 'GDI-NO-10001','GDI');"); - datasetReferenceInsert.executeUpdate(); - connection.close(); - JSONArray tokens = Unirest.get("http://localhost:8000/tokens").asJson().getBody().getArray(); validToken = tokens.getString(0); invalidToken = tokens.getString(1); @@ -93,7 +52,7 @@ public static void setup() { @SneakyThrows @AfterEach public void tearDown() { - File exportFolder = new File("requester@elixir-europe.org"); + File exportFolder = new File("outbox/requester@elixir-europe.org"); if (exportFolder.exists() && exportFolder.isDirectory()) { FileUtils.deleteDirectory(exportFolder); } @@ -186,8 +145,8 @@ void testStreamingValidTokenValidFileRangePlain() { @SneakyThrows @Test void testStreamingValidTokenValidFileFullEncrypted() { - String publicKey = Files.readString(new File("test/crypt4gh.pub.pem").toPath()); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh.sec.pem"), "password".toCharArray()); + String publicKey = Files.readString(new File("test/crypt4gh/crypt4gh.pub.pem").toPath()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/crypt4gh.sec.pem"), "password".toCharArray()); HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); @@ -201,8 +160,8 @@ void testStreamingValidTokenValidFileFullEncrypted() { @SneakyThrows @Test void testStreamingValidTokenValidFileRangeEncrypted() { - String publicKey = Files.readString(new File("test/crypt4gh.pub.pem").toPath()); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh.sec.pem"), "password".toCharArray()); + String publicKey = Files.readString(new File("test/crypt4gh/crypt4gh.pub.pem").toPath()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/crypt4gh.sec.pem"), "password".toCharArray()); HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200&destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); @@ -221,8 +180,8 @@ void testPOSIXExportRequestFileValidToken() { return; } export("EGAF00000000014", false, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -237,8 +196,8 @@ void testPOSIXExportRequestDatasetValidToken() { return; } export("EGAD00010000919", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -253,7 +212,7 @@ void testS3ExportRequestFileValidToken() { return; } export("EGAF00000000014", false, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -269,7 +228,7 @@ void testS3ExportRequestDatasetValidToken() { return; } export("EGAD00010000919", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -285,7 +244,7 @@ void testS3ExportRequestReferenceValidToken() { return; } export("GDI-NO-10001", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -301,8 +260,8 @@ void testPOSIXExportRequestReferenceValidToken() { return; } export("GDI-NO-10001", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -317,8 +276,8 @@ void testPOSIXExportRequestFileValidVisaToken() { return; } export("EGAF00000000014", false, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -333,8 +292,8 @@ void testPOSIXExportRequestDatasetValidVisaToken() { return; } export("EGAD00010000919", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -349,7 +308,7 @@ void testS3ExportRequestFileValidVisaToken() { return; } export("EGAF00000000014", false, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -365,7 +324,7 @@ void testS3ExportRequestDatasetValidVisaToken() { return; } export("EGAD00010000919", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -381,8 +340,8 @@ void testPOSIXExportRequestReferenceValidVisaToken() { return; } export("GDI-NO-10001", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -397,7 +356,7 @@ void testS3ExportRequestReferenceValidVisaToken() { return; } export("GDI-NO-10001", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -406,7 +365,7 @@ void testS3ExportRequestReferenceValidVisaToken() { } @SneakyThrows void export(String id, boolean dataset, String token) { - String mqConnectionString = "amqps://admin:guest@localhost:5671/sda"; + String mqConnectionString = "amqps://guest:guest@localhost:5671/sda"; ConnectionFactory factory = new ConnectionFactory(); factory.setUri(mqConnectionString); com.rabbitmq.client.Connection connectionFactory = factory.newConnection(); @@ -427,7 +386,7 @@ void export(String id, boolean dataset, String token) { token, dataset ? "datasetId" : "fileId", id, - FileUtils.readFileToString(new File("test/my.pub.pem"), Charset.defaultCharset())); + FileUtils.readFileToString(new File("test/crypt4gh/my.pub.pem"), Charset.defaultCharset())); channel.basicPublish("", "exportRequests", properties, @@ -442,8 +401,4 @@ MinioClient getMinioClient() { return MinioClient.builder().endpoint("localhost", 9000, false).region("us-west-1").credentials("minio", "miniostorage").build(); } - - private static String prepareInsertQueryDatasetEvent(String datasetId, String event, String msgType) { - return String.format("INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('%s','%s','{\"type\": \"%s\"}');", datasetId, event, msgType); - } } diff --git a/sda-doa/test/mock_auth.py b/sda-doa/test/mock_auth.py index 9e90ad1aa5..a1beee039d 100644 --- a/sda-doa/test/mock_auth.py +++ b/sda-doa/test/mock_auth.py @@ -19,7 +19,7 @@ def generate_token(): # See available claims here: https://www.iana.org/assignments/jwt/jwt.xhtml # the important claim is the "authorities" header = { - "jku": "http://localhost:8000/jwk", + "jku": "http://mockauth:8000/jwk", "kid": "rsa1", "alg": "RS256", "typ": "JWT" @@ -29,14 +29,14 @@ def generate_token(): "aud": ["aud2", "aud3"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "http://localhost:8000/", + "iss": "http://mockauth:8000/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102" } empty_payload = { "sub": "requester@elixir-europe.org", - "iss": "http://localhost:8000/", + "iss": "http://mockauth:8000/", "exp": 99999999999, "iat": 1547794655, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102" @@ -44,7 +44,7 @@ def generate_token(): # Craft 4 passports, 2 for bona fide status and 2 for dataset permissions # passport for bona fide: terms passport_terms = { - "iss": "http://localhost:8000/", + "iss": "http://mockauth:8000/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "AcceptedTermsAndPolicies", @@ -59,7 +59,7 @@ def generate_token(): } # passport for bona fide: status passport_status = { - "iss": "http://localhost:8000/", + "iss": "http://mockauth:8000/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ResearcherStatus", @@ -74,7 +74,7 @@ def generate_token(): } # passport for dataset permissions 1 passport_dataset1 = { - "iss": "http://localhost:8000/", + "iss": "http://mockauth:8000/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -89,7 +89,7 @@ def generate_token(): } # passport for dataset permissions 2 passport_dataset2 = { - "iss": "http://localhost:8000/", + "iss": "http://mockauth:8000/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -168,7 +168,7 @@ async def userinfo(request): async def openid_configuration(request): """Mock ELIXIR AAI OpenID configuration endpoint.""" data = { - "jwks_uri": "http://localhost:8000/jwk" + "jwks_uri": "http://mockauth:8000/jwk" } return web.json_response(data) diff --git a/sda-doa/test/run/run_sda_doa_tests.sh b/sda-doa/test/run/run_sda_doa_tests.sh new file mode 100644 index 0000000000..27b9fff648 --- /dev/null +++ b/sda-doa/test/run/run_sda_doa_tests.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +cd "$(dirname "$0")/../.." || exit 1 + +PR_NUMBER=$(date +%F) +export PR_NUMBER + +storage_types=("posix" "s3") + +show_spinner() { + printf "\033[33;7mStarting up services...\033[0m\n" + for ((i = 0; i < 25; i++)); do + for s in / - \\ \|; do + printf "\r\033[33m%s\033[0m" "$s" + sleep 0.1 + done + done + echo "" +} + +for storage_type in "${storage_types[@]}"; do + printf "\033[0;35mRunning test for %s \033[0m\n" "$storage_type" + docker compose -f ../.github/integration/sda-doa-"$storage_type"-outbox.yml -p sda-doa up -d + show_spinner + + + for script in "test/setup"/*.sh; do + echo "Running $script..." + bash "$script" + done + + if [[ $storage_type == "posix" ]]; then + export OUTBOX_TYPE="POSIX" + else + export OUTBOX_TYPE="S3" + fi + + if ! mvn test; then + echo "Tests failed for $storage_type. Stopping." + fi + + docker compose -f ../.github/integration/sda-doa-"$storage_type"-outbox.yml down -v + unset OUTBOX_TYPE + [[ $storage_type == "posix" ]] && rm -rf outbox + + rm -rf test/crypt4gh +done +unset PR_NUMBER + diff --git a/sda-doa/test/setup/make_crypt4gh_files.sh b/sda-doa/test/setup/make_crypt4gh_files.sh new file mode 100644 index 0000000000..5797b560dd --- /dev/null +++ b/sda-doa/test/setup/make_crypt4gh_files.sh @@ -0,0 +1,41 @@ +#!/bin/sh + +mkdir -p ./test/crypt4gh + +cat << 'EOF' > ./test/crypt4gh/crypt4gh.pub.pem +-----BEGIN CRYPT4GH PUBLIC KEY----- +PQFOuEiuAOlghTGU0Z7u2EH5FwBtSlnukpiMAji/ml0= +-----END CRYPT4GH PUBLIC KEY----- +EOF + +chmod 444 ./test/crypt4gh/crypt4gh.pub.pem + +cat << 'EOF' > ./test/crypt4gh/my.pub.pem +-----BEGIN CRYPT4GH PUBLIC KEY----- +cUSN6pHzlIFgoclIfDSaDtUgAXRa+DUHBhTodeNL52w= +-----END CRYPT4GH PUBLIC KEY----- +EOF + +chmod 444 ./test/crypt4gh/my.pub.pem + +cat << 'EOF' > ./test/crypt4gh/my.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAOK6Q2g1KxcUELMt/RMhr9wARY2hhY2hhMjBf +cG9seTEzMDUAPL3l1Mt/LvDwD+yffT09Jog2AJO3uG0DaGAPDTPbOfTBKr/gWRik +PcF6893CkScij1pO3n9Ub8p1H4yLAQ== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./test/crypt4gh/my.sec.pem + +cat << 'EOF' > ./test/crypt4gh/crypt4gh.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf +cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J +kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./test/crypt4gh/crypt4gh.sec.pem + +echo "CRYPT4GH files created successfully" diff --git a/sda-doa/test/setup/make_crypt4gh_private_key.sh b/sda-doa/test/setup/make_crypt4gh_private_key.sh new file mode 100644 index 0000000000..cf90bfd049 --- /dev/null +++ b/sda-doa/test/setup/make_crypt4gh_private_key.sh @@ -0,0 +1,19 @@ +#!/bin/sh +docker exec -it --user root doa sh -c ' +mkdir -p test && \ +cat << EOF > test/crypt4gh.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf +cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J +kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 test/crypt4gh.sec.pem + +printf "password" > test/crypt4gh.pass + +chmod 444 test/crypt4gh.pass + +echo "CRYPT4GH private key created successfully" +' 2>/dev/null diff --git a/sda-doa/test/setup/run_db_command.sh b/sda-doa/test/setup/run_db_command.sh new file mode 100644 index 0000000000..3fc117ef03 --- /dev/null +++ b/sda-doa/test/setup/run_db_command.sh @@ -0,0 +1,65 @@ +#!/bin/sh + +echo "Insert File" +docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ + -e PGPASSWORD=password \ + -e PGSSLMODE=verify-ca \ + -e PGSSLCERT=/certs/client.crt \ + -e PGSSLKEY=/certs/client.key \ + -e PGSSLROOTCERT=/certs/ca.crt \ + postgres:latest \ + bash -c "psql -h postgres -U lega_in -d sda -c \"SELECT local_ega.insert_file('body.enc', 'requester@elixir-europe.org');\"" +echo "Set Header For The File" +docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ + -e PGPASSWORD=password \ + -e PGSSLMODE=verify-ca \ + -e PGSSLCERT=/certs/client.crt \ + -e PGSSLKEY=/certs/client.key \ + -e PGSSLROOTCERT=/certs/ca.crt \ + postgres:latest \ + bash -c "psql -h postgres -U lega_in -d sda -c \"UPDATE local_ega.files SET header = '637279707434676801000000010000006c00000000000000aa7ad1bb4f93bf5e4fb3bc28a95bc4d80bf2fd8075e69eb2ee15e0a4f08f1d78ab98c8fd9b50e675f71311936e8d0c6f73538962b836355d5d4371a12eae46addb43518b5236fb9554249710a473026f34b264a61d2ba52ed11abc1efa1d3478fa40a710' WHERE id = 1;\"" +echo "Set File Data" +docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ + -e PGPASSWORD=password \ + -e PGSSLMODE=verify-ca \ + -e PGSSLCERT=/certs/client.crt \ + -e PGSSLKEY=/certs/client.key \ + -e PGSSLROOTCERT=/certs/ca.crt \ + postgres:latest \ + bash -c "psql -h postgres -U lega_in -d sda -c \"UPDATE local_ega.files SET archive_path = 'test/body.enc', status = 'READY', stable_id = 'EGAF00000000014' WHERE id = 1;\"" +echo "Insert Dataset" +docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ + -e PGPASSWORD=password \ + -e PGSSLMODE=verify-ca \ + -e PGSSLCERT=/certs/client.crt \ + -e PGSSLKEY=/certs/client.key \ + -e PGSSLROOTCERT=/certs/ca.crt \ + postgres:latest \ + bash -c "psql -h postgres -U lega_out -d sda -c \"INSERT INTO local_ega_ebi.filedataset(file_id, dataset_stable_id) values(1, 'EGAD00010000919');\"" +echo "Insert Event Log REGISTERED" +docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ + -e PGPASSWORD=password \ + -e PGSSLMODE=verify-ca \ + -e PGSSLCERT=/certs/client.crt \ + -e PGSSLKEY=/certs/client.key \ + -e PGSSLROOTCERT=/certs/ca.crt \ + postgres:latest \ + bash -c "psql -h postgres -U lega_out -d sda -c \"INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'registered', '{\\\"type\\\": \\\"mapping\\\"}')\"" +echo "Insert Event Log RELEASED" +docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ + -e PGPASSWORD=password \ + -e PGSSLMODE=verify-ca \ + -e PGSSLCERT=/certs/client.crt \ + -e PGSSLKEY=/certs/client.key \ + -e PGSSLROOTCERT=/certs/ca.crt \ + postgres:latest \ + bash -c "psql -h postgres -U lega_out -d sda -c \"INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'released', '{\\\"type\\\": \\\"release\\\"}')\"" +echo "Insert Dataset Reference" +docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ + -e PGPASSWORD=rootpasswd \ + -e PGSSLMODE=verify-ca \ + -e PGSSLCERT=/certs/client.crt \ + -e PGSSLKEY=/certs/client.key \ + -e PGSSLROOTCERT=/certs/ca.crt \ + postgres:latest \ + bash -c "psql -h postgres -U postgres -d sda -c \"INSERT INTO sda.dataset_references(dataset_id, reference_id, reference_scheme) values('1', 'GDI-NO-10001','GDI');\"" From 3be9891e2645792f37f25ebed0356d080856f485 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 4 Feb 2025 14:43:48 +0100 Subject: [PATCH 045/250] -change sda-doa to doa in build-sda-doa and add all targets for sda-doa -run the tests in integration_test container and change Test file accordingly -move test related files to .github/integration --- .github/integration/sda-doa-posix-outbox.yml | 32 +++++++-- .github/integration/sda-doa-s3-outbox.yml | 30 +++++++-- .../tests/doa/1.make_crypt4gh_files.sh | 56 ++++++++++++++++ .../tests/doa/2.insert_test_data.sh | 37 +++++++++++ .github/integration/tests/doa/3.run_tests.sh | 2 + Makefile | 22 ++++++- .../doa/LocalEGADOAApplicationTests.java | 34 +++++----- sda-doa/test/run/run_sda_doa_tests.sh | 49 -------------- sda-doa/test/setup/make_crypt4gh_files.sh | 41 ------------ .../test/setup/make_crypt4gh_private_key.sh | 19 ------ sda-doa/test/setup/run_db_command.sh | 65 ------------------- 11 files changed, 179 insertions(+), 208 deletions(-) create mode 100755 .github/integration/tests/doa/1.make_crypt4gh_files.sh create mode 100755 .github/integration/tests/doa/2.insert_test_data.sh create mode 100644 .github/integration/tests/doa/3.run_tests.sh delete mode 100644 sda-doa/test/run/run_sda_doa_tests.sh delete mode 100644 sda-doa/test/setup/make_crypt4gh_files.sh delete mode 100644 sda-doa/test/setup/make_crypt4gh_private_key.sh delete mode 100644 sda-doa/test/setup/run_db_command.sh diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index fada2ffd63..c717ab5a7a 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -48,8 +48,6 @@ services: command: > sh -c "pip install --upgrade pip && pip install aiohttp Authlib && python -u /mock_auth.py 0.0.0.0 8000" - - rabbitmq: build: context: ../../rabbitmq @@ -86,7 +84,6 @@ services: container_name: doa build: context: ../../sda-doa - dockerfile: ../../sda-doa/Dockerfile image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-sda-doa depends_on: postgres: @@ -95,9 +92,6 @@ services: condition: service_healthy certfixer: condition: service_completed_successfully - links: - - rabbitmq - - postgres ports: - "8080:8080" environment: @@ -124,7 +118,31 @@ services: volumes: - client_certs:/certs - ../../sda-doa/test/body.enc:/test/body.enc - - ../../sda-doa/outbox:/outbox + - ./test_file:/outbox + - ./encryption_files:/test + + integration_test: + container_name: integration_test + image: maven:3.9.9-eclipse-temurin-21 + profiles: [test] + volumes: + - ../../sda-doa:/sda-doa + - ../../sda-doa/settings.xml:/root/.m2/settings.xml + - ./test_file:/sda-doa/outbox + - ./tests:/tests + - ./encryption_files:/test + - client_certs:/certs + + depends_on: + - doa + - mockauth + environment: + - OUTBOX_TYPE=POSIX + + command: + - "/bin/sh" + - "/tests/run_scripts.sh" + - "/tests/doa" volumes: certs: diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 437ae05f01..662f1415e2 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -112,7 +112,6 @@ services: container_name: doa build: context: ../../sda-doa - dockerfile: ../../sda-doa/Dockerfile image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-sda-doa depends_on: postgres: @@ -121,9 +120,6 @@ services: condition: service_healthy certfixer: condition: service_completed_successfully - links: - - rabbitmq - - postgres ports: - "8080:8080" environment: @@ -152,12 +148,32 @@ services: volumes: - client_certs:/certs - ../../sda-doa/test/body.enc:/test/body.enc + - ./encryption_files:/test + integration_test: + container_name: integration_test + image: maven:3.9.9-eclipse-temurin-21 + profiles: [test] + volumes: + - ../../sda-doa:/sda-doa + - ../../sda-doa/settings.xml:/root/.m2/settings.xml + - ./tests:/tests + - ./encryption_files:/test + - client_certs:/certs + + depends_on: + - doa + - mockauth + - init-bucket + environment: + - OUTBOX_TYPE=S3 + + command: + - "/bin/sh" + - "/tests/run_scripts.sh" + - "/tests/doa" volumes: - postgres: certs: client_certs: - postgres_data: rabbitmq_data: - shared: \ No newline at end of file diff --git a/.github/integration/tests/doa/1.make_crypt4gh_files.sh b/.github/integration/tests/doa/1.make_crypt4gh_files.sh new file mode 100755 index 0000000000..568508b786 --- /dev/null +++ b/.github/integration/tests/doa/1.make_crypt4gh_files.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +mkdir -p ./sda-doa/test/crypt4gh + +cat << EOF > ./sda-doa/test/crypt4gh/crypt4gh.pub.pem +-----BEGIN CRYPT4GH PUBLIC KEY----- +PQFOuEiuAOlghTGU0Z7u2EH5FwBtSlnukpiMAji/ml0= +-----END CRYPT4GH PUBLIC KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/crypt4gh.pub.pem + +cat << EOF > ./sda-doa/test/crypt4gh/my.pub.pem +-----BEGIN CRYPT4GH PUBLIC KEY----- +cUSN6pHzlIFgoclIfDSaDtUgAXRa+DUHBhTodeNL52w= +-----END CRYPT4GH PUBLIC KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/my.pub.pem + +cat << EOF > ./sda-doa/test/crypt4gh/my.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAOK6Q2g1KxcUELMt/RMhr9wARY2hhY2hhMjBf +cG9seTEzMDUAPL3l1Mt/LvDwD+yffT09Jog2AJO3uG0DaGAPDTPbOfTBKr/gWRik +PcF6893CkScij1pO3n9Ub8p1H4yLAQ== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/my.sec.pem + +cat << EOF > ./sda-doa/test/crypt4gh/crypt4gh.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf +cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J +kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/crypt4gh.sec.pem + + +cat << EOF > ./test/crypt4gh.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf +cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J +kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./test/crypt4gh.sec.pem + +printf "password" > ./test/crypt4gh.pass + +chmod 444 ./test/crypt4gh.pass + +echo "CRYPT4GH files created successfully" \ No newline at end of file diff --git a/.github/integration/tests/doa/2.insert_test_data.sh b/.github/integration/tests/doa/2.insert_test_data.sh new file mode 100755 index 0000000000..0bcc28df28 --- /dev/null +++ b/.github/integration/tests/doa/2.insert_test_data.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +export PGPASSWORD="password" +export PGSSLMODE="verify-ca" +export PGSSLCERT="/certs/client.crt" +export PGSSLKEY="/certs/client.key" +export PGSSLROOTCERT="/certs/ca.crt" + +if ! psql -h postgres -U lega_in -d sda -c "SELECT local_ega.insert_file('body.enc', 'requester@elixir-europe.org');"; then + echo "Insert File failed" +fi + +if ! psql -h postgres -U lega_in -d sda -c "UPDATE local_ega.files SET header = '637279707434676801000000010000006c00000000000000aa7ad1bb4f93bf5e4fb3bc28a95bc4d80bf2fd8075e69eb2ee15e0a4f08f1d78ab98c8fd9b50e675f71311936e8d0c6f73538962b836355d5d4371a12eae46addb43518b5236fb9554249710a473026f34b264a61d2ba52ed11abc1efa1d3478fa40a710' +WHERE id = 1;"; then + echo "Set Header For The File failed" +fi + +if ! psql -h postgres -U lega_in -d sda -c "UPDATE local_ega.files SET archive_path = 'test/body.enc', status = 'READY', stable_id = 'EGAF00000000014' WHERE id = 1;"; then + echo "Set File Data failed" +fi + +if ! psql -h postgres -U lega_out -d sda -c "INSERT INTO local_ega_ebi.filedataset(file_id, dataset_stable_id) values(1, 'EGAD00010000919');"; then + echo "Insert Dataset failed" +fi + +if ! psql -h postgres -U lega_out -d sda -c "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'registered', '{\"type\": \"mapping\"}');"; then + echo "Insert Event Log RELEASED failed" +fi + +if ! psql -h postgres -U lega_out -d sda -c "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'released', '{\"type\": \"release\"}');"; then + echo "Insert Event Log RELEASE failed" +fi + +export PGPASSWORD="rootpasswd" +if ! psql -h postgres -U postgres -d sda -c "INSERT INTO sda.dataset_references(dataset_id, reference_id, reference_scheme) values('1', 'GDI-NO-10001','GDI');"; then + echo "Insert Dataset Reference failed" +fi diff --git a/.github/integration/tests/doa/3.run_tests.sh b/.github/integration/tests/doa/3.run_tests.sh new file mode 100644 index 0000000000..ba6e81d912 --- /dev/null +++ b/.github/integration/tests/doa/3.run_tests.sh @@ -0,0 +1,2 @@ +cd sda-doa || exit +mvn test \ No newline at end of file diff --git a/Makefile b/Makefile index 618337f598..278d0bfa91 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ build-sda-sftp-inbox: build-sda-admin: @cd sda-admin && go build build-sda-doa: - @cd sda-doa && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-sda-doa . + @cd sda-doa && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-doa . go-version-check: SHELL:=/bin/bash @@ -129,8 +129,24 @@ integrationtest-sda-sync-run: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml run integration_test integrationtest-sda-sync-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans -integrationtest-sda-doa: - @cd sda-doa/test/run && sh run_sda_doa_tests.sh + +integrationtest-sda-doa-posix: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml down -v --remove-orphans +integrationtest-sda-doa-posix-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml run integration_test +integrationtest-sda-doa-posix-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml down -v --remove-orphans + +integrationtest-sda-doa-s3: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml down -v --remove-orphans +integrationtest-sda-doa-s3-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml run integration_test +integrationtest-sda-doa-s3-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml down -v --remove-orphans + + # lint go code lint-all: lint-sda lint-sda-download lint-sda-admin diff --git a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java index a84a4ef545..993228c498 100644 --- a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java +++ b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java @@ -43,7 +43,7 @@ class LocalEGADOAApplicationTests { @SneakyThrows @BeforeAll public static void setup() { - JSONArray tokens = Unirest.get("http://localhost:8000/tokens").asJson().getBody().getArray(); + JSONArray tokens = Unirest.get("http://mockauth:8000/tokens").asJson().getBody().getArray(); validToken = tokens.getString(0); invalidToken = tokens.getString(1); validVisaToken = tokens.getString(2); @@ -60,19 +60,19 @@ public void tearDown() { @Test void testMetadataDatasetsNoToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets").asJson().getStatus(); + int status = Unirest.get("http://doa:8080/metadata/datasets").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataDatasetsInvalidToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get("http://doa:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataDatasetsValidToken() { - HttpResponse response = Unirest.get("http://localhost:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get("http://doa:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); JSONArray datasets = response.getBody().getArray(); @@ -82,26 +82,26 @@ void testMetadataDatasetsValidToken() { @Test void testMetadataFilesNoToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000919/files").asJson().getStatus(); + int status = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000919/files").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataFilesInvalidToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataFilesValidTokenInvalidDataset() { - HttpResponse response = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000920/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000920/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.FORBIDDEN.value(), status); } @Test void testMetadataFilesValidTokenValidDataset() { - HttpResponse response = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("[{\"fileId\":\"EGAF00000000014\",\"datasetId\":\"EGAD00010000919\",\"displayFileName\":\"body.enc\",\"fileName\":\"test/body.enc\",\"fileSize\":null,\"unencryptedChecksum\":null,\"unencryptedChecksumType\":null,\"decryptedFileSize\":null,\"decryptedFileChecksum\":null,\"decryptedFileChecksumType\":null,\"fileStatus\":\"READY\"}]", response.getBody().toString()); @@ -109,26 +109,26 @@ void testMetadataFilesValidTokenValidDataset() { @Test void testStreamingNoToken() { - int status = Unirest.get("http://localhost:8080/files/EGAF00000000014").asJson().getStatus(); + int status = Unirest.get("http://doa:8080/files/EGAF00000000014").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testStreamingInvalidToken() { - int status = Unirest.get("http://localhost:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get("http://doa:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testStreamingValidTokenInvalidFile() { - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000015").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000015").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.FORBIDDEN.value(), status); } @Test void testStreamingValidTokenValidFileFullPlain() { - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); + HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(response.getBody())); @@ -136,7 +136,7 @@ void testStreamingValidTokenValidFileFullPlain() { @Test void testStreamingValidTokenValidFileRangePlain() { - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); + HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("09fbeae7cce2cd410b471b0a1a265fb53dc54c66c4c7c3111b8b9b95ac0e956f", DigestUtils.sha256Hex(response.getBody())); @@ -147,7 +147,7 @@ void testStreamingValidTokenValidFileRangePlain() { void testStreamingValidTokenValidFileFullEncrypted() { String publicKey = Files.readString(new File("test/crypt4gh/crypt4gh.pub.pem").toPath()); PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/crypt4gh.sec.pem"), "password".toCharArray()); - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); + HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014?destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(response.getBody()); @@ -162,7 +162,7 @@ void testStreamingValidTokenValidFileFullEncrypted() { void testStreamingValidTokenValidFileRangeEncrypted() { String publicKey = Files.readString(new File("test/crypt4gh/crypt4gh.pub.pem").toPath()); PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/crypt4gh.sec.pem"), "password".toCharArray()); - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200&destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); + HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200&destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(response.getBody()); @@ -365,7 +365,7 @@ void testS3ExportRequestReferenceValidVisaToken() { } @SneakyThrows void export(String id, boolean dataset, String token) { - String mqConnectionString = "amqps://guest:guest@localhost:5671/sda"; + String mqConnectionString = "amqps://guest:guest@rabbitmq:5671/sda"; ConnectionFactory factory = new ConnectionFactory(); factory.setUri(mqConnectionString); com.rabbitmq.client.Connection connectionFactory = factory.newConnection(); @@ -398,7 +398,7 @@ void export(String id, boolean dataset, String token) { } MinioClient getMinioClient() { - return MinioClient.builder().endpoint("localhost", 9000, false).region("us-west-1").credentials("minio", "miniostorage").build(); + return MinioClient.builder().endpoint("outbox", 9000, false).region("us-west-1").credentials("minio", "miniostorage").build(); } } diff --git a/sda-doa/test/run/run_sda_doa_tests.sh b/sda-doa/test/run/run_sda_doa_tests.sh deleted file mode 100644 index 27b9fff648..0000000000 --- a/sda-doa/test/run/run_sda_doa_tests.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash - -cd "$(dirname "$0")/../.." || exit 1 - -PR_NUMBER=$(date +%F) -export PR_NUMBER - -storage_types=("posix" "s3") - -show_spinner() { - printf "\033[33;7mStarting up services...\033[0m\n" - for ((i = 0; i < 25; i++)); do - for s in / - \\ \|; do - printf "\r\033[33m%s\033[0m" "$s" - sleep 0.1 - done - done - echo "" -} - -for storage_type in "${storage_types[@]}"; do - printf "\033[0;35mRunning test for %s \033[0m\n" "$storage_type" - docker compose -f ../.github/integration/sda-doa-"$storage_type"-outbox.yml -p sda-doa up -d - show_spinner - - - for script in "test/setup"/*.sh; do - echo "Running $script..." - bash "$script" - done - - if [[ $storage_type == "posix" ]]; then - export OUTBOX_TYPE="POSIX" - else - export OUTBOX_TYPE="S3" - fi - - if ! mvn test; then - echo "Tests failed for $storage_type. Stopping." - fi - - docker compose -f ../.github/integration/sda-doa-"$storage_type"-outbox.yml down -v - unset OUTBOX_TYPE - [[ $storage_type == "posix" ]] && rm -rf outbox - - rm -rf test/crypt4gh -done -unset PR_NUMBER - diff --git a/sda-doa/test/setup/make_crypt4gh_files.sh b/sda-doa/test/setup/make_crypt4gh_files.sh deleted file mode 100644 index 5797b560dd..0000000000 --- a/sda-doa/test/setup/make_crypt4gh_files.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh - -mkdir -p ./test/crypt4gh - -cat << 'EOF' > ./test/crypt4gh/crypt4gh.pub.pem ------BEGIN CRYPT4GH PUBLIC KEY----- -PQFOuEiuAOlghTGU0Z7u2EH5FwBtSlnukpiMAji/ml0= ------END CRYPT4GH PUBLIC KEY----- -EOF - -chmod 444 ./test/crypt4gh/crypt4gh.pub.pem - -cat << 'EOF' > ./test/crypt4gh/my.pub.pem ------BEGIN CRYPT4GH PUBLIC KEY----- -cUSN6pHzlIFgoclIfDSaDtUgAXRa+DUHBhTodeNL52w= ------END CRYPT4GH PUBLIC KEY----- -EOF - -chmod 444 ./test/crypt4gh/my.pub.pem - -cat << 'EOF' > ./test/crypt4gh/my.sec.pem ------BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- -YzRnaC12MQAGc2NyeXB0ABQAAAAAOK6Q2g1KxcUELMt/RMhr9wARY2hhY2hhMjBf -cG9seTEzMDUAPL3l1Mt/LvDwD+yffT09Jog2AJO3uG0DaGAPDTPbOfTBKr/gWRik -PcF6893CkScij1pO3n9Ub8p1H4yLAQ== ------END CRYPT4GH ENCRYPTED PRIVATE KEY----- -EOF - -chmod 444 ./test/crypt4gh/my.sec.pem - -cat << 'EOF' > ./test/crypt4gh/crypt4gh.sec.pem ------BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- -YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf -cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J -kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== ------END CRYPT4GH ENCRYPTED PRIVATE KEY----- -EOF - -chmod 444 ./test/crypt4gh/crypt4gh.sec.pem - -echo "CRYPT4GH files created successfully" diff --git a/sda-doa/test/setup/make_crypt4gh_private_key.sh b/sda-doa/test/setup/make_crypt4gh_private_key.sh deleted file mode 100644 index cf90bfd049..0000000000 --- a/sda-doa/test/setup/make_crypt4gh_private_key.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -docker exec -it --user root doa sh -c ' -mkdir -p test && \ -cat << EOF > test/crypt4gh.sec.pem ------BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- -YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf -cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J -kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== ------END CRYPT4GH ENCRYPTED PRIVATE KEY----- -EOF - -chmod 444 test/crypt4gh.sec.pem - -printf "password" > test/crypt4gh.pass - -chmod 444 test/crypt4gh.pass - -echo "CRYPT4GH private key created successfully" -' 2>/dev/null diff --git a/sda-doa/test/setup/run_db_command.sh b/sda-doa/test/setup/run_db_command.sh deleted file mode 100644 index 3fc117ef03..0000000000 --- a/sda-doa/test/setup/run_db_command.sh +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh - -echo "Insert File" -docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ - -e PGPASSWORD=password \ - -e PGSSLMODE=verify-ca \ - -e PGSSLCERT=/certs/client.crt \ - -e PGSSLKEY=/certs/client.key \ - -e PGSSLROOTCERT=/certs/ca.crt \ - postgres:latest \ - bash -c "psql -h postgres -U lega_in -d sda -c \"SELECT local_ega.insert_file('body.enc', 'requester@elixir-europe.org');\"" -echo "Set Header For The File" -docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ - -e PGPASSWORD=password \ - -e PGSSLMODE=verify-ca \ - -e PGSSLCERT=/certs/client.crt \ - -e PGSSLKEY=/certs/client.key \ - -e PGSSLROOTCERT=/certs/ca.crt \ - postgres:latest \ - bash -c "psql -h postgres -U lega_in -d sda -c \"UPDATE local_ega.files SET header = '637279707434676801000000010000006c00000000000000aa7ad1bb4f93bf5e4fb3bc28a95bc4d80bf2fd8075e69eb2ee15e0a4f08f1d78ab98c8fd9b50e675f71311936e8d0c6f73538962b836355d5d4371a12eae46addb43518b5236fb9554249710a473026f34b264a61d2ba52ed11abc1efa1d3478fa40a710' WHERE id = 1;\"" -echo "Set File Data" -docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ - -e PGPASSWORD=password \ - -e PGSSLMODE=verify-ca \ - -e PGSSLCERT=/certs/client.crt \ - -e PGSSLKEY=/certs/client.key \ - -e PGSSLROOTCERT=/certs/ca.crt \ - postgres:latest \ - bash -c "psql -h postgres -U lega_in -d sda -c \"UPDATE local_ega.files SET archive_path = 'test/body.enc', status = 'READY', stable_id = 'EGAF00000000014' WHERE id = 1;\"" -echo "Insert Dataset" -docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ - -e PGPASSWORD=password \ - -e PGSSLMODE=verify-ca \ - -e PGSSLCERT=/certs/client.crt \ - -e PGSSLKEY=/certs/client.key \ - -e PGSSLROOTCERT=/certs/ca.crt \ - postgres:latest \ - bash -c "psql -h postgres -U lega_out -d sda -c \"INSERT INTO local_ega_ebi.filedataset(file_id, dataset_stable_id) values(1, 'EGAD00010000919');\"" -echo "Insert Event Log REGISTERED" -docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ - -e PGPASSWORD=password \ - -e PGSSLMODE=verify-ca \ - -e PGSSLCERT=/certs/client.crt \ - -e PGSSLKEY=/certs/client.key \ - -e PGSSLROOTCERT=/certs/ca.crt \ - postgres:latest \ - bash -c "psql -h postgres -U lega_out -d sda -c \"INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'registered', '{\\\"type\\\": \\\"mapping\\\"}')\"" -echo "Insert Event Log RELEASED" -docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ - -e PGPASSWORD=password \ - -e PGSSLMODE=verify-ca \ - -e PGSSLCERT=/certs/client.crt \ - -e PGSSLKEY=/certs/client.key \ - -e PGSSLROOTCERT=/certs/ca.crt \ - postgres:latest \ - bash -c "psql -h postgres -U lega_out -d sda -c \"INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'released', '{\\\"type\\\": \\\"release\\\"}')\"" -echo "Insert Dataset Reference" -docker run --rm --network sda-doa_default -v "sda-doa_client_certs:/certs" \ - -e PGPASSWORD=rootpasswd \ - -e PGSSLMODE=verify-ca \ - -e PGSSLCERT=/certs/client.crt \ - -e PGSSLKEY=/certs/client.key \ - -e PGSSLROOTCERT=/certs/ca.crt \ - postgres:latest \ - bash -c "psql -h postgres -U postgres -d sda -c \"INSERT INTO sda.dataset_references(dataset_id, reference_id, reference_scheme) values('1', 'GDI-NO-10001','GDI');\"" From 32fe16a3a8241f22205dd00abf636c4d715f312f Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 4 Feb 2025 15:05:38 +0100 Subject: [PATCH 046/250] add shebang line --- .github/integration/tests/doa/3.run_tests.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/integration/tests/doa/3.run_tests.sh b/.github/integration/tests/doa/3.run_tests.sh index ba6e81d912..3c2f20e6c3 100644 --- a/.github/integration/tests/doa/3.run_tests.sh +++ b/.github/integration/tests/doa/3.run_tests.sh @@ -1,2 +1,4 @@ +#!/bin/bash + cd sda-doa || exit mvn test \ No newline at end of file From 865626dceeb01cd4c68f2015f0dbfd0e7f4f8c0a Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 6 Feb 2025 10:32:13 +0100 Subject: [PATCH 047/250] -change sda-doa image name -update volume config --- .github/integration/sda-doa-posix-outbox.yml | 13 ++++++++----- .github/integration/sda-doa-s3-outbox.yml | 7 ++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index c717ab5a7a..73ad5a05f8 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -82,9 +82,10 @@ services: doa: container_name: doa + user: root build: context: ../../sda-doa - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-sda-doa + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa depends_on: postgres: condition: service_healthy @@ -118,8 +119,8 @@ services: volumes: - client_certs:/certs - ../../sda-doa/test/body.enc:/test/body.enc - - ./test_file:/outbox - - ./encryption_files:/test + - test_file:/outbox + - encryption_files:/test integration_test: container_name: integration_test @@ -128,9 +129,9 @@ services: volumes: - ../../sda-doa:/sda-doa - ../../sda-doa/settings.xml:/root/.m2/settings.xml - - ./test_file:/sda-doa/outbox + - test_file:/sda-doa/outbox - ./tests:/tests - - ./encryption_files:/test + - encryption_files:/test - client_certs:/certs depends_on: @@ -148,3 +149,5 @@ volumes: certs: client_certs: rabbitmq_data: + test_file: + encryption_files: \ No newline at end of file diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 662f1415e2..799e7c81bd 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -112,7 +112,7 @@ services: container_name: doa build: context: ../../sda-doa - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-sda-doa + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa depends_on: postgres: condition: service_healthy @@ -148,7 +148,7 @@ services: volumes: - client_certs:/certs - ../../sda-doa/test/body.enc:/test/body.enc - - ./encryption_files:/test + - encryption_files:/test integration_test: container_name: integration_test @@ -158,7 +158,7 @@ services: - ../../sda-doa:/sda-doa - ../../sda-doa/settings.xml:/root/.m2/settings.xml - ./tests:/tests - - ./encryption_files:/test + - encryption_files:/test - client_certs:/certs depends_on: @@ -177,3 +177,4 @@ volumes: certs: client_certs: rabbitmq_data: + encryption_files: From c237526d5f110271b3eda7b1a95711418bbf0dda Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 6 Feb 2025 14:43:14 +0100 Subject: [PATCH 048/250] Replace hardcoded host values with environment variables for DOA and MockAuth URLs in test --- .github/integration/sda-doa-posix-outbox.yml | 3 ++ .github/integration/sda-doa-s3-outbox.yml | 3 ++ .../doa/LocalEGADOAApplicationTests.java | 35 +++++++++++-------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index 73ad5a05f8..77111f1352 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -139,6 +139,9 @@ services: - mockauth environment: - OUTBOX_TYPE=POSIX + - DOA_URL=http://doa:8080 + - MOCKAUTH_URL=http://mockauth:8000 + - MINIO_HOST=outbox command: - "/bin/sh" diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 799e7c81bd..6b80cf06dc 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -167,6 +167,9 @@ services: - init-bucket environment: - OUTBOX_TYPE=S3 + - DOA_URL=http://doa:8080 + - MOCKAUTH_URL=http://mockauth:8000 + - MINIO_HOST=outbox command: - "/bin/sh" diff --git a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java index 993228c498..91a52cda66 100644 --- a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java +++ b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java @@ -39,11 +39,18 @@ class LocalEGADOAApplicationTests { private static String validToken; private static String invalidToken; private static String validVisaToken; + private static String doaUrl; + private static String mockauthUrl; + private static String minioHost; @SneakyThrows @BeforeAll public static void setup() { - JSONArray tokens = Unirest.get("http://mockauth:8000/tokens").asJson().getBody().getArray(); + doaUrl = System.getenv("DOA_URL"); + mockauthUrl = System.getenv("MOCKAUTH_URL"); + minioHost = System.getenv("MINIO_HOST"); + + JSONArray tokens = Unirest.get(mockauthUrl + "/tokens").asJson().getBody().getArray(); validToken = tokens.getString(0); invalidToken = tokens.getString(1); validVisaToken = tokens.getString(2); @@ -60,19 +67,19 @@ public void tearDown() { @Test void testMetadataDatasetsNoToken() { - int status = Unirest.get("http://doa:8080/metadata/datasets").asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataDatasetsInvalidToken() { - int status = Unirest.get("http://doa:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataDatasetsValidToken() { - HttpResponse response = Unirest.get("http://doa:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); JSONArray datasets = response.getBody().getArray(); @@ -82,26 +89,26 @@ void testMetadataDatasetsValidToken() { @Test void testMetadataFilesNoToken() { - int status = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000919/files").asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000919/files").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataFilesInvalidToken() { - int status = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataFilesValidTokenInvalidDataset() { - HttpResponse response = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000920/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000920/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.FORBIDDEN.value(), status); } @Test void testMetadataFilesValidTokenValidDataset() { - HttpResponse response = Unirest.get("http://doa:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("[{\"fileId\":\"EGAF00000000014\",\"datasetId\":\"EGAD00010000919\",\"displayFileName\":\"body.enc\",\"fileName\":\"test/body.enc\",\"fileSize\":null,\"unencryptedChecksum\":null,\"unencryptedChecksumType\":null,\"decryptedFileSize\":null,\"decryptedFileChecksum\":null,\"decryptedFileChecksumType\":null,\"fileStatus\":\"READY\"}]", response.getBody().toString()); @@ -109,26 +116,26 @@ void testMetadataFilesValidTokenValidDataset() { @Test void testStreamingNoToken() { - int status = Unirest.get("http://doa:8080/files/EGAF00000000014").asJson().getStatus(); + int status = Unirest.get(doaUrl + "/files/EGAF00000000014").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testStreamingInvalidToken() { - int status = Unirest.get("http://doa:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get(doaUrl + "/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testStreamingValidTokenInvalidFile() { - HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000015").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/files/EGAF00000000015").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.FORBIDDEN.value(), status); } @Test void testStreamingValidTokenValidFileFullPlain() { - HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); + HttpResponse response = Unirest.get(doaUrl + "/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(response.getBody())); @@ -136,7 +143,7 @@ void testStreamingValidTokenValidFileFullPlain() { @Test void testStreamingValidTokenValidFileRangePlain() { - HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); + HttpResponse response = Unirest.get(doaUrl + "/files/EGAF00000000014?startCoordinate=100&endCoordinate=200").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("09fbeae7cce2cd410b471b0a1a265fb53dc54c66c4c7c3111b8b9b95ac0e956f", DigestUtils.sha256Hex(response.getBody())); @@ -398,7 +405,7 @@ void export(String id, boolean dataset, String token) { } MinioClient getMinioClient() { - return MinioClient.builder().endpoint("outbox", 9000, false).region("us-west-1").credentials("minio", "miniostorage").build(); + return MinioClient.builder().endpoint(minioHost, 9000, false).region("us-west-1").credentials("minio", "miniostorage").build(); } } From 4d965be2235d5ca13a139a580e4bb96236cbdccf Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 6 Feb 2025 14:57:06 +0100 Subject: [PATCH 049/250] replace System.getenv("OUTBOX_TYPE") with a variable --- .../doa/LocalEGADOAApplicationTests.java | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java index 91a52cda66..824e655b08 100644 --- a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java +++ b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java @@ -42,6 +42,7 @@ class LocalEGADOAApplicationTests { private static String doaUrl; private static String mockauthUrl; private static String minioHost; + private static String outboxType; @SneakyThrows @BeforeAll @@ -49,6 +50,7 @@ public static void setup() { doaUrl = System.getenv("DOA_URL"); mockauthUrl = System.getenv("MOCKAUTH_URL"); minioHost = System.getenv("MINIO_HOST"); + outboxType = System.getenv("OUTBOX_TYPE"); JSONArray tokens = Unirest.get(mockauthUrl + "/tokens").asJson().getBody().getArray(); validToken = tokens.getString(0); @@ -182,7 +184,7 @@ void testStreamingValidTokenValidFileRangeEncrypted() { @SneakyThrows @Test void testPOSIXExportRequestFileValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } @@ -198,7 +200,7 @@ void testPOSIXExportRequestFileValidToken() { @SneakyThrows @Test void testPOSIXExportRequestDatasetValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } @@ -214,7 +216,7 @@ void testPOSIXExportRequestDatasetValidToken() { @SneakyThrows @Test void testS3ExportRequestFileValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } @@ -230,7 +232,7 @@ void testS3ExportRequestFileValidToken() { @SneakyThrows @Test void testS3ExportRequestDatasetValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } @@ -246,7 +248,7 @@ void testS3ExportRequestDatasetValidToken() { @SneakyThrows @Test void testS3ExportRequestReferenceValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } @@ -262,7 +264,7 @@ void testS3ExportRequestReferenceValidToken() { @SneakyThrows @Test void testPOSIXExportRequestReferenceValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } @@ -278,7 +280,7 @@ void testPOSIXExportRequestReferenceValidToken() { @SneakyThrows @Test void testPOSIXExportRequestFileValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } @@ -294,7 +296,7 @@ void testPOSIXExportRequestFileValidVisaToken() { @SneakyThrows @Test void testPOSIXExportRequestDatasetValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } @@ -310,7 +312,7 @@ void testPOSIXExportRequestDatasetValidVisaToken() { @SneakyThrows @Test void testS3ExportRequestFileValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } @@ -326,7 +328,7 @@ void testS3ExportRequestFileValidVisaToken() { @SneakyThrows @Test void testS3ExportRequestDatasetValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } @@ -342,7 +344,7 @@ void testS3ExportRequestDatasetValidVisaToken() { @SneakyThrows @Test void testPOSIXExportRequestReferenceValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } @@ -358,7 +360,7 @@ void testPOSIXExportRequestReferenceValidVisaToken() { @SneakyThrows @Test void testS3ExportRequestReferenceValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } From 126aba2d16456da78cccd14facb5038e5e6184e8 Mon Sep 17 00:00:00 2001 From: Parisa Date: Mon, 17 Feb 2025 09:13:40 +0100 Subject: [PATCH 050/250] add a script to setup outbox with intended permissions --- .github/integration/sda-doa-posix-outbox.yml | 1 - .github/integration/tests/doa/3.setup_outbox.sh | 4 ++++ .../integration/tests/doa/{3.run_tests.sh => 4.run_tests.sh} | 0 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 .github/integration/tests/doa/3.setup_outbox.sh rename .github/integration/tests/doa/{3.run_tests.sh => 4.run_tests.sh} (100%) diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index 77111f1352..99038f8dcf 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -82,7 +82,6 @@ services: doa: container_name: doa - user: root build: context: ../../sda-doa image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa diff --git a/.github/integration/tests/doa/3.setup_outbox.sh b/.github/integration/tests/doa/3.setup_outbox.sh new file mode 100644 index 0000000000..90c7b9f16e --- /dev/null +++ b/.github/integration/tests/doa/3.setup_outbox.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd sda-doa || exit +mkdir -p outbox && chmod -R 750 outbox && chown -R nobody outbox \ No newline at end of file diff --git a/.github/integration/tests/doa/3.run_tests.sh b/.github/integration/tests/doa/4.run_tests.sh similarity index 100% rename from .github/integration/tests/doa/3.run_tests.sh rename to .github/integration/tests/doa/4.run_tests.sh From b5c7a41332b55435e405a1fe7df30d94d992fe5b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 19:35:02 +0000 Subject: [PATCH 051/250] Bump the all-modules group in /sda with 2 updates Bumps the all-modules group in /sda with 2 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.60 to 1.17.62 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.60...feature/s3/manager/v1.17.62) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.76.0 to 1.77.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.76.0...service/s3/v1.77.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 8 ++++---- sda/go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index d686bc3c55..f752639fcb 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,8 +6,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.1 github.com/aws/aws-sdk-go-v2/config v1.29.6 github.com/aws/aws-sdk-go-v2/credentials v1.17.59 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.60 - github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.62 + github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 github.com/aws/smithy-go v1.22.2 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 @@ -49,14 +49,14 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect diff --git a/sda/go.sum b/sda/go.sum index 7c40e58ab6..fcb4d2df75 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -26,16 +26,16 @@ github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E= github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8 h1:zAxi9p3wsZMIaVCdoiQp2uZ9k1LsZvmAnoTBeZPXom0= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.8/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 h1:VZPDrbzdsU1ZxhyWrvROqLY0nxFWgMCAzhn/nYz3X48= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg= github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ= github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.60 h1:ssZzp6JAGAbOYUTppPfKLa3Cbmx0PtnPsjh4RSy06Ao= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.60/go.mod h1:0fi8BNjII7rWunx2Cvezfnu1iZDCw7EWEiSQyC+Kgww= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.62 h1:qzLOdXzKUuMGDzEAzpEz3QHYy5510nEZCzWI4EBaxZw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.62/go.mod h1:hezn6jOdr8sbGMCJmqJF/WOVK9h9H7EXsmu20zXG2m8= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= @@ -46,14 +46,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvr github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6 h1:cCBJaT7EeEojpJ4s7wTDbhZlHVJOgNHN7iw6qVurGaw= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.5.6/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 h1:kT2WeWcFySdYpPgyqJMSUE7781Qucjtn6wBvrgm9P+M= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q= -github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0 h1:ehvUZNVrGA1Usa6yYo8A8pUqrigRelWXSbcCqYpRLeI= -github.com/aws/aws-sdk-go-v2/service/s3 v1.76.0/go.mod h1:KuLNrwYJFaC2AVZ+CVVc12k9NyqwgWsoNNHjwqF6QNk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 h1:RCOi1rDmLqOICym/6UeS2cqKED4T4m966w2rl1HfL+g= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0/go.mod h1:VC4EKSHqT3nzOcU955VWHMGsQ+w67wfAUBSjC8NOo8U= github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ= github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE= From 626dcf562dad086ea1153ace9ce6fab9116ed410 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Feb 2025 19:38:44 +0000 Subject: [PATCH 052/250] Bump the all-modules group in /sda-sftp-inbox with 2 updates Bumps the all-modules group in /sda-sftp-inbox with 2 updates: [com.amazonaws:aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) and [org.yaml:snakeyaml](https://bitbucket.org/snakeyaml/snakeyaml). Updates `com.amazonaws:aws-java-sdk-s3` from 1.12.780 to 1.12.781 - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.780...1.12.781) Updates `org.yaml:snakeyaml` from 2.3 to 2.4 - [Commits](https://bitbucket.org/snakeyaml/snakeyaml/branches/compare/snakeyaml-2.4..snakeyaml-2.3) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: org.yaml:snakeyaml dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 405c9ff8f7..dd64f971ea 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -96,7 +96,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.780 + 1.12.781 com.google.guava @@ -149,7 +149,7 @@ org.yaml snakeyaml - 2.3 + 2.4 junit From 93b59439082c5b3c42cab53ebdec9c616e57aba7 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 18 Feb 2025 13:21:41 +0000 Subject: [PATCH 053/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 6cf63a3456..ca6d490f32 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.75 -appVersion: v0.3.193 +version: 0.8.76 +appVersion: v0.3.196 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index eb5c0d3765..930bc05160 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.76 -appVersion: v0.3.193 +version: 0.7.77 +appVersion: v0.3.196 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 3a391dd55c..40a77f5022 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.4 -appVersion: v0.3.193 +version: 0.30.5 +appVersion: v0.3.196 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 2c23539f68e83d85d96908ce3f54bd1f8a5793aa Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Feb 2025 09:12:45 +0100 Subject: [PATCH 054/250] [charts][sda-svc] Add `SCHEMA_TYPE` to the API config --- charts/sda-svc/templates/api-deploy.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index 5d0486baff..ae607d6011 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -189,6 +189,8 @@ spec: - name: LOG_LEVEL value: {{ .Values.global.log.level | quote }} {{- end }} + - name: SCHEMA_TYPE + value: {{ default "federated" .Values.global.schemaType }} {{- if .Values.global.api.jwtPubKeyName }} - name: SERVER_JWTPUBKEYPATH value: {{ include "jwtPath" . }} From da38269887b0d6759c2d0af7638b8c82e464b58a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Feb 2025 09:14:31 +0100 Subject: [PATCH 055/250] [charts][sda-svc] Fix Auth OIDc provider URL --- charts/sda-svc/templates/download-deploy.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index f7207afdc6..9b1a2134eb 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -94,7 +94,7 @@ spec: value: "{{ .Values.global.archive.volumePath }}" {{- end }} - name: OIDC_CONFIGURATION_URL - value: "{{ .Values.global.oidc.provider }}/.well-known/openid-configuration" + value: "{{ .Values.global.oidc.provider | trimSuffix "/" }}/.well-known/openid-configuration" {{- if .Values.global.download.trusted.iss }} - name: OIDC_TRUSTED_ISS value: {{ include "trustedIssPath" . }}/{{ default "iss.json" .Values.global.download.trusted.configFile }} From a6211b71840c927fc19211b29eb034dd98a7c2f6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 19 Feb 2025 12:11:14 +0100 Subject: [PATCH 056/250] Bump chart version --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 40a77f5022..1ee98f9513 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 0.30.5 +version: 0.30.6 appVersion: v0.3.196 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From fb6697eca6db5b5c6dd0c191705cde335bc5a374 Mon Sep 17 00:00:00 2001 From: Parisa Date: Wed, 19 Feb 2025 15:49:49 +0100 Subject: [PATCH 057/250] configure Dependabot for elixir-no Maven registry --- .github/dependabot.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 11cc977139..bc297600fb 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -1,4 +1,10 @@ version: 2 +registries: + maven-github-elixir-no: + type: maven-repository + url: 'https://maven.pkg.github.com/ELIXIR-NO' + username: '${{secrets.DOA_MAVEN_GITHUB_USERNAME}}' + password: '${{secrets.DOA_MAVEN_GITHUB_TOKEN}}' updates: - package-ecosystem: "github-actions" directory: "/" @@ -85,6 +91,7 @@ updates: - package-ecosystem: maven directory: "/sda-doa" + registries: "*" groups: all-modules: patterns: @@ -175,6 +182,7 @@ updates: - package-ecosystem: maven target-branch: release_v1 directory: "/sda-doa" + registries: "*" groups: all-modules: patterns: From da54285673a3f57ce4d27d46ba6fd2d6c5662b47 Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 20 Feb 2025 09:19:03 +0100 Subject: [PATCH 058/250] add sda-doa to build_pr_container.yaml to build doa pr image and run doa integration test --- .github/workflows/build_pr_container.yaml | 70 +++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index bd78034979..28177e1d9e 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -189,6 +189,47 @@ jobs: sarif_file: 'inbox-results.sarif' category: sftp-inbox + + - name: create maven settings.xml + uses: s4u/maven-settings-action@v3.0.0 + with: + servers: | + [{ + "id":"github-fega-norway", + "username": "${{github.actor}}", + "password": "${{ secrets.GITHUB_TOKEN }}" + }] + + - name: Copy settings.xml to sda-doa root + shell: bash + run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml + + - name: Build container for sda-doa + uses: docker/build-push-action@v6 + with: + context: ./sda-doa + push: true + tags: | + ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-doa + ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-doa + labels: | + org.opencontainers.image.source=${{ github.event.repository.clone_url }} + org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + org.opencontainers.image.revision=${{ github.sha }} + + - name: Run Trivy vulnerability scanner on doa + uses: aquasecurity/trivy-action@0.29.0 + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-doa + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: 'inbox-results.sarif' + severity: "CRITICAL,HIGH" + rabbitmq: needs: - build_go_images @@ -248,6 +289,35 @@ jobs: run: docker compose -f .github/integration/sda-sync-integration.yml run integration_test if: steps.changes.outputs.sync == 'true' || steps.changes.outputs.sync-api == 'true' + sda-doa: + needs: + - build_java_images + - build_server_images + runs-on: ubuntu-latest + strategy: + matrix: + storage: [s3, posix] + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: create maven settings.xml + uses: s4u/maven-settings-action@v3.0.0 + with: + servers: | + [{ + "id":"github-fega-norway", + "username": "${{github.actor}}", + "password": "${{ secrets.GITHUB_TOKEN }}" + }] + + - name: Copy settings.xml to sda-doa root + shell: bash + run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml + + - name: Test sda-doa for ${{ matrix.storage }} storage + run: docker compose -f .github/integration/sda-doa-${{ matrix.storage }}-outbox.yml run integration_test + chart: needs: - build_go_images From 2c211804207514f34c0bfc4b5c1e2ecc728e6652 Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 20 Feb 2025 10:03:22 +0100 Subject: [PATCH 059/250] add sda-doa to publish_container.yml --- .github/workflows/publish_container.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index b2e5aafc01..4531de6bc2 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -56,7 +56,7 @@ jobs: - name: Pull PR containers run : | - for t in -download -postgres -rabbitmq -sftp-inbox; do + for t in -download -postgres -rabbitmq -sftp-inbox -doa; do docker pull ghcr.io/${{ github.repository }}:PR${{ github.event.number }}$t done docker pull ghcr.io/${{ github.repository }}:PR${{ github.event.number }} @@ -86,6 +86,11 @@ jobs: docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }} ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }} docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }} + - name: Retag PR image for doa + run: | + docker tag ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-doa ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-doa + docker push ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-doa + bump_chart_version: needs: [tag_release, push_to_registry] if: ${{ needs.tag_release.outputs.tag != '' && needs.push_to_registry.result == 'success' }} From 4f74db696552164e36456a7484e0628aa7160bb7 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 4 Feb 2025 22:44:58 +0100 Subject: [PATCH 060/250] [auth] serves both raw and resigned tokens along with s3 config files --- sda/cmd/auth/frontend/static/custom.css | 2 +- sda/cmd/auth/frontend/templates/oidc.html | 15 ++++---- sda/cmd/auth/main.go | 46 +++++++++++++++-------- sda/cmd/auth/oidc.go | 12 ++++-- 4 files changed, 48 insertions(+), 27 deletions(-) diff --git a/sda/cmd/auth/frontend/static/custom.css b/sda/cmd/auth/frontend/static/custom.css index d442c23dda..6fa3a05358 100644 --- a/sda/cmd/auth/frontend/static/custom.css +++ b/sda/cmd/auth/frontend/static/custom.css @@ -7,7 +7,7 @@ } #loginbox1 { - max-width: 600px; + max-width: 800px; } #logintext { diff --git a/sda/cmd/auth/frontend/templates/oidc.html b/sda/cmd/auth/frontend/templates/oidc.html index 4d14a5afa2..adf8daac7c 100644 --- a/sda/cmd/auth/frontend/templates/oidc.html +++ b/sda/cmd/auth/frontend/templates/oidc.html @@ -38,19 +38,20 @@ 
{{.}}
{{ end }} {{ end }} - {{if .Token}} + {{if .ResignedToken}}

- Your access token is: + Your access token {{if .ExpDateResigned}}(expires at {{.ExpDateResigned}} UTC){{end}} for uploading to the Inbox is:

- 
{{.Token}}
+ 
{{.ResignedToken}}
{{end}} - {{if .ExpDate}} + {{if .RawToken}}

- Your access token expires (UTC): + Your access token {{if .ExpDateRaw}}(expires at {{.ExpDateRaw}} UTC){{end}} for downloading from the Archive is:

- 
{{.ExpDate}}
+ 
{{.RawToken}}
{{end}} - Download inbox s3cmd credentials + Download credentials for uploading to the Inbox + Download credentials for downloading from the Archive Continue diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index b2b8c17eda..5dab75177d 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -27,7 +27,8 @@ type LoginOption struct { } type OIDCData struct { - S3Conf map[string]string + S3ConfInbox map[string]string + S3ConfDownload map[string]string OIDCID OIDCIdentity } @@ -40,7 +41,10 @@ type AuthHandler struct { pubKey string } -func (auth AuthHandler) getInboxConfig(ctx iris.Context, authType string) { +// getS3Config retrieves S3 config from session flash and serves it as a +// downloadable s3cmd file with the specified fileName. Redirects to home if +// config is missing. +func (auth AuthHandler) getS3Config(ctx iris.Context, authType string, fileName string) { log.Infoln(ctx.Request().URL.Path) @@ -52,7 +56,8 @@ func (auth AuthHandler) getInboxConfig(ctx iris.Context, authType string) { return } s3cfmap := s3conf.(map[string]string) - ctx.ResponseWriter().Header().Set("Content-Disposition", "attachment; filename=s3cmd.conf") + ctx.ResponseWriter().Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fileName)) + s3c := "[default]\n" for k, v := range s3cfmap { @@ -215,7 +220,7 @@ func (auth AuthHandler) getEGALogin(ctx iris.Context) { // getEGAConf returns an s3config file for an oidc login func (auth AuthHandler) getEGAConf(ctx iris.Context) { - auth.getInboxConfig(ctx, "ega") + auth.getS3Config(ctx, "ega", "s3cmd.conf") } // getOIDC redirects to the oidc page defined in auth.Config @@ -270,6 +275,7 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { } if auth.Config.ResignJwt { + log.Debugf("Resigning token for user %s", idStruct.User) claims := map[string]interface{}{ jwt.ExpirationKey: time.Now().UTC().Add(time.Duration(auth.Config.JwtTTL) * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), @@ -280,14 +286,15 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { if err != nil { log.Errorf("error when generating token: %v", err) } - idStruct.Token = token - idStruct.ExpDate = expDate + idStruct.ResignedToken = token + idStruct.ExpDateResigned = expDate } log.WithFields(log.Fields{"authType": "oidc", "user": idStruct.User}).Infof("User was authenticated") - s3conf := getS3ConfigMap(idStruct.Token, auth.Config.S3Inbox, idStruct.User) + s3confInbox := getS3ConfigMap(idStruct.ResignedToken, auth.Config.S3Inbox, idStruct.User) + s3confDownload := getS3ConfigMap(idStruct.RawToken, auth.Config.S3Inbox, idStruct.User) - return &OIDCData{S3Conf: s3conf, OIDCID: idStruct} + return &OIDCData{S3ConfInbox: s3confInbox, S3ConfDownload: s3confDownload, OIDCID: idStruct} } // getOIDCLogin renders the `oidc.html` template to the given iris context @@ -299,13 +306,16 @@ func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { } s := sessions.Get(ctx) - s.SetFlash("oidc", oidcData.S3Conf) + s.SetFlash("oidcInbox", oidcData.S3ConfInbox) + s.SetFlash("oidcDownload", oidcData.S3ConfDownload) ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) ctx.ViewData("User", oidcData.OIDCID.User) ctx.ViewData("Passport", oidcData.OIDCID.Passport) - ctx.ViewData("Token", oidcData.OIDCID.Token) - ctx.ViewData("ExpDate", oidcData.OIDCID.ExpDate) + ctx.ViewData("RawToken", oidcData.OIDCID.RawToken) + ctx.ViewData("ResignedToken", oidcData.OIDCID.ResignedToken) + ctx.ViewData("ExpDateRaw", oidcData.OIDCID.ExpDateRaw) + ctx.ViewData("ExpDateResigned", oidcData.OIDCID.ExpDateResigned) err := ctx.View("oidc.html") if err != nil { @@ -331,9 +341,14 @@ func (auth AuthHandler) getOIDCCORSLogin(ctx iris.Context) { } } -// getOIDCConf returns an s3config file for an oidc login -func (auth AuthHandler) getOIDCConf(ctx iris.Context) { - auth.getInboxConfig(ctx, "oidc") +// getOIDCConfInbox returns an s3config file for uploading to the Inbox +func (auth AuthHandler) getOIDCConfInbox(ctx iris.Context) { + auth.getS3Config(ctx, "oidcInbox", "s3cmd-inbox.conf") +} + +// getOIDCConfDownload returns an s3config file for downloading from the Archive +func (auth AuthHandler) getOIDCConfDownload(ctx iris.Context) { + auth.getS3Config(ctx, "oidcDownload", "s3cmd-download.conf") } // globalHeaders presets common response headers @@ -427,7 +442,8 @@ func main() { // OIDC endpoints app.Get("/oidc", authHandler.getOIDC) - app.Get("/oidc/s3conf", authHandler.getOIDCConf) + app.Get("/oidc/s3conf-inbox", authHandler.getOIDCConfInbox) + app.Get("/oidc/s3conf-download", authHandler.getOIDCConfDownload) app.Get("/oidc/login", authHandler.getOIDCLogin) app.Get("/oidc/cors_login", authHandler.getOIDCCORSLogin) diff --git a/sda/cmd/auth/oidc.go b/sda/cmd/auth/oidc.go index 5f30858e79..6637c403ad 100644 --- a/sda/cmd/auth/oidc.go +++ b/sda/cmd/auth/oidc.go @@ -18,11 +18,13 @@ import ( type OIDCIdentity struct { User string Passport []string - Token string + RawToken string + ResignedToken string Profile string Email string EdupersonEntitlement []string - ExpDate string + ExpDateRaw string + ExpDateResigned string } // Configure an OpenID Connect aware OAuth2 client. @@ -104,12 +106,14 @@ func authenticateWithOidc(oauth2Config oauth2.Config, provider *oidc.Provider, c idStruct = OIDCIdentity{ User: userInfo.Subject, - Token: rawAccessToken, + RawToken: rawAccessToken, + ResignedToken: rawAccessToken, Passport: claims.PassportClaim, Profile: claims.ProfileClaim, Email: claims.EmailClaim, EdupersonEntitlement: claims.EdupersonEntitlement, - ExpDate: rawExpDate, + ExpDateRaw: rawExpDate, + ExpDateResigned: rawExpDate, } return idStruct, err From f90e7f20a08fe00744f5118cd62a3683a3696254 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 5 Feb 2025 14:27:34 +0100 Subject: [PATCH 061/250] update gitignore --- .gitignore | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.gitignore b/.gitignore index 3c3f6f5a29..e3ca9a9c37 100644 --- a/.gitignore +++ b/.gitignore @@ -189,3 +189,13 @@ buildNumber.properties # End of https://www.gitignore.io/api/java,maven,eclipse,intellij+all db/ + +# MacOS desktop service store +.DS_Store + +# crypt4gh key files +*.pub.pem +*.sec.pem + +# sda-admin binary +sda-admin/sda-admin \ No newline at end of file From ddc91bceef330b46fcbd34d852a07d39deba1fa5 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 5 Feb 2025 16:22:43 +0100 Subject: [PATCH 062/250] [auth] do not show visas on OIDC page --- sda/cmd/auth/frontend/templates/oidc.html | 8 -------- 1 file changed, 8 deletions(-) diff --git a/sda/cmd/auth/frontend/templates/oidc.html b/sda/cmd/auth/frontend/templates/oidc.html index adf8daac7c..ef36d01e6a 100644 --- a/sda/cmd/auth/frontend/templates/oidc.html +++ b/sda/cmd/auth/frontend/templates/oidc.html @@ -30,14 +30,6 @@

Welcome, {{.User}}!

- {{if .Passport}} -

- Your visas are the following: -

- {{ range .Passport }} - 
{{.}}
- {{ end }} - {{ end }} {{if .ResignedToken}}

Your access token {{if .ExpDateResigned}}(expires at {{.ExpDateResigned}} UTC){{end}} for uploading to the Inbox is: From 09434b8e29b8eefc393f0d9a40f4f52c626f5c46 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 6 Feb 2025 14:41:13 +0100 Subject: [PATCH 063/250] [auth] fix unit tests for oidc --- sda/cmd/auth/oidc_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sda/cmd/auth/oidc_test.go b/sda/cmd/auth/oidc_test.go index 6a3d395cca..62160aab09 100644 --- a/sda/cmd/auth/oidc_test.go +++ b/sda/cmd/auth/oidc_test.go @@ -69,9 +69,7 @@ func (suite *OIDCTests) TestGetOidcClient() { } func (suite *OIDCTests) TestAuthenticateWithOidc() { - // Create a code to authenticate - session, err := suite.mockServer.SessionStore.NewSession( "openid email profile", "nonce", mockoidc.DefaultUser(), "", "") if err != nil { @@ -84,7 +82,9 @@ func (suite *OIDCTests) TestAuthenticateWithOidc() { elixirIdentity, err := authenticateWithOidc(oauth2Config, provider, code, jwkURL) assert.Nil(suite.T(), err, "Failed to authenticate with OIDC") - assert.NotEqual(suite.T(), "", elixirIdentity.Token, "Empty token returned from OIDC authentication") + // Ensure both RawToken and ResignedToken are not empty + assert.NotEqual(suite.T(), "", elixirIdentity.RawToken, "Empty RawToken returned from OIDC authentication") + assert.NotEqual(suite.T(), "", elixirIdentity.ResignedToken, "Empty ResignedToken returned from OIDC authentication") } func (suite *OIDCTests) TestValidateJwt() { @@ -93,7 +93,7 @@ func (suite *OIDCTests) TestValidateJwt() { oauth2Config, provider := getOidcClient(suite.OIDCConfig) jwkURL := suite.mockServer.JWKSEndpoint() elixirIdentity, _ := authenticateWithOidc(oauth2Config, provider, session.SessionID, jwkURL) - elixirJWT := elixirIdentity.Token + elixirJWT := elixirIdentity.RawToken claims := map[string]interface{}{ jwt.ExpirationKey: time.Now().UTC().Add(2 * time.Hour), @@ -146,7 +146,7 @@ func (suite *OIDCTests) TestValidateJwt() { // sanity check _, expDate, err := validateToken(elixirJWT, suite.mockServer.JWKSEndpoint()) assert.Nil(suite.T(), err) - assert.Equal(suite.T(), expDate, elixirIdentity.ExpDate, "Returned wrong exp date but without returning errors") + assert.Equal(suite.T(), expDate, elixirIdentity.ExpDateRaw, "Returned wrong exp date but without returning errors") // Not a jwk url _, _, err = validateToken(elixirJWT, "http://some/jwk/endpoint") From ddf88410064b53263eef5b18ab82479420a88895 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 6 Feb 2025 16:44:13 +0100 Subject: [PATCH 064/250] update config for running sda-download with go run --- sda-download/dev_utils/config-notls_local.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml index d425077a8f..57cd74d5d4 100644 --- a/sda-download/dev_utils/config-notls_local.yaml +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -32,4 +32,4 @@ db: oidc: # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri" configuration: - url: "http://localhost:8080/.well-known/openid-configuration" + url: "http://localhost:8800/oidc/.well-known/openid-configuration" From 2de433d9e0dfd7bfa7d66677624180deb20eea46 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 6 Feb 2025 22:58:17 +0100 Subject: [PATCH 065/250] Fix linter --- sda/cmd/auth/main.go | 8 ++++---- sda/cmd/auth/oidc.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 5dab75177d..28189a902b 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -27,9 +27,9 @@ type LoginOption struct { } type OIDCData struct { - S3ConfInbox map[string]string + S3ConfInbox map[string]string S3ConfDownload map[string]string - OIDCID OIDCIdentity + OIDCID OIDCIdentity } type AuthHandler struct { @@ -341,12 +341,12 @@ func (auth AuthHandler) getOIDCCORSLogin(ctx iris.Context) { } } -// getOIDCConfInbox returns an s3config file for uploading to the Inbox +// getOIDCConfInbox returns an s3config file for uploading to the Inbox func (auth AuthHandler) getOIDCConfInbox(ctx iris.Context) { auth.getS3Config(ctx, "oidcInbox", "s3cmd-inbox.conf") } -// getOIDCConfDownload returns an s3config file for downloading from the Archive +// getOIDCConfDownload returns an s3config file for downloading from the Archive func (auth AuthHandler) getOIDCConfDownload(ctx iris.Context) { auth.getS3Config(ctx, "oidcDownload", "s3cmd-download.conf") } diff --git a/sda/cmd/auth/oidc.go b/sda/cmd/auth/oidc.go index 6637c403ad..eb6496cb38 100644 --- a/sda/cmd/auth/oidc.go +++ b/sda/cmd/auth/oidc.go @@ -107,7 +107,7 @@ func authenticateWithOidc(oauth2Config oauth2.Config, provider *oidc.Provider, c idStruct = OIDCIdentity{ User: userInfo.Subject, RawToken: rawAccessToken, - ResignedToken: rawAccessToken, + ResignedToken: rawAccessToken, Passport: claims.PassportClaim, Profile: claims.ProfileClaim, Email: claims.EmailClaim, From 4b686de5ad0b99eed0c76f39bc5798e6e8116905 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 7 Feb 2025 11:23:10 +0100 Subject: [PATCH 066/250] [auth] set correct subjectKey for resigning --- sda/cmd/auth/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 28189a902b..903bec1e26 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -280,7 +280,7 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { jwt.ExpirationKey: time.Now().UTC().Add(time.Duration(auth.Config.JwtTTL) * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: auth.Config.JwtIssuer, - jwt.SubjectKey: idStruct.Profile, + jwt.SubjectKey: idStruct.User, } token, expDate, err := generateJwtToken(claims, auth.Config.JwtPrivateKey, auth.Config.JwtSignatureAlg) if err != nil { @@ -311,6 +311,7 @@ func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) ctx.ViewData("User", oidcData.OIDCID.User) + ctx.ViewData("Profile", oidcData.OIDCID.Profile) ctx.ViewData("Passport", oidcData.OIDCID.Passport) ctx.ViewData("RawToken", oidcData.OIDCID.RawToken) ctx.ViewData("ResignedToken", oidcData.OIDCID.ResignedToken) From 16fff5b1decb42ef2188df9aef97c50c9ee5a8ef Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 7 Feb 2025 11:24:02 +0100 Subject: [PATCH 067/250] [auth] show full name instead of username at the welcome page --- sda/cmd/auth/frontend/templates/oidc.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/auth/frontend/templates/oidc.html b/sda/cmd/auth/frontend/templates/oidc.html index ef36d01e6a..3a0c2ff45b 100644 --- a/sda/cmd/auth/frontend/templates/oidc.html +++ b/sda/cmd/auth/frontend/templates/oidc.html @@ -28,7 +28,7 @@

- Welcome, {{.User}}! + Welcome, {{.Profile}}!

{{if .ResignedToken}}

From 51b74604a827587f3558f8ded58a4d686a40e485 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 7 Feb 2025 15:52:42 +0100 Subject: [PATCH 068/250] [auth] refactor: better variable name --- sda/cmd/auth/frontend/templates/oidc.html | 2 +- sda/cmd/auth/main.go | 4 ++-- sda/cmd/auth/oidc.go | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sda/cmd/auth/frontend/templates/oidc.html b/sda/cmd/auth/frontend/templates/oidc.html index 3a0c2ff45b..ecfe129e7e 100644 --- a/sda/cmd/auth/frontend/templates/oidc.html +++ b/sda/cmd/auth/frontend/templates/oidc.html @@ -28,7 +28,7 @@

- Welcome, {{.Profile}}! + Welcome, {{.Fullname}}!

{{if .ResignedToken}}

diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 903bec1e26..794dfd99b3 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -269,7 +269,7 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { return nil } - err = auth.Config.DB.UpdateUserInfo(idStruct.User, idStruct.Profile, idStruct.Email, idStruct.EdupersonEntitlement) + err = auth.Config.DB.UpdateUserInfo(idStruct.User, idStruct.Fullname, idStruct.Email, idStruct.EdupersonEntitlement) if err != nil { log.Warn("Could not log user info.") } @@ -311,7 +311,7 @@ func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) ctx.ViewData("User", oidcData.OIDCID.User) - ctx.ViewData("Profile", oidcData.OIDCID.Profile) + ctx.ViewData("Fullname", oidcData.OIDCID.Fullname) ctx.ViewData("Passport", oidcData.OIDCID.Passport) ctx.ViewData("RawToken", oidcData.OIDCID.RawToken) ctx.ViewData("ResignedToken", oidcData.OIDCID.ResignedToken) diff --git a/sda/cmd/auth/oidc.go b/sda/cmd/auth/oidc.go index eb6496cb38..a95a0e8afb 100644 --- a/sda/cmd/auth/oidc.go +++ b/sda/cmd/auth/oidc.go @@ -20,7 +20,7 @@ type OIDCIdentity struct { Passport []string RawToken string ResignedToken string - Profile string + Fullname string Email string EdupersonEntitlement []string ExpDateRaw string @@ -94,7 +94,7 @@ func authenticateWithOidc(oauth2Config oauth2.Config, provider *oidc.Provider, c // Extract custom passports, name and email claims var claims struct { PassportClaim []string `json:"ga4gh_passport_v1"` - ProfileClaim string `json:"name"` + FullnameClaim string `json:"name"` EmailClaim string `json:"email"` EdupersonEntitlement []string `json:"eduperson_entitlement"` } @@ -109,7 +109,7 @@ func authenticateWithOidc(oauth2Config oauth2.Config, provider *oidc.Provider, c RawToken: rawAccessToken, ResignedToken: rawAccessToken, Passport: claims.PassportClaim, - Profile: claims.ProfileClaim, + Fullname: claims.FullnameClaim, Email: claims.EmailClaim, EdupersonEntitlement: claims.EdupersonEntitlement, ExpDateRaw: rawExpDate, From 65ae7b7c28a8ed2d8596c555be724186aba5fec6 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 7 Feb 2025 16:24:17 +0100 Subject: [PATCH 069/250] [auth] Improve UX for login page --- sda/cmd/auth/frontend/templates/oidc.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sda/cmd/auth/frontend/templates/oidc.html b/sda/cmd/auth/frontend/templates/oidc.html index ecfe129e7e..9dbc309347 100644 --- a/sda/cmd/auth/frontend/templates/oidc.html +++ b/sda/cmd/auth/frontend/templates/oidc.html @@ -32,18 +32,18 @@

{{if .ResignedToken}}

- Your access token {{if .ExpDateResigned}}(expires at {{.ExpDateResigned}} UTC){{end}} for uploading to the Inbox is: + For uploading to the Inbox, your access token {{if .ExpDateResigned}}(expires at {{.ExpDateResigned}} UTC){{end}} is: 

{{.ResignedToken}}
{{end}} + Download credentials to upload to the Inbox {{if .RawToken}}

- Your access token {{if .ExpDateRaw}}(expires at {{.ExpDateRaw}} UTC){{end}} for downloading from the Archive is: + For downloading from the Archive, your access token {{if .ExpDateRaw}}(expires at {{.ExpDateRaw}} UTC){{end}} is: 

{{.RawToken}}
{{end}} - Download credentials for uploading to the Inbox - Download credentials for downloading from the Archive + Download credentials to access the Archive Continue
From a69bb3ac068e5946be348838ba389a9528d4fb40 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 18 Feb 2025 11:19:49 +0100 Subject: [PATCH 070/250] [auth] make sure only the download token is shown when login via AAI as FEGA user --- .github/integration/sda-s3-integration.yml | 3 +++ .../integration/sda/aai-mock/clients/aai-auth.yaml | 2 +- .../sda/aai-mock/clients/cega-auth.yaml | 8 ++++++++ sda/cmd/auth/frontend/templates/oidc.html | 14 ++++++++------ sda/cmd/auth/main.go | 1 + 5 files changed, 21 insertions(+), 7 deletions(-) create mode 100644 .github/integration/sda/aai-mock/clients/cega-auth.yaml diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 5e303114c9..5ae0a4391c 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -282,6 +282,9 @@ services: - AUTH_RESIGNJWT=true - AUTH_CEGA_ID=test - AUTH_CEGA_SECRET=test + - OIDC_REDIRECTURL=http://localhost:8888/oidc/login + - OIDC_ID=XC56EL11zz + - OIDC_SECRET=wHPVQaYXmdDHa - DB_PASSWORD=auth - DB_USER=auth extra_hosts: diff --git a/.github/integration/sda/aai-mock/clients/aai-auth.yaml b/.github/integration/sda/aai-mock/clients/aai-auth.yaml index e785e83cf4..b6ec2ed326 100644 --- a/.github/integration/sda/aai-mock/clients/aai-auth.yaml +++ b/.github/integration/sda/aai-mock/clients/aai-auth.yaml @@ -1,4 +1,4 @@ -client-name: "auth" +client-name: "aai-auth" client-id: "XC56EL11xx" client-secret: "wHPVQaYXmdDHg" redirect-uris: ["http://localhost:8801/oidc/login"] diff --git a/.github/integration/sda/aai-mock/clients/cega-auth.yaml b/.github/integration/sda/aai-mock/clients/cega-auth.yaml new file mode 100644 index 0000000000..c0c28a73f4 --- /dev/null +++ b/.github/integration/sda/aai-mock/clients/cega-auth.yaml @@ -0,0 +1,8 @@ +client-name: "cega-auth" +client-id: "XC56EL11zz" +client-secret: "wHPVQaYXmdDHa" +redirect-uris: ["http://localhost:8888/oidc/login"] +token-endpoint-auth-method: "client_secret_basic" +scope: ["openid", "profile", "email", "ga4gh_passport_v1", "eduperson_entitlement"] +grant-types: ["authorization_code"] +post-logout-redirect-uris: ["http://localhost:8888/oidc/login"] \ No newline at end of file diff --git a/sda/cmd/auth/frontend/templates/oidc.html b/sda/cmd/auth/frontend/templates/oidc.html index 9dbc309347..0123887372 100644 --- a/sda/cmd/auth/frontend/templates/oidc.html +++ b/sda/cmd/auth/frontend/templates/oidc.html @@ -30,13 +30,15 @@

Welcome, {{.Fullname}}!

- {{if .ResignedToken}} -

- For uploading to the Inbox, your access token {{if .ExpDateResigned}}(expires at {{.ExpDateResigned}} UTC){{end}} is: -

- 
{{.ResignedToken}}
+ {{if (not .cegaID)}} + {{if .ResignedToken}} +

+ For uploading to the Inbox, your access token {{if .ExpDateResigned}}(expires at {{.ExpDateResigned}} UTC){{end}} is: +

+ 
{{.ResignedToken}}
+ {{end}} + Download credentials to upload to the Inbox {{end}} - Download credentials to upload to the Inbox {{if .RawToken}}

For downloading from the Archive, your access token {{if .ExpDateRaw}}(expires at {{.ExpDateRaw}} UTC){{end}} is: diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 794dfd99b3..576b4266e1 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -308,6 +308,7 @@ func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { s := sessions.Get(ctx) s.SetFlash("oidcInbox", oidcData.S3ConfInbox) s.SetFlash("oidcDownload", oidcData.S3ConfDownload) + ctx.ViewData("cegaID", auth.Config.Cega.ID) ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) ctx.ViewData("User", oidcData.OIDCID.User) From 705dcdea83048d74c8944a41e9789ec7a3c37b2c Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Tue, 18 Feb 2025 11:44:54 +0100 Subject: [PATCH 071/250] [auth]: uniformed credential page for ega login and AAI login --- sda/cmd/auth/frontend/templates/ega.html | 16 +++++----------- sda/cmd/auth/main.go | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/sda/cmd/auth/frontend/templates/ega.html b/sda/cmd/auth/frontend/templates/ega.html index 6e921f69a5..ac3ec3f0ab 100644 --- a/sda/cmd/auth/frontend/templates/ega.html +++ b/sda/cmd/auth/frontend/templates/ega.html @@ -31,18 +31,12 @@ Welcome, {{.User}}!

{{if .Token}} -

- Your access token is: -

- 
{{.Token}}
- {{end}} - {{if .ExpDate}} -

- Your access token expires (UTC): -

- 
{{.ExpDate}}
+

+ For uploading to the Inbox, your access token {{if .ExpDate}}(expires at {{.ExpDate}} UTC){{end}} is: +

+ 
{{.Token}}
{{end}} - Download inbox s3cmd credentials + Download credentials to upload to the Inbox Continue
diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 576b4266e1..1fe8088461 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -220,7 +220,7 @@ func (auth AuthHandler) getEGALogin(ctx iris.Context) { // getEGAConf returns an s3config file for an oidc login func (auth AuthHandler) getEGAConf(ctx iris.Context) { - auth.getS3Config(ctx, "ega", "s3cmd.conf") + auth.getS3Config(ctx, "ega", "s3cmd-inbox.conf") } // getOIDC redirects to the oidc page defined in auth.Config From 8b8da92250f402ef3274e633993cfd64c4263328 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 21 Feb 2025 09:57:50 +0100 Subject: [PATCH 072/250] Update Go to version 1.24 --- .github/workflows/code-linter.yaml | 2 +- .github/workflows/test.yml | 2 +- sda-admin/go.mod | 2 +- sda-download/.github/workflows/test.yml | 2 +- sda-download/Dockerfile | 2 +- sda-download/go.mod | 2 +- sda/Dockerfile | 2 +- sda/go.mod | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index f10a243b76..6eabc7aba6 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -8,7 +8,7 @@ on: - '**/go.*' env: - go-version: '1.23' + go-version: '1.24' jobs: lint_download: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1a08ef7156..5ad94bcf57 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: branches-ignore: [main] env: - go-version: '1.23' + go-version: '1.24' jobs: diff --git a/sda-admin/go.mod b/sda-admin/go.mod index 90cea71e00..92f89c85de 100644 --- a/sda-admin/go.mod +++ b/sda-admin/go.mod @@ -1,6 +1,6 @@ module github.com/neicnordic/sensitive-data-archive/sda-admin -go 1.23.1 +go 1.24.0 require ( github.com/neicnordic/crypt4gh v1.12.0 diff --git a/sda-download/.github/workflows/test.yml b/sda-download/.github/workflows/test.yml index 8fe495e021..047f9154c0 100644 --- a/sda-download/.github/workflows/test.yml +++ b/sda-download/.github/workflows/test.yml @@ -8,7 +8,7 @@ on: - "schemas/**" env: - go-version: '1.23' + go-version: '1.24' jobs: build: diff --git a/sda-download/Dockerfile b/sda-download/Dockerfile index b9ceec0366..30f9453531 100644 --- a/sda-download/Dockerfile +++ b/sda-download/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine AS builder +FROM golang:1.24-alpine AS builder ENV GOPATH=$PWD ENV CGO_ENABLED=0 diff --git a/sda-download/go.mod b/sda-download/go.mod index c4453bad24..71ed3dd1a6 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -1,6 +1,6 @@ module github.com/neicnordic/sda-download -go 1.23.1 +go 1.24.0 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 diff --git a/sda/Dockerfile b/sda/Dockerfile index c0d360e877..c3415e736e 100644 --- a/sda/Dockerfile +++ b/sda/Dockerfile @@ -1,4 +1,4 @@ -FROM "golang:${GOLANG_VERSION:-1.23}-bullseye" AS Build +FROM "golang:${GOLANG_VERSION:-1.24}-bullseye" AS Build ENV GO111MODULE=on ENV GOPATH=$PWD ENV CGO_ENABLED=0 diff --git a/sda/go.mod b/sda/go.mod index f752639fcb..2888ca96c3 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -1,6 +1,6 @@ module github.com/neicnordic/sensitive-data-archive -go 1.23.1 +go 1.24.0 require ( github.com/aws/aws-sdk-go-v2 v1.36.1 From 7be2d062b2128d59db95dda1f99c26d31cec9d68 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 15 Feb 2025 09:01:33 +0100 Subject: [PATCH 073/250] Add API specification --- sda/cmd/download/swagger_v1.yml | 258 ++++++++++++++++++++++++++++++++ 1 file changed, 258 insertions(+) create mode 100644 sda/cmd/download/swagger_v1.yml diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml new file mode 100644 index 0000000000..3c758953ae --- /dev/null +++ b/sda/cmd/download/swagger_v1.yml @@ -0,0 +1,258 @@ +openapi: 3.0.3 +info: + title: SDA file download API + version: "1.0" +paths: + /file: + get: + description: Returns the requested file to the user + parameters: + - in: header + description: "Public key used to re-encrypt the file header with" + name: public_key + required: true + schema: + type: string + - in: header + description: "The JWT used to authenticate the user" + name: token + required: true + schema: + type: string + - in: query + name: fileId + schema: + type: string + required: false + description: Stable ID of the file to download + - in: query + name: filePath + schema: + type: string + required: false + description: File path of the file to download + - in: query + name: end + required: false + schema: + default: 0 + minimum: 0 + type: integer + - in: query + name: start + required: false + schema: + default: 0 + minimum: 0 + type: integer + responses: + "200": + content: + application/octet-stream: + schema: + type: string + format: binary + description: successful operation + "400": + description: Invalid file reference + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "416": + description: Range Not Satisfiable + "500": + description: Internal application error + security: + - bearerAuth: [] + /file/{fileId}: + get: + description: Returns the requested file to the user + parameters: + - in: header + description: "Public key used to re-encrypt the file header with" + name: public_key + required: true + schema: + type: string + - in: header + description: "The JWT used to authenticate the user" + name: token + required: true + schema: + type: string + - in: path + name: fileId + schema: + type: string + required: true + description: Stable ID of the file to download + - in: query + name: end + required: false + schema: + default: 0 + minimum: 0 + type: integer + - in: query + name: start + required: false + schema: + default: 0 + minimum: 0 + type: integer + responses: + "200": + content: + application/octet-stream: + schema: + type: string + format: binary + description: successful operation + "400": + description: Invalid file reference + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "500": + description: Internal application error + security: + - bearerAuth: [] + /info/datasets: + get: + description: Returns a list of the datasets the user has access to, an empty list is a valid response. + responses: + "200": + description: successful operation + content: + application/json: + example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] + schema: + type: array + "401": + description: Authentication failure + security: + - bearerAuth: [] + /info/datasets/{datasetId}: + get: + description: Returns an array with metadata about the dataset + parameters: + - in: header + description: "Public key used to re-encrypt the file header with" + name: public_key + required: true + schema: + type: string + - in: header + description: "The JWT used to authenticate the user" + name: token + required: true + schema: + type: string + - in: path + name: datasetId + schema: + type: string + required: true + description: Stable ID of the dataset of interest + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/DatasetInfo" + description: successful operation + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "500": + description: Internal application error + security: + - bearerAuth: [] + /info/datasets/{datasetId}/files: + get: + description: Returns an array with metadata about all files in a dataset + parameters: + - in: header + description: "Public key used to re-encrypt the file header with" + name: public_key + required: true + schema: + type: string + - in: header + description: "The JWT used to authenticate the user" + name: token + required: true + schema: + type: string + - in: path + name: datasetId + schema: + type: string + required: true + description: Stable ID of the dataset of interest + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FileInfo" + description: successful operation + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "500": + description: Internal application error + security: + - bearerAuth: [] + /ready: + get: + description: Returns the status of the application, used by the kubernets API to decide if traffic should be routed to the app + responses: + "200": + description: The service is operational + "503": + description: Unhealthy service +components: + schemas: + DatasetInfo: + type: object + properties: + date: + type: string + format: date-time + example: "2025-02-14T14:51:26.639Z" + files: + type: integer + format: int32 + example: 1234 + name: + type: string + example: aa-dataset-123456-asdfgh + size: + type: integer + format: int64 + example: 6597069766656 + FileInfo: + type: object + properties: + filePath: + type: string + example: "samples/controls/sample1.cram.c4gh" + size: + type: integer + format: int64 + example: 56623104 + stableId: + type: string + example: aa-file-123456-asdfgh + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT From 0f280cd8e4c0aa44835c3301f12cd315eb49962e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 15 Feb 2025 09:59:35 +0100 Subject: [PATCH 074/250] Add dataset name as required for the `/file` endpoint --- sda/cmd/download/swagger_v1.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index 3c758953ae..aed2791840 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -19,6 +19,12 @@ paths: required: true schema: type: string + - in: query + name: dataset + schema: + type: string + required: true + description: Stable ID of the dataset - in: query name: fileId schema: From 68ca7e1601c8f20b39663470b3a265ef97bf2c1e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sat, 15 Feb 2025 10:13:59 +0100 Subject: [PATCH 075/250] Add empty go file to silence the build errors --- sda/cmd/download/download.go | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 sda/cmd/download/download.go diff --git a/sda/cmd/download/download.go b/sda/cmd/download/download.go new file mode 100644 index 0000000000..38dd16da61 --- /dev/null +++ b/sda/cmd/download/download.go @@ -0,0 +1,3 @@ +package main + +func main() {} From 37aa842f721d22add26248d6710a89f39091c3a7 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Sun, 16 Feb 2025 10:11:41 +0100 Subject: [PATCH 076/250] Add more response codes --- sda/cmd/download/swagger_v1.yml | 39 ++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index aed2791840..e5d6ce80c4 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -59,14 +59,21 @@ paths: type: string format: binary description: successful operation + "206": + content: + application/octet-stream: + schema: + type: string + format: binary + description: successful partial file delivery "400": - description: Invalid file reference + description: Bad request i.e start larger than end "401": description: Authentication failure "403": description: The user does not have access to the dataset "416": - description: Range Not Satisfiable + description: Unsatisfiable range i.e. end larger than file size "500": description: Internal application error security: @@ -93,20 +100,11 @@ paths: type: string required: true description: Stable ID of the file to download - - in: query - name: end - required: false - schema: - default: 0 - minimum: 0 - type: integer - - in: query - name: start + - in: header + name: Range required: false schema: - default: 0 - minimum: 0 - type: integer + type: string responses: "200": content: @@ -115,12 +113,21 @@ paths: type: string format: binary description: successful operation + "206": + content: + application/octet-stream: + schema: + type: string + format: binary + description: successful partial file delivery "400": - description: Invalid file reference + description: Bad request i.e start larger than end "401": description: Authentication failure "403": description: The user does not have access to the dataset + "416": + description: Unsatisfiable range i.e. end larger than file size "500": description: Internal application error security: @@ -138,6 +145,8 @@ paths: type: array "401": description: Authentication failure + "500": + description: Internal application error security: - bearerAuth: [] /info/datasets/{datasetId}: From e7f2a2f1a20e58145aa97a7ce900ff142362c565 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Feb 2025 08:24:24 +0100 Subject: [PATCH 077/250] Apply suggestions from code review Co-authored-by: Pontus Freyhult --- sda/cmd/download/swagger_v1.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index e5d6ce80c4..26e92fd0ef 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -8,7 +8,7 @@ paths: description: Returns the requested file to the user parameters: - in: header - description: "Public key used to re-encrypt the file header with" + description: "Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key" name: public_key required: true schema: @@ -30,18 +30,20 @@ paths: schema: type: string required: false - description: Stable ID of the file to download + description: Stable ID of the file to download. It is an error to supply both filePath and fileId - in: query name: filePath schema: type: string required: false - description: File path of the file to download + description: File path of the file to download. It is an error to supply both filePath and fileId - in: query name: end + description: 'End of requested data in the form of the 0-index of the first octet not included in the + response. That is, data delivered will have octet indices in the closed-open set + [start,end). Defaults to end of file if not set' required: false schema: - default: 0 minimum: 0 type: integer - in: query From f51bb9d86b88a803cd0a6bc69df87c5b385efe86 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Feb 2025 15:35:16 +0100 Subject: [PATCH 078/250] Apply suggestions from code review Co-authored-by: Malin Klang --- sda/cmd/download/swagger_v1.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index 26e92fd0ef..6007172390 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -60,14 +60,14 @@ paths: schema: type: string format: binary - description: successful operation + description: Successful operation "206": content: application/octet-stream: schema: type: string format: binary - description: successful partial file delivery + description: Successful partial file delivery "400": description: Bad request i.e start larger than end "401": @@ -107,6 +107,7 @@ paths: required: false schema: type: string + description: Byte range of the file that is to be delivered responses: "200": content: @@ -114,14 +115,14 @@ paths: schema: type: string format: binary - description: successful operation + description: Successful operation "206": content: application/octet-stream: schema: type: string format: binary - description: successful partial file delivery + description: Successful partial file delivery "400": description: Bad request i.e start larger than end "401": @@ -139,7 +140,7 @@ paths: description: Returns a list of the datasets the user has access to, an empty list is a valid response. responses: "200": - description: successful operation + description: Successful operation content: application/json: example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] @@ -179,7 +180,7 @@ paths: application/json: schema: $ref: "#/components/schemas/DatasetInfo" - description: successful operation + description: Successful operation "401": description: Authentication failure "403": @@ -218,7 +219,7 @@ paths: type: array items: $ref: "#/components/schemas/FileInfo" - description: successful operation + description: Successful operation "401": description: Authentication failure "403": From ac6c98907ac656d16ca4f2c40db3ce0d3dcf6448 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Feb 2025 16:08:13 +0100 Subject: [PATCH 079/250] Apply suggestions from code review Co-authored-by: Malin Klang Co-authored-by: Panos Chatzopoulos --- sda/cmd/download/swagger_v1.yml | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index 6007172390..c9b1a0c998 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -13,12 +13,6 @@ paths: required: true schema: type: string - - in: header - description: "The JWT used to authenticate the user" - name: token - required: true - schema: - type: string - in: query name: dataset schema: @@ -90,12 +84,6 @@ paths: required: true schema: type: string - - in: header - description: "The JWT used to authenticate the user" - name: token - required: true - schema: - type: string - in: path name: fileId schema: @@ -107,7 +95,8 @@ paths: required: false schema: type: string - description: Byte range of the file that is to be delivered + description: 0-indexed byte range of the file that is to be delivered. + example: "Range: bytes=20-200, 300-400" responses: "200": content: @@ -230,7 +219,7 @@ paths: - bearerAuth: [] /ready: get: - description: Returns the status of the application, used by the kubernets API to decide if traffic should be routed to the app + description: Returns the status of the application, used by the Kubernets API to decide if traffic should be routed to the app responses: "200": description: The service is operational From acfa9a80fe8bbd28f84f20e2ed079872f8148563 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 21 Feb 2025 11:53:15 +0100 Subject: [PATCH 080/250] Update sda/cmd/download/swagger_v1.yml Co-authored-by: Pontus Freyhult --- sda/cmd/download/swagger_v1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index c9b1a0c998..d0a837bb7b 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -95,7 +95,7 @@ paths: required: false schema: type: string - description: 0-indexed byte range of the file that is to be delivered. + description: 0-indexed byte range of the file that is to be delivered as per RFC7233. example: "Range: bytes=20-200, 300-400" responses: "200": From 5fbf56242fdbe019280b6d9d0b9520ed3d3bdd30 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 21 Feb 2025 13:22:38 +0100 Subject: [PATCH 081/250] Apply suggestions from code review Co-authored-by: Pontus Freyhult --- sda/cmd/download/swagger_v1.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index d0a837bb7b..a436830f59 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -151,12 +151,6 @@ paths: required: true schema: type: string - - in: header - description: "The JWT used to authenticate the user" - name: token - required: true - schema: - type: string - in: path name: datasetId schema: @@ -188,12 +182,6 @@ paths: required: true schema: type: string - - in: header - description: "The JWT used to authenticate the user" - name: token - required: true - schema: - type: string - in: path name: datasetId schema: From ce9aa56c533cfcf15bb63c4ec7d96fccee1578bf Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 21 Feb 2025 14:11:29 +0100 Subject: [PATCH 082/250] Update dataset info endpoint to handle identifiers that are URLs. --- sda/cmd/download/swagger_v1.yml | 24 ++++++------------------ 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index a436830f59..4b75ef468f 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -141,18 +141,12 @@ paths: description: Internal application error security: - bearerAuth: [] - /info/datasets/{datasetId}: + /info/dataset: get: description: Returns an array with metadata about the dataset parameters: - - in: header - description: "Public key used to re-encrypt the file header with" - name: public_key - required: true - schema: - type: string - - in: path - name: datasetId + - in: query + name: dataset schema: type: string required: true @@ -172,18 +166,12 @@ paths: description: Internal application error security: - bearerAuth: [] - /info/datasets/{datasetId}/files: + /info/dataset/files: get: description: Returns an array with metadata about all files in a dataset parameters: - - in: header - description: "Public key used to re-encrypt the file header with" - name: public_key - required: true - schema: - type: string - - in: path - name: datasetId + - in: query + name: dataset schema: type: string required: true From c7db8f81603d9af083b58d828b69dc3d953d7b3b Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 24 Feb 2025 10:33:08 +0100 Subject: [PATCH 083/250] Update to match the demo from last Friday. --- sda/cmd/download/swagger_v1.yml | 49 +++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml index 4b75ef468f..aa4d791d49 100644 --- a/sda/cmd/download/swagger_v1.yml +++ b/sda/cmd/download/swagger_v1.yml @@ -2,13 +2,14 @@ openapi: 3.0.3 info: title: SDA file download API version: "1.0" + description: This is the external data out API for the Sensitive Data Archive. Data is delivered encrypted with a user supplied crypth4gh public key. paths: /file: get: - description: Returns the requested file to the user + description: Returns the requested file to the user. Uses query parameters in order to handle files requested by the `filePath` entity. parameters: - in: header - description: "Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key" + description: Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key name: public_key required: true schema: @@ -18,7 +19,7 @@ paths: schema: type: string required: true - description: Stable ID of the dataset + description: Dataset identifier - in: query name: fileId schema: @@ -33,15 +34,14 @@ paths: description: File path of the file to download. It is an error to supply both filePath and fileId - in: query name: end - description: 'End of requested data in the form of the 0-index of the first octet not included in the - response. That is, data delivered will have octet indices in the closed-open set - [start,end). Defaults to end of file if not set' + description: End of requested data in the form of the 0-index of the first octet not included in the response. That is, data delivered will have octet indices in the closed-open set [start,end). Defaults to end of file if not set required: false schema: minimum: 0 type: integer - in: query name: start + description: Start of the requested data. Defaults to start of file if not set required: false schema: default: 0 @@ -63,23 +63,23 @@ paths: format: binary description: Successful partial file delivery "400": - description: Bad request i.e start larger than end + description: Bad request ex. start larger than end "401": description: Authentication failure "403": description: The user does not have access to the dataset "416": - description: Unsatisfiable range i.e. end larger than file size + description: Unsatisfiable range ex. start larger than the size of the file "500": description: Internal application error security: - bearerAuth: [] /file/{fileId}: get: - description: Returns the requested file to the user + description: Returns the requested file to the user. parameters: - in: header - description: "Public key used to re-encrypt the file header with" + description: Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key name: public_key required: true schema: @@ -113,13 +113,13 @@ paths: format: binary description: Successful partial file delivery "400": - description: Bad request i.e start larger than end + description: Bad request ex. start larger than end "401": description: Authentication failure "403": description: The user does not have access to the dataset "416": - description: Unsatisfiable range i.e. end larger than file size + description: Unsatisfiable range ex. start larger than the size of the file "500": description: Internal application error security: @@ -150,7 +150,7 @@ paths: schema: type: string required: true - description: Stable ID of the dataset of interest + description: Dataset identifier responses: "200": content: @@ -175,7 +175,7 @@ paths: schema: type: string required: true - description: Stable ID of the dataset of interest + description: Dataset identifier responses: "200": content: @@ -193,14 +193,29 @@ paths: description: Internal application error security: - bearerAuth: [] - /ready: + /health/ready: get: - description: Returns the status of the application, used by the Kubernets API to decide if traffic should be routed to the app + description: Returns the status of the application. responses: "200": + content: + application/json: + example: + { + "c4ghService": "responding", + "database": "responding", + "oidcServer": "responding", + "storageBackend": "responding", + } description: The service is operational "503": description: Unhealthy service + /health/live: + get: + description: Returns 200 as long as the main thread is running + responses: + "200": + description: The service is operational components: schemas: DatasetInfo: @@ -226,7 +241,7 @@ components: properties: filePath: type: string - example: "samples/controls/sample1.cram.c4gh" + example: samples/controls/sample1.cram.c4gh size: type: integer format: int64 From 80353e0ce71e02570a6ef817d3bf7b375a7dab07 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 19:58:27 +0000 Subject: [PATCH 084/250] Bump s4u/maven-settings-action from 3.0.0 to 3.1.0 Bumps [s4u/maven-settings-action](https://github.com/s4u/maven-settings-action) from 3.0.0 to 3.1.0. - [Release notes](https://github.com/s4u/maven-settings-action/releases) - [Commits](https://github.com/s4u/maven-settings-action/compare/v3.0.0...v3.1.0) --- updated-dependencies: - dependency-name: s4u/maven-settings-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 28177e1d9e..88d33f222e 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -191,7 +191,7 @@ jobs: - name: create maven settings.xml - uses: s4u/maven-settings-action@v3.0.0 + uses: s4u/maven-settings-action@v3.1.0 with: servers: | [{ @@ -302,7 +302,7 @@ jobs: uses: actions/checkout@v4 - name: create maven settings.xml - uses: s4u/maven-settings-action@v3.0.0 + uses: s4u/maven-settings-action@v3.1.0 with: servers: | [{ From 3a15f847f861fde9a1bb57e1fd294a69d5f251ed Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 20:19:11 +0000 Subject: [PATCH 085/250] Bump golang.org/x/crypto in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [golang.org/x/crypto](https://github.com/golang/crypto). Updates `golang.org/x/crypto` from 0.33.0 to 0.35.0 - [Commits](https://github.com/golang/crypto/compare/v0.33.0...v0.35.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index c4453bad24..014a210824 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -15,7 +15,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.33.0 + golang.org/x/crypto v0.35.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 google.golang.org/grpc v1.70.0 google.golang.org/protobuf v1.36.5 diff --git a/sda-download/go.sum b/sda-download/go.sum index 0cecfdfbe3..8b1d8cbd63 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -180,8 +180,8 @@ golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= From 55edd814279cb5e200b56b2bd2bf61b952140c0b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Feb 2025 22:53:16 +0000 Subject: [PATCH 086/250] Bump github.com/go-jose/go-jose/v4 in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [github.com/go-jose/go-jose/v4](https://github.com/go-jose/go-jose). Updates `github.com/go-jose/go-jose/v4` from 4.0.4 to 4.0.5 - [Release notes](https://github.com/go-jose/go-jose/releases) - [Changelog](https://github.com/go-jose/go-jose/blob/main/CHANGELOG.md) - [Commits](https://github.com/go-jose/go-jose/compare/v4.0.4...v4.0.5) --- updated-dependencies: - dependency-name: github.com/go-jose/go-jose/v4 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index f752639fcb..6694fc7afc 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -85,7 +85,7 @@ require ( github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.3 // indirect - github.com/go-jose/go-jose/v4 v4.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index fcb4d2df75..42916ac15b 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -124,8 +124,8 @@ github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= -github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= From 7453d46496dc1bdb94929769fd0cfbfa3ab45569 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 25 Feb 2025 09:57:32 +0000 Subject: [PATCH 087/250] Bump the all-modules group across 1 directory with 9 updates Bumps the all-modules group with 7 updates in the /sda directory: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.36.1` | `1.36.2` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.6` | `1.29.7` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.62` | `1.17.63` | | [github.com/aws/smithy-go](https://github.com/aws/smithy-go) | `1.22.2` | `1.22.3` | | [github.com/lestrrat-go/jwx/v2](https://github.com/lestrrat-go/jwx) | `2.1.3` | `2.1.4` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.33.0` | `0.35.0` | | [golang.org/x/oauth2](https://github.com/golang/oauth2) | `0.26.0` | `0.27.0` | Updates `github.com/aws/aws-sdk-go-v2` from 1.36.1 to 1.36.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.36.1...v1.36.2) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.6 to 1.29.7 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.6...config/v1.29.7) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.59 to 1.17.60 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.59...credentials/v1.17.60) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.62 to 1.17.63 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.62...feature/s3/manager/v1.17.63) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.77.0 to 1.77.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.77.0...service/s3/v1.77.1) Updates `github.com/aws/smithy-go` from 1.22.2 to 1.22.3 - [Release notes](https://github.com/aws/smithy-go/releases) - [Changelog](https://github.com/aws/smithy-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/smithy-go/compare/v1.22.2...v1.22.3) Updates `github.com/lestrrat-go/jwx/v2` from 2.1.3 to 2.1.4 - [Release notes](https://github.com/lestrrat-go/jwx/releases) - [Changelog](https://github.com/lestrrat-go/jwx/blob/v2.1.4/Changes) - [Commits](https://github.com/lestrrat-go/jwx/compare/v2.1.3...v2.1.4) Updates `golang.org/x/crypto` from 0.33.0 to 0.35.0 - [Commits](https://github.com/golang/crypto/compare/v0.33.0...v0.35.0) Updates `golang.org/x/oauth2` from 0.26.0 to 0.27.0 - [Commits](https://github.com/golang/oauth2/compare/v0.26.0...v0.27.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/smithy-go dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/lestrrat-go/jwx/v2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 46 +++++++++++++-------------- sda/go.sum | 92 +++++++++++++++++++++++++++--------------------------- 2 files changed, 69 insertions(+), 69 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 99bfa6ec53..8933ba9582 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,12 +3,12 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.0 require ( - github.com/aws/aws-sdk-go-v2 v1.36.1 - github.com/aws/aws-sdk-go-v2/config v1.29.6 - github.com/aws/aws-sdk-go-v2/credentials v1.17.59 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.62 - github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 - github.com/aws/smithy-go v1.22.2 + github.com/aws/aws-sdk-go-v2 v1.36.2 + github.com/aws/aws-sdk-go-v2/config v1.29.7 + github.com/aws/aws-sdk-go-v2/credentials v1.17.60 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.63 + github.com/aws/aws-sdk-go-v2/service/s3 v1.77.1 + github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 github.com/gin-gonic/gin v1.10.0 @@ -16,7 +16,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 github.com/kataras/iris/v12 v12.2.11 - github.com/lestrrat-go/jwx/v2 v2.1.3 + github.com/lestrrat-go/jwx/v2 v2.1.4 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 @@ -31,8 +31,8 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.33.0 - golang.org/x/oauth2 v0.26.0 + golang.org/x/crypto v0.35.0 + golang.org/x/oauth2 v0.27.0 google.golang.org/grpc v1.70.0 google.golang.org/protobuf v1.36.5 ) @@ -49,19 +49,19 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.29 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.33 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.14 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.24.16 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.15 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect @@ -74,7 +74,7 @@ require ( github.com/containerd/continuity v0.4.3 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/docker/cli v27.2.1+incompatible // indirect github.com/docker/docker v27.2.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 42916ac15b..347145f7c6 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,44 +24,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.36.1 h1:iTDl5U6oAhkNPba0e1t1hrwAo02ZMqbrGq4k5JBWM5E= -github.com/aws/aws-sdk-go-v2 v1.36.1/go.mod h1:5PMILGVKiW32oDzjj6RU52yrNrDPUHcbZQYr1sM7qmM= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9 h1:VZPDrbzdsU1ZxhyWrvROqLY0nxFWgMCAzhn/nYz3X48= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.9/go.mod h1:3XkePX5dSaxveLAYY7nsbsZZrKxCyEuE5pM4ziFxyGg= -github.com/aws/aws-sdk-go-v2/config v1.29.6 h1:fqgqEKK5HaZVWLQoLiC9Q+xDlSp+1LYidp6ybGE2OGg= -github.com/aws/aws-sdk-go-v2/config v1.29.6/go.mod h1:Ft+WLODzDQmCTHDvqAH1JfC2xxbZ0MxpZAcJqmE1LTQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.59 h1:9btwmrt//Q6JcSdgJOLI98sdr5p7tssS9yAsGe8aKP4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.59/go.mod h1:NM8fM6ovI3zak23UISdWidyZuI1ghNe2xjzUZAyT+08= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28 h1:KwsodFKVQTlI5EyhRSugALzsV6mG/SGrdjlMXSZSdso= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.28/go.mod h1:EY3APf9MzygVhKuPXAc5H+MkGb8k/DOSQjWS0LgkKqI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.62 h1:qzLOdXzKUuMGDzEAzpEz3QHYy5510nEZCzWI4EBaxZw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.62/go.mod h1:hezn6jOdr8sbGMCJmqJF/WOVK9h9H7EXsmu20zXG2m8= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32 h1:BjUcr3X3K0wZPGFg2bxOWW3VPN8rkE3/61zhP+IHviA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.32/go.mod h1:80+OGC/bgzzFFTUmcuwD0lb4YutwQeKLFpmt6hoWapU= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32 h1:m1GeXHVMJsRsUAqG6HjZWx9dj7F5TR+cF1bjyfYyBd4= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.32/go.mod h1:IitoQxGfaKdVLNg0hD8/DXmAqNy0H4K2H2Sf91ti8sI= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2 h1:Pg9URiobXy85kgFev3og2CuOZ8JZUBENF+dcgWBaYNk= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.2/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32 h1:OIHj/nAhVzIXGzbAE+4XmZ8FPvro3THr6NlqErJc3wY= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.32/go.mod h1:LiBEsDo34OJXqdDlRGsilhlIiXR7DL+6Cx2f4p1EgzI= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2 h1:D4oz8/CzT9bAEYtVhSBmFj2dNOtaHOtMKc2vHBwYizA= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.2/go.mod h1:Za3IHqTQ+yNcRHxu1OFucBh0ACZT4j4VQFF0BqpZcLY= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0 h1:kT2WeWcFySdYpPgyqJMSUE7781Qucjtn6wBvrgm9P+M= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.0/go.mod h1:WYH1ABybY7JK9TITPnk6ZlP7gQB8psI4c9qDmMsnLSA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13 h1:SYVGSFQHlchIcy6e7x12bsrxClCXSP5et8cqVhL8cuw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.13/go.mod h1:kizuDaLX37bG5WZaoxGPQR/LNFXpxp0vsUnqfkWXfNE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13 h1:OBsrtam3rk8NfBEq7OLOMm5HtQ9Yyw32X4UQMya/wjw= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.13/go.mod h1:3U4gFA5pmoCOja7aq4nSaIAGbaOHv2Yl2ug018cmC+Q= -github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0 h1:RCOi1rDmLqOICym/6UeS2cqKED4T4m966w2rl1HfL+g= -github.com/aws/aws-sdk-go-v2/service/s3 v1.77.0/go.mod h1:VC4EKSHqT3nzOcU955VWHMGsQ+w67wfAUBSjC8NOo8U= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.15 h1:/eE3DogBjYlvlbhd2ssWyeuovWunHLxfgw3s/OJa4GQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.15/go.mod h1:2PCJYpi7EKeA5SkStAmZlF6fi0uUABuhtF8ILHjGc3Y= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14 h1:M/zwXiL2iXUrHputuXgmO94TVNmcenPHxgLXLutodKE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.14/go.mod h1:RVwIw3y/IqxC2YEXSIkAzRDdEU1iRabDPaYjpGCbCGQ= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.14 h1:TzeR06UCMUq+KA3bDkujxK1GVGy+G8qQN/QVYzGLkQE= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.14/go.mod h1:dspXf/oYWGWo6DEvj98wpaTeqt5+DMidZD0A9BYTizc= -github.com/aws/smithy-go v1.22.2 h1:6D9hW43xKFrRx/tXXfAlIZc4JI+yQe6snnWcQyxSyLQ= -github.com/aws/smithy-go v1.22.2/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2 v1.36.2 h1:Ub6I4lq/71+tPb/atswvToaLGVMxKZvjYDVOWEExOcU= +github.com/aws/aws-sdk-go-v2 v1.36.2/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= +github.com/aws/aws-sdk-go-v2/config v1.29.7 h1:71nqi6gUbAUiEQkypHQcNVSFJVUFANpSeUNShiwWX2M= +github.com/aws/aws-sdk-go-v2/config v1.29.7/go.mod h1:yqJQ3nh2HWw/uxd56bicyvmDW4KSc+4wN6lL8pYjynU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.60 h1:1dq+ELaT5ogfmqtV1eocq8SpOK1NRsuUfmhQtD/XAh4= +github.com/aws/aws-sdk-go-v2/credentials v1.17.60/go.mod h1:HDes+fn/xo9VeszXqjBVkxOo/aUy8Mc6QqKvZk32GlE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.29 h1:JO8pydejFKmGcUNiiwt75dzLHRWthkwApIvPoyUtXEg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.29/go.mod h1:adxZ9i9DRmB8zAT0pO0yGnsmu0geomp5a3uq5XpgOJ8= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.63 h1:cTR4L7zlqh2YJjOWF62sMCyJWhm9ItUN3h/eOKh0xlU= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.63/go.mod h1:ryx0BXDm9YKRus5qaDeKcMh+XiEQ5uok/mJHkuGg4to= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 h1:knLyPMw3r3JsU8MFHWctE4/e2qWbPaxDYLlohPvnY8c= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33/go.mod h1:EBp2HQ3f+XCB+5J+IoEbGhoV7CpJbnrsd4asNXmTL0A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 h1:K0+Ne08zqti8J9jwENxZ5NoUyBnaFDTu3apwQJWrwwA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33/go.mod h1:K97stwwzaWzmqxO8yLGHhClbVW1tC6VT1pDLk1pGrq4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.33 h1:/frG8aV09yhCVSOEC2pzktflJJO48NwY3xntHBwxHiA= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.33/go.mod h1:8vwASlAcV366M+qxZnjNzCjeastk1Rt1bpSRaGZanGU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.1 h1:7SuukGpyIgF5EiAbf1dZRxP+xSnY1WjiHBjL08fjJeE= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.1/go.mod h1:k+Vce/8R28tSozjdWphkrNhK8zLmdS9RgiDNZl6p8Rw= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.14 h1:2scbY6//jy/s8+5vGrk7l1+UtHl0h9A4MjOO2k/TM2E= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.14/go.mod h1:bRpZPHZpSe5YRHmPfK3h1M7UBFCn2szHzyx0rw04zro= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.14 h1:fgdkfsxTehqPcIQa24G/Omwv9RocTq2UcONNX/OnrZI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.14/go.mod h1:wMxQ3OE8fiM8z2YRAeb2J8DLTTWMvRyYYuQOs26AbTQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.1 h1:5bI9tJL2Z0FGFtp/LPDv0eyliFBHCn7LAhqpQuL+7kk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.77.1/go.mod h1:njj3tSJONkfdLt4y6X8pyqeM6sJLNZxmzctKKV+n1GM= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.16 h1:YV6xIKDJp6U7YB2bxfud9IENO1LRpGhe2Tv/OKtPrOQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.24.16/go.mod h1:DvbmMKgtpA6OihFJK13gHMZOZrCHttz8wPHGKXqU+3o= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15 h1:kMyK3aKotq1aTBsj1eS8ERJLjqYRRRcsmP33ozlCvlk= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15/go.mod h1:5uPZU7vSNzb8Y0dm75xTikinegPYK3uJmIHQZFq5Aqo= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.15 h1:ht1jVmeeo2anR7zDiYJLSnRYnO/9NILXXu42FP3rJg0= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.15/go.mod h1:xWZ5cOiFe3czngChE4LhCBqUxNwgfwndEF7XlYP/yD8= +github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= +github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= @@ -95,8 +95,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/docker/cli v27.2.1+incompatible h1:U5BPtiD0viUzjGAjV1p0MGB8eVA3L3cbIrnyWmSJI70= github.com/docker/cli v27.2.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= @@ -234,8 +234,8 @@ github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCG github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.3 h1:Ud4lb2QuxRClYAmRleF50KrbKIoM1TddXgBrneT5/Jo= -github.com/lestrrat-go/jwx/v2 v2.1.3/go.mod h1:q6uFgbgZfEmQrfJfrCo90QcQOcXFMfbI/fO0NqRtvZo= +github.com/lestrrat-go/jwx/v2 v2.1.4 h1:uBCMmJX8oRZStmKuMMOFb0Yh9xmEMgNJLgjuKKt4/qc= +github.com/lestrrat-go/jwx/v2 v2.1.4/go.mod h1:nWRbDFR1ALG2Z6GJbBXzfQaYyvn751KuuyySN2yR6is= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -410,8 +410,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -433,8 +433,8 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.26.0 h1:afQXWNNaeC4nvZ0Ed9XvCCzXM6UHJG7iCg0W4fPqSBE= -golang.org/x/oauth2 v0.26.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= +golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From b5fb52211da8164289d8e2f628c44a3c486b3855 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 26 Feb 2025 10:47:54 +0000 Subject: [PATCH 088/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index ca6d490f32..1aacb9238c 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.76 -appVersion: v0.3.196 +version: 0.8.77 +appVersion: v0.3.202 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 930bc05160..74278734e0 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.77 -appVersion: v0.3.196 +version: 0.7.78 +appVersion: v0.3.202 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 1ee98f9513..283ea2c3d4 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.6 -appVersion: v0.3.196 +version: 0.30.7 +appVersion: v0.3.202 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From f3ce6b707e8868787f0ab53c1c28d8ec357ba6dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Feb 2025 22:19:28 +0000 Subject: [PATCH 089/250] Bump github.com/go-jose/go-jose/v3 in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [github.com/go-jose/go-jose/v3](https://github.com/go-jose/go-jose). Updates `github.com/go-jose/go-jose/v3` from 3.0.3 to 3.0.4 - [Release notes](https://github.com/go-jose/go-jose/releases) - [Changelog](https://github.com/go-jose/go-jose/blob/main/CHANGELOG.md) - [Commits](https://github.com/go-jose/go-jose/compare/v3.0.3...v3.0.4) --- updated-dependencies: - dependency-name: github.com/go-jose/go-jose/v3 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 8933ba9582..f201513a78 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -84,7 +84,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect + github.com/go-jose/go-jose/v3 v3.0.4 // indirect github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index 347145f7c6..e33aa5a912 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -122,8 +122,8 @@ github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= -github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= +github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= From a7111ac47cbba85ab89b7dc921d2e294ce24dc87 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 19:27:59 +0000 Subject: [PATCH 090/250] Bump codecov/codecov-action from 5.3.1 to 5.4.0 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.3.1 to 5.4.0. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.3.1...v5.4.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ad94bcf57..6bf9d664dc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.3.1 + uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.3.1 + uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.3.1 + uses: codecov/codecov-action@v5.4.0 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt From 53cd4d292c8de872a75ee5e432601508adc744dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 19:30:47 +0000 Subject: [PATCH 091/250] Bump the all-modules group in /sda with 6 updates Bumps the all-modules group in /sda with 6 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) | `1.36.2` | `1.36.3` | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.7` | `1.29.8` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.60` | `1.17.61` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.63` | `1.17.64` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.77.1` | `1.78.0` | | [github.com/neicnordic/crypt4gh](https://github.com/neicnordic/crypt4gh) | `1.13.0` | `1.14.0` | Updates `github.com/aws/aws-sdk-go-v2` from 1.36.2 to 1.36.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.36.2...v1.36.3) Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.7 to 1.29.8 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.7...config/v1.29.8) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.60 to 1.17.61 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.60...credentials/v1.17.61) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.63 to 1.17.64 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.63...feature/s3/manager/v1.17.64) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.77.1 to 1.78.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.77.1...service/s3/v1.78.0) Updates `github.com/neicnordic/crypt4gh` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/neicnordic/crypt4gh/releases) - [Changelog](https://github.com/neicnordic/crypt4gh/blob/master/.goreleaser.yml) - [Commits](https://github.com/neicnordic/crypt4gh/compare/v1.13.0...v1.14.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/neicnordic/crypt4gh dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 32 +++++++++++++-------------- sda/go.sum | 64 +++++++++++++++++++++++++++--------------------------- 2 files changed, 48 insertions(+), 48 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index f201513a78..bac981a632 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -3,11 +3,11 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.0 require ( - github.com/aws/aws-sdk-go-v2 v1.36.2 - github.com/aws/aws-sdk-go-v2/config v1.29.7 - github.com/aws/aws-sdk-go-v2/credentials v1.17.60 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.63 - github.com/aws/aws-sdk-go-v2/service/s3 v1.77.1 + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/config v1.29.8 + github.com/aws/aws-sdk-go-v2/credentials v1.17.61 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.64 + github.com/aws/aws-sdk-go-v2/service/s3 v1.78.0 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 @@ -20,7 +20,7 @@ require ( github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 - github.com/neicnordic/crypt4gh v1.13.0 + github.com/neicnordic/crypt4gh v1.14.0 github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 github.com/ory/dockertest v3.3.5+incompatible github.com/ory/dockertest/v3 v3.11.0 @@ -50,18 +50,18 @@ require ( github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.29 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.14 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.14 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.16 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.16 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index e33aa5a912..05658b2907 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,42 +24,42 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.36.2 h1:Ub6I4lq/71+tPb/atswvToaLGVMxKZvjYDVOWEExOcU= -github.com/aws/aws-sdk-go-v2 v1.36.2/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.7 h1:71nqi6gUbAUiEQkypHQcNVSFJVUFANpSeUNShiwWX2M= -github.com/aws/aws-sdk-go-v2/config v1.29.7/go.mod h1:yqJQ3nh2HWw/uxd56bicyvmDW4KSc+4wN6lL8pYjynU= -github.com/aws/aws-sdk-go-v2/credentials v1.17.60 h1:1dq+ELaT5ogfmqtV1eocq8SpOK1NRsuUfmhQtD/XAh4= -github.com/aws/aws-sdk-go-v2/credentials v1.17.60/go.mod h1:HDes+fn/xo9VeszXqjBVkxOo/aUy8Mc6QqKvZk32GlE= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.29 h1:JO8pydejFKmGcUNiiwt75dzLHRWthkwApIvPoyUtXEg= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.29/go.mod h1:adxZ9i9DRmB8zAT0pO0yGnsmu0geomp5a3uq5XpgOJ8= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.63 h1:cTR4L7zlqh2YJjOWF62sMCyJWhm9ItUN3h/eOKh0xlU= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.63/go.mod h1:ryx0BXDm9YKRus5qaDeKcMh+XiEQ5uok/mJHkuGg4to= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33 h1:knLyPMw3r3JsU8MFHWctE4/e2qWbPaxDYLlohPvnY8c= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.33/go.mod h1:EBp2HQ3f+XCB+5J+IoEbGhoV7CpJbnrsd4asNXmTL0A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33 h1:K0+Ne08zqti8J9jwENxZ5NoUyBnaFDTu3apwQJWrwwA= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.33/go.mod h1:K97stwwzaWzmqxO8yLGHhClbVW1tC6VT1pDLk1pGrq4= +github.com/aws/aws-sdk-go-v2/config v1.29.8 h1:RpwAfYcV2lr/yRc4lWhUM9JRPQqKgKWmou3LV7UfWP4= +github.com/aws/aws-sdk-go-v2/config v1.29.8/go.mod h1:t+G7Fq1OcO8cXTPPXzxQSnj/5Xzdc9jAAD3Xrn9/Mgo= +github.com/aws/aws-sdk-go-v2/credentials v1.17.61 h1:Hd/uX6Wo2iUW1JWII+rmyCD7MMhOe7ALwQXN6sKDd1o= +github.com/aws/aws-sdk-go-v2/credentials v1.17.61/go.mod h1:L7vaLkwHY1qgW0gG1zG0z/X0sQ5tpIY5iI13+j3qI80= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.64 h1:RTko0AQ0i1vWXDM97DkuW6zskgOxFxm4RqC0kmBJFkE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.64/go.mod h1:ty968MpOa5CoQ/ALWNB8Gmfoehof2nRHDR/DZDPfimE= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.33 h1:/frG8aV09yhCVSOEC2pzktflJJO48NwY3xntHBwxHiA= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.33/go.mod h1:8vwASlAcV366M+qxZnjNzCjeastk1Rt1bpSRaGZanGU= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.1 h1:7SuukGpyIgF5EiAbf1dZRxP+xSnY1WjiHBjL08fjJeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.1/go.mod h1:k+Vce/8R28tSozjdWphkrNhK8zLmdS9RgiDNZl6p8Rw= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.14 h1:2scbY6//jy/s8+5vGrk7l1+UtHl0h9A4MjOO2k/TM2E= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.14/go.mod h1:bRpZPHZpSe5YRHmPfK3h1M7UBFCn2szHzyx0rw04zro= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.14 h1:fgdkfsxTehqPcIQa24G/Omwv9RocTq2UcONNX/OnrZI= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.14/go.mod h1:wMxQ3OE8fiM8z2YRAeb2J8DLTTWMvRyYYuQOs26AbTQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.77.1 h1:5bI9tJL2Z0FGFtp/LPDv0eyliFBHCn7LAhqpQuL+7kk= -github.com/aws/aws-sdk-go-v2/service/s3 v1.77.1/go.mod h1:njj3tSJONkfdLt4y6X8pyqeM6sJLNZxmzctKKV+n1GM= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.16 h1:YV6xIKDJp6U7YB2bxfud9IENO1LRpGhe2Tv/OKtPrOQ= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.16/go.mod h1:DvbmMKgtpA6OihFJK13gHMZOZrCHttz8wPHGKXqU+3o= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15 h1:kMyK3aKotq1aTBsj1eS8ERJLjqYRRRcsmP33ozlCvlk= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15/go.mod h1:5uPZU7vSNzb8Y0dm75xTikinegPYK3uJmIHQZFq5Aqo= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.15 h1:ht1jVmeeo2anR7zDiYJLSnRYnO/9NILXXu42FP3rJg0= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.15/go.mod h1:xWZ5cOiFe3czngChE4LhCBqUxNwgfwndEF7XlYP/yD8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 h1:t/gZFyrijKuSU0elA5kRngP/oU3mc0I+Dvp8HwRE4c0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.78.0 h1:EBm8lXevBWe+kK9VOU/IBeOI189WPRwPUc3LvJK9GOs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.78.0/go.mod h1:4qzsZSzB/KiX2EzDjs9D7A8rI/WGJxZceVJIHqtJjIU= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.0 h1:2U9sF8nKy7UgyEeLiZTRg6ShBS22z8UnYpV6aRFL0is= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.0/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0 h1:wjAdc85cXdQR5uLx5FwWvGIHm4OPJhTyzUHU8craXtE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.16 h1:BHEK2Q/7CMRMCb3nySi/w8UbIcPhKvYP5s1xf8/izn0= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.16/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -275,8 +275,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.13.0 h1:NbSAPx1+zFpG6a8GCVwW80y/TGHfGdXJF/zqQKqlHZ8= -github.com/neicnordic/crypt4gh v1.13.0/go.mod h1:lfNIrhlcQrSf5awgCaW+poCsRBlvKOrNjR3CBvXU5Ek= +github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U= +github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 h1:9bCMuD3TcnjeqjPT2gSlha4asp8NvgcFRYExCaikCxk= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25/go.mod h1:eDjgYHYDJbPLBLsyZ6qRaugP0mX8vePOhZ5id1fdzJw= From 43a3fd9aa7a1215df95bb3422f50ff22978320c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Mar 2025 19:40:38 +0000 Subject: [PATCH 092/250] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/lestrrat-go/jwx/v2](https://github.com/lestrrat-go/jwx) and [github.com/neicnordic/crypt4gh](https://github.com/neicnordic/crypt4gh). Updates `github.com/lestrrat-go/jwx/v2` from 2.1.3 to 2.1.4 - [Release notes](https://github.com/lestrrat-go/jwx/releases) - [Changelog](https://github.com/lestrrat-go/jwx/blob/v2.1.4/Changes) - [Commits](https://github.com/lestrrat-go/jwx/compare/v2.1.3...v2.1.4) Updates `github.com/neicnordic/crypt4gh` from 1.13.0 to 1.14.0 - [Release notes](https://github.com/neicnordic/crypt4gh/releases) - [Changelog](https://github.com/neicnordic/crypt4gh/blob/master/.goreleaser.yml) - [Commits](https://github.com/neicnordic/crypt4gh/compare/v1.13.0...v1.14.0) --- updated-dependencies: - dependency-name: github.com/lestrrat-go/jwx/v2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/neicnordic/crypt4gh dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 6 +++--- sda-download/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 17fc3b6fd8..218ecad99f 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -9,9 +9,9 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 - github.com/lestrrat-go/jwx/v2 v2.1.3 + github.com/lestrrat-go/jwx/v2 v2.1.4 github.com/lib/pq v1.10.9 - github.com/neicnordic/crypt4gh v1.13.0 + github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 @@ -30,7 +30,7 @@ require ( github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 8b1d8cbd63..3a598cf9e2 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -24,8 +24,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84= github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= @@ -92,8 +92,8 @@ github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCG github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.3 h1:Ud4lb2QuxRClYAmRleF50KrbKIoM1TddXgBrneT5/Jo= -github.com/lestrrat-go/jwx/v2 v2.1.3/go.mod h1:q6uFgbgZfEmQrfJfrCo90QcQOcXFMfbI/fO0NqRtvZo= +github.com/lestrrat-go/jwx/v2 v2.1.4 h1:uBCMmJX8oRZStmKuMMOFb0Yh9xmEMgNJLgjuKKt4/qc= +github.com/lestrrat-go/jwx/v2 v2.1.4/go.mod h1:nWRbDFR1ALG2Z6GJbBXzfQaYyvn751KuuyySN2yR6is= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -109,8 +109,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.13.0 h1:NbSAPx1+zFpG6a8GCVwW80y/TGHfGdXJF/zqQKqlHZ8= -github.com/neicnordic/crypt4gh v1.13.0/go.mod h1:lfNIrhlcQrSf5awgCaW+poCsRBlvKOrNjR3CBvXU5Ek= +github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U= +github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= From 5cb63ecce22b8fd46b0004e4ac7632ec69d37cf5 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 25 Feb 2025 08:51:01 +0100 Subject: [PATCH 093/250] Remove junit dependency to avoid conflict --- sda-doa/pom.xml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index a25299e1a0..a6078dcafb 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -107,12 +107,6 @@ - - org.junit.jupiter - junit-jupiter-api - 5.11.4 - test - no.elixir crypt4gh From 95bf0cea79def084613df683c2ce0b141521572d Mon Sep 17 00:00:00 2001 From: Parisa Date: Fri, 28 Feb 2025 08:09:33 +0100 Subject: [PATCH 094/250] Read issuer URL from environment variable --- .github/integration/sda-doa-posix-outbox.yml | 2 ++ .github/integration/sda-doa-s3-outbox.yml | 2 ++ sda-doa/test/mock_auth.py | 21 +++++++++++--------- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index 99038f8dcf..164a69615a 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -42,6 +42,8 @@ services: image: python:3.11-slim ports: - 8000:8000 + environment: + - ISSUER_URL=http://mockauth:8000 volumes: - ../../sda-doa/test/mock_auth.py:/mock_auth.py - client_certs:/client_certs diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 6b80cf06dc..888d9105cd 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -42,6 +42,8 @@ services: image: python:3.11-slim ports: - 8000:8000 + environment: + - ISSUER_URL=http://mockauth:8000 volumes: - ../../sda-doa/test/mock_auth.py:/mock_auth.py - client_certs:/client_certs diff --git a/sda-doa/test/mock_auth.py b/sda-doa/test/mock_auth.py index a1beee039d..f870d32242 100644 --- a/sda-doa/test/mock_auth.py +++ b/sda-doa/test/mock_auth.py @@ -5,10 +5,12 @@ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +import os def generate_token(): """Generate RSA Key pair to be used to sign token and the JWT Token itself.""" + issuer_url = os.getenv('ISSUER_URL', 'http://localhost:8000') private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) public_key = private_key.public_key().public_bytes(encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo) @@ -19,7 +21,7 @@ def generate_token(): # See available claims here: https://www.iana.org/assignments/jwt/jwt.xhtml # the important claim is the "authorities" header = { - "jku": "http://mockauth:8000/jwk", + "jku": f"{issuer_url}/jwk", "kid": "rsa1", "alg": "RS256", "typ": "JWT" @@ -29,14 +31,14 @@ def generate_token(): "aud": ["aud2", "aud3"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "http://mockauth:8000/", + "iss": f"{issuer_url}/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102" } empty_payload = { "sub": "requester@elixir-europe.org", - "iss": "http://mockauth:8000/", + "iss": f"{issuer_url}/", "exp": 99999999999, "iat": 1547794655, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102" @@ -44,7 +46,7 @@ def generate_token(): # Craft 4 passports, 2 for bona fide status and 2 for dataset permissions # passport for bona fide: terms passport_terms = { - "iss": "http://mockauth:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "AcceptedTermsAndPolicies", @@ -59,7 +61,7 @@ def generate_token(): } # passport for bona fide: status passport_status = { - "iss": "http://mockauth:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ResearcherStatus", @@ -74,7 +76,7 @@ def generate_token(): } # passport for dataset permissions 1 passport_dataset1 = { - "iss": "http://mockauth:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -89,7 +91,7 @@ def generate_token(): } # passport for dataset permissions 2 passport_dataset2 = { - "iss": "http://mockauth:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -103,7 +105,7 @@ def generate_token(): "jti": "9fa600d6-4148-47c1-b708-36c4ba2e980e" } passport_dataset_gdi = { - "iss": "http://129.177.177.134:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -167,8 +169,9 @@ async def userinfo(request): async def openid_configuration(request): """Mock ELIXIR AAI OpenID configuration endpoint.""" + issuer_url = os.getenv('ISSUER_URL', 'http://localhost:8000') data = { - "jwks_uri": "http://mockauth:8000/jwk" + "jwks_uri": f"{issuer_url}/jwk" } return web.json_response(data) From 290db2b92b1545688a3df7a3bc9c73ae3775cd40 Mon Sep 17 00:00:00 2001 From: Parisa Date: Fri, 28 Feb 2025 08:22:26 +0100 Subject: [PATCH 095/250] Replace deprecated jwk.dumps with JsonWebKey.import_key per AuthlibDeprecationWarning --- sda-doa/test/mock_auth.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sda-doa/test/mock_auth.py b/sda-doa/test/mock_auth.py index f870d32242..a1eea5d351 100644 --- a/sda-doa/test/mock_auth.py +++ b/sda-doa/test/mock_auth.py @@ -1,7 +1,7 @@ """Mock OAUTH2 aiohttp.web server.""" from aiohttp import web -from authlib.jose import jwt, jwk +from authlib.jose import jwt, JsonWebKey from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa @@ -118,15 +118,15 @@ def generate_token(): "exp": 99999999999, "jti": "2b322848-506b-492c-914f-47f9da967cdd" } - public_jwk = jwk.dumps(public_key, kty='RSA') - private_jwk = jwk.dumps(pem, kty='RSA') - dataset_encoded = jwt.encode(header, dataset_payload, private_jwk).decode('utf-8') - empty_encoded = jwt.encode(header, empty_payload, private_jwk).decode('utf-8') - passport_terms_encoded = jwt.encode(header, passport_terms, private_jwk).decode('utf-8') - passport_status_encoded = jwt.encode(header, passport_status, private_jwk).decode('utf-8') - passport_dataset1_encoded = jwt.encode(header, passport_dataset1, private_jwk).decode('utf-8') - passport_dataset2_encoded = jwt.encode(header, passport_dataset2, private_jwk).decode('utf-8') - passport_dataset_gdi_encoded = jwt.encode(header, passport_dataset_gdi, private_jwk).decode('utf-8') + public_jwk = JsonWebKey.import_key(public_key, {'kty': 'RSA', 'kid': 'rsa1', 'use': 'sig'}).as_dict() + private_key_obj = JsonWebKey.import_key(pem, {'kty': 'RSA', 'kid': 'rsa1', 'use': 'sig'}) + dataset_encoded = jwt.encode(header, dataset_payload, private_key_obj).decode('utf-8') + empty_encoded = jwt.encode(header, empty_payload, private_key_obj).decode('utf-8') + passport_terms_encoded = jwt.encode(header, passport_terms, private_key_obj).decode('utf-8') + passport_status_encoded = jwt.encode(header, passport_status, private_key_obj).decode('utf-8') + passport_dataset1_encoded = jwt.encode(header, passport_dataset1, private_key_obj).decode('utf-8') + passport_dataset2_encoded = jwt.encode(header, passport_dataset2, private_key_obj).decode('utf-8') + passport_dataset_gdi_encoded = jwt.encode(header, passport_dataset_gdi, private_key_obj).decode('utf-8') return (public_jwk, dataset_encoded, empty_encoded, passport_terms_encoded, passport_status_encoded, passport_dataset1_encoded, passport_dataset2_encoded, passport_dataset_gdi_encoded) From 1d18ae0147c5c9ed97c050b7a8ff994b32dcae37 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 5 Mar 2025 09:12:23 +0000 Subject: [PATCH 096/250] Bump the all-modules group across 1 directory with 5 updates Bumps the all-modules group with 5 updates in the /sda-doa directory: | Package | From | To | | --- | --- | --- | | [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot) | `3.4.2` | `3.4.3` | | [com.google.code.gson:gson](https://github.com/google/gson) | `2.11.0` | `2.12.1` | | no.elixir:crypt4gh | `3.0.5` | `3.0.15` | | no.elixir:clearinghouse | `2.0.6` | `2.0.13` | | [org.yaml:snakeyaml](https://bitbucket.org/snakeyaml/snakeyaml) | `2.3` | `2.4` | Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.2 to 3.4.3 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.2...v3.4.3) Updates `com.google.code.gson:gson` from 2.11.0 to 2.12.1 - [Release notes](https://github.com/google/gson/releases) - [Changelog](https://github.com/google/gson/blob/main/CHANGELOG.md) - [Commits](https://github.com/google/gson/compare/gson-parent-2.11.0...gson-parent-2.12.1) Updates `no.elixir:crypt4gh` from 3.0.5 to 3.0.15 Updates `no.elixir:clearinghouse` from 2.0.6 to 2.0.13 Updates `org.yaml:snakeyaml` from 2.3 to 2.4 - [Commits](https://bitbucket.org/snakeyaml/snakeyaml/branches/compare/snakeyaml-2.4..snakeyaml-2.3) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: com.google.code.gson:gson dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: no.elixir:crypt4gh dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: org.yaml:snakeyaml dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index a6078dcafb..045e39e5a2 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.2 + 3.4.3 no.uio.ifi @@ -21,7 +21,7 @@ com.google.code.gson gson - 2.11.0 + 2.12.1 org.springframework.boot @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.5 + 3.0.15 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.6 + 2.0.13 org.slf4j @@ -132,7 +132,7 @@ org.yaml snakeyaml - 2.3 + 2.4 From 362802f572644ff2a107759bb692bf14bca8d4e7 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 5 Mar 2025 12:31:44 +0000 Subject: [PATCH 097/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 1aacb9238c..8f4e3afb00 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.77 -appVersion: v0.3.202 +version: 0.8.78 +appVersion: v0.3.208 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 74278734e0..10afd3c0ca 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.78 -appVersion: v0.3.202 +version: 0.7.79 +appVersion: v0.3.208 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 283ea2c3d4..11320cf5d5 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.7 -appVersion: v0.3.202 +version: 0.30.8 +appVersion: v0.3.208 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 966f3d515a07421927aafc868380691fe897a444 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 25 Feb 2025 11:15:24 +0100 Subject: [PATCH 098/250] Remove obsolete info and add overview and installation sections --- sda-doa/README.md | 59 +++++++---------------------------------------- 1 file changed, 8 insertions(+), 51 deletions(-) diff --git a/sda-doa/README.md b/sda-doa/README.md index 3bfdb667e9..04a9524f88 100644 --- a/sda-doa/README.md +++ b/sda-doa/README.md @@ -1,6 +1,11 @@ -# NeIC sda-doa Data Out API -![Java CI](https://github.com/neicnordic/sda-doa/workflows/Java%20CI/badge.svg) +# Overview +The Data Out API (DOA) is a service that provides an interface for exporting datasets and files from the FEGA archive. It provides secure access to datasets and files, ensuring proper authorization and encryption. +## Installation + +1. Navigate to github packages +2. Select the package [https://github.com/orgs/neicnordic/packages/container/package/sensitive-data-archive](sensitive-data-archive) +3. Find the image you want to install (tag format: `vX.X.XXX-doa`) ## Configuration @@ -53,54 +58,6 @@ Environment variables used: | VISA_PUBLIC_KEY_PATH | /etc/ega/jwt/visa.pem | Path to the public key for visas JWT validation | | CRYPT4GH_PRIVATE_KEY_PATH | /etc/ega/crypt4gh/key.pem | Path to the Crypt4GH private key | | CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH | /etc/ega/crypt4gh/key.pass | Path to the Crypt4GH private key passphrase | -| LOGSTASH_HOST | | Hostname of the Logstash instance (if any) | -| LOGSTASH_PORT | | Port of the Logstash instance (if any) | - -If `LOGSTASH_HOST` or `LOGSTASH_PORT` is empty, Logstash logging will not be enabled. - -In addition, environment variables can be used to configure log level for different packages. Package loggers can be configured using corresponding package names, for example, to turn of logs of Spring, one can set environment variable `LOGGING_LEVEL_ORG_SPRINGFRAMEWORK=OFF`, or to set DOA's own logs to debug: `LOGGING_LEVEL_NO_UIO_IFI=DEBUG`, etc. -## Sample Docker Swarm entry -``` -... - doa: - image: neicnordic/sda-doa:latest - ports: - - 443:8080 - deploy: - restart_policy: - condition: on-failure - delay: 5s - window: 120s - environment: - - S3_PORT - - KEYSTORE_PASSWORD=${SERVER_CERT_PASSWORD} - - DB_INSTANCE=${DB_HOST} - - POSTGRES_DB=${DB_DATABASE_NAME} - - POSTGRES_PASSWORD=${DB_LEGA_OUT_PASSWORD} - - S3_ACCESS_KEY=${MINIO_ACCESS_KEY} - - S3_SECRET_KEY=${MINIO_SECRET_KEY} - - LOGSTASH_HOST - - LOGSTASH_PORT - secrets: - - source: rootCA.pem - target: /etc/ega/ssl/CA.cert - - source: rootCA.pem - target: /etc/ssl/certs/ca-certificates.crt - - source: server.p12 - target: /etc/ega/ssl/server.cert - - source: client.pem - target: /etc/ega/ssl/client.cert - - source: client-key.der - target: /etc/ega/ssl/client.key - - source: jwt.pub.pem - target: /etc/ega/jwt/passport.pem - - source: jwt.pub.pem - target: /etc/ega/jwt/visa.pem - - source: ega.sec.pem - target: /etc/ega/crypt4gh/key.pem - - source: ega.sec.pass - target: /etc/ega/crypt4gh/key.pass -... -``` +In addition, environment variables can be used to configure log level for different packages. Package loggers can be configured using corresponding package names, for example, to turn off logs of Spring, one can set environment variable `LOGGING_LEVEL_ORG_SPRINGFRAMEWORK=OFF`, or to set DOA's own logs to debug: `LOGGING_LEVEL_NO_UIO_IFI=DEBUG`, etc. \ No newline at end of file From c7281ae681dbb89dde8e10566bfac040dea4a515 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 25 Feb 2025 11:45:49 +0100 Subject: [PATCH 099/250] add info related to sda-doa targets --- README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 287875c91f..d8fd226cf0 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,14 @@ This will build all required images, bring up the services, run the integration ```sh make integrationtest-sda-sync ``` +- Integration test for SDA-DOA using POSIX as the storage backend: + ```sh + make integrationtest-sda-doa-posix + ``` +- Integration test for SDA-DOA using S3 as the storage backend: + ```sh + make integrationtest-sda-doa-s3 + ``` #### Running the integration tests without shutting down the services This will run the integration tests and keep the services running after the tests are finished. @@ -110,7 +118,14 @@ This will run the integration tests and keep the services running after the test ```sh make integrationtest-sda-sync-run ``` - +- Integration test for SDA-DOA using POSIX as the storage backend: + ```sh + make integrationtest-sda-doa-posix-run + ``` +- Integration test for SDA-DOA using S3 as the storage backend: + ```sh + make integrationtest-sda-doa-s3-run + ``` After that, you will need to shut down the services manually. - Shut down services for SDA using POSIX as the storage backend @@ -125,6 +140,14 @@ After that, you will need to shut down the services manually. ```sh make integrationtest-sda-sync-down ``` +- Shut down services for SDA-DOA using POSIX as the storage backend + ```sh + make integrationtest-sda-doa-posix-down + ``` +- Shut down services for SDA-DOA using S3 as the storage backend + ```sh + make integrationtest-sda-doa-s3-down + ``` ### Linting the Go code From 4eec403db8b4ce24311d55afa3e677e83e85cb1e Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 25 Feb 2025 13:32:07 +0100 Subject: [PATCH 100/250] add permission info --- sda-doa/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sda-doa/README.md b/sda-doa/README.md index 04a9524f88..9e14329498 100644 --- a/sda-doa/README.md +++ b/sda-doa/README.md @@ -4,8 +4,9 @@ The Data Out API (DOA) is a service that provides an interface for exporting dat ## Installation 1. Navigate to github packages -2. Select the package [https://github.com/orgs/neicnordic/packages/container/package/sensitive-data-archive](sensitive-data-archive) -3. Find the image you want to install (tag format: `vX.X.XXX-doa`) +2. Select the package [sensitive-data-archive](https://github.com/orgs/neicnordic/packages/container/package/sensitive-data-archive) +3. Find the image you want to install (tag format: `vX.X.XXX-doa`) +4. Since the image runs as the _nobody_ user, if your outbox is posix you may need to adjust the ownership and permissions of the mounted volume to ensure the container can access the files. ## Configuration From 5133d4f77869a18c4b1bd6c49809c87714fa78c5 Mon Sep 17 00:00:00 2001 From: Parisa Date: Wed, 5 Mar 2025 12:51:08 +0100 Subject: [PATCH 101/250] correct default values remove unnecessary details --- sda-doa/README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/sda-doa/README.md b/sda-doa/README.md index 9e14329498..c7e61ed41a 100644 --- a/sda-doa/README.md +++ b/sda-doa/README.md @@ -36,7 +36,7 @@ Environment variables used: | CERT_KEY | /etc/ega/ssl/client.key | Path to the client key for database connectivity | | POSTGRES_USER | lega_out | Database username | | POSTGRES_PASSWORD | | Database password | -| S3_ENDPOINT | archive | S3 server hostname | +| S3_ENDPOINT | vault | S3 server hostname | | S3_PORT | 443 | S3 server port | | S3_ACCESS_KEY | minio | S3 access key | | S3_SECRET_KEY | miniostorage | S3 secret key | @@ -59,6 +59,3 @@ Environment variables used: | VISA_PUBLIC_KEY_PATH | /etc/ega/jwt/visa.pem | Path to the public key for visas JWT validation | | CRYPT4GH_PRIVATE_KEY_PATH | /etc/ega/crypt4gh/key.pem | Path to the Crypt4GH private key | | CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH | /etc/ega/crypt4gh/key.pass | Path to the Crypt4GH private key passphrase | - - -In addition, environment variables can be used to configure log level for different packages. Package loggers can be configured using corresponding package names, for example, to turn off logs of Spring, one can set environment variable `LOGGING_LEVEL_ORG_SPRINGFRAMEWORK=OFF`, or to set DOA's own logs to debug: `LOGGING_LEVEL_NO_UIO_IFI=DEBUG`, etc. \ No newline at end of file From 0025e099d857f46d6357f06b60e1772d78fb6a41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 19:18:37 +0000 Subject: [PATCH 102/250] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.15 to 3.0.17 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.15...crypt4gh-3.0.17) Updates `no.elixir:clearinghouse` from 2.0.13 to 2.0.15 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-2.0.13...clearinghouse-2.0.15) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 045e39e5a2..3aa279d379 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.15 + 3.0.17 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.13 + 2.0.15 org.slf4j From eef9e3aabb86d6650fd392754c54c9fc3ca4be1c Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 6 Mar 2025 11:24:09 +0100 Subject: [PATCH 103/250] remove unnecessary lines from log --- .../main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java | 1 + .../main/java/no/uio/ifi/localega/doa/services/AAIService.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java index 9a1859059e..ff43e6eff0 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java @@ -12,6 +12,7 @@ @Data public class ExportRequest { + @ToString.Exclude private String jwtToken; private String datasetId; private String fileId; diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java index 0e6565adc6..e22b223c7e 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java @@ -67,7 +67,7 @@ public Collection getDatasetIds(String accessToken) { .collect(Collectors.toList()); if (CollectionUtils.isEmpty(controlledAccessGrantsVisas)) { - log.info("Unauthorized access attempt: user doesn't have any valid visas. Access token: {}", accessToken); + log.info("Unauthorized access attempt: user doesn't have any valid visas."); } String subject = controlledAccessGrantsVisas.stream().findFirst().orElseThrow(RuntimeException::new).getSub(); From cf4b0e6c2c961b7cb73ff92530d71929893a8de9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 19:22:16 +0000 Subject: [PATCH 104/250] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [golang.org/x/crypto](https://github.com/golang/crypto) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `golang.org/x/crypto` from 0.35.0 to 0.36.0 - [Commits](https://github.com/golang/crypto/compare/v0.35.0...v0.36.0) Updates `google.golang.org/grpc` from 1.70.0 to 1.71.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.70.0...v1.71.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 12 +++++------ sda-download/go.sum | 50 +++++++++++++++++++++++---------------------- 2 files changed, 32 insertions(+), 30 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 218ecad99f..ff04576a80 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -15,9 +15,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.35.0 + golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - google.golang.org/grpc v1.70.0 + google.golang.org/grpc v1.71.0 google.golang.org/protobuf v1.36.5 ) @@ -71,11 +71,11 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect golang.org/x/tools v0.25.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 3a598cf9e2..2741595eda 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -164,24 +164,26 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -193,8 +195,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -209,23 +211,23 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -235,10 +237,10 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From b55176f52e8400b9cf787925c25cf1b908626b46 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Mar 2025 19:27:58 +0000 Subject: [PATCH 105/250] Bump the all-modules group in /sda with 7 updates Bumps the all-modules group in /sda with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.8` | `1.29.9` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.61` | `1.17.62` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.64` | `1.17.65` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.78.0` | `1.78.1` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.35.0` | `0.36.0` | | [golang.org/x/oauth2](https://github.com/golang/oauth2) | `0.27.0` | `0.28.0` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.70.0` | `1.71.0` | Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.8 to 1.29.9 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.8...config/v1.29.9) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.61 to 1.17.62 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.61...credentials/v1.17.62) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.64 to 1.17.65 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.64...feature/s3/manager/v1.17.65) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.78.0 to 1.78.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.78.0...service/s3/v1.78.1) Updates `golang.org/x/crypto` from 0.35.0 to 0.36.0 - [Commits](https://github.com/golang/crypto/compare/v0.35.0...v0.36.0) Updates `golang.org/x/oauth2` from 0.27.0 to 0.28.0 - [Commits](https://github.com/golang/oauth2/compare/v0.27.0...v0.28.0) Updates `google.golang.org/grpc` from 1.70.0 to 1.71.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.70.0...v1.71.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 28 +++++++++---------- sda/go.sum | 82 ++++++++++++++++++++++++++++-------------------------- 2 files changed, 56 insertions(+), 54 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index bac981a632..c8d882515d 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,10 +4,10 @@ go 1.24.0 require ( github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.8 - github.com/aws/aws-sdk-go-v2/credentials v1.17.61 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.64 - github.com/aws/aws-sdk-go-v2/service/s3 v1.78.0 + github.com/aws/aws-sdk-go-v2/config v1.29.9 + github.com/aws/aws-sdk-go-v2/credentials v1.17.62 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65 + github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.103.0 github.com/coreos/go-oidc/v3 v3.12.0 @@ -31,9 +31,9 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.35.0 - golang.org/x/oauth2 v0.27.0 - google.golang.org/grpc v1.70.0 + golang.org/x/crypto v0.36.0 + golang.org/x/oauth2 v0.28.0 + google.golang.org/grpc v1.71.0 google.golang.org/protobuf v1.36.5 ) @@ -59,9 +59,9 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.0 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.16 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect @@ -157,11 +157,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.33.0 // indirect - golang.org/x/sys v0.30.0 // indirect - golang.org/x/text v0.22.0 // indirect + golang.org/x/net v0.34.0 // indirect + golang.org/x/sys v0.31.0 // indirect + golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.6.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 05658b2907..30a3b0d9b2 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -28,14 +28,14 @@ github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38y github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.8 h1:RpwAfYcV2lr/yRc4lWhUM9JRPQqKgKWmou3LV7UfWP4= -github.com/aws/aws-sdk-go-v2/config v1.29.8/go.mod h1:t+G7Fq1OcO8cXTPPXzxQSnj/5Xzdc9jAAD3Xrn9/Mgo= -github.com/aws/aws-sdk-go-v2/credentials v1.17.61 h1:Hd/uX6Wo2iUW1JWII+rmyCD7MMhOe7ALwQXN6sKDd1o= -github.com/aws/aws-sdk-go-v2/credentials v1.17.61/go.mod h1:L7vaLkwHY1qgW0gG1zG0z/X0sQ5tpIY5iI13+j3qI80= +github.com/aws/aws-sdk-go-v2/config v1.29.9 h1:Kg+fAYNaJeGXp1vmjtidss8O2uXIsXwaRqsQJKXVr+0= +github.com/aws/aws-sdk-go-v2/config v1.29.9/go.mod h1:oU3jj2O53kgOU4TXq/yipt6ryiooYjlkqqVaZk7gY/U= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBajbBWGqjNTCa79ocU= +github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.64 h1:RTko0AQ0i1vWXDM97DkuW6zskgOxFxm4RqC0kmBJFkE= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.64/go.mod h1:ty968MpOa5CoQ/ALWNB8Gmfoehof2nRHDR/DZDPfimE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65 h1:03zF9oWZyXvw08Say761JGpE9PbeGPd4FAmdpgDAm/I= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65/go.mod h1:hBobvLKm46Igpcw6tkq9hFUmU14iAOrC5KL6EyYYckA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -52,14 +52,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.78.0 h1:EBm8lXevBWe+kK9VOU/IBeOI189WPRwPUc3LvJK9GOs= -github.com/aws/aws-sdk-go-v2/service/s3 v1.78.0/go.mod h1:4qzsZSzB/KiX2EzDjs9D7A8rI/WGJxZceVJIHqtJjIU= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.0 h1:2U9sF8nKy7UgyEeLiZTRg6ShBS22z8UnYpV6aRFL0is= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.0/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0 h1:wjAdc85cXdQR5uLx5FwWvGIHm4OPJhTyzUHU8craXtE= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.16 h1:BHEK2Q/7CMRMCb3nySi/w8UbIcPhKvYP5s1xf8/izn0= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.16/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1 h1:1M0gSbyP6q06gl3384wpoKPaH9G16NPqZFieEhLboSU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1/go.mod h1:4qzsZSzB/KiX2EzDjs9D7A8rI/WGJxZceVJIHqtJjIU= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 h1:8JdC7Gr9NROg1Rusk25IcZeTO59zLxsKgE0gkh5O6h0= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.1/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 h1:KwuLovgQPcdjNMfFt9OhUd9a2OwcOKhxfvF4glTzLuA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -387,16 +387,18 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -410,8 +412,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= -golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= +golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -431,10 +433,10 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= -golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= +golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -459,16 +461,16 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= +golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= -golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= +golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= +golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -476,8 +478,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= +golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= +golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -493,10 +495,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a h1:hgh8P4EuoxpsuKMXX/To36nOFD7vixReXgn8lPGnt+o= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241202173237-19429a94021a/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= +google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= +google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From b14afd9e9b7ce91b02b6622a3375df9606daad1b Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 11 Mar 2025 13:41:34 +0000 Subject: [PATCH 106/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 8f4e3afb00..1b84f7962e 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.78 -appVersion: v0.3.208 +version: 0.8.79 +appVersion: v0.3.212 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 10afd3c0ca..a2d281400c 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.79 -appVersion: v0.3.208 +version: 0.7.80 +appVersion: v0.3.212 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 11320cf5d5..26682eab67 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.8 -appVersion: v0.3.208 +version: 0.30.9 +appVersion: v0.3.212 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 09c403585849d7c91e7e2a582616b305c7593cd0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 02:02:31 +0000 Subject: [PATCH 107/250] Bump golang.org/x/net in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.34.0 to 0.36.0 - [Commits](https://github.com/golang/net/compare/v0.34.0...v0.36.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index c8d882515d..1412279ac4 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -157,7 +157,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.34.0 // indirect + golang.org/x/net v0.36.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/time v0.6.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 30a3b0d9b2..1946d11b4f 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -433,8 +433,8 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From e6eaf20e3a052c9c2929ff8d303a75f65623aa1f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 13 Mar 2025 02:09:03 +0000 Subject: [PATCH 108/250] Bump golang.org/x/net in /sda-download in the go_modules group Bumps the go_modules group in /sda-download with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.34.0 to 0.36.0 - [Commits](https://github.com/golang/net/compare/v0.34.0...v0.36.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index ff04576a80..3afaaa15b2 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -71,7 +71,7 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect - golang.org/x/net v0.34.0 // indirect + golang.org/x/net v0.36.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect golang.org/x/tools v0.25.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 2741595eda..3065bfd569 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -195,8 +195,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= -golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= +golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= +golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 9370a3005d7d897ae7eff1299480868bd6bccc98 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Fri, 14 Mar 2025 09:06:16 +0000 Subject: [PATCH 109/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 1b84f7962e..4356103e17 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.79 -appVersion: v0.3.212 +version: 0.8.80 +appVersion: v0.3.214 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index a2d281400c..9a59323680 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.80 -appVersion: v0.3.212 +version: 0.7.81 +appVersion: v0.3.214 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 26682eab67..4788fbe579 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.9 -appVersion: v0.3.212 +version: 0.30.10 +appVersion: v0.3.214 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From dde2179e3d7c83d550a9334f4f631535337712cd Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Wed, 26 Feb 2025 13:02:48 +0100 Subject: [PATCH 110/250] doc: fix typos in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d8fd226cf0..9a75c7f33b 100644 --- a/README.md +++ b/README.md @@ -252,9 +252,9 @@ Bootstrap the dependencies with the command: `make k3d-deploy-dependencies`. Start by building and importing the required containers using the `make k3d-import-images`. -The Postgres and RabbitMQ Needs to be deployed first using the following commands: `make k3d-deplploy-postgres` and `make k3d-deploy-rabbitmq`. +The Postgres and RabbitMQ Needs to be deployed first using the following commands: `make k3d-deploy-postgres` and `make k3d-deploy-rabbitmq`. -Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`), `make k3d-deplpoy-sda-posix` or `make k3d-deplpoy-sda-s3`. +Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`), `make k3d-deploy-sda-posix` or `make k3d-deploy-sda-s3`. #### Testing with ingress From ab126a22624ee2555c203d9bfe59b8e1c14f2e9f Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Thu, 27 Feb 2025 14:14:13 +0100 Subject: [PATCH 111/250] fix: show error msg and standard page when ega login fails setting a viewModel in .View("file.html", viewModel) overrides any previous viewdata calls. --- sda/cmd/auth/cega.go | 5 ----- sda/cmd/auth/main.go | 4 +++- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/sda/cmd/auth/cega.go b/sda/cmd/auth/cega.go index 54f2919bf5..57f26fa72c 100644 --- a/sda/cmd/auth/cega.go +++ b/sda/cmd/auth/cega.go @@ -11,11 +11,6 @@ import ( bcrypt "golang.org/x/crypto/bcrypt" ) -// EGALoginError is used to store message errors -type EGALoginError struct { - Reason string -} - // CegaUserResponse captures the response list type CegaUserResponse struct { PasswordHash string `json:"passwordHash"` diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 1fe8088461..f8822a416f 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -160,6 +160,7 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { s3conf := getS3ConfigMap(token, auth.Config.S3Inbox, username) s.SetFlash("ega", s3conf) + ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) ctx.ViewData("User", username) @@ -210,7 +211,8 @@ func (auth AuthHandler) getEGALogin(ctx iris.Context) { } ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) - err := ctx.View("loginform.html", EGALoginError{Reason: message}) + ctx.ViewData("Reason", message) + err := ctx.View("loginform.html") if err != nil { log.Error("Failed to view invalid credentials form: ", err) From 84c323c5e07f6d95f878aa0688a9b343bd9fbf94 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 7 Mar 2025 14:03:28 +0100 Subject: [PATCH 112/250] [auth] fix: handle cega-nss being down --- sda/cmd/auth/cega.go | 3 +-- sda/cmd/auth/main.go | 9 ++++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/sda/cmd/auth/cega.go b/sda/cmd/auth/cega.go index 57f26fa72c..8f99aa9977 100644 --- a/sda/cmd/auth/cega.go +++ b/sda/cmd/auth/cega.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/neicnordic/sensitive-data-archive/internal/config" - log "github.com/sirupsen/logrus" bcrypt "golang.org/x/crypto/bcrypt" ) @@ -44,7 +43,7 @@ func authenticateWithCEGA(conf config.CegaConfig, username string) (*http.Respon req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", strings.TrimSuffix(conf.AuthURL, "/"), username), payload) if err != nil { - log.Fatal(err) + return nil, err } req.Header.Add("Authorization", "Basic "+getb64Credentials(conf.ID, conf.Secret)) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index f8822a416f..4e8404fc4c 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -118,13 +118,16 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { res, err := authenticateWithCEGA(auth.Config.Cega, username) + var statusCode int if err != nil { log.Error(err) + statusCode = 404 + } else { + statusCode = res.StatusCode + defer res.Body.Close() } - defer res.Body.Close() - - switch res.StatusCode { + switch statusCode { case 200: if err != nil { log.Error(err) From 4c3761254986b53f035a1091ebe45491dd22ef7a Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 10 Mar 2025 08:49:08 +0100 Subject: [PATCH 113/250] [auth] add case for bad service credentials to ega server --- sda/cmd/auth/main.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 4e8404fc4c..1eca5d9e5d 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -188,6 +188,10 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { s.SetFlash("message", "EGA authentication server could not be contacted") ctx.Redirect("/ega/login", iris.StatusSeeOther) + case 401: + log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Failed to authenticate service (auth_cega_id/secret)") + s.SetFlash("message", "Problems connecting to EGA authentication server") + ctx.Redirect("/ega/login", iris.StatusSeeOther) default: log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Failed to authenticate user") s.SetFlash("message", "Provided credentials are not valid") From a7999d3bff14da6c026b58ca4a2e05609b334877 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 10 Mar 2025 13:32:32 +0100 Subject: [PATCH 114/250] [auth] refactor error handeling --- sda/cmd/auth/main.go | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 1eca5d9e5d..f27a9ba6c2 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -118,25 +118,19 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { res, err := authenticateWithCEGA(auth.Config.Cega, username) - var statusCode int if err != nil { - log.Error(err) - statusCode = 404 - } else { - statusCode = res.StatusCode - defer res.Body.Close() + log.Errorf("No response from cega, error: %v", err) + res = &http.Response{ + Body: io.NopCloser(nil), + StatusCode: http.StatusInternalServerError, + } } + defer res.Body.Close() - switch statusCode { + switch res.StatusCode { case 200: - if err != nil { - log.Error(err) - - return - } - var ur CegaUserResponse - err = json.NewDecoder(res.Body).Decode(&ur) + err := json.NewDecoder(res.Body).Decode(&ur) if err != nil { log.Error("Failed to parse response: ", err) From b456ef0d70d084ec69004ef07f10dcd8ad50961f Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 10 Mar 2025 13:34:21 +0100 Subject: [PATCH 115/250] [auth] handle more errors --- sda/cmd/auth/main.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index f27a9ba6c2..23cf006816 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -133,7 +133,9 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { err := json.NewDecoder(res.Body).Decode(&ur) if err != nil { - log.Error("Failed to parse response: ", err) + log.Error("Failed to parse cega response: ", err) + s.SetFlash("message", "Problems connecting to EGA authentication server") + ctx.Redirect("/ega/login", iris.StatusSeeOther) return } @@ -153,6 +155,10 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { token, expDate, err := generateJwtToken(claims, auth.Config.JwtPrivateKey, auth.Config.JwtSignatureAlg) if err != nil { log.Errorf("error when generating token: %v", err) + s.SetFlash("message", "Unexpected error, please try again.") + ctx.Redirect("/ega/login", iris.StatusSeeOther) + + return } s3conf := getS3ConfigMap(token, auth.Config.S3Inbox, username) @@ -163,12 +169,17 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { ctx.ViewData("User", username) ctx.ViewData("Token", token) ctx.ViewData("ExpDate", expDate) - err = ctx.View("ega.html") - if err != nil { - log.Error("Failed to parse response: ", err) - return + if err != nil { + log.Error("Failed to create view: ", err) + + // Since the context has already started writing the response to + // the client, the resulting page will be ugly, but at least + // show an error message, and an oppertunity to log in again. + ctx.ViewData("Reason", "Unexpected error, please try again.") + err = ctx.View("loginform.html") + log.Error("Failed to create backup view: ", err) } } else { @@ -177,7 +188,7 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { ctx.Redirect("/ega/login", iris.StatusSeeOther) } - case 404: + case 500, 502, 503: log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Failed to authenticate user") s.SetFlash("message", "EGA authentication server could not be contacted") ctx.Redirect("/ega/login", iris.StatusSeeOther) From 7c7e03afc35aa9d846070cc9ea7f8e75eeee3792 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 14 Mar 2025 11:54:13 +0100 Subject: [PATCH 116/250] auth: refactor --- sda/cmd/auth/main.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 23cf006816..3eac088b64 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -208,27 +208,15 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { func (auth AuthHandler) getEGALogin(ctx iris.Context) { s := sessions.Get(ctx) - message := s.GetFlashString("message") - if message == "" { - ctx.ViewData("infoUrl", auth.Config.InfoURL) - ctx.ViewData("infoText", auth.Config.InfoText) - err := ctx.View("loginform.html") - if err != nil { - log.Error("Failed to return to login form: ", err) - - return - } - - return - } ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) - ctx.ViewData("Reason", message) + message := s.GetFlashString("message") + if message != "" { + ctx.ViewData("Reason", message) + } err := ctx.View("loginform.html") if err != nil { log.Error("Failed to view invalid credentials form: ", err) - - return } } From 165bc18cbc274661ec0c131e87cdf533c5bfc9d7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:12:16 +0000 Subject: [PATCH 117/250] Bump aquasecurity/trivy-action from 0.29.0 to 0.30.0 Bumps [aquasecurity/trivy-action](https://github.com/aquasecurity/trivy-action) from 0.29.0 to 0.30.0. - [Release notes](https://github.com/aquasecurity/trivy-action/releases) - [Commits](https://github.com/aquasecurity/trivy-action/compare/0.29.0...0.30.0) --- updated-dependencies: - dependency-name: aquasecurity/trivy-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/build_pr_container.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 88d33f222e..a1fffb78bd 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -105,7 +105,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@0.30.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -123,7 +123,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@0.30.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -172,7 +172,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@0.30.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -218,7 +218,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@0.30.0 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true From 60cfcd63f14e65b42aeb9a6d2ca13dd0b910b597 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:12:23 +0000 Subject: [PATCH 118/250] Bump golangci/golangci-lint-action from 6.5.0 to 6.5.1 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.5.0 to 6.5.1. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v6.5.0...v6.5.1) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 6eabc7aba6..b8e169d8cd 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.0 + uses: golangci/golangci-lint-action@v6.5.1 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-download @@ -48,7 +48,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.0 + uses: golangci/golangci-lint-action@v6.5.1 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.0 + uses: golangci/golangci-lint-action@v6.5.1 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-admin From 4b76de334f1d79d7a12d426c8a4891f50a9ca0a9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:24:06 +0000 Subject: [PATCH 119/250] Bump github.com/spf13/viper in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [github.com/spf13/viper](https://github.com/spf13/viper). Updates `github.com/spf13/viper` from 1.19.0 to 1.20.0 - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.19.0...v1.20.0) --- updated-dependencies: - dependency-name: github.com/spf13/viper dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 18 +++++++----------- sda-download/go.sum | 36 ++++++++++++++---------------------- 2 files changed, 21 insertions(+), 33 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 3afaaa15b2..d9146d2f46 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -13,7 +13,7 @@ require ( github.com/lib/pq v1.10.9 github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.19.0 + github.com/spf13/viper v1.20.0 github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 @@ -32,14 +32,14 @@ require ( github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.1 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.3 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect @@ -49,23 +49,20 @@ require ( github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect @@ -76,7 +73,6 @@ require ( golang.org/x/text v0.23.0 // indirect golang.org/x/tools v0.25.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect - gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 3065bfd569..1a27f371df 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -34,8 +34,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -54,6 +54,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -63,8 +65,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -98,12 +98,8 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -122,10 +118,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= @@ -136,14 +130,14 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= +github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -246,8 +240,6 @@ google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojt gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From cfb0a389d5caed8b2a10c54187ec82ef01b7d503 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Mar 2025 09:28:18 +0100 Subject: [PATCH 120/250] Add swagger file for Auths info endpoint --- sda/cmd/auth/swagger_v1.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 sda/cmd/auth/swagger_v1.yml diff --git a/sda/cmd/auth/swagger_v1.yml b/sda/cmd/auth/swagger_v1.yml new file mode 100644 index 0000000000..937f801703 --- /dev/null +++ b/sda/cmd/auth/swagger_v1.yml @@ -0,0 +1,33 @@ +openapi: 3.0.3 +info: + title: SDA auth frontend + version: "1.0" + description: This is the auth frontend for the sensitive data archive. +paths: + /info: + get: + description: Returns the fino JSON + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Info" + description: Successful operation +components: + schemas: + Info: + type: object + properties: + client_id: + example: 7daee13a-cd72-4b92-8a59-89c0e063b20f + type: string + inbox_uri: + example: inbox.demo.org + type: string + oidc_uri: + example: https://aai.provider.org/oidc + type: string + public_key: + example: LS0tLS1CRUdJTiBDUllQVDRHSCBQVUJMSUMgS0VZLS0t + type: string \ No newline at end of file From c5f412a196a757fa24cf77fe85f686db57b046ab Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Mar 2025 09:28:31 +0100 Subject: [PATCH 121/250] Add swagger file for the admin API --- sda/cmd/api/swagger_v1.yml | 410 +++++++++++++++++++++++++++++++++++++ 1 file changed, 410 insertions(+) create mode 100644 sda/cmd/api/swagger_v1.yml diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml new file mode 100644 index 0000000000..beca0a7a3e --- /dev/null +++ b/sda/cmd/api/swagger_v1.yml @@ -0,0 +1,410 @@ +openapi: 3.0.3 +info: + title: SDA admin API + version: "1.0" + description: This is the admin API for the sensitive data archive. +paths: + /c4gh-keys/add: + post: + description: Registeres an crypt4gh public key hash in the database + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/C4ghKeyAdd" + responses: + "200": + description: Successful operation. + "400": + description: Bad payload + "401": + description: Authentication failure. + "409": + description: Key hash already exists in the database. + "500": + description: Internal application error. + /c4gh-keys/deprecate/{keyHash}: + post: + description: Deprecate a given key hash + parameters: + - in: path + name: keyHash + schema: + type: string + required: true + responses: + "200": + description: Successful operation + "400": + description: Bad key hash + "401": + description: Authentication failure + "500": + description: Internal application error + /c4gh-keys/list: + get: + description: Returns a list of all registered encryptio ns keys and thei statuses. + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/C4ghKeysList" + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /dataset/create: + post: + description: Create a new dataset + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DatasetCreate" + responses: + "200": + description: Successful operation + "400": + description: Bad key hash + "401": + description: Authentication failure + "500": + description: Internal application error + /dataset/release/{datasetID}: + post: + description: Create a new dataset + parameters: + - in: path + name: datasetID + schema: + type: string + required: true + responses: + "200": + description: Successful operation + "400": + description: Bad dataset ID + "401": + description: Authentication failure + "500": + description: Internal application error + /dataset/verify/{datasetID}: + put: + description: Triggers reverification of all files in the dataset. + parameters: + - in: path + name: datasetID + schema: + type: string + required: true + responses: + "200": + description: Successful operation + "400": + description: Bad dataset ID + "401": + description: Authentication failure + "500": + description: Internal application error + /datasets: + get: + description: Lists datasets beloing to the calling userName + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatasetInfo" + description: Successful operation + "500": + description: Internal application error + /datasets/list: + get: + description: Lists all datasets with their status. + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatasetInfo" + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /datasets/list/{userName}: + get: + description: Lists all datasets with their status for a specific user. + parameters: + - in: path + name: userName + schema: + type: string + required: true + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatasetInfo" + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /file/accession: + post: + description: Assigns accession ID to a given file. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/FileAccession" + responses: + "200": + description: Successful operation. + "400": + description: Bad payload + "401": + description: Authentication failure. + "500": + description: Internal application error. + /file/ingest: + post: + description: Trigger ingestion of a given file. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/FileIngest" + responses: + "200": + description: Successful operation. + "400": + description: Bad payload + "401": + description: Authentication failure. + "500": + description: Internal application error. + /file/{userName}/{fileID}: + delete: + description: Delete a file from the inbox + parameters: + - in: path + name: fileID + schema: + type: string + required: true + - in: path + name: userName + schema: + type: string + required: true + responses: + "200": + description: Successful operation. + "400": + description: No file Id supplied + "401": + description: Authentication failure. + "404": + description: Bad user file ID combination + "500": + description: Internal application error. + /file/verify/{accessionID}: + post: + description: Trigger re-verification of a file with a given accession ID. + parameters: + - in: path + name: accessionID + schema: + type: string + required: true + responses: + "200": + description: Successful operation. + "400": + description: Bad payload + "401": + description: Authentication failure. + "500": + description: Internal application error. + /files: + get: + description: List all files belonging to the calling user that is not part of a dataset. + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FileInfo" + description: Successful operation + "400": + description: Bad username + "401": + description: Authentication failure + "500": + description: Internal application error + /ready: + get: + description: Returns the status of the application. + responses: + "200": + content: + application/json: + example: + {} + description: The service is operational + "503": + content: + application/json: + example: + {} + description: Unhealthy service + security: [] + /users: + get: + description: Lists all users with ongoing submissions. + responses: + "200": + content: + application/json: + schema: + example: ["test.user@demo.org","demo.user@demo.org"] + type: array + items: + type: string + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /users/{userName}/files: + get: + description: Lists all users with ongoing submissions. + parameters: + - in: path + name: userName + schema: + type: string + required: true + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FileInfo" + description: Successful operation + "400": + description: Bad username + "401": + description: Authentication failure + "500": + description: Internal application error +components: + schemas: + C4ghKeyAdd: + type: object + properties: + description: + type: string + example: pubkey description + pubkey: + type: string + example: 19GT1JNQVRgIGNhbiBiZSBzZXQgdG8gYGpzb25gIHRvIGdldCBsb2dzIGluIEpTT04gZm9ybWF0= + C4ghKeysList: + type: object + properties: + hash: + type: string + example: cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23 + description: + type: string + example: This is an example + createdAt: + type: string + example: "2025-03-01T12:13:14.123Z" + deprecatedAt: + type: string + example: "2025-03-02T13:14:15.123Z" + DatasetCreate: + type: object + properties: + accession_ids: + example: ["zz-file-123456-asdfgh", "zz-file-123456-qwerty"] + type: array + items: + type: string + dataset_id: + type: string + example: zz-dataset-123456-asdfgh + user: + type: string + example: test.user@dummy.org + DatasetInfo: + type: object + properties: + DatasetId: + type: string + example: zz-dataset-123456-asdfgh + Status: + type: string + example: registered + Timestamp: + type: string + example: 2025-03-30T09:10:11.321Z + FileAccession: + type: object + properties: + accession_id: + type: string + example: zz-file-123456-zxcvbn + filepath: + type: string + example: uploads/file-1.c4gh + user: + type: string + example: test.user@dummy.org + FileIngest: + type: object + properties: + filepath: + type: string + example: uploads/file-1.c4gh + user: + type: string + example: test.user@dummy.org + FileInfo: + type: object + properties: + FileID: + type: string + example: e996e130-c08b-4b33-98d1-9aebbbf75850 + InboxPath: + type: string + example: uploads/file-001.c4gh + Status: + type: string + example: verified + CreatedAt: + type: string + example: "2025-03-02T13:14:15.123Z" + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT +security: + - bearerAuth: [] \ No newline at end of file From d487f2e29ae63bc4eaff64ca91528fe27f2f1521 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 17 Mar 2025 19:48:27 +0000 Subject: [PATCH 122/250] Bump the all-modules group in /sda with 5 updates Bumps the all-modules group in /sda with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.65` | `1.17.66` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.78.1` | `1.78.2` | | [github.com/coreos/go-oidc/v3](https://github.com/coreos/go-oidc) | `3.12.0` | `3.13.0` | | [github.com/pkg/sftp](https://github.com/pkg/sftp) | `1.13.7` | `1.13.8` | | [github.com/spf13/viper](https://github.com/spf13/viper) | `1.19.0` | `1.20.0` | Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.65 to 1.17.66 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.65...feature/s3/manager/v1.17.66) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.78.1 to 1.78.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.78.1...service/s3/v1.78.2) Updates `github.com/coreos/go-oidc/v3` from 3.12.0 to 3.13.0 - [Release notes](https://github.com/coreos/go-oidc/releases) - [Commits](https://github.com/coreos/go-oidc/compare/v3.12.0...v3.13.0) Updates `github.com/pkg/sftp` from 1.13.7 to 1.13.8 - [Release notes](https://github.com/pkg/sftp/releases) - [Commits](https://github.com/pkg/sftp/compare/v1.13.7...v1.13.8) Updates `github.com/spf13/viper` from 1.19.0 to 1.20.0 - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.19.0...v1.20.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/coreos/go-oidc/v3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/pkg/sftp dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/spf13/viper dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 32 +++++++++---------- sda/go.sum | 92 +++++++++++++++++++++++++++++++----------------------- 2 files changed, 67 insertions(+), 57 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 1412279ac4..f6c49e9f4c 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,11 +6,11 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.9 github.com/aws/aws-sdk-go-v2/credentials v1.17.62 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65 - github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66 + github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.103.0 - github.com/coreos/go-oidc/v3 v3.12.0 + github.com/coreos/go-oidc/v3 v3.13.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 @@ -25,11 +25,11 @@ require ( github.com/ory/dockertest v3.3.5+incompatible github.com/ory/dockertest/v3 v3.11.0 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.7 + github.com/pkg/sftp v1.13.8 github.com/rabbitmq/amqp091-go v1.10.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.19.0 + github.com/spf13/viper v1.20.0 github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.36.0 golang.org/x/oauth2 v0.28.0 @@ -56,7 +56,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 // indirect @@ -81,7 +81,7 @@ require ( github.com/docker/go-units v0.5.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.4 // indirect @@ -89,7 +89,7 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.1 // indirect - github.com/go-viper/mapstructure/v2 v2.1.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect @@ -98,7 +98,6 @@ require ( github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -116,13 +115,11 @@ require ( github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mailgun/raymond/v2 v2.0.48 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -134,14 +131,13 @@ require ( github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tdewolff/minify/v2 v2.20.37 // indirect github.com/tdewolff/parse/v2 v2.7.15 // indirect @@ -157,10 +153,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.36.0 // indirect + golang.org/x/net v0.37.0 // indirect golang.org/x/sys v0.31.0 // indirect golang.org/x/text v0.23.0 // indirect - golang.org/x/time v0.6.0 // indirect + golang.org/x/time v0.8.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 1946d11b4f..9cba17c71c 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBa github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65 h1:03zF9oWZyXvw08Say761JGpE9PbeGPd4FAmdpgDAm/I= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.65/go.mod h1:hBobvLKm46Igpcw6tkq9hFUmU14iAOrC5KL6EyYYckA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66 h1:MTLivtC3s89de7Fe3P8rzML/8XPNRfuyJhlRTsCEt0k= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66/go.mod h1:NAuQ2s6gaFEsuTIb2+P5t6amB1w5MhvJFxppoezGWH0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -46,14 +46,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcu github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2 h1:t/gZFyrijKuSU0elA5kRngP/oU3mc0I+Dvp8HwRE4c0= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.6.2/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1 h1:1M0gSbyP6q06gl3384wpoKPaH9G16NPqZFieEhLboSU= -github.com/aws/aws-sdk-go-v2/service/s3 v1.78.1/go.mod h1:4qzsZSzB/KiX2EzDjs9D7A8rI/WGJxZceVJIHqtJjIU= +github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2 h1:jIiopHEV22b4yQP2q36Y0OmwLbsxNWdWwfZRR5QRRO4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 h1:8JdC7Gr9NROg1Rusk25IcZeTO59zLxsKgE0gkh5O6h0= github.com/aws/aws-sdk-go-v2/service/sso v1.25.1/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 h1:KwuLovgQPcdjNMfFt9OhUd9a2OwcOKhxfvF4glTzLuA= @@ -85,8 +85,8 @@ github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/coreos/go-oidc/v3 v3.12.0 h1:sJk+8G2qq94rDI6ehZ71Bol3oUHy63qNYmkiSjrc/Jo= -github.com/coreos/go-oidc/v3 v3.12.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= +github.com/coreos/go-oidc/v3 v3.13.0 h1:M66zd0pcc5VxvBNM4pB331Wrsanby+QomQYjN8HamW8= +github.com/coreos/go-oidc/v3 v3.13.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -114,8 +114,8 @@ github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0H github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -140,8 +140,8 @@ github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27 github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= -github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= @@ -177,8 +177,6 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= @@ -240,8 +238,6 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -261,8 +257,6 @@ github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= @@ -294,8 +288,8 @@ github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNH github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM= -github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY= +github.com/pkg/sftp v1.13.8 h1:Xt7eJ/xqXv7s0VuzFw7JXhZj6Oc1zI6l4GK8KP9sFB0= +github.com/pkg/sftp v1.13.8/go.mod h1:DmvEkvKE2lshEeuo2JMp06yqcx9HVnR7e3zqQl42F3U= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -305,10 +299,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -327,14 +319,14 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= +github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -410,8 +402,10 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= @@ -421,6 +415,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -433,8 +430,11 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= +golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -443,6 +443,10 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -459,16 +463,21 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -477,11 +486,14 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -491,6 +503,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 21541fa051fa5dc7dbbde9805cc10bbe9093399e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Mar 2025 11:42:05 +0100 Subject: [PATCH 123/250] Apply suggestions from code review Co-authored-by: Malin Klang --- sda/cmd/api/swagger_v1.yml | 16 +++++++++------- sda/cmd/auth/swagger_v1.yml | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index beca0a7a3e..9e5481a797 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -6,7 +6,7 @@ info: paths: /c4gh-keys/add: post: - description: Registeres an crypt4gh public key hash in the database + description: Registers an crypt4gh public key hash in the database requestBody: content: application/json: @@ -14,7 +14,7 @@ paths: $ref: "#/components/schemas/C4ghKeyAdd" responses: "200": - description: Successful operation. + description: Successful operation "400": description: Bad payload "401": @@ -43,13 +43,15 @@ paths: description: Internal application error /c4gh-keys/list: get: - description: Returns a list of all registered encryptio ns keys and thei statuses. + description: Returns a list of all registered encryptions keys and their statuses. responses: "200": content: application/json: schema: - $ref: "#/components/schemas/C4ghKeysList" + type: array + items: + $ref: "#/components/schemas/C4ghKeysList" description: Successful operation "401": description: Authentication failure @@ -74,7 +76,7 @@ paths: description: Internal application error /dataset/release/{datasetID}: post: - description: Create a new dataset + description: Release a dataset, so that it is accessible for downloading parameters: - in: path name: datasetID @@ -110,7 +112,7 @@ paths: description: Internal application error /datasets: get: - description: Lists datasets beloing to the calling userName + description: Lists datasets belonging to the calling userName responses: "200": content: @@ -291,7 +293,7 @@ paths: description: Internal application error /users/{userName}/files: get: - description: Lists all users with ongoing submissions. + description: List all files belonging to the given user that is not part of a dataset. parameters: - in: path name: userName diff --git a/sda/cmd/auth/swagger_v1.yml b/sda/cmd/auth/swagger_v1.yml index 937f801703..c0123d02bf 100644 --- a/sda/cmd/auth/swagger_v1.yml +++ b/sda/cmd/auth/swagger_v1.yml @@ -6,7 +6,7 @@ info: paths: /info: get: - description: Returns the fino JSON + description: Returns the info JSON responses: "200": content: From 48996560287bcfa3f276439e7d57a45ab355957d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Mar 2025 11:59:04 +0100 Subject: [PATCH 124/250] Apply suggestions from code review --- sda/cmd/api/swagger_v1.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index 9e5481a797..d6f54da431 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -69,7 +69,7 @@ paths: "200": description: Successful operation "400": - description: Bad key hash + description: Bad request bode content "401": description: Authentication failure "500": @@ -142,7 +142,7 @@ paths: description: Internal application error /datasets/list/{userName}: get: - description: Lists all datasets with their status for a specific user. + description: Lists all datasets with their status submitted by a specific user. parameters: - in: path name: userName From 7dcc0519d63ab8b9c86e7c70ea8094f8676a3ed7 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Mar 2025 12:04:24 +0100 Subject: [PATCH 125/250] Update sda/cmd/api/swagger_v1.yml Co-authored-by: Dimitris Bampalikis --- sda/cmd/api/swagger_v1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index d6f54da431..9765ef7ffe 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -69,7 +69,7 @@ paths: "200": description: Successful operation "400": - description: Bad request bode content + description: Bad request body content "401": description: Authentication failure "500": From 8c5d3286e96c8a41f577c098fccc278fe3744b35 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Mar 2025 13:56:08 +0100 Subject: [PATCH 126/250] Expand the SubmissionFileInfo struct to also contain the file stable ID --- sda/internal/database/database.go | 9 +++++---- sda/internal/database/db_functions.go | 6 ++++-- sda/internal/database/db_functions_test.go | 3 +++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index eed386a8c7..4f8f3685eb 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -46,10 +46,11 @@ type SyncData struct { } type SubmissionFileInfo struct { - FileID string `json:"fileID"` - InboxPath string `json:"inboxPath"` - Status string `json:"fileStatus"` - CreateAt string `json:"createAt"` + AccessionID string `json:"accessionID,omitempty"` + FileID string `json:"fileID"` + InboxPath string `json:"inboxPath"` + Status string `json:"fileStatus"` + CreateAt string `json:"createAt"` } type DatasetInfo struct { diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 9e643b1a1c..85ed597f3b 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -715,7 +715,7 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { db := dbs.DB // select all files (that are not part of a dataset) of the user, each one annotated with its latest event - const query = "SELECT f.id, f.submission_file_path, e.event, f.created_at FROM sda.files f " + + const query = "SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f " + "LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " + "AND f.id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id); " @@ -728,12 +728,14 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { // Iterate rows for rows.Next() { + var accessionID sql.NullString // Read rows into struct fi := &SubmissionFileInfo{} - err := rows.Scan(&fi.FileID, &fi.InboxPath, &fi.Status, &fi.CreateAt) + err := rows.Scan(&fi.FileID, &fi.InboxPath, &accessionID, &fi.Status, &fi.CreateAt) if err != nil { return nil, err } + fi.AccessionID = accessionID.String // Add instance of struct (file) to array if the status is not disabled if fi.Status != "disabled" { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 2c744d724e..97c5692467 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -456,6 +456,8 @@ func (suite *DatabaseTests) TestGetUserFiles() { assert.NoError(suite.T(), err, "failed to register file in database") err = db.UpdateFileEventLog(fileID, "uploaded", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") + err = db.SetAccessionID(fmt.Sprintf("stableID-00%d", i), fileID) + assert.NoError(suite.T(), err, "failed to update satus of file in database") err = db.UpdateFileEventLog(fileID, "ready", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") } @@ -469,6 +471,7 @@ func (suite *DatabaseTests) TestGetUserFiles() { for _, fileInfo := range filelist { assert.Equal(suite.T(), "ready", fileInfo.Status, "incorrect file status") + assert.Contains(suite.T(), fileInfo.AccessionID, "stableID-00", "incorrect file accession ID") } } From 074542db2a6d85f63dd74e5cd8c8243d5576e40b Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Mar 2025 13:56:51 +0100 Subject: [PATCH 127/250] [internal/database] Add bool to selectively return the accession ID --- sda/internal/database/db_functions.go | 9 ++++++--- sda/internal/database/db_functions_test.go | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 85ed597f3b..0ce9b056bb 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -690,14 +690,14 @@ func (dbs *SDAdb) getArchivePath(stableID string) (string, error) { } // GetUserFiles retrieves all the files a user submitted -func (dbs *SDAdb) GetUserFiles(userID string) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) GetUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { var err error files := []*SubmissionFileInfo{} // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { - files, err = dbs.getUserFiles(userID) + files, err = dbs.getUserFiles(userID, allData) if err == nil { break } @@ -708,7 +708,7 @@ func (dbs *SDAdb) GetUserFiles(userID string) ([]*SubmissionFileInfo, error) { } // getUserFiles is the actual function performing work for GetUserFiles -func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) getUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { dbs.checkAndReconnectIfNeeded() files := []*SubmissionFileInfo{} @@ -735,7 +735,10 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { if err != nil { return nil, err } + + if allData { fi.AccessionID = accessionID.String + } // Add instance of struct (file) to array if the status is not disabled if fi.Status != "disabled" { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 97c5692467..4dc2c6c0a2 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -461,11 +461,11 @@ func (suite *DatabaseTests) TestGetUserFiles() { err = db.UpdateFileEventLog(fileID, "ready", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") } - filelist, err := db.GetUserFiles("unknownuser") + filelist, err := db.GetUserFiles("unknownuser", true) assert.NoError(suite.T(), err, "failed to get (empty) file list of unknown user") assert.Empty(suite.T(), filelist, "file list of unknown user is not empty") - filelist, err = db.GetUserFiles(testUser) + filelist, err = db.GetUserFiles(testUser, true) assert.NoError(suite.T(), err, "failed to get file list") assert.Equal(suite.T(), testCases, len(filelist), "file list is of incorrect length") From 31d721e4904e60252d46317eab7a8690c1e49880 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Mar 2025 13:58:49 +0100 Subject: [PATCH 128/250] [API] don't return the accession ID of files to a submitter --- sda/cmd/api/api.go | 4 ++-- sda/cmd/api/api_test.go | 13 ++++++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 338b9cf007..7ed8ded538 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -232,7 +232,7 @@ func getFiles(c *gin.Context) { return } - files, err := Conf.API.DB.GetUserFiles(token.Subject()) + files, err := Conf.API.DB.GetUserFiles(token.Subject(), false) if err != nil { // something went wrong with querying or parsing rows c.JSON(502, err.Error()) @@ -544,7 +544,7 @@ func listUserFiles(c *gin.Context) { username = strings.TrimPrefix(username, "/") username = strings.TrimSuffix(username, "/files") log.Debugln(username) - files, err := Conf.API.DB.GetUserFiles(username) + files, err := Conf.API.DB.GetUserFiles(username, true) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 946d019d22..2e67f6b27f 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -504,6 +504,10 @@ func (suite *TestSuite) TestAPIGetFiles() { assert.NoError(suite.T(), err) // Update the file's status and make sure only the lastest status is listed + err = Conf.API.DB.SetAccessionID("stableID", fileID) + if err != nil { + suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) + } latestStatus = "ready" err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, suite.User, "{}", "{}") assert.NoError(suite.T(), err, "got (%v) when trying to update file status") @@ -513,13 +517,15 @@ func (suite *TestSuite) TestAPIGetFiles() { assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + assert.NoError(suite.T(), err) + assert.NotContains(suite.T(), string(data), "accessionID") - err = json.NewDecoder(resp.Body).Decode(&filesData) + err = json.Unmarshal(data, &filesData) assert.NoError(suite.T(), err) assert.Equal(suite.T(), len(filesData), 1) assert.Equal(suite.T(), filesData[0].Status, latestStatus) - - assert.NoError(suite.T(), err) + assert.Empty(suite.T(), filesData[0].AccessionID) // Insert a second file and make sure it is listed file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", suite.User) @@ -1548,6 +1554,7 @@ func (suite *TestSuite) TestListUserFiles() { err = json.NewDecoder(okResponse.Body).Decode(&files) assert.NoError(suite.T(), err, "failed to list users from DB") assert.Equal(suite.T(), 2, len(files)) + assert.Contains(suite.T(), files[0].AccessionID, "accession_user_example.org_0") } func (suite *TestSuite) TestAddC4ghHash() { From 4b224c540741c958cfe18da70e024409fa32c740 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 21 Mar 2025 13:49:14 +0100 Subject: [PATCH 129/250] Apply suggestions from code review --- sda/cmd/api/swagger_v1.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index 9765ef7ffe..f0efc433de 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -6,7 +6,7 @@ info: paths: /c4gh-keys/add: post: - description: Registers an crypt4gh public key hash in the database + description: Registers an crypt4gh public key in the database requestBody: content: application/json: From a002f9c422be69c014318cd8e2d59191ff751db0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 21 Mar 2025 22:27:28 +0000 Subject: [PATCH 130/250] Bump github.com/golang-jwt/jwt/v5 in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [github.com/golang-jwt/jwt/v5](https://github.com/golang-jwt/jwt). Updates `github.com/golang-jwt/jwt/v5` from 5.2.1 to 5.2.2 - [Release notes](https://github.com/golang-jwt/jwt/releases) - [Changelog](https://github.com/golang-jwt/jwt/blob/main/VERSION_HISTORY.md) - [Commits](https://github.com/golang-jwt/jwt/compare/v5.2.1...v5.2.2) --- updated-dependencies: - dependency-name: github.com/golang-jwt/jwt/v5 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index f6c49e9f4c..6e12899b25 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -92,7 +92,7 @@ require ( github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect diff --git a/sda/go.sum b/sda/go.sum index 9cba17c71c..3810568de9 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -148,8 +148,8 @@ github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= From 94ec330a00c06836fad5b221cb69650dedd3e655 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 20:50:12 +0000 Subject: [PATCH 131/250] Bump golangci/golangci-lint-action from 6.5.1 to 7.0.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 6.5.1 to 7.0.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v6.5.1...v7.0.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index b8e169d8cd..4387291d4f 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.1 + uses: golangci/golangci-lint-action@v7.0.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-download @@ -48,7 +48,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.1 + uses: golangci/golangci-lint-action@v7.0.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.1 + uses: golangci/golangci-lint-action@v7.0.0 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m working-directory: sda-admin From d9dd809cfff1eb95e8ad6e2353d7d6bf77d47df1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 20:57:39 +0000 Subject: [PATCH 132/250] Bump the all-modules group in /sda with 5 updates Bumps the all-modules group in /sda with 5 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.9` | `1.29.10` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.62` | `1.17.63` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.66` | `1.17.67` | | [github.com/casbin/casbin/v2](https://github.com/casbin/casbin) | `2.103.0` | `2.104.0` | | google.golang.org/protobuf | `1.36.5` | `1.36.6` | Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.9 to 1.29.10 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.9...config/v1.29.10) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.62 to 1.17.63 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.62...credentials/v1.17.63) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.66 to 1.17.67 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.66...feature/s3/manager/v1.17.67) Updates `github.com/casbin/casbin/v2` from 2.103.0 to 2.104.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.103.0...v2.104.0) Updates `google.golang.org/protobuf` from 1.36.5 to 1.36.6 --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/casbin/casbin/v2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 12 ++++++------ sda/go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 6e12899b25..2ddebed478 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -4,12 +4,12 @@ go 1.24.0 require ( github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.9 - github.com/aws/aws-sdk-go-v2/credentials v1.17.62 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66 + github.com/aws/aws-sdk-go-v2/config v1.29.10 + github.com/aws/aws-sdk-go-v2/credentials v1.17.63 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.67 github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2 github.com/aws/smithy-go v1.22.3 - github.com/casbin/casbin/v2 v2.103.0 + github.com/casbin/casbin/v2 v2.104.0 github.com/coreos/go-oidc/v3 v3.13.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 @@ -34,7 +34,7 @@ require ( golang.org/x/crypto v0.36.0 golang.org/x/oauth2 v0.28.0 google.golang.org/grpc v1.71.0 - google.golang.org/protobuf v1.36.5 + google.golang.org/protobuf v1.36.6 ) require ( @@ -60,7 +60,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.2 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index 3810568de9..33dbadb6c6 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -28,14 +28,14 @@ github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38y github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.9 h1:Kg+fAYNaJeGXp1vmjtidss8O2uXIsXwaRqsQJKXVr+0= -github.com/aws/aws-sdk-go-v2/config v1.29.9/go.mod h1:oU3jj2O53kgOU4TXq/yipt6ryiooYjlkqqVaZk7gY/U= -github.com/aws/aws-sdk-go-v2/credentials v1.17.62 h1:fvtQY3zFzYJ9CfixuAQ96IxDrBajbBWGqjNTCa79ocU= -github.com/aws/aws-sdk-go-v2/credentials v1.17.62/go.mod h1:ElETBxIQqcxej++Cs8GyPBbgMys5DgQPTwo7cUPDKt8= +github.com/aws/aws-sdk-go-v2/config v1.29.10 h1:yNjgjiGBp4GgaJrGythyBXg2wAs+Im9fSWIUwvi1CAc= +github.com/aws/aws-sdk-go-v2/config v1.29.10/go.mod h1:A0mbLXSdtob/2t59n1X0iMkPQ5d+YzYZB4rwu7SZ7aA= +github.com/aws/aws-sdk-go-v2/credentials v1.17.63 h1:rv1V3kIJ14pdmTu01hwcMJ0WAERensSiD9rEWEBb1Tk= +github.com/aws/aws-sdk-go-v2/credentials v1.17.63/go.mod h1:EJj+yDf0txT26Ulo0VWTavBl31hOsaeuMxIHu2m0suY= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66 h1:MTLivtC3s89de7Fe3P8rzML/8XPNRfuyJhlRTsCEt0k= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.66/go.mod h1:NAuQ2s6gaFEsuTIb2+P5t6amB1w5MhvJFxppoezGWH0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.67 h1:V5KBNdfgTNFd8aLQDXKgHtDbiX5Z0AbH6HibzDx2CWU= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.67/go.mod h1:yut3GOtsk0hs3wnkOnpSmy+l+TxGC86/faMixuNiQLA= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -56,8 +56,8 @@ github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2 h1:jIiopHEV22b4yQP2q36Y0OmwLbsxN github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 h1:8JdC7Gr9NROg1Rusk25IcZeTO59zLxsKgE0gkh5O6h0= github.com/aws/aws-sdk-go-v2/service/sso v1.25.1/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1 h1:KwuLovgQPcdjNMfFt9OhUd9a2OwcOKhxfvF4glTzLuA= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.2 h1:wK8O+j2dOolmpNVY1EWIbLgxrGCHJKVPm08Hv/u80M8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.2/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc= github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= @@ -71,8 +71,8 @@ github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKz github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/casbin/casbin/v2 v2.103.0 h1:dHElatNXNrr8XcseUov0ZSiWjauwmZZE6YMV3eU1yic= -github.com/casbin/casbin/v2 v2.103.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.104.0 h1:qDakyBZ4jUg1VskF1+UzIwkg+uXWcp0u0M9PMm1RsTA= +github.com/casbin/casbin/v2 v2.104.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -513,8 +513,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From 4501c021fc3402ae26a79c4025b0daa80a311b5a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 21:00:05 +0000 Subject: [PATCH 133/250] Bump google.golang.org/protobuf Bumps the all-modules group in /sda-download with 1 update: google.golang.org/protobuf. Updates `google.golang.org/protobuf` from 1.36.5 to 1.36.6 --- updated-dependencies: - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index d9146d2f46..c1d9955067 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -18,7 +18,7 @@ require ( golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 google.golang.org/grpc v1.71.0 - google.golang.org/protobuf v1.36.5 + google.golang.org/protobuf v1.36.6 ) require ( diff --git a/sda-download/go.sum b/sda-download/go.sum index 1a27f371df..a2327baff4 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -235,8 +235,8 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From f754aa66296030936061d9e1da04687d1f84531d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 24 Mar 2025 21:20:40 +0000 Subject: [PATCH 134/250] Bump the all-modules group in /sda-doa with 3 updates Bumps the all-modules group in /sda-doa with 3 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot), [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.3 to 3.4.4 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.3...v3.4.4) Updates `no.elixir:crypt4gh` from 3.0.17 to 3.0.19 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.17...crypt4gh-3.0.19) Updates `no.elixir:clearinghouse` from 2.0.15 to 2.0.16 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-2.0.15...clearinghouse-2.0.16) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:crypt4gh dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 3aa279d379..c99d7795e1 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.3 + 3.4.4 no.uio.ifi @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.17 + 3.0.19 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.15 + 2.0.16 org.slf4j From 66c896f88ccca2fcfa0e87388f031199f892163b Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 17 Mar 2025 13:49:18 +0100 Subject: [PATCH 135/250] Allow delay in GetFileSize from storage leaving time for s3 to complete writing, when file is just uploaded --- sda/cmd/finalize/finalize.go | 2 +- sda/cmd/ingest/ingest.go | 4 ++-- sda/cmd/sync/sync.go | 2 +- sda/cmd/verify/verify.go | 2 +- sda/internal/storage/storage.go | 11 ++++++----- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 3a2bd33cd0..48fd1a4f9b 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -244,7 +244,7 @@ func backupFile(delivered amqp.Delivery) error { } // Get size on disk, will also give some time for the file to appear if it has not already - diskFileSize, err := archive.GetFileSize(filePath) + diskFileSize, err := archive.GetFileSize(filePath, false) if err != nil { return fmt.Errorf("failed to get size info for archived file, reason: %v", err) } diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index cfd4bd62ba..d64ec55ce5 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -311,7 +311,7 @@ func main() { continue } - fileSize, err := inbox.GetFileSize(message.FilePath) + fileSize, err := inbox.GetFileSize(message.FilePath, false) if err != nil { log.Errorf("Failed to get file size of file to ingest, reason: (%s)", err.Error()) // Nack message so the server gets notified that something is wrong and requeue the message. @@ -496,7 +496,7 @@ func main() { fileInfo := database.FileInfo{} fileInfo.Path = fileID fileInfo.Checksum = fmt.Sprintf("%x", hash.Sum(nil)) - fileInfo.Size, err = archive.GetFileSize(fileID) + fileInfo.Size, err = archive.GetFileSize(fileID, true) if err != nil { log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) if err = delivered.Nack(false, true); err != nil { diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 4f120a0184..618b7944ee 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -172,7 +172,7 @@ func syncFiles(stableID string) error { return fmt.Errorf("failed to get archive path for file with stable ID: %s", stableID) } - fileSize, err := archive.GetFileSize(archivePath) + fileSize, err := archive.GetFileSize(archivePath, false) if err != nil { return err } diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index e8a723d420..9651eea3b5 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -157,7 +157,7 @@ func main() { } var file database.FileInfo - file.Size, err = archive.GetFileSize(message.ArchivePath) + file.Size, err = archive.GetFileSize(message.ArchivePath, false) if err != nil { //nolint:nestif log.Errorf("Failed to get archived file size, reson: (%s)", err.Error()) if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index d3edcb10fe..fc39d16885 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -31,7 +31,7 @@ import ( // Backend defines methods to be implemented by PosixBackend, S3Backend and sftpBackend type Backend interface { - GetFileSize(filePath string) (int64, error) + GetFileSize(filePath string, expectDelay bool) (int64, error) RemoveFile(filePath string) error NewFileReader(filePath string) (io.ReadCloser, error) NewFileWriter(filePath string) (io.WriteCloser, error) @@ -114,7 +114,7 @@ func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { } // GetFileSize returns the size of the file -func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { +func (pb *posixBackend) GetFileSize(filePath string, _ bool) (int64, error) { if pb == nil { return 0, fmt.Errorf("invalid posixBackend") } @@ -289,7 +289,7 @@ func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { } // GetFileSize returns the size of a specific object -func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { +func (sb *s3Backend) GetFileSize(filePath string, expectDelay bool) (int64, error) { r, err := sb.Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ Bucket: &sb.Bucket, Key: &filePath, @@ -305,10 +305,11 @@ func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { // Retry on error up to five minutes to allow for // "slow writes' or s3 eventual consistency for err != nil && time.Since(start) < retryTime { - if strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { + if !expectDelay && (strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:")) { return 0, err } time.Sleep(1 * time.Second) + r, err = sb.Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ Bucket: &sb.Bucket, Key: &filePath, @@ -449,7 +450,7 @@ func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { } // GetFileSize returns the size of the file -func (sfb *sftpBackend) GetFileSize(filePath string) (int64, error) { +func (sfb *sftpBackend) GetFileSize(filePath string, _ bool) (int64, error) { if sfb == nil { return 0, fmt.Errorf("invalid sftpBackend") } From 2c0b57faab4ff8558f8b36bbdbac04cc80b8d62a Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 17 Mar 2025 13:54:37 +0100 Subject: [PATCH 136/250] [storage] tests: add expectedDelay in storage tests --- sda/internal/storage/storage_test.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index d07d5e905f..5aaff86120 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -256,7 +256,7 @@ func (suite *StorageTestSuite) TestPosixBackend() { assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") assert.Nil(suite.T(), err, "unexpected error when reading back data") - size, err := backend.GetFileSize("testFile") + size, err := backend.GetFileSize("testFile", false) assert.Nil(suite.T(), err, "posix NewFileReader failed when it should work") assert.NotNil(suite.T(), size, "Got a nil size for posix") @@ -270,7 +270,7 @@ func (suite *StorageTestSuite) TestPosixBackend() { assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") buf.Reset() - _, err = backend.GetFileSize("posixDoesNotExist") + _, err = backend.GetFileSize("posixDoesNotExist", false) assert.Error(suite.T(), err, "posix GetFileSize worked when it should not") assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") @@ -296,7 +296,7 @@ func (suite *StorageTestSuite) TestPosixBackend() { assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") - _, err = dummyBackend.GetFileSize("/") + _, err = dummyBackend.GetFileSize("/", false) assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") err = dummyBackend.RemoveFile("/") @@ -323,11 +323,17 @@ func (suite *StorageTestSuite) TestS3Backend() { assert.NoError(suite.T(), err, "s3 NewFileReader failed when it should work") assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") - size, err := s3back.GetFileSize("s3Creatable") + size, err := s3back.GetFileSize("s3Creatable", false) assert.Nil(suite.T(), err, "s3 GetFileSize failed when it should work") assert.NotNil(suite.T(), size, "Got a nil size for s3") assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") + // make sure expectDelay=true works + size, err = s3back.GetFileSize("s3Creatable", true) + assert.Nil(suite.T(), err, "s3 GetFileSize with expected delay failed when it should work") + assert.NotNil(suite.T(), size, "Got a nil size for s3") + assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") + var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") @@ -339,7 +345,7 @@ func (suite *StorageTestSuite) TestS3Backend() { err = s3back.RemoveFile("s3Creatable") assert.Nil(suite.T(), err, "s3 RemoveFile failed when it should work") - _, err = s3back.GetFileSize("s3DoesNotExist") + _, err = s3back.GetFileSize("s3DoesNotExist", false) assert.NotNil(suite.T(), err, "s3 GetFileSize worked when it should not") assert.Error(suite.T(), err) @@ -377,7 +383,7 @@ func (suite *StorageTestSuite) TestSftpBackend() { assert.Nil(suite.T(), err, "sftp NewFileReader failed when it should work") assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") - size, err := sftpBack.GetFileSize(sftpCreatable) + size, err := sftpBack.GetFileSize(sftpCreatable, false) assert.Nil(suite.T(), err, "sftp GetFileSize failed when it should work") assert.NotNil(suite.T(), size, "Got a nil size for sftp") assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") @@ -398,7 +404,7 @@ func (suite *StorageTestSuite) TestSftpBackend() { assert.Nil(suite.T(), err, "unexpected error when reading back data") } - _, err = sftpBack.GetFileSize(sftpDoesNotExist) + _, err = sftpBack.GetFileSize(sftpDoesNotExist, false) assert.EqualError(suite.T(), err, "failed to get file size with sftp, file does not exist") reader, err = sftpBack.NewFileReader(sftpDoesNotExist) assert.EqualError(suite.T(), err, "failed to open file with sftp, file does not exist") @@ -442,7 +448,7 @@ func (suite *StorageTestSuite) TestSftpBackend() { assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") - _, err = dummyBackend.GetFileSize("/") + _, err = dummyBackend.GetFileSize("/", false) assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") err = dummyBackend.RemoveFile("/") From bf5f2b9a6669baa5d66d7437df57c8a0f3a01525 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 21 Mar 2025 10:14:59 +0100 Subject: [PATCH 137/250] [storage] tests: getfilesize waits for s3 to write --- sda/internal/storage/storage_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index 5aaff86120..78ff9085e8 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -329,6 +329,19 @@ func (suite *StorageTestSuite) TestS3Backend() { assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") // make sure expectDelay=true works + // delete file and make sure the file size can not be retrieved anymore + err = s3back.RemoveFile("s3Creatable") + assert.Nil(suite.T(), err, "s3 RemoveFile failed when it should work") + _, err = s3back.GetFileSize("s3Creatable", true) + assert.NotNil(suite.T(), err, "s3 GetFileSize worked when it should not") + assert.Error(suite.T(), err) + // rewrite file, do not wait before retrieving file size + writer, err = s3back.NewFileWriter("s3Creatable") + assert.Nil(suite.T(), err, "s3 NewFileWriter failed when it shouldn't") + written, err = writer.Write(writeData) + assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") + assert.Nil(suite.T(), err, "Failure when writing to s3 writer") + writer.Close() size, err = s3back.GetFileSize("s3Creatable", true) assert.Nil(suite.T(), err, "s3 GetFileSize with expected delay failed when it should work") assert.NotNil(suite.T(), size, "Got a nil size for s3") From 587f443190c6958c81dc5dc7ce10b252166145a9 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Wed, 26 Mar 2025 09:29:43 +0000 Subject: [PATCH 138/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 4356103e17..a3c81180ab 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.80 -appVersion: v0.3.214 +version: 0.8.81 +appVersion: v0.3.223 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 9a59323680..b04f94b984 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.81 -appVersion: v0.3.214 +version: 0.7.82 +appVersion: v0.3.223 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 4788fbe579..e7fbf0a37c 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.10 -appVersion: v0.3.214 +version: 0.30.11 +appVersion: v0.3.223 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 97bbe0553a518e9a29e7de7728658f9484953aa8 Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 27 Mar 2025 12:56:54 +0100 Subject: [PATCH 139/250] add junit bom for dependency management to sftp pom file --- sda-sftp-inbox/pom.xml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index dd64f971ea..b462490db6 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -24,6 +24,18 @@ 19 + + + + org.junit + junit-bom + 5.12.1 + pom + import + + + + net.i2p.crypto @@ -160,10 +172,8 @@ org.junit.vintage junit-vintage-engine - 5.11.4 test - From 01f58531d12110a8ca07cb398e81ece7b926efaf Mon Sep 17 00:00:00 2001 From: Parisa Date: Thu, 27 Mar 2025 13:35:49 +0100 Subject: [PATCH 140/250] add tag context to tag action to release new tags based on the branch not repo(default behavior) --- .github/workflows/publish_container.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index 4531de6bc2..80432bf452 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -34,6 +34,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true DEFAULT_BUMP: patch + TAG_CONTEXT: branch push_to_registry: needs: tag_release From f23889887339e28bafc6fcf54265f0587aaa57f9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Mar 2025 13:10:58 +0000 Subject: [PATCH 141/250] Bump the all-modules group across 1 directory with 4 updates Bumps the all-modules group with 4 updates in the /sda-sftp-inbox directory: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot), [org.apache.sshd:sshd-sftp](https://github.com/apache/mina-sshd), [com.amazonaws:aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) and [com.google.guava:guava](https://github.com/google/guava). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.2 to 3.4.4 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.2...v3.4.4) Updates `org.apache.sshd:sshd-sftp` from 2.14.0 to 2.15.0 - [Release notes](https://github.com/apache/mina-sshd/releases) - [Changelog](https://github.com/apache/mina-sshd/blob/master/CHANGES.md) - [Commits](https://github.com/apache/mina-sshd/compare/sshd-2.14.0...sshd-2.15.0) Updates `com.amazonaws:aws-java-sdk-s3` from 1.12.781 to 1.12.782 - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.781...1.12.782) Updates `com.google.guava:guava` from 33.4.0-jre to 33.4.6-jre - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: org.apache.sshd:sshd-sftp dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index b462490db6..f229e3a9f2 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.2 + 3.4.4 @@ -98,7 +98,7 @@ org.apache.sshd sshd-sftp - 2.14.0 + 2.15.0 commons-io @@ -108,12 +108,12 @@ com.amazonaws aws-java-sdk-s3 - 1.12.781 + 1.12.782 com.google.guava guava - 33.4.0-jre + 33.4.6-jre net.logstash.logback From 08653d834169b4976bb23ecbf04c88b2ca8e9b20 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 20 Mar 2025 15:08:27 +0100 Subject: [PATCH 142/250] [internal/database] add test step to append files to a dataset. --- sda/internal/database/db_functions_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 4dc2c6c0a2..c50c271c81 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -286,6 +286,15 @@ func (suite *DatabaseTests) TestMapFilesToDataset() { err := db.MapFilesToDataset(di, acs) assert.NoError(suite.T(), err, "failed to map file to dataset") } + + // Append files to an existing dataset + assert.NoError(suite.T(), db.MapFilesToDataset("dataset1", accessions[9:11]), "failed to append files to a dataset") + var dsMembers int + const q1 = "SELECT count(file_id) from sda.file_dataset WHERE dataset_id = (SELECT id from sda.datasets WHERE stable_id = $1);" + if err := db.DB.QueryRow(q1, "dataset1").Scan(&dsMembers); err != nil { + suite.FailNow("failed to get dataset members from database") + } + assert.Equal(suite.T(), 5, dsMembers) } func (suite *DatabaseTests) TestGetInboxPath() { From cebcd3aed3aa656077f6688cfb92d0158ef87a5b Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 26 Mar 2025 11:42:40 +0100 Subject: [PATCH 143/250] Update API RBAC file --- .github/integration/sda/rbac.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 4995a3449c..7d37530658 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -12,8 +12,8 @@ }, { "role": "admin", - "path": "/dataset/verify/:dataset", - "action": "PUT" + "path": "/dataset/*", + "action": "(POST)|(PUT)" }, { "role": "submission", From 47bd1a2c75c00cb5416b764aa9fde98f07424800 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 26 Mar 2025 11:43:56 +0100 Subject: [PATCH 144/250] Revert golangci-lint to version 1.64.8 --- .github/workflows/code-linter.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 4387291d4f..67fcf88850 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -27,9 +27,10 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v7.0.0 + uses: golangci/golangci-lint-action@v6.5.2 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107,G115 --timeout 5m + version: v1.64.8 working-directory: sda-download lint_sda: @@ -48,9 +49,10 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v7.0.0 + uses: golangci/golangci-lint-action@v6.5.2 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m + version: v1.64.8 working-directory: sda lint_sda_admin: @@ -69,7 +71,8 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v7.0.0 + uses: golangci/golangci-lint-action@v6.5.2 with: args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m + version: v1.64.8 working-directory: sda-admin From 1aca0625f5ee45c4259f48adf8127a4e305e8b6a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Mar 2025 19:12:51 +0000 Subject: [PATCH 145/250] Bump github.com/spf13/viper in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [github.com/spf13/viper](https://github.com/spf13/viper). Updates `github.com/spf13/viper` from 1.20.0 to 1.20.1 - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.20.0...v1.20.1) --- updated-dependencies: - dependency-name: github.com/spf13/viper dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 4 +++- sda-download/go.sum | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index c1d9955067..440b55cf64 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -2,6 +2,8 @@ module github.com/neicnordic/sda-download go 1.24.0 +toolchain go1.24.1 + require ( github.com/DATA-DOG/go-sqlmock v1.5.2 github.com/aws/aws-sdk-go v1.55.6 @@ -13,7 +15,7 @@ require ( github.com/lib/pq v1.10.9 github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.0 + github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 diff --git a/sda-download/go.sum b/sda-download/go.sum index a2327baff4..0cb7ecaa6f 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -136,8 +136,8 @@ github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= -github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= From fe167c7f8f69622ed46d638664155f34c5ccad1b Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Tue, 1 Apr 2025 10:09:38 +0200 Subject: [PATCH 146/250] make check robust to library changes --- sda/internal/storage/storage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index 78ff9085e8..a4c87402a3 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -405,7 +405,7 @@ func (suite *StorageTestSuite) TestSftpBackend() { assert.Nil(suite.T(), err, "sftp RemoveFile failed when it should work") err = sftpBack.RemoveFile(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "failed to remove file with sftp, file does not exist") + assert.ErrorContains(suite.T(), err, "file does not exist") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) From dbd963466ccd76e2488019c944e1a5a93ec0c6b8 Mon Sep 17 00:00:00 2001 From: Parisa Date: Wed, 2 Apr 2025 14:55:19 +0200 Subject: [PATCH 147/250] add update section for github actions on release_v1 --- .github/dependabot.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index bc297600fb..cd2f66ea0f 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -111,6 +111,16 @@ updates: - "neicnordic/sensitive-data-development-collaboration" ## release v1 branch + - package-ecosystem: "github-actions" + target-branch: release_v1 + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 10 + reviewers: + - "neicnordic/sensitive-data-development-collaboration" + commit-message: + prefix: "[release_v1]" ### Docker - package-ecosystem: docker target-branch: release_v1 From 802a0ab807df78019ace9821b703f18e51fb3468 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:48:05 +0000 Subject: [PATCH 148/250] Bump the all-modules group in /sda-sftp-inbox with 2 updates Bumps the all-modules group in /sda-sftp-inbox with 2 updates: [org.apache.maven.plugins:maven-surefire-plugin](https://github.com/apache/maven-surefire) and [org.apache.maven.plugins:maven-failsafe-plugin](https://github.com/apache/maven-surefire). Updates `org.apache.maven.plugins:maven-surefire-plugin` from 3.5.2 to 3.5.3 - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-3.5.2...surefire-3.5.3) Updates `org.apache.maven.plugins:maven-failsafe-plugin` from 3.5.2 to 3.5.3 - [Release notes](https://github.com/apache/maven-surefire/releases) - [Commits](https://github.com/apache/maven-surefire/compare/surefire-3.5.2...surefire-3.5.3) --- updated-dependencies: - dependency-name: org.apache.maven.plugins:maven-surefire-plugin dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: org.apache.maven.plugins:maven-failsafe-plugin dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index f229e3a9f2..9c50b6779f 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -184,12 +184,12 @@ maven-surefire-plugin - 3.5.2 + 3.5.3 org.apache.maven.plugins maven-failsafe-plugin - 3.5.2 + 3.5.3 From 8133036f57b1e50f5af2d844468720fecf86f61e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:48:05 +0000 Subject: [PATCH 149/250] Bump google.golang.org/grpc in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `google.golang.org/grpc` from 1.71.0 to 1.71.1 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.71.0...v1.71.1) --- updated-dependencies: - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 440b55cf64..1c98aba7ac 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -19,7 +19,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.36.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - google.golang.org/grpc v1.71.0 + google.golang.org/grpc v1.71.1 google.golang.org/protobuf v1.36.6 ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 0cb7ecaa6f..4614234dbf 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -233,8 +233,8 @@ golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= -google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= -google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 3024247666e0d3eb02c7bac6a2f15eff5796c1b1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:52:20 +0000 Subject: [PATCH 150/250] Bump the all-modules group across 1 directory with 8 updates Bumps the all-modules group with 6 updates in the /sda directory: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.10` | `1.29.12` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.67` | `1.17.70` | | [github.com/ory/dockertest/v3](https://github.com/ory/dockertest) | `3.11.0` | `3.12.0` | | [github.com/pkg/sftp](https://github.com/pkg/sftp) | `1.13.8` | `1.13.9` | | [github.com/spf13/viper](https://github.com/spf13/viper) | `1.20.0` | `1.20.1` | | [google.golang.org/grpc](https://github.com/grpc/grpc-go) | `1.71.0` | `1.71.1` | Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.10 to 1.29.12 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.10...config/v1.29.12) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.63 to 1.17.65 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.63...credentials/v1.17.65) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.67 to 1.17.70 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.67...feature/s3/manager/v1.17.70) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.78.2 to 1.79.0 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.78.2...service/s3/v1.79.0) Updates `github.com/ory/dockertest/v3` from 3.11.0 to 3.12.0 - [Release notes](https://github.com/ory/dockertest/releases) - [Commits](https://github.com/ory/dockertest/compare/v3.11.0...v3.12.0) Updates `github.com/pkg/sftp` from 1.13.8 to 1.13.9 - [Release notes](https://github.com/pkg/sftp/releases) - [Commits](https://github.com/pkg/sftp/compare/v1.13.8...v1.13.9) Updates `github.com/spf13/viper` from 1.20.0 to 1.20.1 - [Release notes](https://github.com/spf13/viper/releases) - [Commits](https://github.com/spf13/viper/compare/v1.20.0...v1.20.1) Updates `google.golang.org/grpc` from 1.71.0 to 1.71.1 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.71.0...v1.71.1) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/ory/dockertest/v3 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/pkg/sftp dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/spf13/viper dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 29 ++++++++++++++++------------- sda/go.sum | 54 ++++++++++++++++++++++++++++-------------------------- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 2ddebed478..97a847196a 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -2,12 +2,14 @@ module github.com/neicnordic/sensitive-data-archive go 1.24.0 +toolchain go1.24.1 + require ( github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.10 - github.com/aws/aws-sdk-go-v2/credentials v1.17.63 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.67 - github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2 + github.com/aws/aws-sdk-go-v2/config v1.29.12 + github.com/aws/aws-sdk-go-v2/credentials v1.17.65 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.70 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.104.0 github.com/coreos/go-oidc/v3 v3.13.0 @@ -23,17 +25,17 @@ require ( github.com/neicnordic/crypt4gh v1.14.0 github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 github.com/ory/dockertest v3.3.5+incompatible - github.com/ory/dockertest/v3 v3.11.0 + github.com/ory/dockertest/v3 v3.12.0 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.8 + github.com/pkg/sftp v1.13.9 github.com/rabbitmq/amqp091-go v1.10.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.0 + github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.36.0 golang.org/x/oauth2 v0.28.0 - google.golang.org/grpc v1.71.0 + google.golang.org/grpc v1.71.1 google.golang.org/protobuf v1.36.6 ) @@ -59,8 +61,8 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect @@ -71,11 +73,11 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/containerd/continuity v0.4.3 // indirect + github.com/containerd/continuity v0.4.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect - github.com/docker/cli v27.2.1+incompatible // indirect + github.com/docker/cli v27.4.1+incompatible // indirect github.com/docker/docker v27.2.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -121,12 +123,13 @@ require ( github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/user v0.3.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.1.14 // indirect + github.com/opencontainers/runc v1.2.3 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 33dbadb6c6..c2799dd01e 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -28,14 +28,14 @@ github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38y github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.10 h1:yNjgjiGBp4GgaJrGythyBXg2wAs+Im9fSWIUwvi1CAc= -github.com/aws/aws-sdk-go-v2/config v1.29.10/go.mod h1:A0mbLXSdtob/2t59n1X0iMkPQ5d+YzYZB4rwu7SZ7aA= -github.com/aws/aws-sdk-go-v2/credentials v1.17.63 h1:rv1V3kIJ14pdmTu01hwcMJ0WAERensSiD9rEWEBb1Tk= -github.com/aws/aws-sdk-go-v2/credentials v1.17.63/go.mod h1:EJj+yDf0txT26Ulo0VWTavBl31hOsaeuMxIHu2m0suY= +github.com/aws/aws-sdk-go-v2/config v1.29.12 h1:Y/2a+jLPrPbHpFkpAAYkVEtJmxORlXoo5k2g1fa2sUo= +github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI= +github.com/aws/aws-sdk-go-v2/credentials v1.17.65 h1:q+nV2yYegofO/SUXruT+pn4KxkxmaQ++1B/QedcKBFM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.67 h1:V5KBNdfgTNFd8aLQDXKgHtDbiX5Z0AbH6HibzDx2CWU= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.67/go.mod h1:yut3GOtsk0hs3wnkOnpSmy+l+TxGC86/faMixuNiQLA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.70 h1:pgaM86/BFt7dR0b/Jj+OU+taT34nkQlKPkjkYH1POAo= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.70/go.mod h1:vnoXXAU4FFW5JqLC/ZPF67IA5N0f8gah0t0aGI+N9+4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -52,12 +52,12 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2 h1:jIiopHEV22b4yQP2q36Y0OmwLbsxNWdWwfZRR5QRRO4= -github.com/aws/aws-sdk-go-v2/service/s3 v1.78.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.1 h1:8JdC7Gr9NROg1Rusk25IcZeTO59zLxsKgE0gkh5O6h0= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.1/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.2 h1:wK8O+j2dOolmpNVY1EWIbLgxrGCHJKVPm08Hv/u80M8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.29.2/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0 h1:OIw2nryEApESTYI5deCZGcq4Gvz8DBAt4tJlNyg3v5o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 h1:pdgODsAhGo4dvzC3JAG5Ce0PX8kWXrTZGx+jxADD+5E= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 h1:90uX0veLKcdHVfvxhkWUQSCi5VabtwMLFutYiRke4oo= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc= github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= @@ -83,8 +83,8 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= -github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/coreos/go-oidc/v3 v3.13.0 h1:M66zd0pcc5VxvBNM4pB331Wrsanby+QomQYjN8HamW8= github.com/coreos/go-oidc/v3 v3.13.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= @@ -97,8 +97,8 @@ github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFn github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= -github.com/docker/cli v27.2.1+incompatible h1:U5BPtiD0viUzjGAjV1p0MGB8eVA3L3cbIrnyWmSJI70= -github.com/docker/cli v27.2.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= +github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= github.com/docker/docker v27.2.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -259,6 +259,8 @@ github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQ github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/mocktools/go-smtp-mock v1.10.0 h1:glrRmjNqASyy+jf1IJ2nCWgEbJScD3Amf2IGcXgdEVg= @@ -278,18 +280,18 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= -github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= +github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= +github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= -github.com/ory/dockertest/v3 v3.11.0 h1:OiHcxKAvSDUwsEVh2BjxQQc/5EHz9n0va9awCtNGuyA= -github.com/ory/dockertest/v3 v3.11.0/go.mod h1:VIPxS1gwT9NpPOrfD3rACs8Y9Z7yhzO4SB194iUDnUI= +github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= +github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.8 h1:Xt7eJ/xqXv7s0VuzFw7JXhZj6Oc1zI6l4GK8KP9sFB0= -github.com/pkg/sftp v1.13.8/go.mod h1:DmvEkvKE2lshEeuo2JMp06yqcx9HVnR7e3zqQl42F3U= +github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= +github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -325,8 +327,8 @@ github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.0 h1:zrxIyR3RQIOsarIrgL8+sAvALXul9jeEPa06Y0Ph6vY= -github.com/spf13/viper v1.20.0/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -511,8 +513,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= -google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= -google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= +google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 96aacc4ad0673bfa982c2f8bb660014049ba3cc3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 2 Apr 2025 13:48:28 +0000 Subject: [PATCH 151/250] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `no.elixir:crypt4gh` from 3.0.19 to 3.0.21 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.19...crypt4gh-3.0.21) Updates `no.elixir:clearinghouse` from 2.0.16 to 2.0.17 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-2.0.16...clearinghouse-2.0.17) --- updated-dependencies: - dependency-name: no.elixir:crypt4gh dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index c99d7795e1..d79e45dd83 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.19 + 3.0.21 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.16 + 2.0.17 org.slf4j From 8ae9be37654b61f31a72cc14cf12aafc22409082 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 1 Apr 2025 14:22:17 +0200 Subject: [PATCH 152/250] [s3inbox] send overwrite message when file is reuploaded --- sda/cmd/s3inbox/proxy.go | 77 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 76 insertions(+), 1 deletion(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 50940e6c9e..47337f3af3 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -21,6 +21,7 @@ import ( "github.com/minio/minio-go/v6/pkg/signer" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" @@ -156,8 +157,9 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } - // register file in database if it's the start of an upload + // if this is an upload request if p.detectRequestType(r) == Put && p.fileIds[r.URL.Path] == "" { + // register file in database log.Debugf("registering file %v in the database", r.URL.Path) p.fileIds[r.URL.Path], err = p.database.RegisterFile(filepath, username) log.Debugf("fileId: %v", p.fileIds[r.URL.Path]) @@ -166,6 +168,16 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } + + // check if the file already exists, in that case send an overwrite message, + // so that the FEGA portal is informed that a new version of the file exists. + err = p.sendMessageOnOverwrite(r, rawFilepath, token) + if err != nil { + p.internalServerError(w, r, err.Error()) + + return + } + } log.Debug("Forwarding to backend") @@ -285,18 +297,21 @@ func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { func (p *Proxy) uploadFinishedSuccessfully(req *http.Request, response *http.Response) bool { if response.StatusCode != 200 { + return false } switch req.Method { case http.MethodPut: if !strings.Contains(req.URL.String(), "partNumber") { + return true } return false case http.MethodPost: if strings.Contains(req.URL.String(), "uploadId") { + return true } @@ -511,6 +526,66 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { } +// checkFileExists makes a request to the S3 to check whether the file already +// is uploaded. Returns a bool indicating whether the file was found. +func (p *Proxy) checkFileExists(fullPath string) (bool, error) { + filePath := strings.Replace(fullPath, "/"+p.s3.Bucket+"/", "", 1) + client, err := storage.NewS3Client(p.s3) + if err != nil { + return false, fmt.Errorf("could not connect to s3: %v", err) + } + + result, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: &p.s3.Bucket, + Key: &filePath, + }) + + if err != nil && strings.Contains(err.Error(), "StatusCode: 404") { + log.Debugf("s3 could not find file %s: %s", fullPath, err.Error()) + + return false, nil + } + + return result != nil, err +} + +func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, token jwt.Token) error { + + exist, err := p.checkFileExists(r.URL.Path) + if err != nil { + + return err + } + if exist { + log.Error("create rewrite message") + + username := token.Subject() + msg := schema.InboxRemove{ + User: username, + FilePath: rawFilepath, + Operation: "remove", + } + + privateClaims := token.PrivateClaims() + log.Info("user ", msg.User, " with pilot ", privateClaims["pilot"], " will overwrite file ", msg.FilePath, " at ", time.Now()) + + jsonMessage, err := json.Marshal(msg) + if err != nil { + + return err + } + + err = p.checkAndSendMessage(jsonMessage, r) + if err != nil { + + return err + } + + } + + return nil +} + // FormatUploadFilePath ensures that path separators are "/", and returns error if the // filepath contains a disallowed character matched with regex func formatUploadFilePath(filePath string) (string, error) { From a28e20e2e7111dc99abe8127791a4a318d7e0060 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 1 Apr 2025 14:51:30 +0200 Subject: [PATCH 153/250] [tests] check that an extra message is sent One extra message for deletion of the overwritten file is expected. --- .../integration/tests/sda/10_upload_test.sh | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh index 802bb8db0b..8c045f66ea 100644 --- a/.github/integration/tests/sda/10_upload_test.sh +++ b/.github/integration/tests/sda/10_upload_test.sh @@ -33,13 +33,9 @@ done ## reupload a file under a different name s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh -## reupload a file with the same name -s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ - - echo "waiting for upload to complete" RETRY_TIMES=0 -until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 5 ]; do echo "waiting for upload to complete" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then @@ -49,6 +45,23 @@ until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messag sleep 2 done +## reupload a file with the same name +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ + +## expect 2 new messages, one for deletion of the overwritten file, one for the new upload +echo "waiting for re-upload to complete" +RETRY_TIMES=0 +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do + echo "waiting for re-upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "::error::Time out while waiting for re-upload to complete" + exit 1 + fi + sleep 2 +done + + num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") if [ "$num_rows" -ne 5 ]; then echo "database queries for register_files failed, expected 5 got $num_rows" From 259bd7de338f408cc8983795a7e6204aaa7a442e Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 4 Apr 2025 10:12:41 +0200 Subject: [PATCH 154/250] [s3inbox]tests: add minio container --- sda/cmd/s3inbox/healthchecks_test.go | 12 +-- sda/cmd/s3inbox/proxy_test.go | 132 ++++++++++++++++++++++++--- sda/cmd/s3inbox/s3inbox_test.go | 59 +++++++++++- 3 files changed, 181 insertions(+), 22 deletions(-) diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index f5e8a933b6..0f77348b86 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -33,7 +33,7 @@ func (suite *HealthcheckTestSuite) TearDownTest() { } func (suite *HealthcheckTestSuite) TestHttpsGetCheck() { - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) + p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) url, _ := p.getS3ReadyPath() assert.NoError(suite.T(), p.httpsGetCheck(url)) @@ -41,7 +41,7 @@ func (suite *HealthcheckTestSuite) TestHttpsGetCheck() { } func (suite *HealthcheckTestSuite) TestS3URL() { - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) + p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) _, err := p.getS3ReadyPath() assert.NoError(suite.T(), err) @@ -57,7 +57,7 @@ func (suite *HealthcheckTestSuite) TestHealthchecks() { database, _ := database.NewSDAdb(suite.DBConf) messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) @@ -72,7 +72,7 @@ func (suite *HealthcheckTestSuite) TestClosedDBHealthchecks() { database, _ := database.NewSDAdb(suite.DBConf) messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) // Check that 200 is reported w := httptest.NewRecorder() @@ -95,7 +95,7 @@ func (suite *HealthcheckTestSuite) TestNoS3Healthchecks() { database, _ := database.NewSDAdb(suite.DBConf) messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) // S3 unavailable, check that 503 is reported w := httptest.NewRecorder() @@ -111,7 +111,7 @@ func (suite *HealthcheckTestSuite) TestNoMQHealthchecks() { database, _ := database.NewSDAdb(suite.DBConf) messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) // Messenger unavailable, check that 503 is reported p.messenger.Conf.Port = 123456 diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 08d44de61f..78f0919966 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -10,6 +10,7 @@ import ( "net/http/httptest" "net/url" "os" + "strings" "testing" "github.com/lestrrat-go/jwx/v2/jwk" @@ -20,13 +21,15 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type ProxyTests struct { suite.Suite - S3conf storage.S3Conf + S3Fakeconf storage.S3Conf // fakeserver + S3conf storage.S3Conf // actual s3 container DBConf database.DBConf fakeServer *FakeServer MQConf broker.MQConf @@ -46,7 +49,7 @@ func (suite *ProxyTests) SetupTest() { suite.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - suite.S3conf = storage.S3Conf{ + suite.S3Fakeconf = storage.S3Conf{ URL: "http://127.0.0.1", Port: 9024, AccessKey: "someAccess", @@ -55,6 +58,15 @@ func (suite *ProxyTests) SetupTest() { Region: "us-east-1", } + suite.S3conf = storage.S3Conf{ + URL: "http://127.0.0.1", + Port: s3Port, + AccessKey: "access", + SecretKey: "secretKey", + Bucket: "buckbuck", + Region: "us-east-1", + } + // Create a configuration for the fake MQ suite.MQConf = broker.MQConf{ Host: "127.0.0.1", @@ -125,10 +137,15 @@ func startFakeServer(port string) *FakeServer { if err != nil { panic(fmt.Errorf("can't create mock server for testing: %s", err)) } + log.Warnf("fake server running") f := FakeServer{} foo := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { f.pinged = true - fmt.Fprint(w, f.resp) + log.Warnf("hello fake will return %s", f.resp) + if f.resp != "" { + log.Warnf("fake writes %s", f.resp) + fmt.Fprint(w, f.resp) + } }) ts := httptest.NewUnstartedServer(foo) ts.Listener.Close() @@ -173,11 +190,12 @@ func (m *MockMessenger) SendMessage(uuid string, body []byte) error { // nolint:bodyclose func (suite *ProxyTests) TestServeHTTP_disallowed() { - proxy := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) + proxy := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() + log.Warnf("using proxy on port %d", suite.S3Fakeconf.Port) // Remove bucket disallowed r.Method = "DELETE" r.URL, _ = url.Parse("/asdf/") @@ -192,6 +210,7 @@ func (suite *ProxyTests) TestServeHTTP_disallowed() { assert.Equal(suite.T(), 403, w.Result().StatusCode) assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + log.Warnf("getting to not allowed stuff") // Policy methods are not allowed w = httptest.NewRecorder() r.Method = "GET" @@ -225,7 +244,7 @@ func (suite *ProxyTests) TestServeHTTP_disallowed() { assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) // Not authorized user get 401 response - proxy = NewProxy(suite.S3conf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) + proxy = NewProxy(suite.S3Fakeconf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) w = httptest.NewRecorder() r.Method = "GET" r.URL, _ = url.Parse("/username/file") @@ -259,7 +278,7 @@ func (suite *ProxyTests) TestServeHTTP_MQConnectionClosed() { messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) // Test that the mq connection will be restored when needed proxy.messenger.Connection.Close() @@ -277,7 +296,7 @@ func (suite *ProxyTests) TestServeHTTP_MQChannelClosed() { messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) // Test that the mq channel will be restored when needed proxy.messenger.Channel.Close() @@ -295,7 +314,7 @@ func (suite *ProxyTests) TestServeHTTP_MQ_Unavailable() { messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) // Test that the correct status code is returned when mq connection can't be created proxy.messenger.Conf.Port = 123456 @@ -313,7 +332,7 @@ func (suite *ProxyTests) TestServeHTTP_allowed() { messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) // List files works r, err := http.NewRequest("GET", "/dummy/file", nil) @@ -419,7 +438,7 @@ func (suite *ProxyTests) TestMessageFormatting() { assert.NoError(suite.T(), claims.Set("sub", "user@host.domain")) // start proxy that denies everything - proxy := NewProxy(suite.S3conf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) + proxy := NewProxy(suite.S3Fakeconf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) suite.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" msg, err := proxy.CreateMessageFromRequest(r, claims) assert.Nil(suite.T(), err) @@ -451,11 +470,15 @@ func (suite *ProxyTests) TestDatabaseConnection() { assert.NoError(suite.T(), err) defer messenger.Connection.Close() // Start proxy that allows everything - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) // PUT a file into the system filename := "/dummy/db-test-file" - r, _ := http.NewRequest("PUT", filename, nil) + + myString := "Hello, world!" + stringReader := strings.NewReader(myString) + + r, _ := http.NewRequest("PUT", filename, stringReader) w := httptest.NewRecorder() suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" proxy.allowedResponse(w, r, suite.token) @@ -503,3 +526,88 @@ func (suite *ProxyTests) TestFormatUploadFilePath() { _, err = formatUploadFilePath(weirdPath) assert.EqualError(suite.T(), err, "filepath contains disallowed characters: :, *, ?, \", <, >, |, !, ', (, ), ;, @, &, =, +, $, ,, #, [, ], %") } + +func (suite *ProxyTests) TestCheckFileExists() { + filepath := "somefile" + suite.fakeServer.resp = "fileExists!" + + database, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + defer database.Close() + messenger, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer messenger.Connection.Close() + proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + + res, err := proxy.checkFileExists(filepath) + assert.True(suite.T(), res) + assert.Nil(suite.T(), err) +} + +func (suite *ProxyTests) TestCheckFileExists2() { + database, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + defer database.Close() + messenger, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer messenger.Connection.Close() + proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + + filepath := "/dummy/file" + r, err := http.NewRequest("POST", filepath, nil) + assert.NoError(suite.T(), err) + w := httptest.NewRecorder() + r.Method = "PUT" + proxy.allowedResponse(w, r, suite.token) + assert.Equal(suite.T(), 200, w.Result().StatusCode) + res, err := proxy.checkFileExists(filepath) + assert.True(suite.T(), res) + assert.NotNil(suite.T(), err) + /* + filePath := "existingFile" + proxy.client.open(filePath, 0x00000001) // readonly mode toPflags(os.O_RDONLY)) + file, err := proxy.client.Open(filePath) + res, err := proxy.checkFileExists(filePath) + + assert.True(suite.T(), res) + assert.NotNil(suite.T(), err) + */ + +} + +func (suite *ProxyTests) TestCheckFileDoNotExists() { + database, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + defer database.Close() + messenger, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer messenger.Connection.Close() + proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + + res, err := proxy.checkFileExists("nonexistingfilepath") + assert.False(suite.T(), res) + assert.Nil(suite.T(), err) + +} + +func (suite *ProxyTests) TestCheckFileExistsUnresponsive() { + database, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + defer database.Close() + messenger, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer messenger.Connection.Close() + s3conf := storage.S3Conf{ + URL: "http://localhost:40211", + AccessKey: "someAccess", + SecretKey: "someSecret", + Bucket: "buckbuck", + Region: "us-east-1", + } + proxy := NewProxy(s3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + + res, err := proxy.checkFileExists("nonexistingfilepath") + assert.False(suite.T(), res) + assert.NotNil(suite.T(), err) + +} diff --git a/sda/cmd/s3inbox/s3inbox_test.go b/sda/cmd/s3inbox/s3inbox_test.go index 224cc80b7d..3837fa5bde 100644 --- a/sda/cmd/s3inbox/s3inbox_test.go +++ b/sda/cmd/s3inbox/s3inbox_test.go @@ -11,14 +11,14 @@ import ( "testing" "time" + "github.com/neicnordic/sensitive-data-archive/internal/helper" log "github.com/sirupsen/logrus" - "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" ) -var DBport, MQport, OIDCport int +var DBport, MQport, OIDCport, s3Port int func TestMain(m *testing.M) { if _, err := os.Stat("/.dockerenv"); err == nil { @@ -39,6 +39,54 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to Docker: %s", err) } + // pulls an image, creates a container based on it and runs it + minio, err := pool.RunWithOptions(&dockertest.RunOptions{ + Name: "s3test", + Repository: "minio/minio", + Tag: "RELEASE.2023-05-18T00-05-36Z", + Cmd: []string{"server", "/data", "--console-address", ":9001"}, + Env: []string{ + "MINIO_ROOT_USER=access", + "MINIO_ROOT_PASSWORD=secretKey", + "MINIO_SERVER_URL=http://127.0.0.1:9000", + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + log.Panicf("Could not start resource: %s", err) + } + + s3HostAndPort := minio.GetHostPort("9000/tcp") + s3Port, _ = strconv.Atoi(minio.GetPort("9000/tcp")) + log.Warnf("found host+port %s and port %d", s3HostAndPort, s3Port) + + client := http.Client{Timeout: 5 * time.Second} + req, err := http.NewRequest(http.MethodGet, "http://"+s3HostAndPort+"/minio/health/live", http.NoBody) + log.Warnf("request to %v", req) + if err != nil { + log.Panic(err) + } + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + if err := pool.Retry(func() error { + res, err := client.Do(req) + if err != nil { + return err + } + res.Body.Close() + + return nil + }); err != nil { + if err := pool.Purge(minio); err != nil { + log.Panicf("Could not purge resource: %s", err) + } + log.Panicf("Could not connect to minio: %s", err) + } // pulls an image, creates a container based on it and runs it postgres, err := pool.RunWithOptions(&dockertest.RunOptions{ Repository: "postgres", @@ -100,8 +148,8 @@ func TestMain(m *testing.M) { MQport, _ = strconv.Atoi(rabbitmq.GetPort("5672/tcp")) mqHostAndPort := rabbitmq.GetHostPort("15672/tcp") - client := http.Client{Timeout: 5 * time.Second} - req, err := http.NewRequest(http.MethodPut, "http://"+mqHostAndPort+"/api/queues/%2F/inbox", http.NoBody) + client = http.Client{Timeout: 5 * time.Second} + req, err = http.NewRequest(http.MethodPut, "http://"+mqHostAndPort+"/api/queues/%2F/inbox", http.NoBody) if err != nil { log.Fatal(err) } @@ -193,6 +241,9 @@ func TestMain(m *testing.M) { if err := pool.Purge(oidc); err != nil { log.Fatalf("Could not purge resource: %s", err) } + if err := pool.Purge(minio); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } os.Exit(code) } From 8b3b6f1a99b944689eab75edf0b846be0079be9c Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 4 Apr 2025 15:29:08 +0200 Subject: [PATCH 155/250] [s3inbox] tests: real S3 for proxy tests --- sda/cmd/s3inbox/proxy_test.go | 66 ++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 78f0919966..2a7df38d00 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -1,6 +1,7 @@ package main import ( + "context" "crypto/tls" "database/sql" "encoding/json" @@ -13,6 +14,11 @@ import ( "strings" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/lestrrat-go/jwx/v2/jwk" "github.com/lestrrat-go/jwx/v2/jws" "github.com/lestrrat-go/jwx/v2/jwt" @@ -119,6 +125,33 @@ func (suite *ProxyTests) SetupTest() { if err != nil { suite.T().FailNow() } + + s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(suite.S3conf.AccessKey, suite.S3conf.SecretKey, ""))) + if err != nil { + suite.FailNow("bad") + } + s3Client := s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", suite.S3conf.URL, suite.S3conf.Port)) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(suite.S3conf.URL, "http:") + o.Region = suite.S3conf.Region + o.UsePathStyle = true + }, + ) + _, _ = s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(suite.S3conf.Bucket)}) + if err != nil { + fmt.Println(err.Error()) + } + + output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("This is a test"), + Bucket: aws.String(suite.S3conf.Bucket), + Key: aws.String("/dummy/file"), + ContentEncoding: aws.String("application/octet-stream"), + }) + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), output, output) } func (suite *ProxyTests) TearDownTest() { @@ -544,7 +577,7 @@ func (suite *ProxyTests) TestCheckFileExists() { assert.Nil(suite.T(), err) } -func (suite *ProxyTests) TestCheckFileExists2() { +func (suite *ProxyTests) TestCheckFileExists_realBackend() { database, err := database.NewSDAdb(suite.DBConf) assert.NoError(suite.T(), err) defer database.Close() @@ -553,29 +586,12 @@ func (suite *ProxyTests) TestCheckFileExists2() { defer messenger.Connection.Close() proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) - filepath := "/dummy/file" - r, err := http.NewRequest("POST", filepath, nil) - assert.NoError(suite.T(), err) - w := httptest.NewRecorder() - r.Method = "PUT" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - res, err := proxy.checkFileExists(filepath) + res, err := proxy.checkFileExists("/dummy/file") assert.True(suite.T(), res) - assert.NotNil(suite.T(), err) - /* - filePath := "existingFile" - proxy.client.open(filePath, 0x00000001) // readonly mode toPflags(os.O_RDONLY)) - file, err := proxy.client.Open(filePath) - res, err := proxy.checkFileExists(filePath) - - assert.True(suite.T(), res) - assert.NotNil(suite.T(), err) - */ - + assert.Nil(suite.T(), err) } -func (suite *ProxyTests) TestCheckFileDoNotExists() { +func (suite *ProxyTests) TestCheckFileExists_nonExistingFile() { database, err := database.NewSDAdb(suite.DBConf) assert.NoError(suite.T(), err) defer database.Close() @@ -586,11 +602,12 @@ func (suite *ProxyTests) TestCheckFileDoNotExists() { res, err := proxy.checkFileExists("nonexistingfilepath") assert.False(suite.T(), res) - assert.Nil(suite.T(), err) - + assert.NotNil(suite.T(), err) + assert.Contains(suite.T(), err.Error(), "NotFound") } -func (suite *ProxyTests) TestCheckFileExistsUnresponsive() { +func (suite *ProxyTests) TestCheckFileExists_unresponsive() { + suite.fakeServer.resp = "fileExists!" database, err := database.NewSDAdb(suite.DBConf) assert.NoError(suite.T(), err) defer database.Close() @@ -609,5 +626,4 @@ func (suite *ProxyTests) TestCheckFileExistsUnresponsive() { res, err := proxy.checkFileExists("nonexistingfilepath") assert.False(suite.T(), res) assert.NotNil(suite.T(), err) - } From 1039af2134335147069268873b53745f9e6bf296 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 4 Apr 2025 15:29:20 +0200 Subject: [PATCH 156/250] [s3inbox] tests: Fix healthcheck tests --- sda/cmd/s3inbox/healthchecks_test.go | 54 +++++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index 0f77348b86..fe7448aba8 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -9,13 +9,21 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" + "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type HealthcheckTestSuite struct { - ProxyTests + suite.Suite + S3Fakeconf storage.S3Conf // fakeserver + S3conf storage.S3Conf // actual s3 container + DBConf database.DBConf + fakeServer *FakeServer + MQConf broker.MQConf + messenger *broker.AMQPBroker + database *database.SDAdb } func TestHealthTestSuite(t *testing.T) { @@ -23,13 +31,49 @@ func TestHealthTestSuite(t *testing.T) { } func (suite *HealthcheckTestSuite) SetupTest() { - // Reuse the setup from Proxy - suite.ProxyTests.SetupTest() + suite.fakeServer = startFakeServer("9024") + + // Create an s3config for the fake server + suite.S3Fakeconf = storage.S3Conf{ + URL: "http://127.0.0.1", + Port: 9024, + AccessKey: "someAccess", + SecretKey: "someSecret", + Bucket: "buckbuck", + Region: "us-east-1", + } + + // Create a configuration for the fake MQ + suite.MQConf = broker.MQConf{ + Host: "127.0.0.1", + Port: MQport, + User: "guest", + Password: "guest", + Vhost: "/", + Exchange: "", + } + + suite.messenger = &broker.AMQPBroker{} + + // Create a database configuration for the fake database + suite.DBConf = database.DBConf{ + Host: "127.0.0.1", + Port: DBport, + User: "postgres", + Password: "rootpasswd", + Database: "sda", + CACert: "", + SslMode: "disable", + ClientCert: "", + ClientKey: "", + } + + suite.database = &database.SDAdb{} } func (suite *HealthcheckTestSuite) TearDownTest() { - // Reuse the teardown from Proxy - suite.ProxyTests.TearDownTest() + suite.fakeServer.Close() + suite.database.Close() } func (suite *HealthcheckTestSuite) TestHttpsGetCheck() { From 000c8ad0a262c9e192b3a8e961050e4bd8398158 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Fri, 4 Apr 2025 16:16:12 +0200 Subject: [PATCH 157/250] [s3inbox] tests: improve tests --- sda/cmd/s3inbox/proxy_test.go | 46 ++++++++++++++--------------------- 1 file changed, 18 insertions(+), 28 deletions(-) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 2a7df38d00..1429a54f4e 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -561,23 +561,6 @@ func (suite *ProxyTests) TestFormatUploadFilePath() { } func (suite *ProxyTests) TestCheckFileExists() { - filepath := "somefile" - suite.fakeServer.resp = "fileExists!" - - database, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) - defer database.Close() - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - defer messenger.Connection.Close() - proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) - - res, err := proxy.checkFileExists(filepath) - assert.True(suite.T(), res) - assert.Nil(suite.T(), err) -} - -func (suite *ProxyTests) TestCheckFileExists_realBackend() { database, err := database.NewSDAdb(suite.DBConf) assert.NoError(suite.T(), err) defer database.Close() @@ -592,6 +575,8 @@ func (suite *ProxyTests) TestCheckFileExists_realBackend() { } func (suite *ProxyTests) TestCheckFileExists_nonExistingFile() { + // Check that looking for a non-existing file gives (false, nil) + // from checkFileExists database, err := database.NewSDAdb(suite.DBConf) assert.NoError(suite.T(), err) defer database.Close() @@ -602,28 +587,33 @@ func (suite *ProxyTests) TestCheckFileExists_nonExistingFile() { res, err := proxy.checkFileExists("nonexistingfilepath") assert.False(suite.T(), res) - assert.NotNil(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "NotFound") + assert.Nil(suite.T(), err) } func (suite *ProxyTests) TestCheckFileExists_unresponsive() { - suite.fakeServer.resp = "fileExists!" + // Check that errors when connecting to S3 are forwarded + // and that checkFileExists return (false, someError) database, err := database.NewSDAdb(suite.DBConf) assert.NoError(suite.T(), err) defer database.Close() messenger, err := broker.NewMQ(suite.MQConf) assert.NoError(suite.T(), err) defer messenger.Connection.Close() - s3conf := storage.S3Conf{ - URL: "http://localhost:40211", - AccessKey: "someAccess", - SecretKey: "someSecret", - Bucket: "buckbuck", - Region: "us-east-1", - } - proxy := NewProxy(s3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + + // Unaccessible S3 (wrong port) + proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + proxy.s3.Port = 1111 res, err := proxy.checkFileExists("nonexistingfilepath") assert.False(suite.T(), res) assert.NotNil(suite.T(), err) + assert.Contains(suite.T(), err.Error(), "S3: HeadObject") + + // Bad access key gives 403 + proxy.s3.Port = suite.S3conf.Port + proxy.s3.AccessKey = "invaild" + res, err = proxy.checkFileExists("nonexistingfilepath") + assert.False(suite.T(), res) + assert.NotNil(suite.T(), err) + assert.Contains(suite.T(), err.Error(), "StatusCode: 403") } From b3b7ec992d18840e9ac540374087ca1b0cae4678 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Mon, 7 Apr 2025 08:32:22 +0200 Subject: [PATCH 158/250] [s3inbox] remove newlines etc --- sda/cmd/s3inbox/proxy.go | 4 ---- sda/cmd/s3inbox/proxy_test.go | 5 +---- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 47337f3af3..f6f9ba94c8 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -550,15 +550,12 @@ func (p *Proxy) checkFileExists(fullPath string) (bool, error) { } func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, token jwt.Token) error { - exist, err := p.checkFileExists(r.URL.Path) if err != nil { return err } if exist { - log.Error("create rewrite message") - username := token.Subject() msg := schema.InboxRemove{ User: username, @@ -580,7 +577,6 @@ func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, toke return err } - } return nil diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 1429a54f4e..0eef37afbe 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -18,7 +18,6 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" - "github.com/lestrrat-go/jwx/v2/jwk" "github.com/lestrrat-go/jwx/v2/jws" "github.com/lestrrat-go/jwx/v2/jwt" @@ -508,9 +507,7 @@ func (suite *ProxyTests) TestDatabaseConnection() { // PUT a file into the system filename := "/dummy/db-test-file" - myString := "Hello, world!" - stringReader := strings.NewReader(myString) - + stringReader := strings.NewReader("a brand new string") r, _ := http.NewRequest("PUT", filename, stringReader) w := httptest.NewRecorder() suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" From 4c4f2735fc0d7adcedb4c95e02aa50ea8e5092fb Mon Sep 17 00:00:00 2001 From: Malin Klang Date: Mon, 7 Apr 2025 08:36:36 +0200 Subject: [PATCH 159/250] Apply suggestions from code review Co-authored-by: Joakim Bygdell --- sda/cmd/s3inbox/proxy.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index f6f9ba94c8..3ac9cc987a 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -552,7 +552,6 @@ func (p *Proxy) checkFileExists(fullPath string) (bool, error) { func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, token jwt.Token) error { exist, err := p.checkFileExists(r.URL.Path) if err != nil { - return err } if exist { @@ -568,13 +567,11 @@ func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, toke jsonMessage, err := json.Marshal(msg) if err != nil { - return err } err = p.checkAndSendMessage(jsonMessage, r) if err != nil { - return err } } From f916fda5ae80b9c1776b036a1602c718f7df71d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 20:21:33 +0000 Subject: [PATCH 160/250] Bump golang.org/x/crypto in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [golang.org/x/crypto](https://github.com/golang/crypto). Updates `golang.org/x/crypto` from 0.36.0 to 0.37.0 - [Commits](https://github.com/golang/crypto/compare/v0.36.0...v0.37.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.37.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 6 +++--- sda-download/go.sum | 16 ++++++++-------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 1c98aba7ac..09767f9169 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -17,7 +17,7 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.36.0 + golang.org/x/crypto v0.37.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 google.golang.org/grpc v1.71.1 google.golang.org/protobuf v1.36.6 @@ -71,8 +71,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/net v0.36.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect golang.org/x/tools v0.25.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 4614234dbf..3069d973ef 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -176,8 +176,8 @@ golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -205,23 +205,23 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From b43c330bfd9f8165714a8d08ca880ee441861262 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 20:51:03 +0000 Subject: [PATCH 161/250] Bump the all-modules group in /sda with 7 updates Bumps the all-modules group in /sda with 7 updates: | Package | From | To | | --- | --- | --- | | [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2) | `1.29.12` | `1.29.13` | | [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2) | `1.17.65` | `1.17.66` | | [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) | `1.17.70` | `1.17.71` | | [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) | `1.79.0` | `1.79.1` | | [github.com/coreos/go-oidc/v3](https://github.com/coreos/go-oidc) | `3.13.0` | `3.14.1` | | [golang.org/x/crypto](https://github.com/golang/crypto) | `0.36.0` | `0.37.0` | | [golang.org/x/oauth2](https://github.com/golang/oauth2) | `0.28.0` | `0.29.0` | Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.12 to 1.29.13 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.12...config/v1.29.13) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.65 to 1.17.66 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.65...credentials/v1.17.66) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.70 to 1.17.71 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.70...feature/s3/manager/v1.17.71) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.79.0 to 1.79.1 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.79.0...service/s3/v1.79.1) Updates `github.com/coreos/go-oidc/v3` from 3.13.0 to 3.14.1 - [Release notes](https://github.com/coreos/go-oidc/releases) - [Commits](https://github.com/coreos/go-oidc/compare/v3.13.0...v3.14.1) Updates `golang.org/x/crypto` from 0.36.0 to 0.37.0 - [Commits](https://github.com/golang/crypto/compare/v0.36.0...v0.37.0) Updates `golang.org/x/oauth2` from 0.28.0 to 0.29.0 - [Commits](https://github.com/golang/oauth2/compare/v0.28.0...v0.29.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.29.13 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.17.66 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.71 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.79.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/coreos/go-oidc/v3 dependency-version: 3.14.1 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.37.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-version: 0.29.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 24 ++++++++++++------------ sda/go.sum | 52 ++++++++++++++++++++++++++-------------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 97a847196a..cbd24674d7 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,13 +6,13 @@ toolchain go1.24.1 require ( github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.12 - github.com/aws/aws-sdk-go-v2/credentials v1.17.65 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.70 - github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0 + github.com/aws/aws-sdk-go-v2/config v1.29.13 + github.com/aws/aws-sdk-go-v2/credentials v1.17.66 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.104.0 - github.com/coreos/go-oidc/v3 v3.13.0 + github.com/coreos/go-oidc/v3 v3.14.1 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 @@ -33,8 +33,8 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.36.0 - golang.org/x/oauth2 v0.28.0 + golang.org/x/crypto v0.37.0 + golang.org/x/oauth2 v0.29.0 google.golang.org/grpc v1.71.1 google.golang.org/protobuf v1.36.6 ) @@ -61,9 +61,9 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect @@ -157,8 +157,8 @@ require ( golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.37.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/sys v0.32.0 // indirect + golang.org/x/text v0.24.0 // indirect golang.org/x/time v0.8.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/sda/go.sum b/sda/go.sum index c2799dd01e..16de6a52ad 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -28,14 +28,14 @@ github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38y github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.12 h1:Y/2a+jLPrPbHpFkpAAYkVEtJmxORlXoo5k2g1fa2sUo= -github.com/aws/aws-sdk-go-v2/config v1.29.12/go.mod h1:xse1YTjmORlb/6fhkWi8qJh3cvZi4JoVNhc+NbJt4kI= -github.com/aws/aws-sdk-go-v2/credentials v1.17.65 h1:q+nV2yYegofO/SUXruT+pn4KxkxmaQ++1B/QedcKBFM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.65/go.mod h1:4zyjAuGOdikpNYiSGpsGz8hLGmUzlY8pc8r9QQ/RXYQ= +github.com/aws/aws-sdk-go-v2/config v1.29.13 h1:RgdPqWoE8nPpIekpVpDJsBckbqT4Liiaq9f35pbTh1Y= +github.com/aws/aws-sdk-go-v2/config v1.29.13/go.mod h1:NI28qs/IOUIRhsR7GQ/JdexoqRN9tDxkIrYZq0SOF44= +github.com/aws/aws-sdk-go-v2/credentials v1.17.66 h1:aKpEKaTy6n4CEJeYI1MNj97oSDLi4xro3UzQfwf5RWE= +github.com/aws/aws-sdk-go-v2/credentials v1.17.66/go.mod h1:xQ5SusDmHb/fy55wU0QqTy0yNfLqxzec59YcsRZB+rI= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.70 h1:pgaM86/BFt7dR0b/Jj+OU+taT34nkQlKPkjkYH1POAo= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.70/go.mod h1:vnoXXAU4FFW5JqLC/ZPF67IA5N0f8gah0t0aGI+N9+4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 h1:s43gLuY+zGmtpx+KybfFP4IckopmTfDOPdlf/L++N5I= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71/go.mod h1:KH6wWmY3O3c/jVAjHk0MGzVAFDxkOSt42Eoe4ZO4ge0= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -52,14 +52,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0 h1:OIw2nryEApESTYI5deCZGcq4Gvz8DBAt4tJlNyg3v5o= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.0/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.2 h1:pdgODsAhGo4dvzC3JAG5Ce0PX8kWXrTZGx+jxADD+5E= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.2/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0 h1:90uX0veLKcdHVfvxhkWUQSCi5VabtwMLFutYiRke4oo= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.0/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.17 h1:PZV5W8yk4OtH1JAuhV2PXwwO9v5G5Aoj+eMCn4T+1Kc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.17/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 h1:2Ku1xwAohSSXHR1tpAnyVDSQSxoDMA+/NZBytW+f4qg= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3QasqBfuAFHY4Pk5g= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= @@ -85,8 +85,8 @@ github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= -github.com/coreos/go-oidc/v3 v3.13.0 h1:M66zd0pcc5VxvBNM4pB331Wrsanby+QomQYjN8HamW8= -github.com/coreos/go-oidc/v3 v3.13.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= +github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= +github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -408,8 +408,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -437,8 +437,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= -golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -469,8 +469,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -480,8 +480,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -492,8 +492,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From 57c04be7a317fd5604c583c84b29994885c15bdd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 21:00:04 +0000 Subject: [PATCH 162/250] Bump net.logstash.logback:logstash-logback-encoder Bumps the all-modules group in /sda-sftp-inbox with 1 update: [net.logstash.logback:logstash-logback-encoder](https://github.com/logfellow/logstash-logback-encoder). Updates `net.logstash.logback:logstash-logback-encoder` from 8.0 to 8.1 - [Release notes](https://github.com/logfellow/logstash-logback-encoder/releases) - [Commits](https://github.com/logfellow/logstash-logback-encoder/compare/logstash-logback-encoder-8.0...logstash-logback-encoder-8.1) --- updated-dependencies: - dependency-name: net.logstash.logback:logstash-logback-encoder dependency-version: '8.1' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 9c50b6779f..43fd4b0067 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -118,7 +118,7 @@ net.logstash.logback logstash-logback-encoder - 8.0 + 8.1 org.bouncycastle From a17fb2d5c33ed25f419d14d2a61a982bc027246f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 7 Apr 2025 20:21:57 +0000 Subject: [PATCH 163/250] Bump net.logstash.logback:logstash-logback-encoder Bumps the all-modules group in /sda-doa with 1 update: [net.logstash.logback:logstash-logback-encoder](https://github.com/logfellow/logstash-logback-encoder). Updates `net.logstash.logback:logstash-logback-encoder` from 8.0 to 8.1 - [Release notes](https://github.com/logfellow/logstash-logback-encoder/releases) - [Commits](https://github.com/logfellow/logstash-logback-encoder/compare/logstash-logback-encoder-8.0...logstash-logback-encoder-8.1) --- updated-dependencies: - dependency-name: net.logstash.logback:logstash-logback-encoder dependency-version: '8.1' dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index d79e45dd83..fef4252b1b 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -89,7 +89,7 @@ net.logstash.logback logstash-logback-encoder - 8.0 + 8.1 org.springframework.boot From 4ece2daae18a1849b101e0ff16e4e043d9951259 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 27 Feb 2025 13:10:52 +0100 Subject: [PATCH 164/250] [Ingest] break apart the main work loop to make it more testable #minor --- sda/cmd/ingest/ingest.go | 780 +++++++++++++++++++-------------------- 1 file changed, 377 insertions(+), 403 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index d64ec55ce5..b3e1527161 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -28,7 +28,17 @@ import ( log "github.com/sirupsen/logrus" ) -func main() { +type Ingest struct { + Archive storage.Backend + ArchiveKeyList []*[32]byte + Conf *config.Config + DB *database.SDAdb + Inbox storage.Backend + MQ *broker.AMQPBroker +} + +func (app *Ingest) main() { + var err error sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) // Create a function to handle panic and exit gracefully @@ -39,71 +49,71 @@ func main() { }() forever := make(chan bool) - conf, err := config.NewConfig("ingest") + + app.Conf, err = config.NewConfig("ingest") if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - mq, err := broker.NewMQ(conf.Broker) + app.MQ, err = broker.NewMQ(app.Conf.Broker) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - db, err := database.NewSDAdb(conf.Database) + app.DB, err = database.NewSDAdb(app.Conf.Database) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - if db.Version < 8 { + if app.DB.Version < 8 { log.Error("database schema v8 is required") sigc <- syscall.SIGINT panic(err) } - archiveKeyList, err := config.GetC4GHprivateKeys() + app.ArchiveKeyList, err = config.GetC4GHprivateKeys() if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - archive, err := storage.NewBackend(conf.Archive) + app.Archive, err = storage.NewBackend(app.Conf.Archive) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - inbox, err := storage.NewBackend(conf.Inbox) + app.Inbox, err = storage.NewBackend(app.Conf.Inbox) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() + defer app.MQ.Channel.Close() + defer app.MQ.Connection.Close() + defer app.DB.Close() go func() { - connError := mq.ConnectionWatcher() + connError := app.MQ.ConnectionWatcher() log.Error(connError) forever <- false }() go func() { - connError := mq.ChannelWatcher() + connError := app.MQ.ChannelWatcher() log.Error(connError) forever <- false }() log.Info("starting ingest service") - var message schema.IngestionTrigger go func() { start := time.Now() for i := 1; i > 0; i++ { - h, err := db.ListKeyHashes() + h, err := app.DB.ListKeyHashes() if err != nil { log.Errorln(err.Error()) } @@ -119,14 +129,15 @@ func main() { log.Errorln("no crypt4gh key hash registered") } - messages, err := mq.GetMessages(conf.Broker.Queue) + messages, err := app.MQ.GetMessages(app.Conf.Broker.Queue) if err != nil { - log.Fatal(err) + log.Panic(err) } - mainWorkLoop: + for delivered := range messages { log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", conf.Broker.SchemasPath), delivered.Body) + message := schema.IngestionTrigger{} + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.Conf.Broker.SchemasPath), delivered.Body) if err != nil { log.Errorf("validation of incoming message (ingestion-trigger) failed, reason: (%s)", err.Error()) // Send the message to an error queue so it can be analyzed. @@ -137,440 +148,403 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - // Restart on new message continue } // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) + log.Infof("Received work (corr-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) - log.Infof( - "Received work (corr-id: %s, filepath: %s, user: %s)", - delivered.CorrelationId, message.FilePath, message.User, - ) - + ackNack := "" switch message.Type { case "cancel": - fileUUID, err := db.GetFileID(delivered.CorrelationId) - if err != nil || fileUUID == "" { - log.Errorf("failed to get ID for file from message: %v", delivered.CorrelationId) - - if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - if err := db.UpdateFileEventLog(fileUUID, "disabled", delivered.CorrelationId, "ingest", "{}", string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue + ackNack = app.cancelFile(delivered.CorrelationId, message) + case "ingest": + ackNack = app.ingestFile(delivered.CorrelationId, message) + default: + log.Errorln("unexpected ingest message type") + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message for reason: (%s)", err.Error()) } + } + switch ackNack { + case "ack": if err := delivered.Ack(false); err != nil { log.Errorf("failed to ack message for reason: (%s)", err.Error()) } - - continue - case "ingest": - var fileID string - status, err := db.GetFileStatus(delivered.CorrelationId) - if err != nil && err.Error() != "sql: no rows in result set" { - log.Errorf("failed to get status for file, reason: (%s)", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue + case "nack": + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) } - - switch status { - case "disabled": - fileID, err := db.GetFileID(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - fileInfo, err := db.GetFileInfo(fileID) - if err != nil { - log.Errorf("failed to get info for file: %s", fileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - if err = db.UpdateFileEventLog(fileInfo.Path, "enabled", delivered.CorrelationId, "ingest", "{}", string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - if fileInfo.Checksum != "" { - msg := schema.IngestionVerification{ - User: message.User, - FilePath: message.FilePath, - FileID: fileID, - ArchivePath: fileInfo.Path, - EncryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fileInfo.Checksum}, - }, - } - archivedMsg, _ := json.Marshal(&msg) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), archivedMsg) - if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) - - continue - } - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, archivedMsg); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - - continue - } - - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } - - continue - } - case "": - // Catch all for inboxes that doesn't update the DB - fileID, err = db.RegisterFile(message.FilePath, message.User) - if err != nil { - log.Errorf("InsertFile failed, reason: (%s)", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - default: - fileID, err = db.GetFileID(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } + default: + log.Errorln("unexpected ingest message type") + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message for reason: (%s)", err.Error()) } + } + } + }() - file, err := inbox.NewFileReader(message.FilePath) - if err != nil { //nolint:nestif - log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) - if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") { - jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - if err := db.UpdateFileEventLog(fileID, "error", delivered.CorrelationId, "ingest", string(jsonMsg), string(delivered.Body)); err != nil { - log.Errorf("failed to set error status for file from message: %v, reason: %s", delivered.CorrelationId, err.Error()) - } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to open file to ingest", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - if err = delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - // Restart on new message - continue - } + <-forever +} - fileSize, err := inbox.GetFileSize(message.FilePath, false) - if err != nil { - log.Errorf("Failed to get file size of file to ingest, reason: (%s)", err.Error()) - // Nack message so the server gets notified that something is wrong and requeue the message. - // Since reading the file worked, this should eventually succeed so it is ok to requeue. - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to get file size of file to ingest", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err = mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - - // Restart on new message - continue - } +func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { + fileUUID, err := app.DB.GetFileID(correlationID) + if err != nil { + log.Errorf("failed to get ID for file from message: %s, reason: %s", correlationID, err.Error()) + if strings.Contains(err.Error(), "sql: no rows in result set") { + return "reject" + } - if err = db.UpdateFileEventLog(fileID, "submitted", delivered.CorrelationId, message.User, "{}", string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) - } + return "nack" + } - dest, err := archive.NewFileWriter(fileID) - if err != nil { - log.Errorf("Failed to create archive file, reason: (%s)", err.Error()) - // Nack message so the server gets notified that something is wrong and requeue the message. - // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileUUID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set event log status for file: %s", correlationID) - continue - } + return "nack" + } - // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock - var bufSize int - if bufSize = 4 * 1024 * 1024; conf.Inbox.S3.Chunksize > 4*1024*1024 { - bufSize = conf.Inbox.S3.Chunksize - } - readBuffer := make([]byte, bufSize) - hash := sha256.New() - var bytesRead int64 - var byteBuf bytes.Buffer - - for bytesRead < fileSize { - i, _ := io.ReadFull(file, readBuffer) - if i == 0 { - return - } - // truncate the readbuffer if the file is smaller than the buffer size - if i < len(readBuffer) { - readBuffer = readBuffer[:i] - } - - bytesRead += int64(i) - - h := bytes.NewReader(readBuffer) - if _, err = io.Copy(hash, h); err != nil { - log.Errorf("Copy to hash failed while reading file, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - //nolint:nestif - if bytesRead <= int64(len(readBuffer)) { - var privateKey *[32]byte - var header []byte - - // Iterate over the key list to try decryption - for _, key := range archiveKeyList { - header, err = tryDecrypt(key, readBuffer) - if err == nil { - privateKey = key - - break - } - log.Warnf("Decryption failed with key, trying next key. Reason: (%s)", err.Error()) - } - - // Check if decryption was successful with any key - if privateKey == nil { - log.Errorf("All keys failed to decrypt the submitted file") - if err := db.UpdateFileEventLog(fileID, "error", delivered.CorrelationId, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(delivered.Body)); err != nil { - log.Errorf("Failed to set ingestion status for file from message: %v", delivered.CorrelationId) - } - - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) - } - - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Trying to decrypt the submitted file failed", - Reason: "Decryption failed with the available key(s)", - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - // Proceed with the successful key - // Set the file's hex encoded public key - publicKey := keys.DerivePublicKey(*privateKey) - keyhash := hex.EncodeToString(publicKey[:]) - err = db.SetKeyHash(keyhash, fileID) - if err != nil { - log.Errorf("Key hash %s could not be set for fileID %s: (%s)", keyhash, fileID, err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - log.Debugln("store header") - if err := db.StoreHeader(header, fileID); err != nil { - log.Errorf("StoreHeader failed, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for header read, reason: %v)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - // Strip header from buffer - h := make([]byte, len(header)) - if _, err = byteBuf.Read(h); err != nil { - log.Errorf("Failed to strip header from buffer, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - } else { - if i < len(readBuffer) { - readBuffer = readBuffer[:i] - } - if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for full read, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - } - - // Write data to file - if _, err = byteBuf.WriteTo(dest); err != nil { - log.Errorf("Failed to write to archive file, reason: (%s)", err.Error()) - - continue mainWorkLoop - } - } + return "ack" +} - file.Close() - dest.Close() +func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrigger) string { + var fileID string + status, err := app.DB.GetFileStatus(correlationID) + if err != nil && err.Error() != "sql: no rows in result set" { + log.Errorf("failed to get status for file, reason: (%s)", err.Error()) - // At this point we should do checksum comparison, but that requires updating the AWS library + return "nack" + } - fileInfo := database.FileInfo{} - fileInfo.Path = fileID - fileInfo.Checksum = fmt.Sprintf("%x", hash.Sum(nil)) - fileInfo.Size, err = archive.GetFileSize(fileID, true) - if err != nil { - log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } + switch status { + case "disabled": + fileID, err = app.DB.GetFileID(correlationID) + if err != nil { + log.Errorf("failed to get ID for file, reason: %s", err.Error()) - continue - } + return "nack" + } + + fileInfo, err := app.DB.GetFileInfo(fileID) + if err != nil { + log.Errorf("failed to get info for file: %s, reason: %s", fileID, err.Error()) - log.Debugf("Wrote archived file (corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", - delivered.CorrelationId, message.User, message.FilePath, fileID, fileInfo.Size) + return "nack" + } - status, err = db.GetFileStatus(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - } - if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping ingestion", delivered.CorrelationId) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + // What if the file in the inbox is different this time? + // Check uploaded checksum in the DB against the checksum of the file. + // Would be easy if the inbox message had the checksum or that the s3inbox added the checksum to the DB. + + if fileInfo.Checksum != "" { + msg := schema.IngestionVerification{ + User: message.User, + FilePath: message.FilePath, + FileID: fileID, + ArchivePath: fileInfo.Path, + EncryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fileInfo.Checksum}, + }, + } + archivedMsg, _ := json.Marshal(&msg) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) + if err != nil { + log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) - continue - } + return "nack" + } - if err := db.SetArchived(fileInfo, fileID, delivered.CorrelationId); err != nil { - log.Errorf("SetArchived failed, reason: (%s)", err.Error()) - } + m, _ := json.Marshal(message) + if err = app.DB.UpdateFileEventLog(fileInfo.Path, "enabled", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set ingestion status for file from message: %v", correlationID) - log.Debugf("File marked as archived (corr-id: %s, user: %s, filepath: %s, archivepath: %s)", - delivered.CorrelationId, message.User, message.FilePath, fileID) - - // Send message to archived - msg := schema.IngestionVerification{ - User: message.User, - FilePath: message.FilePath, - FileID: fileID, - ArchivePath: fileID, - EncryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fmt.Sprintf("%x", hash.Sum(nil))}, - }, - } - archivedMsg, _ := json.Marshal(&msg) + return "nack" + } + + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + return "reject" + } + + return "ack" + } + case "": + // Catch all for inboxes that doesn't update the DB + fileID, err = app.DB.RegisterFile(message.FilePath, message.User) + if err != nil { + log.Errorf("InsertFile failed, reason: (%s)", err.Error()) + + return "nack" + } + case "uploaded": + fileID, err = app.DB.GetFileID(correlationID) + if err != nil { + log.Errorf("failed to get ID for file, reason: %s", err.Error()) + + return "nack" + } + default: + return "reject" + } + + file, err := app.Inbox.NewFileReader(message.FilePath) + if err != nil { + switch { + case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): + log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { + log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) + } + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to open file to ingest", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + return "reject" + } + + return "ack" + default: + return "nack" + } + } + + fileSize, err := app.Inbox.GetFileSize(message.FilePath, false) + if err != nil { + log.Errorf("Failed to get file size of file to ingest, reason: (%s)", err.Error()) + // Since reading the file worked, this should eventually succeed so it is ok to requeue. + return "nack" + } + + dest, err := app.Archive.NewFileWriter(fileID) + if err != nil { + log.Errorf("Failed to create archive file, reason: (%s)", err.Error()) + // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. + return "nack" + } - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), archivedMsg) - if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + m, _ := json.Marshal(message) + if err = app.DB.UpdateFileEventLog(fileID, "submitted", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set ingestion status for file from message: %v", correlationID) + } + + // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock + var bufSize int + if bufSize = 4 * 1024 * 1024; app.Conf.Inbox.S3.Chunksize > 4*1024*1024 { + bufSize = app.Conf.Inbox.S3.Chunksize + } + readBuffer := make([]byte, bufSize) + hash := sha256.New() + var bytesRead int64 + var byteBuf bytes.Buffer + + for bytesRead < fileSize { + i, _ := io.ReadFull(file, readBuffer) + if i == 0 { + log.Debugln("this should not happen") + + return "reject" + } + // truncate the readbuffer if the file is smaller than the buffer size + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } + + bytesRead += int64(i) - continue + h := bytes.NewReader(readBuffer) + if _, err = io.Copy(hash, h); err != nil { + log.Errorf("Copy to hash failed while reading file, reason: (%s)", err.Error()) + + return "nack" + } + + switch { + case bytesRead <= int64(len(readBuffer)): + var privateKey *[32]byte + var header []byte + + // Iterate over the key list to try decryption + for _, key := range app.ArchiveKeyList { + header, err = tryDecrypt(key, readBuffer) + if err == nil { + privateKey = key + + break } + log.Warnf("Decryption failed with key, trying next key. Reason: (%s)", err.Error()) + } - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, archivedMsg); err != nil { - // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + // Check if decryption was successful with any key + if privateKey == nil { + log.Errorf("All keys failed to decrypt the submitted file") + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { + log.Errorf("Failed to set ingestion status for file from message: %v", correlationID) + } - // Do not try to ACK message to make sure we have another go - continue + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Trying to decrypt the submitted file failed", + Reason: "Decryption failed with the available key(s)", + OriginalMessage: message, } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } + + return "ack" + } + + // Proceed with the successful key + // Set the file's hex encoded public key + publicKey := keys.DerivePublicKey(*privateKey) + keyhash := hex.EncodeToString(publicKey[:]) + err = app.DB.SetKeyHash(keyhash, fileID) + if err != nil { + log.Errorf("Key hash %s could not be set for fileID %s: (%s)", keyhash, fileID, err.Error()) + + return "nack" + } + + log.Debugln("store header") + if err := app.DB.StoreHeader(header, fileID); err != nil { + log.Errorf("StoreHeader failed, reason: (%s)", err.Error()) + + return "nack" + } + + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for header read, reason: %v)", err.Error()) + + return "nack" + } + + // Strip header from buffer + h := make([]byte, len(header)) + if _, err = byteBuf.Read(h); err != nil { + log.Errorf("Failed to strip header from buffer, reason: (%s)", err.Error()) + + return "nack" + } + default: + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for full read, reason: (%s)", err.Error()) + + return "nack" } } - }() - <-forever + // Write data to file + if _, err = byteBuf.WriteTo(dest); err != nil { + log.Errorf("Failed to write to archive file, reason: (%s)", err.Error()) + + _ = file.Close() + _ = dest.Close() + if err := app.Archive.RemoveFile(fileID); err != nil { + log.Errorf("error when removing uncompleted file: %s", fileID) + } + + return "nack" + } + } + + file.Close() + dest.Close() + + // At this point we should do checksum comparison, but that requires updating the AWS library + + fileInfo := database.FileInfo{} + fileInfo.Path = fileID + fileInfo.UploadedChecksum = fmt.Sprintf("%x", hash.Sum(nil)) + fileInfo.Size, err = app.Archive.GetFileSize(fileID, true) + if err != nil { + log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) + + return "nack" + } + + log.Debugf("Wrote archived file (corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", correlationID, message.User, message.FilePath, fileID, fileInfo.Size) + + status, err = app.DB.GetFileStatus(correlationID) + if err != nil { + log.Errorf("failed to get file status, reason: (%s)", err.Error()) + + return "nack" + } + + if status == "disabled" { + log.Infof("file with correlation ID: %s is disabled, stopping ingestion", correlationID) + + return "ack" + } + + if err := app.DB.SetArchived(fileInfo, fileID, correlationID); err != nil { + log.Errorf("SetArchived failed, reason: (%s)", err.Error()) + + return "nack" + } + + if err := app.DB.UpdateFileEventLog(fileID, "archived", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set event log status for file: %s", correlationID) + + return "nack" + } + log.Debugf("File marked as archived (corr-id: %s, user: %s, filepath: %s, archivepath: %s)", correlationID, message.User, message.FilePath, fileID) + + // Send message to archived + msg := schema.IngestionVerification{ + User: message.User, + FilePath: message.FilePath, + FileID: fileID, + ArchivePath: fileID, + EncryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fmt.Sprintf("%x", hash.Sum(nil))}, + }, + } + archivedMsg, _ := json.Marshal(&msg) + + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) + if err != nil { + log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + + return "nack" + } + + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + // TODO fix resend mechanism + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + return "reject" + } + + return "ack" } // tryDecrypt tries to decrypt the start of buf. func tryDecrypt(key *[32]byte, buf []byte) ([]byte, error) { - log.Debugln("Try decrypting the first data block") a := bytes.NewReader(buf) b, err := streaming.NewCrypt4GHReader(a, *key, nil) From 797eaf0d2f2305531af06c806815ee7644150be4 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 27 Feb 2025 13:11:52 +0100 Subject: [PATCH 165/250] [ingest_test] Set up suite for live testing --- sda/cmd/ingest/ingest_test.go | 285 ++++++++++++++++++++++++++++++---- 1 file changed, 259 insertions(+), 26 deletions(-) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 48c55a3da0..dd418671e0 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -1,66 +1,300 @@ package main import ( + "crypto/rand" + "database/sql" + "encoding/hex" "fmt" "io" + "net/http" "os" + "path" + "path/filepath" + "runtime" + "strconv" "testing" + "time" + log "github.com/sirupsen/logrus" + + "github.com/google/uuid" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/crypt4gh/streaming" + "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" + "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) -type TestSuite struct { - suite.Suite -} +var DBport, MQport int +var BrokerAPI string -var pubKeyList = [][32]byte{} +func TestMain(m *testing.M) { + if _, err := os.Stat("/.dockerenv"); err == nil { + m.Run() + } + _, b, _, _ := runtime.Caller(0) + rootDir := path.Join(path.Dir(b), "../../../") -func TestConfigTestSuite(t *testing.T) { + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not construct pool: %s", err) + } + + // uses pool to try to connect to Docker + err = pool.Client.Ping() + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) + } + + // pulls an image, creates a container based on it and runs it + postgres, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: "15.2-alpine3.17", + Env: []string{ + "POSTGRES_PASSWORD=rootpasswd", + "POSTGRES_DB=sda", + }, + Mounts: []string{ + fmt.Sprintf("%s/postgresql/initdb.d:/docker-entrypoint-initdb.d", rootDir), + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + dbHostAndPort := postgres.GetHostPort("5432/tcp") + DBport, _ = strconv.Atoi(postgres.GetPort("5432/tcp")) + databaseURL := fmt.Sprintf("postgres://postgres:rootpasswd@%s/sda?sslmode=disable", dbHostAndPort) + + pool.MaxWait = 120 * time.Second + if err = pool.Retry(func() error { + db, err := sql.Open("postgres", databaseURL) + if err != nil { + log.Println(err) + + return err + } + + query := "SELECT MAX(version) FROM sda.dbschema_version;" + var dbVersion int + + return db.QueryRow(query).Scan(&dbVersion) + }); err != nil { + log.Fatalf("Could not connect to postgres: %s", err) + } + + // pulls an image, creates a container based on it and runs it + rabbitmq, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "ghcr.io/neicnordic/sensitive-data-archive", + Tag: "v0.3.89-rabbitmq", + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + log.Fatalf("Could not start resource: %s", err) + } + + MQport, _ = strconv.Atoi(rabbitmq.GetPort("5672/tcp")) + BrokerAPI = rabbitmq.GetHostPort("15672/tcp") + + client := http.Client{Timeout: 30 * time.Second} + req, err := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/", http.NoBody) + if err != nil { + log.Fatal(err) + } + req.SetBasicAuth("guest", "guest") + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + if err := pool.Retry(func() error { + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + return err + } + res.Body.Close() + + return nil + }); err != nil { + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + if err := pool.Purge(rabbitmq); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + log.Fatalf("Could not connect to rabbitmq: %s", err) + } + + log.Println("starting tests") + code := m.Run() + + log.Println("tests completed") + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + if err := pool.Purge(rabbitmq); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + + os.Exit(code) +} + +func TestIngestTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { - viper.Set("log.level", "debug") - archive := suite.T().TempDir() - defer os.RemoveAll(archive) - viper.Set("archive.location", archive) +type TestSuite struct { + suite.Suite + filePath string + pubKeyList [][32]byte + ingest Ingest +} +func (suite *TestSuite) SetupSuite() { + viper.Set("log.level", "debug") tempDir := suite.T().TempDir() keyFile1 := fmt.Sprintf("%s/c4gh1.key", tempDir) keyFile2 := fmt.Sprintf("%s/c4gh2.key", tempDir) publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test") - assert.NoError(suite.T(), err) + if err != nil { + suite.FailNow("Failed to create c4gh key") + } // Add only the first public key to the list - pubKeyList = append(pubKeyList, publicKey) + suite.pubKeyList = append(suite.pubKeyList, publicKey) _, err = helper.CreatePrivateKeyFile(keyFile2, "test") - assert.NoError(suite.T(), err) + if err != nil { + suite.FailNow("Failed to create c4gh key") + } viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ {FilePath: keyFile1, Passphrase: "test"}, {FilePath: keyFile2, Passphrase: "test"}, }) + viper.Set("archive.type", "posix") + viper.Set("archive.location", "/tmp/") + viper.Set("broker.host", "localhost") + viper.Set("broker.port", MQport) + viper.Set("broker.user", "guest") + viper.Set("broker.password", "guest") + viper.Set("broker.queue", "ingest") + viper.Set("broker.routingkey", "verify") + viper.Set("broker.vhost", "sda") + viper.Set("db.host", "localhost") + viper.Set("db.port", DBport) + viper.Set("db.user", "postgres") + viper.Set("db.password", "rootpasswd") + viper.Set("db.database", "sda") + viper.Set("db.sslMode", "disable") + viper.Set("inbox.type", "posix") + viper.Set("inbox.location", "/tmp/") + viper.Set("schema.path", "../../schemas/isolated/") + + suite.ingest.Conf, err = config.NewConfig("ingest") + if err != nil { + suite.FailNowf("failed to init config: %s", err.Error()) + } + suite.ingest.DB, err = database.NewSDAdb(suite.ingest.Conf.Database) + if err != nil { + suite.FailNowf("failed to setup database connection: %s", err.Error()) + } + suite.ingest.MQ, err = broker.NewMQ(suite.ingest.Conf.Broker) + if err != nil { + suite.FailNowf("failed to setup rabbitMQ connection: %s", err.Error()) + } + suite.ingest.ArchiveKeyList, err = config.GetC4GHprivateKeys() + if err != nil { + suite.FailNow("no private keys configured") + } + + if err := suite.ingest.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), "the test key"); err != nil { + suite.FailNow("failed to register the public key") + } +} - viper.Set("broker.host", "test") - viper.Set("broker.port", 123) - viper.Set("broker.user", "test") - viper.Set("broker.password", "test") - viper.Set("broker.queue", "test") - viper.Set("broker.routingkey", "test") - viper.Set("db.host", "test") - viper.Set("db.port", 123) - viper.Set("db.user", "test") - viper.Set("db.password", "test") - viper.Set("db.database", "test") +func (suite *TestSuite) TearDownSuite() { + _ = os.RemoveAll(suite.ingest.Conf.Archive.Posix.Location) + _ = os.RemoveAll(suite.ingest.Conf.Inbox.Posix.Location) } +func (suite *TestSuite) SetupTest() { + var err error + suite.ingest.Conf.Archive.Posix.Location, err = os.MkdirTemp("", "archive") + if err != nil { + suite.FailNow("failed to create temp folder") + } + + suite.ingest.Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") + if err != nil { + suite.FailNow("failed to create temp folder") + } + + f, err := os.CreateTemp(suite.ingest.Conf.Inbox.Posix.Location, "") + if err != nil { + suite.FailNow("failed to create test file") + } + defer f.Close() + + _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + suite.FailNow("failed to write data to test file") + } + + outFileName := f.Name() + ".c4gh" + outFile, err := os.Create(outFileName) + if err != nil { + suite.FailNow("failed to create encrypted test file") + } + defer outFile.Close() + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + suite.FailNow("failed to create private c4gh key") + } + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, suite.pubKeyList, nil) + if err != nil { + suite.FailNow("failed to create c4gh writer") + } + + _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + suite.FailNow("failed to write data to encrypted test file") + } + crypt4GHWriter.Close() + + suite.filePath = filepath.Base(outFileName) + + suite.ingest.Archive, err = storage.NewBackend(suite.ingest.Conf.Archive) + if err != nil { + suite.FailNow("failed to setup archive backend") + } + suite.ingest.Inbox, err = storage.NewBackend(suite.ingest.Conf.Inbox) + if err != nil { + suite.FailNow("failed to setup inbox backend") + } +} func (suite *TestSuite) TestTryDecrypt_wrongFile() { tempDir := suite.T().TempDir() err := os.WriteFile(fmt.Sprintf("%s/dummy.file", tempDir), []byte("hello\ngo\n"), 0600) @@ -80,7 +314,6 @@ func (suite *TestSuite) TestTryDecrypt_wrongFile() { assert.Nil(suite.T(), header) assert.EqualError(suite.T(), err, "not a Crypt4GH file") } - func (suite *TestSuite) TestTryDecrypt() { _, signingKey, err := keys.GenerateKeyPair() assert.NoError(suite.T(), err) @@ -96,7 +329,7 @@ func (suite *TestSuite) TestTryDecrypt() { encryptedFile, err := os.CreateTemp(tempDir, "encryptedFile-") assert.NoError(suite.T(), err) - crypt4GHWriter, err := streaming.NewCrypt4GHWriter(encryptedFile, signingKey, pubKeyList, nil) + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(encryptedFile, signingKey, suite.pubKeyList, nil) assert.NoError(suite.T(), err) _, err = io.Copy(crypt4GHWriter, unencryptedFile) From 31cda547f5574b4a7f576a464d0fe5fb7fdae15b Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 27 Feb 2025 13:14:23 +0100 Subject: [PATCH 166/250] [ingest_test] add cases for `cancel` messages --- sda/cmd/ingest/ingest_test.go | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index dd418671e0..597c932bd8 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -357,3 +357,45 @@ func (suite *TestSuite) TestTryDecrypt() { } } } + +// messages of type `cancel` +func (suite *TestSuite) TestCancelFile() { + // prepare the DB entries + UserName := "test-cancel" + file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", UserName) + fileID, err := suite.ingest.DB.RegisterFile(file1, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "cancel", + FilePath: file1, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.cancelFile(corrID, message)) +} +func (suite *TestSuite) TestCancelFile_wrongCorrelationID() { + // prepare the DB entries + UserName := "test-cancel" + file1 := fmt.Sprintf("/%v/TestCancelMessage_wrongCorrelationID.c4gh", UserName) + fileID, err := suite.ingest.DB.RegisterFile(file1, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "cancel", + FilePath: file1, + User: UserName, + } + + assert.Equal(suite.T(), "reject", suite.ingest.cancelFile(uuid.New().String(), message)) +} From 168b23588c0629f7ac8e35090d570d17ca471df1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 27 Feb 2025 13:45:00 +0100 Subject: [PATCH 167/250] [ingest_test] add file ingest scenarios --- sda/cmd/ingest/ingest_test.go | 151 ++++++++++++++++++++++++++++++++++ 1 file changed, 151 insertions(+) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 597c932bd8..8b34de039c 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -2,6 +2,7 @@ package main import ( "crypto/rand" + "crypto/sha256" "database/sql" "encoding/hex" "fmt" @@ -399,3 +400,153 @@ func (suite *TestSuite) TestCancelFile_wrongCorrelationID() { assert.Equal(suite.T(), "reject", suite.ingest.cancelFile(uuid.New().String(), message)) } + +// messages of type `ingest` +func (suite *TestSuite) TestIngestFile() { + // prepare the DB entries + UserName := "test-ingest" + fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) +} +func (suite *TestSuite) TestIngestFile_secondTime() { + // prepare the DB entries + UserName := "test-ingest" + fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + // file is already in `archived` state + assert.Equal(suite.T(), "reject", suite.ingest.ingestFile(corrID, message)) +} +func (suite *TestSuite) TestIngestFile_unknownInboxType() { + UserName := "test-ingest-unknown" + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(uuid.New().String(), message)) +} +func (suite *TestSuite) TestIngestFile_reingestCancelledFile() { + // prepare the DB entries + UserName := "test-ingest" + fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) +} +func (suite *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { + // prepare the DB entries + UserName := "test-ingest" + fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + // over write the encrypted file to generate new checksum + f, err := os.CreateTemp(suite.ingest.Conf.Inbox.Posix.Location, "") + if err != nil { + suite.FailNow("failed to create test file") + } + defer f.Close() + + _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + suite.FailNow("failed to write data to test file") + } + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + suite.FailNow("failed to create private c4gh key") + } + + outFile, err := os.Create(path.Join(suite.ingest.Conf.Inbox.Posix.Location, suite.filePath)) + if err != nil { + suite.FailNowf("failed to create encrypted test file: %s", err.Error()) + } + defer outFile.Close() + + sha256hash := sha256.New() + mr := io.MultiWriter(outFile, sha256hash) + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, suite.pubKeyList, nil) + if err != nil { + suite.FailNowf("failed to create c4gh writer: %s", err.Error()) + } + + _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + suite.FailNow("failed to write data to encrypted test file") + } + crypt4GHWriter.Close() + + // reingestion should work + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + // DB should have the new checksum + var dbChecksum string + const q = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" + if err := suite.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { + suite.FailNow("failed to get checksum from database") + } + + assert.Equal(suite.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) +} \ No newline at end of file From 3a40f00fc22154152a31dc433f76053d5da73a23 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 27 Feb 2025 13:46:43 +0100 Subject: [PATCH 168/250] [internal/database] move `SetArchvied` from SQL to GO --- sda/cmd/api/api_test.go | 12 ++--- sda/cmd/ingest/ingest.go | 2 +- sda/cmd/sync/sync_test.go | 2 +- sda/internal/database/db_functions.go | 41 +++++++++-------- sda/internal/database/db_functions_test.go | 51 +++++++++++----------- 5 files changed, 55 insertions(+), 53 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 2e67f6b27f..2ee2e8a764 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -857,7 +857,7 @@ func (suite *TestSuite) TestSetAccession() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) + err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) @@ -1011,7 +1011,7 @@ func (suite *TestSuite) TestCreateDataset() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) + err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) @@ -1090,7 +1090,7 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) + err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) @@ -1212,7 +1212,7 @@ func (suite *TestSuite) TestCreateDataset_WrongUser() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) + err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) @@ -1974,7 +1974,7 @@ func (suite *TestSuite) TestReVerifyFile() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - if err := Conf.API.DB.SetArchived(fileInfo, fileID, fileID); err != nil { + if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as Archived") } @@ -2076,7 +2076,7 @@ func (suite *TestSuite) TestReVerifyDataset() { DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - if err := Conf.API.DB.SetArchived(fileInfo, fileID, fileID); err != nil { + if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as Archived") } diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index b3e1527161..b4e94b7624 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -501,7 +501,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "ack" } - if err := app.DB.SetArchived(fileInfo, fileID, correlationID); err != nil { + if err := app.DB.SetArchived(fileInfo, fileID); err != nil { log.Errorf("SetArchived failed, reason: (%s)", err.Error()) return "nack" diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index 3a86ca9222..ef8d3518cd 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -168,7 +168,7 @@ func (suite *SyncTest) TestBuildSyncDatasetJSON() { fileInfo := database.FileInfo{Checksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = db.SetVerified(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "failed to mark file as Verified") diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 0ce9b056bb..3073c2c820 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -202,34 +202,37 @@ func (dbs *SDAdb) storeHeader(header []byte, id string) error { } // SetArchived marks the file as 'ARCHIVED' -func (dbs *SDAdb) SetArchived(file FileInfo, fileID, corrID string) error { - var ( - err error - count int - ) +func (dbs *SDAdb) SetArchived(file FileInfo, fileID string) error { + var err error - for count == 0 || (err != nil && count < RetryTimes) { - err = dbs.setArchived(file, fileID, corrID) - count++ + for count := 1; count <= RetryTimes; count++ { + err = dbs.setArchived(file, fileID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) } return err } -func (dbs *SDAdb) setArchived(file FileInfo, fileID, corrID string) error { +func (dbs *SDAdb) setArchived(file FileInfo, fileID string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "SELECT sda.set_archived($1, $2, $3, $4, $5, $6);" - _, err := db.Exec(query, - fileID, - corrID, - file.Path, - file.Size, - file.Checksum, - "SHA256", - ) + const setArchived = "UPDATE sda.files SET archive_file_path = $1, archive_file_size = $2 WHERE id = $3;" + if _, err := db.Exec(setArchived, file.Path, file.Size, fileID); err != nil { + return fmt.Errorf("setArchived error: %s", err.Error()) + } - return err + log.Debugf("checksum: %s", file.Checksum) + const addChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + + "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source)" + + "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + if _, err := db.Exec(addChecksum, fileID, file.Checksum, "SHA256"); err != nil { + return fmt.Errorf("addChecksum error: %s", err.Error()) + } + + return nil } func (dbs *SDAdb) GetFileStatus(corrID string) (string, error) { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index c50c271c81..e791aa8aeb 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -105,16 +105,15 @@ func (suite *DatabaseTests) TestSetArchived() { fileID, err := db.RegisterFile("/testuser/TestSetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), -1} - corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), -1, fmt.Sprintf("%x", sha256.New())} + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = db.SetArchived(fileInfo, "00000000-0000-0000-0000-000000000000", corrID) + err = db.SetArchived(fileInfo, "00000000-0000-0000-0000-000000000000") assert.ErrorContains(suite.T(), err, "violates foreign key constraint") - err = db.SetArchived(fileInfo, fileID, "00000000-0000-0000-0000-000000000000") - assert.ErrorContains(suite.T(), err, "duplicate key value violates unique constraint") + err = db.SetArchived(fileInfo, fileID) + assert.NoError(suite.T(), err, "failed to mark file as Archived") } func (suite *DatabaseTests) TestGetFileStatus() { @@ -159,7 +158,7 @@ func (suite *DatabaseTests) TestSetVerified() { assert.NoError(suite.T(), err, "failed to register file in database") corrID := uuid.New().String() - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/testuser/TestSetVerified.c4gh", fmt.Sprintf("%x", sha256.New()), 948} + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/testuser/TestSetVerified.c4gh", fmt.Sprintf("%x", sha256.New()), 948, fmt.Sprintf("%x", sha256.New())} err = db.SetVerified(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) } @@ -174,7 +173,7 @@ func (suite *DatabaseTests) TestGetArchived() { fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987} corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -194,7 +193,7 @@ func (suite *DatabaseTests) TestSetAccessionID() { assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987} corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -212,7 +211,7 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987} corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -247,7 +246,7 @@ func (suite *DatabaseTests) TestGetFileInfo() { fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/tmp/TestGetFileInfo.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") err = db.SetVerified(fileInfo, fileID, corrID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) @@ -383,7 +382,7 @@ func (suite *DatabaseTests) TestGetSyncData() { checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/tmp/TestGetGetSyncData.c4gh", checksum, 999} corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = db.SetVerified(fileInfo, fileID, corrID) @@ -442,8 +441,8 @@ func (suite *DatabaseTests) TestGetArchivePath() { checksum := fmt.Sprintf("%x", sha256.New()) corrID := uuid.New().String() - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1234, corrID, checksum, 999} - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1234, corrID, checksum, 999, fmt.Sprintf("%x", sha256.New())} + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = db.SetAccessionID("acession-0001", fileID) @@ -517,8 +516,8 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { } checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, fileID, checksum, 999} - if err := db.SetArchived(fileInfo, fileID, fileID); err != nil { + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, fileID, checksum, 999, fmt.Sprintf("%x", sha256.New())} + if err := db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as archived") } if err := db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}"); err != nil { @@ -604,8 +603,8 @@ func (suite *DatabaseTests) TestListActiveUsers() { assert.Equal(suite.T(), fileID, corrID) checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999} - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999, fmt.Sprintf("%x", sha256.New())} + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -668,7 +667,7 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { checksum, 999, } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -852,7 +851,7 @@ func (suite *DatabaseTests) TestListDatasets() { checksum, 999, } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -935,7 +934,7 @@ func (suite *DatabaseTests) TestListUserDatasets() { checksum, 999, } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } @@ -1070,7 +1069,7 @@ func (suite *DatabaseTests) TestGetReVerificationData() { fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} corrID := uuid.New().String() - if err = db.SetArchived(fileInfo, fileID, corrID); err != nil { + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { @@ -1109,7 +1108,7 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} corrID := uuid.New().String() - if err = db.SetArchived(fileInfo, fileID, corrID); err != nil { + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { @@ -1146,9 +1145,9 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { suite.FailNow("failed to generate checksum") } - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetDecryptedChecksum.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetDecryptedChecksum.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() - if err = db.SetArchived(fileInfo, fileID, corrID); err != nil { + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { @@ -1190,7 +1189,7 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { checksum, 999, } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } From 6e60415e7a6487578ece50e6d098e5aea286e4d3 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 27 Feb 2025 13:47:24 +0100 Subject: [PATCH 169/250] [internal/database] GetFileInfo needs to be able to handle null values --- sda/internal/database/db_functions.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 3073c2c820..070a0446b4 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -572,9 +572,12 @@ func (dbs *SDAdb) getFileInfo(id string) (FileInfo, error) { return FileInfo{}, err } - if err := db.QueryRow(checkSum, id).Scan(&info.Checksum, &info.DecryptedChecksum); err != nil { + var checksum, decryptedChecksum sql.NullString + if err := db.QueryRow(checkSum, id).Scan(&checksum, &decryptedChecksum); err != nil { return FileInfo{}, err } + info.Checksum = checksum.String + info.DecryptedChecksum = decryptedChecksum.String return info, nil } From 4951e97e89eec2b595ffdb95e59e7f1fec6926be Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 27 Feb 2025 14:17:16 +0100 Subject: [PATCH 170/250] [ingest_test] ingest verified file --- sda/cmd/ingest/ingest_test.go | 161 +++++++++++++++++++++++++++++++++- 1 file changed, 160 insertions(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 8b34de039c..753d4fe6f5 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -549,4 +549,163 @@ func (suite *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { } assert.Equal(suite.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) -} \ No newline at end of file +} +func (suite *TestSuite) TestIngestFile_reingestVerifiedFile() { + // prepare the DB entries + UserName := "test-ingest" + fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + // fake file verification + sha256hash := sha256.New() + var fi database.FileInfo + fi.Checksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedSize = 10 * 1024 * 1024 + fi.Size = (10 * 1024 * 1024) + 456 + if err := suite.ingest.DB.SetVerified(fi, fileID, corrID); err != nil { + suite.Fail("failed to mark file as verified") + } + + assert.Equal(suite.T(), "reject", suite.ingest.ingestFile(corrID, message)) +} +func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { + // prepare the DB entries + UserName := "test-ingest" + fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + // fake file verification + sha256hash := sha256.New() + var fi database.FileInfo + fi.Checksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedSize = 10 * 1024 * 1024 + fi.Size = (10 * 1024 * 1024) + 456 + if err := suite.ingest.DB.SetVerified(fi, fileID, corrID); err != nil { + suite.Fail("failed to mark file as verified") + } + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) +} +func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { + // prepare the DB entries + UserName := "test-ingest" + fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) + assert.NoError(suite.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: suite.filePath, + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + var firstDbChecksum string + const q1 = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" + if err := suite.ingest.DB.DB.QueryRow(q1, fileID).Scan(&firstDbChecksum); err != nil { + suite.FailNow("failed to get checksum from database") + } + + // fake file verification + verifiedSha256 := sha256.New() + var fi database.FileInfo + fi.ArchiveChecksum = hex.EncodeToString(verifiedSha256.Sum(nil)) + fi.DecryptedChecksum = hex.EncodeToString(verifiedSha256.Sum(nil)) + fi.DecryptedSize = 10 * 1024 * 1024 + fi.Size = (10 * 1024 * 1024) + 456 + if err := suite.ingest.DB.SetVerified(fi, fileID, corrID); err != nil { + suite.Fail("failed to mark file as verified") + } + + if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + suite.Fail("failed to update file event log") + } + + // over write the encrypted file to generate new checksum + f, err := os.CreateTemp(suite.ingest.Conf.Inbox.Posix.Location, "") + if err != nil { + suite.FailNow("failed to create test file") + } + defer f.Close() + + _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + suite.FailNow("failed to write data to test file") + } + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + suite.FailNow("failed to create private c4gh key") + } + + outFile, err := os.Create(path.Join(suite.ingest.Conf.Inbox.Posix.Location, suite.filePath)) + if err != nil { + suite.FailNowf("failed to create encrypted test file: %s", err.Error()) + } + defer outFile.Close() + + sha256hash := sha256.New() + mr := io.MultiWriter(outFile, sha256hash) + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, suite.pubKeyList, nil) + if err != nil { + suite.FailNowf("failed to create c4gh writer: %s", err.Error()) + } + + _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + suite.FailNow("failed to write data to encrypted test file") + } + crypt4GHWriter.Close() + + // reingestion should work + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + + // DB should have the new checksum + var dbChecksum string + const q = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" + if err := suite.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { + suite.FailNow("failed to get checksum from database") + } + + assert.Equal(suite.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) + + assert.NotEqual(suite.T(), dbChecksum, firstDbChecksum) +} From 945e3b7b5e3782ba7a12379670283a1e22260f45 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Mar 2025 10:42:35 +0100 Subject: [PATCH 171/250] [intetrnal/database] expand file info struct with all checksums --- sda/internal/database/database.go | 3 ++- sda/internal/database/db_functions.go | 17 ++++++++++------- sda/internal/database/db_functions_test.go | 20 ++++++++++++-------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 4f8f3685eb..8e016f27f8 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -32,11 +32,12 @@ type SDAdb struct { // FileInfo is used by ingest for file metadata (path, size, checksum) type FileInfo struct { - Checksum string + ArchiveChecksum string Size int64 Path string DecryptedChecksum string DecryptedSize int64 + UploadedChecksum string } type SyncData struct { diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 070a0446b4..2abd874afe 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -224,11 +224,11 @@ func (dbs *SDAdb) setArchived(file FileInfo, fileID string) error { return fmt.Errorf("setArchived error: %s", err.Error()) } - log.Debugf("checksum: %s", file.Checksum) + log.Debugf("checksum: %s", file.UploadedChecksum) const addChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source)" + "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" - if _, err := db.Exec(addChecksum, fileID, file.Checksum, "SHA256"); err != nil { + if _, err := db.Exec(addChecksum, fileID, file.UploadedChecksum, "SHA256"); err != nil { return fmt.Errorf("addChecksum error: %s", err.Error()) } @@ -319,7 +319,7 @@ func (dbs *SDAdb) setVerified(file FileInfo, fileID, corrID string) error { _, err := db.Exec(completed, fileID, corrID, - file.Checksum, + file.ArchiveChecksum, "SHA256", file.DecryptedSize, file.DecryptedChecksum, @@ -566,18 +566,21 @@ func (dbs *SDAdb) getFileInfo(id string) (FileInfo, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB const getFileID = "SELECT archive_file_path, archive_file_size from sda.files where id = $1;" - const checkSum = "SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted from sda.checksums where file_id = $1;" + const checkSum = "SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, " + + "MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted, " + + "MAX(checksum) FILTER(where source = 'UPLOADED') as Uploaded from sda.checksums where file_id = $1;" var info FileInfo if err := db.QueryRow(getFileID, id).Scan(&info.Path, &info.Size); err != nil { return FileInfo{}, err } - var checksum, decryptedChecksum sql.NullString - if err := db.QueryRow(checkSum, id).Scan(&checksum, &decryptedChecksum); err != nil { + var archivedChecksum, decryptedChecksum, uploadedChecksum sql.NullString + if err := db.QueryRow(checkSum, id).Scan(&archivedChecksum, &decryptedChecksum, &uploadedChecksum); err != nil { return FileInfo{}, err } - info.Checksum = checksum.String + info.ArchiveChecksum = archivedChecksum.String info.DecryptedChecksum = decryptedChecksum.String + info.UploadedChecksum = uploadedChecksum.String return info, nil } diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index e791aa8aeb..ef05d75954 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -171,7 +171,7 @@ func (suite *DatabaseTests) TestGetArchived() { fileID, err := db.RegisterFile("/testuser/TestGetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987} + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") @@ -191,7 +191,7 @@ func (suite *DatabaseTests) TestSetAccessionID() { // register a file in the database fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987} + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") @@ -209,7 +209,7 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { // register a file in the database fileID, err := db.RegisterFile("/testuser/TestCheckAccessionIDExists.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987} + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") @@ -244,7 +244,7 @@ func (suite *DatabaseTests) TestGetFileInfo() { _, err = decSha.Write([]byte("DecryptedChecksum")) assert.NoError(suite.T(), err) - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/tmp/TestGetFileInfo.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/tmp/TestGetFileInfo.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") @@ -255,7 +255,7 @@ func (suite *DatabaseTests) TestGetFileInfo() { assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) assert.Equal(suite.T(), int64(2000), info.Size) assert.Equal(suite.T(), "/tmp/TestGetFileInfo.c4gh", info.Path) - assert.Equal(suite.T(), "11c94bc7fb13afeb2b3fb16c1dbe9206dc09560f1b31420f2d46210ca4ded0a8", info.Checksum) + assert.Equal(suite.T(), "11c94bc7fb13afeb2b3fb16c1dbe9206dc09560f1b31420f2d46210ca4ded0a8", info.ArchiveChecksum) assert.Equal(suite.T(), "a671218c2418aa51adf97e33c5c91a720289ba3c9fd0d36f6f4bf9610730749f", info.DecryptedChecksum) } @@ -380,7 +380,7 @@ func (suite *DatabaseTests) TestGetSyncData() { assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/tmp/TestGetGetSyncData.c4gh", checksum, 999} + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/tmp/TestGetGetSyncData.c4gh", checksum, 999, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") @@ -666,6 +666,7 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } err = db.SetArchived(fileInfo, fileID) if err != nil { @@ -850,6 +851,7 @@ func (suite *DatabaseTests) TestListDatasets() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } err = db.SetArchived(fileInfo, fileID) if err != nil { @@ -933,6 +935,7 @@ func (suite *DatabaseTests) TestListUserDatasets() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } err = db.SetArchived(fileInfo, fileID) if err != nil { @@ -1067,7 +1070,7 @@ func (suite *DatabaseTests) TestGetReVerificationData() { suite.FailNow("failed to generate checksum") } - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") @@ -1106,7 +1109,7 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { suite.FailNow("failed to generate checksum") } - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} corrID := uuid.New().String() if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") @@ -1188,6 +1191,7 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } err = db.SetArchived(fileInfo, fileID) if err != nil { From ff364a21cff5f37cb159ab1b9032d310ca8bdf09 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Mar 2025 10:44:01 +0100 Subject: [PATCH 172/250] [ingest] use proper checksum names --- sda/cmd/ingest/ingest.go | 2 +- sda/cmd/ingest/ingest_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index b4e94b7624..51cbe494f6 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -253,7 +253,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig FileID: fileID, ArchivePath: fileInfo.Path, EncryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fileInfo.Checksum}, + {Type: "sha256", Value: fileInfo.ArchiveChecksum}, }, } archivedMsg, _ := json.Marshal(&msg) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 753d4fe6f5..776ca9818b 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -572,7 +572,7 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedFile() { // fake file verification sha256hash := sha256.New() var fi database.FileInfo - fi.Checksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.ArchiveChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 @@ -604,7 +604,7 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // fake file verification sha256hash := sha256.New() var fi database.FileInfo - fi.Checksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.ArchiveChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 From c457cb22c1a4fa268f942078c821ffccea72d037 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Mar 2025 10:45:07 +0100 Subject: [PATCH 173/250] [ingest] calculate checksum if file is cancelled. --- sda/cmd/ingest/ingest.go | 39 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 51cbe494f6..bd92fa34fe 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -245,8 +245,45 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // What if the file in the inbox is different this time? // Check uploaded checksum in the DB against the checksum of the file. // Would be easy if the inbox message had the checksum or that the s3inbox added the checksum to the DB. + file, err := app.Inbox.NewFileReader(message.FilePath) + if err != nil { + switch { + case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): + log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { + log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) + } + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to open file to ingest", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + return "reject" + } + + return "ack" + default: + return "nack" + } + } + defer file.Close() + + inboxChecksum := sha256.New() + _, err = io.Copy(inboxChecksum, file) + if err != nil { + log.Errorf("failed to calculate cheksum for file: %s, reason: %s", fileID, err.Error()) + + return "nack" + } - if fileInfo.Checksum != "" { + if fileInfo.UploadedChecksum == string(inboxChecksum.Sum(nil)) && fileInfo.ArchiveChecksum != "" { msg := schema.IngestionVerification{ User: message.User, FilePath: message.FilePath, From 7de98c85493e47f78d7a86d43cd96ed62cff5dd7 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Mar 2025 11:38:29 +0100 Subject: [PATCH 174/250] [api_test] update to use new checksum names --- sda/cmd/api/api_test.go | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 2ee2e8a764..c9c273e50a 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -411,12 +411,13 @@ func (suite *TestSuite) SetupTest() { assert.NoError(suite.T(), err) Conf.Broker = broker.MQConf{ - Host: "localhost", - Port: mqPort, - User: "guest", - Password: "guest", - Exchange: "sda", - Vhost: "/sda", + Host: "localhost", + Port: mqPort, + User: "guest", + Password: "guest", + Exchange: "sda", + Vhost: "/sda", + SchemasPath: "../../schemas/isolated", } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) assert.NoError(suite.T(), err) @@ -851,7 +852,7 @@ func (suite *TestSuite) TestSetAccession() { assert.NoError(suite.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), @@ -1005,7 +1006,7 @@ func (suite *TestSuite) TestCreateDataset() { assert.NoError(suite.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), @@ -1084,7 +1085,7 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() { assert.NoError(suite.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), @@ -1206,7 +1207,7 @@ func (suite *TestSuite) TestCreateDataset_WrongUser() { assert.NoError(suite.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), @@ -1968,11 +1969,12 @@ func (suite *TestSuite) TestReVerifyFile() { } fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), - Size: 1000, - Path: filePath, + ArchiveChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, + Path: filePath, + Size: 1000, + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as Archived") @@ -1993,7 +1995,6 @@ func (suite *TestSuite) TestReVerifyFile() { gin.SetMode(gin.ReleaseMode) assert.NoError(suite.T(), setupJwtAuth()) - Conf.Broker.SchemasPath = "../../schemas/isolated" // Mock request and response holders w := httptest.NewRecorder() @@ -2070,11 +2071,12 @@ func (suite *TestSuite) TestReVerifyDataset() { } fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), - Size: 1000, - Path: filePath, + ArchiveChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, + Path: filePath, + Size: 1000, + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as Archived") From 17fd6900769751a3cb998701c372d6e2fa8b6059 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 18 Mar 2025 11:39:21 +0100 Subject: [PATCH 175/250] [verify] use full checksum names --- sda/cmd/verify/verify.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 9651eea3b5..82b623a71c 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -257,7 +257,7 @@ func main() { // At this point we should do checksum comparison - file.Checksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) + file.ArchiveChecksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) file.DecryptedChecksum = fmt.Sprintf("%x", sha256hash.Sum(nil)) switch { @@ -289,8 +289,8 @@ func main() { continue } - if file.Checksum != message.EncryptedChecksums[0].Value { - log.Errorf("encrypted checksum don't match for file: %s, expected %s, got %s", message.FilePath, message.EncryptedChecksums[0].Value, file.Checksum) + if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { + log.Errorf("encrypted checksum don't match for file: %s, expected %s, got %s", message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { log.Errorf("set status ready failed, reason: (%v)", err) if err := delivered.Nack(false, true); err != nil { From 476ee166e5452cd280cebba532e84b0d0f91c427 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 31 Mar 2025 10:56:05 +0200 Subject: [PATCH 176/250] [sync_test] use full checksum names --- sda/cmd/sync/sync_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index ef8d3518cd..91a0cecdd9 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -165,7 +165,7 @@ func (suite *SyncTest) TestBuildSyncDatasetJSON() { assert.NoError(suite.T(), err) checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := database.FileInfo{Checksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} + fileInfo := database.FileInfo{ArchiveChecksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} corrID := uuid.New().String() err = db.SetArchived(fileInfo, fileID) From 04899d7aa689686db7d46650dc0a07263d92b3fc Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 31 Mar 2025 12:39:47 +0200 Subject: [PATCH 177/250] [ingest] app struct needs to be declared inside of the main function --- sda/cmd/ingest/ingest.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index bd92fa34fe..c6e8230215 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -37,7 +37,8 @@ type Ingest struct { MQ *broker.AMQPBroker } -func (app *Ingest) main() { +func main() { + app := Ingest{} var err error sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) From efb7d1f4ad27119bb4b6212a434ccfd73e8fd642 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 1 Apr 2025 09:21:20 +0200 Subject: [PATCH 178/250] [ingest] improve error messges --- sda/cmd/ingest/ingest.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index c6e8230215..fecc7eb3db 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -186,7 +186,7 @@ func main() { log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) } default: - log.Errorln("unexpected ingest message type") + log.Errorln("better message needed") if err := delivered.Reject(false); err != nil { log.Errorf("failed to reject message for reason: (%s)", err.Error()) } @@ -319,6 +319,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } case "": // Catch all for inboxes that doesn't update the DB + log.Warnln("ingesting unregisterd file") fileID, err = app.DB.RegisterFile(message.FilePath, message.User) if err != nil { log.Errorf("InsertFile failed, reason: (%s)", err.Error()) @@ -333,13 +334,15 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "nack" } default: + log.Warnf("unsupported file status: %s", status) + return "reject" } file, err := app.Inbox.NewFileReader(message.FilePath) if err != nil { switch { - case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): + case (strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:")): log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) @@ -361,6 +364,8 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "ack" default: + log.Errorf("unexpected eror when opening file for reading: %s", err.Error()) + return "nack" } } From 51926cf304b0b36404e3321e2982f75b1cea86c1 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 1 Apr 2025 09:21:54 +0200 Subject: [PATCH 179/250] [ingest_test] add test for missing file in the inbox --- sda/cmd/ingest/ingest_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 776ca9818b..581561c898 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -709,3 +709,17 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum( assert.NotEqual(suite.T(), dbChecksum, firstDbChecksum) } +func (suite *TestSuite) TestIngestFile_missingFile() { + // prepare the DB entries + UserName := "test-ingest" + corrID := uuid.New().String() + basepath := filepath.Dir(suite.filePath) + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: fmt.Sprintf("%s/missing.file.c4gh", basepath), + User: UserName, + } + + assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) +} From 0156cf3dd43dc22b487029c63ba3cbe355703389 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 1 Apr 2025 09:34:56 +0200 Subject: [PATCH 180/250] [verify] move SetVerified from SQL to GO --- sda/cmd/verify/verify.go | 11 ++++- sda/internal/database/db_functions.go | 37 ++++++++++------- sda/internal/database/db_functions_test.go | 47 +++++++++++----------- 3 files changed, 56 insertions(+), 39 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 82b623a71c..549b49a2c7 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -383,7 +383,7 @@ func main() { } } - if err := db.SetVerified(file, message.FileID, delivered.CorrelationId); err != nil { + if err := db.SetVerified(file, message.FileID); err != nil { log.Errorf("SetVerified failed, reason: (%s)", err.Error()) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) @@ -392,6 +392,15 @@ func main() { continue } + if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { + log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + } + + continue + } + // Send message to verified queue if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { // TODO fix resend mechanism diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 2abd874afe..d7264fb18c 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -298,35 +298,42 @@ func (dbs *SDAdb) getHeader(fileID string) ([]byte, error) { } // MarkCompleted marks the file as "COMPLETED" -func (dbs *SDAdb) SetVerified(file FileInfo, fileID, corrID string) error { +func (dbs *SDAdb) SetVerified(file FileInfo, fileID string) error { var ( err error count int ) for count == 0 || (err != nil && count < RetryTimes) { - err = dbs.setVerified(file, fileID, corrID) + err = dbs.setVerified(file, fileID) count++ } return err } -func (dbs *SDAdb) setVerified(file FileInfo, fileID, corrID string) error { +func (dbs *SDAdb) setVerified(file FileInfo, fileID string) error { dbs.checkAndReconnectIfNeeded() - db := dbs.DB - const completed = "SELECT sda.set_verified($1, $2, $3, $4, $5, $6, $7);" - _, err := db.Exec(completed, - fileID, - corrID, - file.ArchiveChecksum, - "SHA256", - file.DecryptedSize, - file.DecryptedChecksum, - "SHA256", - ) + const verified = "UPDATE sda.files SET decrypted_file_size = $1 WHERE id = $2;" + if _, err := dbs.DB.Exec(verified, file.DecryptedSize, fileID); err != nil { + return fmt.Errorf("setVerified error: %s", err.Error()) + } - return err + const addArchiveChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + + "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source)" + + "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + if _, err := dbs.DB.Exec(addArchiveChecksum, fileID, file.ArchiveChecksum, "SHA256"); err != nil { + return fmt.Errorf("addArchiveChecksum error: %s", err.Error()) + } + + const addUnencryptedChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + + "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source)" + + "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + if _, err := dbs.DB.Exec(addUnencryptedChecksum, fileID, file.DecryptedChecksum, "SHA256"); err != nil { + return fmt.Errorf("addUnencryptedChecksum error: %s", err.Error()) + } + + return nil } // GetArchived retrieves the location and size of archive diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index ef05d75954..efca0f43fb 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -157,9 +157,11 @@ func (suite *DatabaseTests) TestSetVerified() { fileID, err := db.RegisterFile("/testuser/TestSetVerified.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - corrID := uuid.New().String() fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/testuser/TestSetVerified.c4gh", fmt.Sprintf("%x", sha256.New()), 948, fmt.Sprintf("%x", sha256.New())} - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) + assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) } @@ -172,10 +174,10 @@ func (suite *DatabaseTests) TestGetArchived() { assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) filePath, fileSize, err := db.GetArchived(fileID) @@ -192,10 +194,10 @@ func (suite *DatabaseTests) TestSetAccessionID() { fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) stableID := "TEST:000-1234-4567" err = db.SetAccessionID(stableID, fileID) @@ -210,10 +212,10 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { fileID, err := db.RegisterFile("/testuser/TestCheckAccessionIDExists.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) stableID := "TEST:111-1234-4567" err = db.SetAccessionID(stableID, fileID) @@ -245,10 +247,10 @@ func (suite *DatabaseTests) TestGetFileInfo() { assert.NoError(suite.T(), err) fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/tmp/TestGetFileInfo.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) info, err := db.GetFileInfo(fileID) @@ -381,11 +383,11 @@ func (suite *DatabaseTests) TestGetSyncData() { checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/tmp/TestGetGetSyncData.c4gh", checksum, 999, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Verified") stableID := "TEST:000-1111-2222" @@ -609,7 +611,7 @@ func (suite *DatabaseTests) TestListActiveUsers() { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -673,7 +675,7 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -858,7 +860,7 @@ func (suite *DatabaseTests) TestListDatasets() { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -942,7 +944,7 @@ func (suite *DatabaseTests) TestListUserDatasets() { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -1071,11 +1073,10 @@ func (suite *DatabaseTests) TestGetReVerificationData() { } fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } - if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { + if err = db.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as verified") } accession := "acession-001" @@ -1110,11 +1111,11 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { } fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } - if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { + if err = db.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as verified") } accession := "acession-001" @@ -1149,11 +1150,11 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { } fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetDecryptedChecksum.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} - corrID := uuid.New().String() + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } - if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { + if err = db.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as verified") } @@ -1198,7 +1199,7 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } From ed88d21d3cc0cb6b122dd2987a52e84b49241838 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 1 Apr 2025 09:35:16 +0200 Subject: [PATCH 181/250] [tests] fix SetVerified --- sda/cmd/api/api_test.go | 12 ++++++------ sda/cmd/ingest/ingest_test.go | 6 +++--- sda/cmd/sync/sync_test.go | 4 +--- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index c9c273e50a..606ff59c6d 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -861,7 +861,7 @@ func (suite *TestSuite) TestSetAccession() { err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) + err = Conf.API.DB.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) gin.SetMode(gin.ReleaseMode) @@ -1015,7 +1015,7 @@ func (suite *TestSuite) TestCreateDataset() { err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) + err = Conf.API.DB.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) @@ -1094,7 +1094,7 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() { err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) + err = Conf.API.DB.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) @@ -1216,7 +1216,7 @@ func (suite *TestSuite) TestCreateDataset_WrongUser() { err = Conf.API.DB.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) + err = Conf.API.DB.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) @@ -1980,7 +1980,7 @@ func (suite *TestSuite) TestReVerifyFile() { suite.FailNow("failed to mark file as Archived") } - if err := Conf.API.DB.SetVerified(fileInfo, fileID, fileID); err != nil { + if err := Conf.API.DB.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as Verified") } @@ -2082,7 +2082,7 @@ func (suite *TestSuite) TestReVerifyDataset() { suite.FailNow("failed to mark file as Archived") } - if err := Conf.API.DB.SetVerified(fileInfo, fileID, fileID); err != nil { + if err := Conf.API.DB.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as Verified") } diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 581561c898..af1dbc8c75 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -576,7 +576,7 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedFile() { fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 - if err := suite.ingest.DB.SetVerified(fi, fileID, corrID); err != nil { + if err := suite.ingest.DB.SetVerified(fi, fileID); err != nil { suite.Fail("failed to mark file as verified") } @@ -608,7 +608,7 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 - if err := suite.ingest.DB.SetVerified(fi, fileID, corrID); err != nil { + if err := suite.ingest.DB.SetVerified(fi, fileID); err != nil { suite.Fail("failed to mark file as verified") } @@ -650,7 +650,7 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum( fi.DecryptedChecksum = hex.EncodeToString(verifiedSha256.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 - if err := suite.ingest.DB.SetVerified(fi, fileID, corrID); err != nil { + if err := suite.ingest.DB.SetVerified(fi, fileID); err != nil { suite.Fail("failed to mark file as verified") } diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index 91a0cecdd9..ef2ba65735 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -14,7 +14,6 @@ import ( "testing" "time" - "github.com/google/uuid" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/ory/dockertest/v3" @@ -166,11 +165,10 @@ func (suite *SyncTest) TestBuildSyncDatasetJSON() { checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := database.FileInfo{ArchiveChecksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} - corrID := uuid.New().String() err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Verified") accessions := []string{"ed6af454-d910-49e3-8cda-488a6f246e67"} From cb574c4ef2ba5b59700cb7b55515263d90fb73c4 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 1 Apr 2025 09:23:33 +0200 Subject: [PATCH 182/250] [integration_tests] Ensure messages correlation ids are unique --- .../integration/tests/sda/92_handle_file_errors.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/integration/tests/sda/92_handle_file_errors.sh b/.github/integration/tests/sda/92_handle_file_errors.sh index f715ebc90e..bc28ef3f35 100644 --- a/.github/integration/tests/sda/92_handle_file_errors.sh +++ b/.github/integration/tests/sda/92_handle_file_errors.sh @@ -40,7 +40,7 @@ encrypted_checksums=$( --arg md5 "$(echo "aa" | md5sum | cut -d ' ' -f1)" \ '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))' ) - + bad_file_payload=$( jq -r -c -n \ --arg type ingest \ @@ -72,7 +72,6 @@ if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/ exit 1 fi - missing_file_payload=$( jq -r -c -n \ --arg type ingest \ @@ -82,6 +81,16 @@ missing_file_payload=$( '$ARGS.named|@base64' ) +CORRID=${CORRID//a/b} +properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg correlation_id "$CORRID" \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' +) + missing_file=$( jq -c -n \ --arg vhost sda \ @@ -99,7 +108,6 @@ curl -s -u guest:guest 'http://rabbitmq:15672/api/exchanges/sda/sda/publish' \ sleep 10 - if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/error_stream | jq '.messages_ready')" ]; then echo "missing file not moved to error" exit 1 From 9a97917b1a3b3c705d90569a1c985d5fc90210d2 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 8 Apr 2025 08:34:40 +0200 Subject: [PATCH 183/250] Apply suggestions from code review Co-authored-by: Nanjiang Shu --- sda/cmd/ingest/ingest.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index fecc7eb3db..c184ecaf64 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -186,7 +186,6 @@ func main() { log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) } default: - log.Errorln("better message needed") if err := delivered.Reject(false); err != nil { log.Errorf("failed to reject message for reason: (%s)", err.Error()) } @@ -200,7 +199,7 @@ func main() { func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { fileUUID, err := app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file from message: %s, reason: %s", correlationID, err.Error()) + log.Errorf("failed to get ID for file from message (correlationID: %s), reason: %s", correlationID, err.Error()) if strings.Contains(err.Error(), "sql: no rows in result set") { return "reject" } From 4f1afb0cac7ec8e0fc5a7f93290b71d715076727 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 8 Apr 2025 12:49:26 +0200 Subject: [PATCH 184/250] Apply suggestions from code review Co-authored-by: Malin Klang --- sda/cmd/ingest/ingest.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index c184ecaf64..0c436ee6d5 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -186,6 +186,7 @@ func main() { log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) } default: + // will catch `reject`s, failures that should not be requeued. if err := delivered.Reject(false); err != nil { log.Errorf("failed to reject message for reason: (%s)", err.Error()) } @@ -401,7 +402,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig for bytesRead < fileSize { i, _ := io.ReadFull(file, readBuffer) if i == 0 { - log.Debugln("this should not happen") + log.Errorln("readBuffer returned 0 bytes, this should not happen") return "reject" } From 3758fe975b91c305f229df9986499ab070330354 Mon Sep 17 00:00:00 2001 From: Parisa Date: Tue, 8 Apr 2025 14:28:28 +0200 Subject: [PATCH 185/250] update compose volume mounts for sda-doa src and pom.xml --- .github/integration/sda-doa-posix-outbox.yml | 3 ++- .github/integration/sda-doa-s3-outbox.yml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/integration/sda-doa-posix-outbox.yml b/.github/integration/sda-doa-posix-outbox.yml index 164a69615a..9fba0fb7e2 100644 --- a/.github/integration/sda-doa-posix-outbox.yml +++ b/.github/integration/sda-doa-posix-outbox.yml @@ -128,7 +128,8 @@ services: image: maven:3.9.9-eclipse-temurin-21 profiles: [test] volumes: - - ../../sda-doa:/sda-doa + - ../../sda-doa/src:/sda-doa/src + - ../../sda-doa/pom.xml:/sda-doa/pom.xml - ../../sda-doa/settings.xml:/root/.m2/settings.xml - test_file:/sda-doa/outbox - ./tests:/tests diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml index 888d9105cd..ada7422362 100644 --- a/.github/integration/sda-doa-s3-outbox.yml +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -157,7 +157,8 @@ services: image: maven:3.9.9-eclipse-temurin-21 profiles: [test] volumes: - - ../../sda-doa:/sda-doa + - ../../sda-doa/src:/sda-doa/src + - ../../sda-doa/pom.xml:/sda-doa/pom.xml - ../../sda-doa/settings.xml:/root/.m2/settings.xml - ./tests:/tests - encryption_files:/test From 6383ce5e73bd24b67ee9d1820fe5de1fadb6c76c Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 7 Apr 2025 13:50:13 +0200 Subject: [PATCH 186/250] [s3inbox] add function to get size of object from backend --- sda/cmd/s3inbox/proxy.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 3ac9cc987a..1db4b8fb96 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -16,6 +16,8 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/minio/minio-go/v6/pkg/signer" @@ -624,3 +626,28 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { log.Error(err) } } + +func (p *Proxy) storeObjectSizeInDB(path, fileID string) error { + s3cfg, _ := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(p.s3.AccessKey, p.s3.SecretKey, ""))) + client := s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(p.s3.URL, "http:") + o.Region = p.s3.Region + o.UsePathStyle = true + }, + ) + o, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: &p.s3.Bucket, + Key: &path, + }) + if err != nil { + return err + } + + const setObjectSize = "UPDATE sda.files set submission_file_size = $1 where id = $2;" + _, err = p.database.DB.Exec(setObjectSize, *o.ContentLength, fileID) + + return err +} From a6f98cf9a69ba4e728aa3015d83a8b1b3c89a468 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 7 Apr 2025 13:50:58 +0200 Subject: [PATCH 187/250] [s3inbox_tests] Add tests for get object size function --- sda/cmd/s3inbox/proxy_test.go | 65 +++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 0eef37afbe..315f2fbefa 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -614,3 +614,68 @@ func (suite *ProxyTests) TestCheckFileExists_unresponsive() { assert.NotNil(suite.T(), err) assert.Contains(suite.T(), err.Error(), "StatusCode: 403") } + +func (suite *ProxyTests) TestStoreObjectSizeInDB() { + db, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + defer db.Close() + + mq, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer mq.Connection.Close() + + p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/dummy/file", "test-user") + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), fileID) + + assert.NoError(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + + const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" + var objectSize int64 + assert.NoError(suite.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) + assert.Equal(suite.T(), int64(14), objectSize) +} + +func (suite *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { + db, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + + mq, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer mq.Connection.Close() + + p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/dummy/file", "test-user") + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), fileID) + + db.Close() + assert.NoError(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) +} + +func (suite *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { + db, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + defer db.Close() + mq, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer mq.Connection.Close() + + p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/dummy/file", "test-user") + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), fileID) + + p.s3.AccessKey = "badKey" + assert.Error(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + + p.s3.Port = 1234 + assert.Error(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) +} From 2158387f012d4a2b62e98273ef98ec668b395c5a Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 7 Apr 2025 13:51:40 +0200 Subject: [PATCH 188/250] [s3inbox] get object size, handle DB reconnects --- sda/cmd/s3inbox/proxy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 1db4b8fb96..863a06d64c 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -646,6 +646,11 @@ func (p *Proxy) storeObjectSizeInDB(path, fileID string) error { return err } + if err := p.database.DB.Ping(); err != nil { + p.database.Close() + _ = p.database.Connect() + } + const setObjectSize = "UPDATE sda.files set submission_file_size = $1 where id = $2;" _, err = p.database.DB.Exec(setObjectSize, *o.ContentLength, fileID) From 30aba3a369ae5a472390d0f021143c3b60e01a83 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 8 Apr 2025 07:59:46 +0200 Subject: [PATCH 189/250] [s3inbox_test] Add test for rapid check of file size Upload a file and directly check the object size. --- sda/cmd/s3inbox/proxy_test.go | 47 +++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 315f2fbefa..d5cfc4656c 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -679,3 +679,50 @@ func (suite *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { p.s3.Port = 1234 assert.Error(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) } + +func (suite *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { + db, err := database.NewSDAdb(suite.DBConf) + assert.NoError(suite.T(), err) + defer db.Close() + + mq, err := broker.NewMQ(suite.MQConf) + assert.NoError(suite.T(), err) + defer mq.Connection.Close() + + p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/test/new_file", "test-user") + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), fileID) + + s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(suite.S3conf.AccessKey, suite.S3conf.SecretKey, ""))) + if err != nil { + suite.FailNow("bad") + } + s3Client := s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", suite.S3conf.URL, suite.S3conf.Port)) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(suite.S3conf.URL, "http:") + o.Region = suite.S3conf.Region + o.UsePathStyle = true + }, + ) + + output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader(strings.Repeat("A", 10*1024*1024)), + Bucket: aws.String(suite.S3conf.Bucket), + Key: aws.String("/test/new_file"), + ContentEncoding: aws.String("application/octet-stream"), + }) + assert.NoError(suite.T(), err) + assert.NotNil(suite.T(), output, output) + + assert.NoError(suite.T(), p.storeObjectSizeInDB("/test/new_file", fileID)) + + const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" + var objectSize int64 + assert.NoError(suite.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) + assert.Equal(suite.T(), int64(10*1024*1024), objectSize) +} From e88d3b0510adf99c5379aeb370f94d9cbd11702d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Mon, 7 Apr 2025 13:54:55 +0200 Subject: [PATCH 190/250] [s3inbox] store upload size in DB on completion #minor --- sda/cmd/s3inbox/proxy.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 863a06d64c..3c35e525d0 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -227,6 +227,13 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw log.Debugf("resuming work on file with fileId: %v", p.fileIds[r.URL.Path]) } + if err := p.storeObjectSizeInDB(rawFilepath, p.fileIds[r.URL.Path]); err != nil { + log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) + p.internalServerError(w, r, "storeObjectSizeInDB failed") + + return + } + log.Debugf("marking file %v as 'uploaded' in database", p.fileIds[r.URL.Path]) err = p.database.UpdateFileEventLog(p.fileIds[r.URL.Path], "uploaded", p.fileIds[r.URL.Path], "inbox", "{}", string(jsonMessage)) if err != nil { From d053cf5bba21fce917e87d3ab4617cff2c967741 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Apr 2025 12:55:14 +0200 Subject: [PATCH 191/250] Add test to ensure 2 submitter's can upload to the same relative path --- .../integration/tests/postgres/20_inbox_queries.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/integration/tests/postgres/20_inbox_queries.sh b/.github/integration/tests/postgres/20_inbox_queries.sh index e79abfa98b..f2e8d51bee 100644 --- a/.github/integration/tests/postgres/20_inbox_queries.sh +++ b/.github/integration/tests/postgres/20_inbox_queries.sh @@ -10,6 +10,17 @@ for host in migrate postgres; do exit 1 fi + newFileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', 'other-user');") + if [ -z "$newFileID" ]; then + echo "register_file failed" + exit 1 + fi + + if [ "$fileID" = "$newFileID" ]; then + echo "File IDs should not be the same" + exit 1 + fi + resp=$(psql -U inbox -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, user_id, message) VALUES('$fileID', 'uploaded', 'test-user', '{\"uploaded\": \"message\"}');") if [ "$resp" != "INSERT 0 1" ]; then echo "Mark file as uploaded failed" From 98350ad82ebc06d80bcacfd58deabee893525e81 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Thu, 10 Apr 2025 12:56:00 +0200 Subject: [PATCH 192/250] [postgres] update `unique_ingested`constraint Add the submission user as part of the constraint --- postgresql/initdb.d/01_main.sql | 2 +- postgresql/migratedb.d/17.sql | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) create mode 100644 postgresql/migratedb.d/17.sql diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index a7cacf3ae4..fc931b49e3 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -74,7 +74,7 @@ CREATE TABLE files ( created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(), last_modified TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(), - CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path) + CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user) ); -- The user info is used by auth to be able to link users to their name and email diff --git a/postgresql/migratedb.d/17.sql b/postgresql/migratedb.d/17.sql new file mode 100644 index 0000000000..6ac20f256c --- /dev/null +++ b/postgresql/migratedb.d/17.sql @@ -0,0 +1,21 @@ +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 16; + changes VARCHAR := 'Add submission user to constraint'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + ALTER TABLE sda.files DROP CONSTRAINT unique_ingested; + ALTER TABLE sda.files ADD CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user); + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ \ No newline at end of file From 124810104a405c614db91cc197db78392313ecce Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 11 Apr 2025 11:25:24 +0200 Subject: [PATCH 193/250] [s3inbox] Use Predefined S3 client --- sda/cmd/s3inbox/proxy.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 3c35e525d0..d1b73bf9a0 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -16,8 +16,6 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/aws" - "github.com/aws/aws-sdk-go-v2/config" - "github.com/aws/aws-sdk-go-v2/credentials" "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lestrrat-go/jwx/v2/jwt" "github.com/minio/minio-go/v6/pkg/signer" @@ -635,16 +633,11 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { } func (p *Proxy) storeObjectSizeInDB(path, fileID string) error { - s3cfg, _ := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(p.s3.AccessKey, p.s3.SecretKey, ""))) - client := s3.NewFromConfig( - s3cfg, - func(o *s3.Options) { - o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", p.s3.URL, p.s3.Port)) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(p.s3.URL, "http:") - o.Region = p.s3.Region - o.UsePathStyle = true - }, - ) + client, err := storage.NewS3Client(p.s3) + if err != nil { + return err + } + o, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ Bucket: &p.s3.Bucket, Key: &path, From 153fa6799f8bdfb3c50500c9f06ff8a74b194281 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Fri, 11 Apr 2025 11:27:21 +0200 Subject: [PATCH 194/250] [s3inbox test] Add comments to S3 backend test --- sda/cmd/s3inbox/proxy_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index d5cfc4656c..3aa5ed932c 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -673,13 +673,17 @@ func (suite *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { assert.NoError(suite.T(), err) assert.NotNil(suite.T(), fileID) + // Detect autentication failure p.s3.AccessKey = "badKey" assert.Error(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + // Detect unresponsive backend service p.s3.Port = 1234 assert.Error(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) } +// This test is intended to try to catch some issues we sometimes see when a query to the S3 backend +// happens to fast so that it is not ready and returns a false 404. func (suite *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { db, err := database.NewSDAdb(suite.DBConf) assert.NoError(suite.T(), err) @@ -723,6 +727,7 @@ func (suite *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" var objectSize int64 + // If the S3 backend haven't had the time to process the request above correctly this will generate an error. assert.NoError(suite.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) assert.Equal(suite.T(), int64(10*1024*1024), objectSize) } From 130150cddb35d5011e2a3edf9472465a30302b92 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 18:58:34 +0000 Subject: [PATCH 195/250] Bump golang.org/x/crypto Bumps the go_modules group with 1 update in the /sda-admin directory: [golang.org/x/crypto](https://github.com/golang/crypto). Updates `golang.org/x/crypto` from 0.31.0 to 0.35.0 - [Commits](https://github.com/golang/crypto/compare/v0.31.0...v0.35.0) --- updated-dependencies: - dependency-name: golang.org/x/crypto dependency-version: 0.35.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-admin/go.mod | 4 ++-- sda-admin/go.sum | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/sda-admin/go.mod b/sda-admin/go.mod index 92f89c85de..bb272c942d 100644 --- a/sda-admin/go.mod +++ b/sda-admin/go.mod @@ -14,7 +14,7 @@ require ( github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-admin/go.sum b/sda-admin/go.sum index 1ed0c599f2..b244f10f24 100644 --- a/sda-admin/go.sum +++ b/sda-admin/go.sum @@ -14,12 +14,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= From 03f7582643c2eb28f3783a93f7c5a7303d8e41df Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 19:59:33 +0000 Subject: [PATCH 196/250] Bump the all-modules group in /sda-doa with 2 updates Bumps the all-modules group in /sda-doa with 2 updates: [com.google.code.gson:gson](https://github.com/google/gson) and [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `com.google.code.gson:gson` from 2.12.1 to 2.13.0 - [Release notes](https://github.com/google/gson/releases) - [Changelog](https://github.com/google/gson/blob/main/CHANGELOG.md) - [Commits](https://github.com/google/gson/compare/gson-parent-2.12.1...gson-parent-2.13.0) Updates `no.elixir:crypt4gh` from 3.0.21 to 3.0.22 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.21...crypt4gh-3.0.22) --- updated-dependencies: - dependency-name: com.google.code.gson:gson dependency-version: 2.13.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.22 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index fef4252b1b..b5cb4bddbf 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -21,7 +21,7 @@ com.google.code.gson gson - 2.12.1 + 2.13.0 org.springframework.boot @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.21 + 3.0.22 org.slf4j From 0d11ce806bc84a36bc3bdf93afe816db7e1273a2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 20:04:32 +0000 Subject: [PATCH 197/250] Bump codecov/codecov-action from 5.4.0 to 5.4.2 Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 5.4.0 to 5.4.2. - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v5.4.0...v5.4.2) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-version: 5.4.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6bf9d664dc..c488d2a90b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.0 + uses: codecov/codecov-action@v5.4.2 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.0 + uses: codecov/codecov-action@v5.4.2 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.0 + uses: codecov/codecov-action@v5.4.2 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt From 431d2feccd9549bfdb17c507293ef71cc9eae19c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 20:12:41 +0000 Subject: [PATCH 198/250] Bump the all-modules group in /sda with 4 updates Bumps the all-modules group in /sda with 4 updates: [github.com/aws/aws-sdk-go-v2/config](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/credentials](https://github.com/aws/aws-sdk-go-v2), [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2). Updates `github.com/aws/aws-sdk-go-v2/config` from 1.29.13 to 1.29.14 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/config/v1.29.13...config/v1.29.14) Updates `github.com/aws/aws-sdk-go-v2/credentials` from 1.17.66 to 1.17.67 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/credentials/v1.17.66...credentials/v1.17.67) Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.71 to 1.17.72 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.71...feature/s3/manager/v1.17.72) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.79.1 to 1.79.2 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.79.1...service/s3/v1.79.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/config dependency-version: 1.29.14 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/credentials dependency-version: 1.17.67 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.72 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.79.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 10 +++++----- sda/go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index cbd24674d7..318bcd39e0 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -6,10 +6,10 @@ toolchain go1.24.1 require ( github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.13 - github.com/aws/aws-sdk-go-v2/credentials v1.17.66 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 - github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.104.0 github.com/coreos/go-oidc/v3 v3.14.1 @@ -63,7 +63,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect diff --git a/sda/go.sum b/sda/go.sum index 16de6a52ad..f7bb47b95f 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -28,14 +28,14 @@ github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38y github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.13 h1:RgdPqWoE8nPpIekpVpDJsBckbqT4Liiaq9f35pbTh1Y= -github.com/aws/aws-sdk-go-v2/config v1.29.13/go.mod h1:NI28qs/IOUIRhsR7GQ/JdexoqRN9tDxkIrYZq0SOF44= -github.com/aws/aws-sdk-go-v2/credentials v1.17.66 h1:aKpEKaTy6n4CEJeYI1MNj97oSDLi4xro3UzQfwf5RWE= -github.com/aws/aws-sdk-go-v2/credentials v1.17.66/go.mod h1:xQ5SusDmHb/fy55wU0QqTy0yNfLqxzec59YcsRZB+rI= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71 h1:s43gLuY+zGmtpx+KybfFP4IckopmTfDOPdlf/L++N5I= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.71/go.mod h1:KH6wWmY3O3c/jVAjHk0MGzVAFDxkOSt42Eoe4ZO4ge0= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 h1:PcKMOZfp+kNtJTw2HF2op6SjDvwPBYRvz0Y24PQLUR4= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72/go.mod h1:vq7/m7dahFXcdzWVOvvjasDI9RcsD3RsTfHmDundJYg= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -52,14 +52,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1 h1:2Ku1xwAohSSXHR1tpAnyVDSQSxoDMA+/NZBytW+f4qg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.1/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 h1:tWUG+4wZqdMl/znThEk9tcCy8tTMxq8dW0JTgamohrY= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.18 h1:xz7WvTMfSStb9Y8NpCT82FXLNC3QasqBfuAFHY4Pk5g= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.18/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= From 303ebc232433fa28196dbc9656d99638ea0425c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Apr 2025 20:20:05 +0000 Subject: [PATCH 199/250] Bump the all-modules group in /sda-sftp-inbox with 3 updates Bumps the all-modules group in /sda-sftp-inbox with 3 updates: [org.junit:junit-bom](https://github.com/junit-team/junit5), commons-io:commons-io and [com.google.guava:guava](https://github.com/google/guava). Updates `org.junit:junit-bom` from 5.12.1 to 5.12.2 - [Release notes](https://github.com/junit-team/junit5/releases) - [Commits](https://github.com/junit-team/junit5/compare/r5.12.1...r5.12.2) Updates `commons-io:commons-io` from 2.18.0 to 2.19.0 Updates `com.google.guava:guava` from 33.4.6-jre to 33.4.8-jre - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: org.junit:junit-bom dependency-version: 5.12.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: commons-io:commons-io dependency-version: 2.19.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: com.google.guava:guava dependency-version: 33.4.8-jre dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 43fd4b0067..b30137a36c 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -29,7 +29,7 @@ org.junit junit-bom - 5.12.1 + 5.12.2 pom import @@ -103,7 +103,7 @@ commons-io commons-io - 2.18.0 + 2.19.0 com.amazonaws @@ -113,7 +113,7 @@ com.google.guava guava - 33.4.6-jre + 33.4.8-jre net.logstash.logback From 1d6d752947128efa5a6a2cfd709f51c053abc5c0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Wed, 26 Mar 2025 12:24:38 +0100 Subject: [PATCH 200/250] [actions] make linter action work with v2 of golangci-lint --- .github/workflows/code-linter.yaml | 15 ++++++--------- .golangci.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 .golangci.yml diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 67fcf88850..26c3741630 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -27,10 +27,9 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.2 + uses: golangci/golangci-lint-action@v7.0.0 with: - args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107,G115 --timeout 5m - version: v1.64.8 + args: --timeout 5m working-directory: sda-download lint_sda: @@ -49,10 +48,9 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.2 + uses: golangci/golangci-lint-action@v7.0.0 with: - args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m - version: v1.64.8 + args: --timeout 5m working-directory: sda lint_sda_admin: @@ -71,8 +69,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.5.2 + uses: golangci/golangci-lint-action@v7.0.0 with: - args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m - version: v1.64.8 + args: --timeout 5m working-directory: sda-admin diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000..1704a256ca --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,26 @@ +version: "2" +formatters: + enable: + - gofmt + - goimports +linters: + default: none + enable: + - bodyclose + - gocritic + # - gosec + - govet + - nestif + - nlreturn + # - revive + - rowserrcheck + - staticcheck + settings: + gosec: + excludes: + - G107 + - G115 + - G302 + - G304 + - G401 + - G501 From 28ba3b07c11a4ad31e45721ef64a5a79bb50fa02 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 08:32:09 +0200 Subject: [PATCH 201/250] [gosec] fix false positives --- .golangci.yml | 5 ++--- sda/cmd/verify/verify.go | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 1704a256ca..bd04b6b0be 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -8,7 +8,7 @@ linters: enable: - bodyclose - gocritic - # - gosec + - gosec - govet - nestif - nlreturn @@ -18,8 +18,7 @@ linters: settings: gosec: excludes: - - G107 - - G115 + - G104 - G302 - G304 - G401 diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 549b49a2c7..ebdd7977e3 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -4,7 +4,7 @@ package main import ( "bytes" - "crypto/md5" // #nosec + "crypto/md5" "crypto/sha256" "encoding/json" "fmt" @@ -229,7 +229,7 @@ func main() { continue } - md5hash := md5.New() // #nosec + md5hash := md5.New() sha256hash := sha256.New() stream := io.TeeReader(c4ghr, md5hash) From 2804c8bfe4c54cb7a1f7f4bf7685ab0e1a135e53 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 08:43:12 +0200 Subject: [PATCH 202/250] [revive] use any instead of interface{} --- sda-download/internal/database/database_test.go | 2 +- sda/cmd/auth/jwt.go | 2 +- sda/cmd/auth/jwt_test.go | 2 +- sda/cmd/auth/main.go | 4 ++-- sda/cmd/auth/oidc_test.go | 2 +- sda/cmd/intercept/intercept.go | 2 +- sda/cmd/mapper/mapper.go | 2 +- sda/cmd/notify/notify.go | 2 +- sda/cmd/orchestrate/orchestrate.go | 16 ++++++++-------- sda/cmd/s3inbox/proxy.go | 12 ++++++------ sda/cmd/syncapi/syncapi.go | 2 +- sda/internal/broker/broker.go | 6 +++--- sda/internal/helper/helper.go | 16 ++++++++-------- sda/internal/schema/schema.go | 16 ++++++++-------- 14 files changed, 43 insertions(+), 43 deletions(-) diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index 6b9922282a..22e97f89df 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -59,7 +59,7 @@ func TestBuildConnInfo(t *testing.T) { } // testLogFatalf -func testLogFatalf(f string, args ...interface{}) { +func testLogFatalf(f string, args ...any) { s := fmt.Sprintf(f, args...) panic(s) } diff --git a/sda/cmd/auth/jwt.go b/sda/cmd/auth/jwt.go index b38e02c5f6..cc3a15521c 100644 --- a/sda/cmd/auth/jwt.go +++ b/sda/cmd/auth/jwt.go @@ -10,7 +10,7 @@ import ( "github.com/lestrrat-go/jwx/v2/jwt" ) -func generateJwtToken(tokenClaims map[string]interface{}, keyPath, alg string) (string, string, error) { +func generateJwtToken(tokenClaims map[string]any, keyPath, alg string) (string, string, error) { prKey, err := os.ReadFile(filepath.Clean(keyPath)) if err != nil { return "", "", err diff --git a/sda/cmd/auth/jwt_test.go b/sda/cmd/auth/jwt_test.go index 5d74b5f675..7b923a29ec 100644 --- a/sda/cmd/auth/jwt_test.go +++ b/sda/cmd/auth/jwt_test.go @@ -68,7 +68,7 @@ func (suite *JWTTests) TestGenerateJwtToken() { {Algorithm: "ES256", Keyfile: suite.TempDir + "/ec", Pubfile: suite.TempDir + "/ec.pub"}, } - claims := map[string]interface{}{ + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(2 * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: "http://local.issuer", diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 3eac088b64..d0719f03d0 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -146,7 +146,7 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { if ok { log.WithFields(log.Fields{"authType": "cega", "user": username}).Info("Valid password entered by user") - claims := map[string]interface{}{ + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(time.Duration(auth.Config.JwtTTL) * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: auth.Config.JwtIssuer, @@ -278,7 +278,7 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { if auth.Config.ResignJwt { log.Debugf("Resigning token for user %s", idStruct.User) - claims := map[string]interface{}{ + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(time.Duration(auth.Config.JwtTTL) * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: auth.Config.JwtIssuer, diff --git a/sda/cmd/auth/oidc_test.go b/sda/cmd/auth/oidc_test.go index 62160aab09..7042447649 100644 --- a/sda/cmd/auth/oidc_test.go +++ b/sda/cmd/auth/oidc_test.go @@ -95,7 +95,7 @@ func (suite *OIDCTests) TestValidateJwt() { elixirIdentity, _ := authenticateWithOidc(oauth2Config, provider, session.SessionID, jwkURL) elixirJWT := elixirIdentity.RawToken - claims := map[string]interface{}{ + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(2 * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: "http://local.issuer", diff --git a/sda/cmd/intercept/intercept.go b/sda/cmd/intercept/intercept.go index 22239850be..64fd0e4dc4 100644 --- a/sda/cmd/intercept/intercept.go +++ b/sda/cmd/intercept/intercept.go @@ -106,7 +106,7 @@ func main() { // typeFromMessage returns the type value given a JSON structure for the message // supplied in body func typeFromMessage(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index a7455342f2..215e52cedb 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -154,7 +154,7 @@ func main() { // schemaFromDatasetOperation returns the operation done with dataset supplied in body of the message func schemaFromDatasetOperation(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err diff --git a/sda/cmd/notify/notify.go b/sda/cmd/notify/notify.go index 0466e06833..8e6a923524 100644 --- a/sda/cmd/notify/notify.go +++ b/sda/cmd/notify/notify.go @@ -93,7 +93,7 @@ func getUser(queue string, orgMsg []byte) string { _ = json.Unmarshal(orgMsg, ¬ify) orgMsg, _ := base64.StdEncoding.DecodeString(notify.OriginalMessage.(string)) - var message map[string]interface{} + var message map[string]any _ = json.Unmarshal(orgMsg, &message) return fmt.Sprint(message["user"]) diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index cc117af046..fda6ba6692 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -137,7 +137,7 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * } var publishMsg []byte - var publishType interface{} + var publishType any routingSchema, err := schemaNameFromQueue(routingKey, nil, conf) @@ -232,7 +232,7 @@ func schemaNameFromQueue(queue string, body []byte, conf *config.Config) (string // schemaFromInboxOperation returns the operation done in inbox // supplied in body of the message func schemaFromInboxOperation(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err @@ -264,7 +264,7 @@ func schemaFromInboxOperation(body []byte) (string, error) { // schemaFromDatasetOperation returns the operation done with dataset // supplied in body of the message func schemaFromDatasetOperation(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err @@ -293,7 +293,7 @@ func schemaFromDatasetOperation(body []byte) (string, error) { } -func ingestMessage(body []byte) ([]byte, interface{}) { +func ingestMessage(body []byte) ([]byte, any) { var message upload err := json.Unmarshal(body, &message) if err != nil { @@ -312,7 +312,7 @@ func ingestMessage(body []byte) ([]byte, interface{}) { return publish, new(trigger) } -func finalizeMessage(body []byte, conf *config.Config) ([]byte, interface{}) { +func finalizeMessage(body []byte, conf *config.Config) ([]byte, any) { var message request err := json.Unmarshal(body, &message) if err != nil { @@ -335,7 +335,7 @@ func finalizeMessage(body []byte, conf *config.Config) ([]byte, interface{}) { return publish, new(finalize) } -func mappingMessage(body []byte, conf *config.Config) ([]byte, interface{}) { +func mappingMessage(body []byte, conf *config.Config) ([]byte, any) { var message finalize if err := json.Unmarshal(body, &message); err != nil { return nil, nil @@ -355,7 +355,7 @@ func mappingMessage(body []byte, conf *config.Config) ([]byte, interface{}) { return publish, new(mapping) } -func releaseMessage(body []byte, conf *config.Config) ([]byte, interface{}) { +func releaseMessage(body []byte, conf *config.Config) ([]byte, any) { var message finalize if err := json.Unmarshal(body, &message); err != nil { return nil, nil @@ -374,7 +374,7 @@ func releaseMessage(body []byte, conf *config.Config) ([]byte, interface{}) { return publish, new(mapping) } -func validateMsg(delivered *amqp091.Delivery, mq *broker.AMQPBroker, routingKey string, routingSchema string, publishMsg []byte, publishType interface{}) error { +func validateMsg(delivered *amqp091.Delivery, mq *broker.AMQPBroker, routingKey string, routingSchema string, publishMsg []byte, publishType any) error { err := schema.ValidateJSON(fmt.Sprintf("%s/%s.json", mq.Conf.SchemasPath, routingSchema), delivered.Body) if err != nil { return err diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index d1b73bf9a0..ee717be774 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -39,11 +39,11 @@ type Proxy struct { // The Event struct type Event struct { - Operation string `json:"operation"` - Username string `json:"user"` - Filepath string `json:"filepath"` - Filesize int64 `json:"filesize"` - Checksum []interface{} `json:"encrypted_checksums"` + Operation string `json:"operation"` + Username string `json:"user"` + Filepath string `json:"filepath"` + Filesize int64 `json:"filesize"` + Checksum []any `json:"encrypted_checksums"` } // Checksum used in the message @@ -500,7 +500,7 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token) (Eve event.Filepath = strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) event.Username = claims.Subject() checksum.Type = "sha256" - event.Checksum = []interface{}{checksum} + event.Checksum = []any{checksum} privateClaims := claims.PrivateClaims() log.Info("user ", event.Username, " with pilot ", privateClaims["pilot"], " uploaded file ", event.Filepath, " with checksum ", checksum.Value, " at ", time.Now()) diff --git a/sda/cmd/syncapi/syncapi.go b/sda/cmd/syncapi/syncapi.go index 04a1bafd09..1a17789e6d 100644 --- a/sda/cmd/syncapi/syncapi.go +++ b/sda/cmd/syncapi/syncapi.go @@ -210,7 +210,7 @@ func respondWithError(w http.ResponseWriter, code int, message string) { respondWithJSON(w, code, map[string]string{"error": message}) } -func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) { +func respondWithJSON(w http.ResponseWriter, code int, payload any) { log.Infoln(payload) response, _ := json.Marshal(payload) diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index ff7a9b87f0..48a9c094e4 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -44,9 +44,9 @@ type MQConf struct { // The empty interface allows for appending various json msgs but also broken json msgs as strings. // It is ok as long as we do not need to access fields in the msg, which we don't. type InfoError struct { - Error string `json:"error"` - Reason string `json:"reason"` - OriginalMessage interface{} `json:"original-message"` + Error string `json:"error"` + Reason string `json:"reason"` + OriginalMessage any `json:"original-message"` } // buildMQURI builds the MQ connection URI diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index bc54713733..abdf190276 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -26,14 +26,14 @@ import ( // Global variables for test token creation var ( - DefaultTokenClaims = map[string]interface{}{ + DefaultTokenClaims = map[string]any{ "iss": "https://dummy.ega.nbis.se", "sub": "dummy", "exp": time.Now().Add(time.Hour * 2).Unix(), "pilot": "dummy-pilot", } - WrongUserClaims = map[string]interface{}{ + WrongUserClaims = map[string]any{ "sub": "c5773f41d17d27bd53b1e6794aedc32d7906e779@elixir-europe.org", "aud": "15137645-3153-4d49-9ddb-594027cd4ca7", "azp": "15137645-3153-4d49-9ddb-594027cd4ca7", @@ -44,7 +44,7 @@ var ( "jti": "5e6c6d24-42eb-408e-ba20-203904e388a1", } - ExpiredClaims = map[string]interface{}{ + ExpiredClaims = map[string]any{ "sub": "dummy", "aud": "15137645-3153-4d49-9ddb-594027cd4ca7", "azp": "15137645-3153-4d49-9ddb-594027cd4ca7", @@ -55,7 +55,7 @@ var ( "jti": "5e6c6d24-42eb-408e-ba20-203904e388a1", } - NonValidClaims = map[string]interface{}{ + NonValidClaims = map[string]any{ "sub": "c5773f41d17d27bd53b1e6794aedc32d7906e779@elixir-europe.org", "aud": "15137645-3153-4d49-9ddb-594027cd4ca7", "azp": "15137645-3153-4d49-9ddb-594027cd4ca7", @@ -66,7 +66,7 @@ var ( "jti": "5e6c6d24-42eb-408e-ba20-203904e388a1", } - WrongTokenAlgClaims = map[string]interface{}{ + WrongTokenAlgClaims = map[string]any{ "iss": "Online JWT Builder", "iat": time.Now().Unix(), "exp": time.Now().Add(time.Hour * 2).Unix(), @@ -208,7 +208,7 @@ func CreateSSHKey(path string) error { } // CreateRSAToken creates an RSA token -func CreateRSAToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]interface{}) (string, error) { +func CreateRSAToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]any) (string, error) { if err := jwk.AssignKeyID(jwtKey); err != nil { return "AssignKeyID failed", err } @@ -232,7 +232,7 @@ func CreateRSAToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]int } // CreateECToken creates an EC token -func CreateECToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]interface{}) (string, error) { +func CreateECToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]any) (string, error) { if err := jwk.AssignKeyID(jwtKey); err != nil { return "AssignKeyID failed", err } @@ -256,7 +256,7 @@ func CreateECToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]inte } // CreateHSToken creates an HS token -func CreateHSToken(key []byte, tokenClaims map[string]interface{}) (string, error) { +func CreateHSToken(key []byte, tokenClaims map[string]any) (string, error) { token := jwt.New() for key, value := range tokenClaims { if err := token.Set(key, value); err != nil { diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go index 503e7e3088..6f2d771422 100644 --- a/sda/internal/schema/schema.go +++ b/sda/internal/schema/schema.go @@ -22,7 +22,7 @@ func ValidateJSON(reference string, body []byte) error { return err } - var v interface{} + var v any if err := json.Unmarshal(body, &v); err != nil { return err } @@ -34,7 +34,7 @@ func ValidateJSON(reference string, body []byte) error { return nil } -func getStructName(path string) interface{} { +func getStructName(path string) any { switch strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) { case "dataset-deprecate": return new(DatasetDeprecate) @@ -93,9 +93,9 @@ type DatasetRelease struct { } type InfoError struct { - Error string `json:"error"` - Reason string `json:"reason"` - OriginalMessage interface{} `json:"original-message"` + Error string `json:"error"` + Reason string `json:"reason"` + OriginalMessage any `json:"original-message"` } type InboxRemove struct { @@ -172,12 +172,12 @@ type DatasetFiles struct { } type SyncMetadata struct { - DatasetID string `json:"dataset_id"` - Metadata interface{} `json:"metadata"` + DatasetID string `json:"dataset_id"` + Metadata any `json:"metadata"` } type Metadata struct { - Metadata interface{} + Metadata any } type C4ghPubKey struct { From 05f2d97fe450f543653ab96a4add7c2ac4a3b4a0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 08:50:08 +0200 Subject: [PATCH 203/250] [deep-exit] ignore for now, refactoring required --- sda-download/internal/storage/storage.go | 2 +- sda/cmd/api/api.go | 2 +- sda/cmd/auth/oidc.go | 2 +- sda/cmd/orchestrate/orchestrate.go | 2 +- sda/internal/helper/helper.go | 14 +++++++------- sda/internal/storage/storage.go | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sda-download/internal/storage/storage.go b/sda-download/internal/storage/storage.go index 358287e440..22f938f990 100644 --- a/sda-download/internal/storage/storage.go +++ b/sda-download/internal/storage/storage.go @@ -299,7 +299,7 @@ func transportConfigS3(config S3Conf) http.RoundTripper { if config.Cacert != "" { cacert, e := os.ReadFile(config.Cacert) // #nosec this file comes from our config if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) + log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main } if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { log.Debug("no certs appended, using system certs only") diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 7ed8ded538..555cf5b590 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -91,7 +91,7 @@ func setup(config *config.Config) *http.Server { e, err := casbin.NewEnforcer(model, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) if err != nil { shutdown() - log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) + log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) // nolint # FIXME Fatal should only be called from main } r := gin.Default() diff --git a/sda/cmd/auth/oidc.go b/sda/cmd/auth/oidc.go index a95a0e8afb..96cf971eb9 100644 --- a/sda/cmd/auth/oidc.go +++ b/sda/cmd/auth/oidc.go @@ -32,7 +32,7 @@ func getOidcClient(conf config.OIDCConfig) (oauth2.Config, *oidc.Provider) { contx := context.Background() provider, err := oidc.NewProvider(contx, conf.Provider) if err != nil { - log.Fatal(err) + log.Fatal(err) // nolint # FIXME Fatal should only be called from main } oauth2Config := oauth2.Config{ diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index fda6ba6692..be94da4458 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -101,7 +101,7 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * messages, err := mq.GetMessages(queue) if err != nil { - log.Fatal(err) + log.Fatal(err) // nolint # FIXME Fatal should only be called from main } for delivered := range messages { log.Debugf("Received a message: %s", delivered.Body) diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index abdf190276..1dcacb72b1 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -343,28 +343,28 @@ func MakeCerts(outDir string) { // create our private and public key caPrivKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } // create the CA certificate caBytes, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caPrivKey.PublicKey, caPrivKey) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } err = TLScertToFile(outDir+"/ca.crt", caBytes) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } tlsKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } err = TLSkeyToFile(outDir+"/tls.key", tlsKey) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } // set up our server certificate @@ -386,12 +386,12 @@ func MakeCerts(outDir string) { // create the TLS certificate certBytes, err := x509.CreateCertificate(rand.Reader, certTemplate, caTemplate, &tlsKey.PublicKey, caPrivKey) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } err = TLScertToFile(outDir+"/tls.crt", certBytes) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } log.Printf("certificartes written to: %s", outDir) } diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index fc39d16885..4a1fd41b43 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -353,7 +353,7 @@ func transportConfigS3(conf S3Conf) http.RoundTripper { if conf.CAcert != "" { cacert, e := os.ReadFile(conf.CAcert) // #nosec this file comes from our config if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) + log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main } if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { log.Debug("no certs appended, using system certs only") From 5e39998395b908729c29c435210b5527529b7520 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 09:16:46 +0200 Subject: [PATCH 204/250] [revive] fix blank lines at block boundaries --- sda-admin/helpers/helpers.go | 1 - sda-download/api/api_test.go | 1 - sda-download/api/middleware/middleware.go | 2 -- .../api/middleware/middleware_test.go | 13 ---------- sda-download/api/s3/s3.go | 2 -- sda-download/api/s3/s3_test.go | 9 ------- sda-download/api/sda/sda.go | 8 ------- sda-download/api/sda/sda_test.go | 24 ------------------- sda-download/internal/config/config.go | 1 - sda-download/internal/config/config_test.go | 9 ------- sda-download/internal/database/database.go | 2 -- .../internal/database/database_test.go | 13 ---------- sda-download/internal/session/session.go | 1 - sda-download/internal/session/session_test.go | 6 ----- sda-download/internal/storage/seekable.go | 9 ------- .../internal/storage/seekable_test.go | 5 ---- sda-download/internal/storage/storage.go | 3 --- sda-download/internal/storage/storage_test.go | 8 ------- sda-download/pkg/auth/auth.go | 4 ---- sda-download/pkg/auth/auth_test.go | 21 ---------------- sda-download/pkg/request/request_test.go | 6 ----- sda/cmd/api/api.go | 1 - sda/cmd/auth/info_test.go | 1 - sda/cmd/auth/main.go | 13 ---------- sda/cmd/auth/oidc_test.go | 1 - sda/cmd/auth/s3conf_test.go | 2 -- sda/cmd/ingest/ingest.go | 1 - sda/cmd/mapper/mapper.go | 1 - sda/cmd/notify/notify_test.go | 2 -- sda/cmd/orchestrate/orchestrate.go | 3 --- sda/cmd/reencrypt/reencrypt.go | 2 -- sda/cmd/reencrypt/reencrypt_test.go | 2 -- sda/cmd/s3inbox/healthchecks.go | 2 -- sda/cmd/s3inbox/healthchecks_test.go | 1 - sda/cmd/s3inbox/proxy.go | 5 ---- sda/cmd/s3inbox/proxy_test.go | 1 - sda/cmd/verify/verify.go | 1 - sda/internal/broker/broker_test.go | 1 - sda/internal/config/config.go | 1 - sda/internal/database/database.go | 2 -- sda/internal/database/db_functions.go | 3 --- sda/internal/database/db_functions_test.go | 4 ---- sda/internal/database/db_test.go | 5 ---- sda/internal/helper/helper.go | 1 - sda/internal/jsonadapter/jsonadapter_test.go | 1 - sda/internal/storage/storage.go | 1 - sda/internal/userauth/userauth.go | 2 -- 47 files changed, 208 deletions(-) diff --git a/sda-admin/helpers/helpers.go b/sda-admin/helpers/helpers.go index 31b0289cc7..352ee75b6b 100644 --- a/sda-admin/helpers/helpers.go +++ b/sda-admin/helpers/helpers.go @@ -87,7 +87,6 @@ func CheckValidChars(filename string) error { re := regexp.MustCompile(`[\\<>"\|\x00-\x1F\x7F\!\*\'\(\)\;\:\@\&\=\+\$\,\?\%\#\[\]]`) disallowedChars := re.FindAllString(filename, -1) if disallowedChars != nil { - return fmt.Errorf( "filepath '%v' contains disallowed characters: %+v", filename, diff --git a/sda-download/api/api_test.go b/sda-download/api/api_test.go index 6d7ab59750..d52fef9361 100644 --- a/sda-download/api/api_test.go +++ b/sda-download/api/api_test.go @@ -7,7 +7,6 @@ import ( ) func TestSetup(t *testing.T) { - // Create web server app config.Config.App.Host = "localhost" config.Config.App.Port = 8080 diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index e63f00048e..de6a9846d4 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -20,7 +20,6 @@ const requestContextKey = "requestContextKey" // The datasets are stored into a session cache for subsequent requests, and also // to the current request context for use in the endpoints. func TokenMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { // Check if dataset permissions are cached to session sessionCookie, err := c.Cookie(config.Config.Session.Name) @@ -84,7 +83,6 @@ func TokenMiddleware() gin.HandlerFunc { // Forward request to the next endpoint handler c.Next() } - } // GetCacheFromContext is a helper function that endpoints can use to get data diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index 302568c2a1..3aec599b3f 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -23,7 +23,6 @@ const token string = "token" func testEndpoint(_ *gin.Context) {} func TestTokenMiddleware_Fail_GetToken(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken @@ -60,11 +59,9 @@ func TestTokenMiddleware_Fail_GetToken(t *testing.T) { // Return mock functions to originals auth.GetToken = originalGetToken - } func TestTokenMiddleware_Fail_GetVisas(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken originalGetVisas := auth.GetVisas @@ -106,11 +103,9 @@ func TestTokenMiddleware_Fail_GetVisas(t *testing.T) { // Return mock functions to originals auth.GetToken = originalGetToken auth.GetVisas = originalGetVisas - } func TestTokenMiddleware_Fail_GetPermissions(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken originalGetVisas := auth.GetVisas @@ -149,11 +144,9 @@ func TestTokenMiddleware_Fail_GetPermissions(t *testing.T) { auth.GetToken = originalGetToken auth.GetVisas = originalGetVisas auth.GetPermissions = originalGetPermissions - } func TestTokenMiddleware_Success_NoCache(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken originalGetVisas := auth.GetVisas @@ -215,11 +208,9 @@ func TestTokenMiddleware_Success_NoCache(t *testing.T) { auth.GetVisas = originalGetVisas auth.GetPermissions = originalGetPermissions session.NewSessionKey = originalNewSessionKey - } func TestTokenMiddleware_Success_FromCache(t *testing.T) { - // Save original to-be-mocked functions originalGetCache := session.Get @@ -275,7 +266,6 @@ func TestTokenMiddleware_Success_FromCache(t *testing.T) { // Return mock functions to originals session.Get = originalGetCache - } func TestStoreDatasets(t *testing.T) { @@ -294,11 +284,9 @@ func TestStoreDatasets(t *testing.T) { if !reflect.DeepEqual(datasets, storedDatasets) { t.Errorf("TestStoreDatasets failed, got %s, expected %s", storedDatasets, datasets) } - } func TestGetDatasets(t *testing.T) { - // Get a request context for testing if data is saved w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -314,5 +302,4 @@ func TestGetDatasets(t *testing.T) { if !reflect.DeepEqual(datasets, storedDatasets) { t.Errorf("TestStoreDatasets failed, got %s, expected %s", storedDatasets, datasets) } - } diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go index 4bea88bbc4..35cb03667e 100644 --- a/sda-download/api/s3/s3.go +++ b/sda-download/api/s3/s3.go @@ -259,7 +259,6 @@ func GetEcnryptedObject(c *gin.Context) { // which can include slashes, so finding the separation between filename and // dataset name is done by comparing to accessible datasets. func parseParams(c *gin.Context) *gin.Context { - // When doing list requests from s3cmd, the tool will assume that the first // slash delimits the bucket, and the rest is the prefix. We use this to // "restore" the path in case the prefix param is set. @@ -319,7 +318,6 @@ func Download(c *gin.Context) { // S3 request types are described here: // https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html switch { - case strings.Contains(c.Request.URL.String(), "?location"): GetBucketLocation(c) diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go index c490c65d29..bf08274211 100644 --- a/sda-download/api/s3/s3_test.go +++ b/sda-download/api/s3/s3_test.go @@ -24,7 +24,6 @@ type S3TestSuite struct { } func (suite *S3TestSuite) SetupTest() { - var err error var db *sql.DB @@ -61,7 +60,6 @@ func TestS3TestSuite(t *testing.T) { } func (suite *S3TestSuite) TestGetBucketLocation() { - // Send a request through the middleware to get datasets w := httptest.NewRecorder() @@ -80,7 +78,6 @@ func (suite *S3TestSuite) TestGetBucketLocation() { } func (suite *S3TestSuite) TestListBuckets() { - // Setup a mock database to handle queries query := `SELECT stable_id, created_at FROM sda.datasets WHERE stable_id = \$1` @@ -117,7 +114,6 @@ func (suite *S3TestSuite) TestListBuckets() { } func (suite *S3TestSuite) TestListByPrefix() { - // Setup a mock database to handle queries fileInfo := &database.FileInfo{ FileID: "file1", @@ -195,7 +191,6 @@ func (suite *S3TestSuite) TestListByPrefix() { } func (suite *S3TestSuite) TestListObjects() { - // Setup a mock database to handlequeries fileInfo := &database.FileInfo{ FileID: "file1", @@ -272,7 +267,6 @@ func (suite *S3TestSuite) TestListObjects() { } func (suite *S3TestSuite) TestParseParams() { - type paramTest struct { Path string Dataset string @@ -291,7 +285,6 @@ func (suite *S3TestSuite) TestParseParams() { } for _, params := range testParams { - // response function to check parameter parsing testParseParams := func(c *gin.Context) { parseParams(c) @@ -313,7 +306,5 @@ func (suite *S3TestSuite) TestParseParams() { defer response.Body.Close() assert.Equal(suite.T(), http.StatusAccepted, response.StatusCode, "Request failed") - } - } diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index 76a25340a1..4998efa6bc 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -135,7 +135,6 @@ func find(datasetID string, datasets []string) bool { // getFiles returns files belonging to a dataset var getFiles = func(datasetID string, ctx *gin.Context) ([]*database.FileInfo, int, error) { - // Retrieve dataset list from request context // generated by the authentication middleware cache := middleware.GetCacheFromContext(ctx) @@ -160,7 +159,6 @@ var getFiles = func(datasetID string, ctx *gin.Context) ([]*database.FileInfo, i // Files serves a list of files belonging to a dataset func Files(c *gin.Context) { - // get dataset parameter dataset := c.Param("dataset") @@ -347,7 +345,6 @@ func Download(c *gin.Context) { } if c.Request.Method == http.MethodHead { - // Create headers for htsget, containing size of the crypt4gh header reencKey := c.GetHeader("Client-Public-Key") headerSize := bytes.NewReader(fileDetails.Header).Size() @@ -466,17 +463,14 @@ func Download(c *gin.Context) { var adjustSeekPos = func(fileStream io.ReadSeeker, start, end int64) (int64, int64, error) { if start != 0 { - // We don't want to read from start, skip ahead to where we should be _, err := fileStream.Seek(start, 0) if err != nil { - return 0, 0, fmt.Errorf("error occurred while finding sending start: %v", err) } // adjust end to reflect that the file start has been moved end -= start start = 0 - } return start, end, nil @@ -484,7 +478,6 @@ var adjustSeekPos = func(fileStream io.ReadSeeker, start, end int64) (int64, int // used from: https://github.com/neicnordic/crypt4gh/blob/master/examples/reader/main.go#L48C1-L113C1 var sendStream = func(reader io.Reader, writer http.ResponseWriter, start, end int64) error { - // Calculate how much we should read (if given) togo := end - start @@ -572,5 +565,4 @@ var calculateCoords = func(start, end int64, htsget_range string, fileDetails *d } return start, headlength.Size() + bodyEnd, nil - } diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go index 3b2a06a673..087ca86f3c 100644 --- a/sda-download/api/sda/sda_test.go +++ b/sda-download/api/sda/sda_test.go @@ -35,7 +35,6 @@ import ( ) func TestDatasets(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext @@ -72,11 +71,9 @@ func TestDatasets(t *testing.T) { // Return mock functions to originals middleware.GetCacheFromContext = originalGetCacheFromContext - } func TestFind_Found(t *testing.T) { - // Test case datasets := []string{"dataset1", "dataset2", "dataset3"} @@ -89,11 +86,9 @@ func TestFind_Found(t *testing.T) { if found != expectedFound { t.Errorf("TestFind_Found failed, got %t expected %t", found, expectedFound) } - } func TestFind_NotFound(t *testing.T) { - // Test case datasets := []string{"dataset1", "dataset2", "dataset3"} @@ -106,11 +101,9 @@ func TestFind_NotFound(t *testing.T) { if found != expectedFound { t.Errorf("TestFind_Found failed, got %t expected %t", found, expectedFound) } - } func TestGetFiles_Fail_Database(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext originalGetFilesDB := database.GetFiles @@ -147,11 +140,9 @@ func TestGetFiles_Fail_Database(t *testing.T) { // Return mock functions to originals middleware.GetCacheFromContext = originalGetCacheFromContext database.GetFiles = originalGetFilesDB - } func TestGetFiles_Fail_NotFound(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext @@ -186,7 +177,6 @@ func TestGetFiles_Fail_NotFound(t *testing.T) { } func TestGetFiles_Success(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext originalGetFilesDB := database.GetFiles @@ -229,11 +219,9 @@ func TestGetFiles_Success(t *testing.T) { // Return mock functions to originals middleware.GetCacheFromContext = originalGetCacheFromContext database.GetFiles = originalGetFilesDB - } func TestFiles_Fail(t *testing.T) { - // Save original to-be-mocked functions originalGetFiles := getFiles @@ -272,11 +260,9 @@ func TestFiles_Fail(t *testing.T) { // Return mock functions to originals getFiles = originalGetFiles - } func TestFiles_Success(t *testing.T) { - // Save original to-be-mocked functions originalGetFiles := getFiles @@ -335,11 +321,9 @@ func TestFiles_Success(t *testing.T) { // Return mock functions to originals getFiles = originalGetFiles - } func TestDownload_Fail_UnencryptedDownloadNotAllowed(t *testing.T) { - // Save original to-be-mocked config originalServeUnencryptedDataTrigger := config.Config.C4GH.PublicKeyB64 config.Config.C4GH.PublicKeyB64 = "" @@ -396,7 +380,6 @@ func TestDownload_Fail_UnencryptedDownloadNotAllowed(t *testing.T) { } func TestDownload_Fail_FileNotFound(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -445,11 +428,9 @@ func TestDownload_Fail_FileNotFound(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - } func TestDownload_Fail_NoPermissions(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -504,11 +485,9 @@ func TestDownload_Fail_NoPermissions(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - } func TestDownload_Fail_GetFile(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -569,11 +548,9 @@ func TestDownload_Fail_GetFile(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - } func TestDownload_Fail_OpenFile(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -742,7 +719,6 @@ func (f *fakeGRPC) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func TestDownload_Whole_Range_Encrypted(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index af468ee1d7..515af2ef3c 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -380,7 +380,6 @@ func (c *Map) appConfig() error { var err error if viper.GetString("c4gh.transientKeyPath") != "" { - if !viper.IsSet("c4gh.transientPassphrase") { return errors.New("c4gh.transientPassphrase is not set") } diff --git a/sda-download/internal/config/config_test.go b/sda-download/internal/config/config_test.go index d493c683ac..7f4c473c14 100644 --- a/sda-download/internal/config/config_test.go +++ b/sda-download/internal/config/config_test.go @@ -64,7 +64,6 @@ func (suite *TestSuite) TestMissingRequiredConfVar() { } func (suite *TestSuite) TestAppConfig() { - // Generate a crypth4gh private key file _, privateKey, err := keys.GenerateKeyPair() assert.NoError(suite.T(), err) @@ -134,11 +133,9 @@ func (suite *TestSuite) TestArchiveConfig() { c := &Map{} c.configArchive() assert.Equal(suite.T(), "/test", c.Archive.Posix.Location) - } func (suite *TestSuite) TestSessionConfig() { - viper.Set("session.expiration", 3600) viper.Set("session.domain", "test") viper.Set("session.secure", false) @@ -152,11 +149,9 @@ func (suite *TestSuite) TestSessionConfig() { assert.Equal(suite.T(), "test", c.Session.Domain) assert.Equal(suite.T(), false, c.Session.Secure) assert.Equal(suite.T(), false, c.Session.HTTPOnly) - } func (suite *TestSuite) TestDatabaseConfig() { - // Test error on missing SSL vars viper.Set("db.sslmode", "verify-full") c := &Map{} @@ -191,11 +186,9 @@ func (suite *TestSuite) TestDatabaseConfig() { assert.Equal(suite.T(), "test", c.DB.CACert) assert.Equal(suite.T(), "test", c.DB.ClientCert) assert.Equal(suite.T(), "test", c.DB.ClientKey) - } func (suite *TestSuite) TestOIDC() { - // Test wrong file viper.Set("oidc.trusted.iss", "../../iss.json") c := &Map{} @@ -222,7 +215,6 @@ func (suite *TestSuite) TestOIDC() { assert.Equal(suite.T(), "test", c.OIDC.CACert) assert.Equal(suite.T(), trustedList, c.OIDC.TrustedList) assert.Equal(suite.T(), whitelist, c.OIDC.Whitelist) - } func (suite *TestSuite) TestConfigReencrypt() { @@ -243,5 +235,4 @@ func (suite *TestSuite) TestConfigReencrypt() { // it will fail if certificate is set to a folder viper.Set("grpc.clientcert", tempDir) assert.ErrorContains(suite.T(), c.configReencrypt(), "is a folder") - } diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index b3470ddf1e..e7c8a58ae7 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -127,7 +127,6 @@ func (dbs *SQLdb) checkAndReconnectIfNeeded() { log.Debugln("Reconnecting to DB") dbs.DB, _ = sqlOpen("postgres", dbs.ConnInfo) } - } // GetFiles retrieves the file details @@ -212,7 +211,6 @@ func (dbs *SQLdb) getFiles(datasetID string) ([]*FileInfo, error) { // Iterate rows for rows.Next() { - // Read rows into struct fi := &FileInfo{} err := rows.Scan(&fi.FileID, &fi.DatasetID, &fi.DisplayFileName, diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index 22e97f89df..5b45385c40 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -42,7 +42,6 @@ func TestMain(m *testing.M) { } func TestBuildConnInfo(t *testing.T) { - s := buildConnInfo(testPgconf) assert.Equalf(t, s, testConnInfo, "Bad string for verify-full: '%s' while expecting '%s'", s, testConnInfo) @@ -55,7 +54,6 @@ func TestBuildConnInfo(t *testing.T) { assert.Equalf(t, s, "host=localhost port=42 user=user password=password dbname=database sslmode=disable", "Bad string for disable: %s", s) - } // testLogFatalf @@ -65,14 +63,12 @@ func testLogFatalf(f string, args ...any) { } func TestCheckAndReconnect(t *testing.T) { - db, mock, _ := sqlmock.New(sqlmock.MonitorPingsOption(true)) mock.ExpectPing().WillReturnError(fmt.Errorf("ping fail for testing bad conn")) err := CatchPanicCheckAndReconnect(SQLdb{db, ""}) assert.Error(t, err, "Should have received error from checkAndReconnectOnNeeded fataling") - } func CatchPanicCheckAndReconnect(db SQLdb) (err error) { @@ -106,7 +102,6 @@ func CatchNewDBPanic() (err error) { } func TestNewDB(t *testing.T) { - // Test failure first sqlOpen = func(_ string, _ string) (*sql.DB, error) { @@ -162,7 +157,6 @@ func TestNewDB(t *testing.T) { err = mock.ExpectationsWereMet() assert.Nilf(t, err, "there were unfulfilled expectations: %s", err) - } // Helper function for "simple" sql tests @@ -187,7 +181,6 @@ func sqlTesterHelper(t *testing.T, f func(sqlmock.Sqlmock, *SQLdb) error) error func TestClose(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - mock.ExpectClose() testDb.Close() @@ -199,7 +192,6 @@ func TestClose(t *testing.T) { func TestCheckFilePermission(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := "dataset1" query := ` SELECT datasets.stable_id FROM sda.file_dataset @@ -230,7 +222,6 @@ func TestCheckFilePermission(t *testing.T) { func TestCheckDataset(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := true query := `SELECT stable_id FROM sda.datasets WHERE stable_id = \$1` mock.ExpectQuery(query). @@ -256,7 +247,6 @@ func TestCheckDataset(t *testing.T) { func TestGetDatasetInfo(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := &DatasetInfo{ DatasetID: "dataset1", CreatedAt: "now", @@ -285,7 +275,6 @@ func TestGetDatasetInfo(t *testing.T) { func TestGetDatasetFileInfo(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := &FileInfo{ FileID: "file1", DatasetID: "dataset1", @@ -355,7 +344,6 @@ func TestGetDatasetFileInfo(t *testing.T) { func TestGetFile(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := &FileDownload{ ArchivePath: "file.txt", ArchiveSize: 32, @@ -403,7 +391,6 @@ func TestGetFile(t *testing.T) { func TestGetFiles(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := []*FileInfo{} fileInfo := &FileInfo{ FileID: "file1", diff --git a/sda-download/internal/session/session.go b/sda-download/internal/session/session.go index 83b40519b7..5b38d08baa 100644 --- a/sda-download/internal/session/session.go +++ b/sda-download/internal/session/session.go @@ -74,7 +74,6 @@ var NewSessionKey = func() string { var sessionKey string exists := true for exists { - // Generate the key key := uuid.New().String() sessionKey = key diff --git a/sda-download/internal/session/session_test.go b/sda-download/internal/session/session_test.go index b760eed186..ffccc3fe06 100644 --- a/sda-download/internal/session/session_test.go +++ b/sda-download/internal/session/session_test.go @@ -9,7 +9,6 @@ import ( ) func TestNewSessionKey(t *testing.T) { - // Initialise a cache for testing cache, _ := InitialiseSessionCache() SessionCache = cache @@ -24,11 +23,9 @@ func TestNewSessionKey(t *testing.T) { if len(key) != expectedLen { t.Errorf("TestNewSessionKey failed, expected key length %d but received %d", expectedLen, len(key)) } - } func TestGetSetCache_Found(t *testing.T) { - // Set expiration time config.Config.Session.Expiration = time.Duration(60 * time.Second) @@ -50,11 +47,9 @@ func TestGetSetCache_Found(t *testing.T) { if expectedExists != exists { t.Errorf("TestGetSetCache_Found failed, expected %t but received %t", expectedExists, exists) } - } func TestGetSetCache_NotFound(t *testing.T) { - // Set expiration time config.Config.Session.Expiration = time.Duration(60 * time.Second) @@ -76,5 +71,4 @@ func TestGetSetCache_NotFound(t *testing.T) { if expectedExists != exists { t.Errorf("TestGetSetCache_NotFound failed, expected %t but received %t", expectedExists, exists) } - } diff --git a/sda-download/internal/storage/seekable.go b/sda-download/internal/storage/seekable.go index 6c572f3661..a6ba50da4b 100644 --- a/sda-download/internal/storage/seekable.go +++ b/sda-download/internal/storage/seekable.go @@ -14,7 +14,6 @@ import ( // NewFileReadSeeker returns an io.ReadSeeker instance func (pb *posixBackend) NewFileReadSeeker(filePath string) (io.ReadSeekCloser, error) { - reader, err := pb.NewFileReader(filePath) if err != nil { return nil, err @@ -85,7 +84,6 @@ func (r *s3Reader) pruneCache() { // Prune the cache keepfrom := len(r.local) - 8 r.local = r.local[keepfrom:] - } func (r *s3Reader) prefetchSize() int64 { @@ -383,7 +381,6 @@ type seekableMultiReader struct { // SeekableMultiReader constructs a multireader that supports seeking. Requires // all passed readers to be seekable func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { - r := make([]io.Reader, len(readers)) sizes := make([]int64, len(readers)) @@ -403,14 +400,12 @@ func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { sizes[i] = size totalSize += size - } return &seekableMultiReader{r, sizes, 0, totalSize}, nil } func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { - switch whence { case io.SeekStart: r.currentOffset = offset @@ -418,21 +413,17 @@ func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { r.currentOffset += offset case io.SeekEnd: r.currentOffset = r.totalSize + offset - default: return 0, fmt.Errorf("Unsupported whence") - } return r.currentOffset, nil } func (r *seekableMultiReader) Read(dst []byte) (int, error) { - var readerStartAt int64 for i, reader := range r.readers { - if r.currentOffset < readerStartAt { // We want data from a previous reader (? HELP ?) readerStartAt += r.sizes[i] diff --git a/sda-download/internal/storage/seekable_test.go b/sda-download/internal/storage/seekable_test.go index adbdf7b33a..7e9211a277 100644 --- a/sda-download/internal/storage/seekable_test.go +++ b/sda-download/internal/storage/seekable_test.go @@ -15,9 +15,7 @@ import ( ) func TestSeekableBackend(t *testing.T) { - for _, backendType := range []string{posixType, s3Type} { - testConf.Type = backendType backend, err := NewBackend(testConf) @@ -94,7 +92,6 @@ func TestSeekableBackend(t *testing.T) { _, err = seeker.Seek(0, 4) assert.NotNil(t, err, "Seek didn't fail when it should") - } offset, err := seeker.Seek(15, io.SeekStart) @@ -171,7 +168,6 @@ func TestSeekableBackend(t *testing.T) { } func TestS3SeekablePrefetchSize(t *testing.T) { - testConf.Type = s3Type chunkSize := testConf.S3.Chunksize testConf.S3.Chunksize = 5 * 1024 * 1024 @@ -297,7 +293,6 @@ func TestS3SeekableSpecial(t *testing.T) { } func TestSeekableMultiReader(t *testing.T) { - readers := make([]io.Reader, 10) for i := 0; i < 10; i++ { readers[i] = bytes.NewReader(writeData) diff --git a/sda-download/internal/storage/storage.go b/sda-download/internal/storage/storage.go index 22f938f990..9a6c3dd159 100644 --- a/sda-download/internal/storage/storage.go +++ b/sda-download/internal/storage/storage.go @@ -162,7 +162,6 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { if err != nil { if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && aerr.Code() != s3.ErrCodeBucketAlreadyExists { log.Error("Unexpected issue while creating bucket", err) @@ -231,7 +230,6 @@ func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { reader, writer := io.Pipe() go func() { - _, err := sb.Uploader.Upload(&s3manager.UploadInput{ Body: reader, Bucket: aws.String(sb.Bucket), @@ -272,7 +270,6 @@ func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { Key: aws.String(filePath)}) time.Sleep(1 * time.Second) - } if err != nil { diff --git a/sda-download/internal/storage/storage_test.go b/sda-download/internal/storage/storage_test.go index 7de3720da8..6bd0921a35 100644 --- a/sda-download/internal/storage/storage_test.go +++ b/sda-download/internal/storage/storage_test.go @@ -80,7 +80,6 @@ func doCleanup() { cleanupFiles = cleanupFilesBack[0:0] } func TestNewBackend(t *testing.T) { - testConf.Type = posixType p, err := NewBackend(testConf) assert.Nil(t, err, "Backend posix failed") @@ -100,7 +99,6 @@ func TestNewBackend(t *testing.T) { } func TestMain(m *testing.M) { - err := setupFakeS3() if err != nil { @@ -113,7 +111,6 @@ func TestMain(m *testing.M) { os.Exit(ret) } func TestPosixBackend(t *testing.T) { - defer doCleanup() testConf.Type = posixType backend, err := NewBackend(testConf) @@ -187,7 +184,6 @@ func TestPosixBackend(t *testing.T) { assert.NotZero(t, buf.Len(), "Expected warning missing") buf.Reset() - } func setupFakeS3() (err error) { @@ -226,7 +222,6 @@ func setupFakeS3() (err error) { if err != nil { if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && aerr.Code() != s3.ErrCodeBucketAlreadyExists { log.Error("Unexpected issue while creating bucket: ", err) @@ -241,7 +236,6 @@ func setupFakeS3() (err error) { } func TestS3Fail(t *testing.T) { - testConf.Type = s3Type tmp := testConf.S3.URL @@ -296,7 +290,6 @@ func TestPOSIXFail(t *testing.T) { } func TestS3Backend(t *testing.T) { - testConf.Type = s3Type backend, err := NewBackend(testConf) assert.Nil(t, err, "Backend failed") @@ -360,5 +353,4 @@ func TestS3Backend(t *testing.T) { } log.SetOutput(os.Stdout) - } diff --git a/sda-download/pkg/auth/auth.go b/sda-download/pkg/auth/auth.go index 0862c17c4f..89ca8a3654 100644 --- a/sda-download/pkg/auth/auth.go +++ b/sda-download/pkg/auth/auth.go @@ -185,7 +185,6 @@ var GetPermissions = func(visas Visas) []string { // Iterate visas for _, v := range visas.Visa { - // Check that visa is of type ControlledAccessGrants if checkVisaType(v, "ControlledAccessGrants") { // Check that visa is valid and return visa token @@ -195,7 +194,6 @@ var GetPermissions = func(visas Visas) []string { datasets = getDatasets(verifiedVisa, datasets) } } - } log.Debugf("matched datasets: %s", datasets) @@ -204,7 +202,6 @@ var GetPermissions = func(visas Visas) []string { } func checkVisaType(visa string, visaType string) bool { - log.Debug("checking visa type") unknownToken, err := jwt.Parse([]byte(visa), jwt.WithVerify(false)) @@ -235,7 +232,6 @@ func checkVisaType(visa string, visaType string) bool { log.Debug("visa type check passed") return true - } func validateVisa(visa string) (jwt.Token, bool) { diff --git a/sda-download/pkg/auth/auth_test.go b/sda-download/pkg/auth/auth_test.go index 41f33ba4e5..76da2703bd 100644 --- a/sda-download/pkg/auth/auth_test.go +++ b/sda-download/pkg/auth/auth_test.go @@ -14,7 +14,6 @@ import ( ) func TestGetOIDCDetails_Fail_MakeRequest(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -46,7 +45,6 @@ func TestGetOIDCDetails_Fail_MakeRequest(t *testing.T) { } func TestGetOIDCDetails_Fail_JSONDecode(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -86,7 +84,6 @@ func TestGetOIDCDetails_Fail_JSONDecode(t *testing.T) { } func TestGetOIDCDetails_Success(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -125,7 +122,6 @@ func TestGetOIDCDetails_Success(t *testing.T) { } func TestGetToken_Fail_EmptyHeader(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "") @@ -145,11 +141,9 @@ func TestGetToken_Fail_EmptyHeader(t *testing.T) { if err.Error() != expectedError { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected %s, received %s", expectedError, err.Error()) } - } func TestGetToken_X_Amz_Security_token(t *testing.T) { - // Expected results expectedToken := "token" expectedCode := 0 @@ -168,11 +162,9 @@ func TestGetToken_X_Amz_Security_token(t *testing.T) { if err != nil { t.Errorf("TestGetToken_X_Amz_Security_token failed, expected nil, received %v", err) } - } func TestGetToken_Fail_WrongScheme(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "Basic token") @@ -192,11 +184,9 @@ func TestGetToken_Fail_WrongScheme(t *testing.T) { if err.Error() != expectedError { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected %s, received %s", expectedError, err.Error()) } - } func TestGetToken_Fail_MissingToken(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "Bearer") @@ -216,11 +206,9 @@ func TestGetToken_Fail_MissingToken(t *testing.T) { if err.Error() != expectedError { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected %s, received %s", expectedError, err.Error()) } - } func TestGetToken_Success(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "Bearer token") @@ -239,11 +227,9 @@ func TestGetToken_Success(t *testing.T) { if err != nil { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected nil, received %v", err) } - } func TestGetVisas_Fail_MakeRequest(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -268,11 +254,9 @@ func TestGetVisas_Fail_MakeRequest(t *testing.T) { // Return mock functions to originals request.MakeRequest = originalMakeRequest - } func TestGetVisas_Fail_JSONDecode(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -305,11 +289,9 @@ func TestGetVisas_Fail_JSONDecode(t *testing.T) { // Return mock functions to originals request.MakeRequest = originalMakeRequest - } func TestGetVisas_Success(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -342,11 +324,9 @@ func TestGetVisas_Success(t *testing.T) { // Return mock functions to originals request.MakeRequest = originalMakeRequest - } func TestValidateTrustedIss(t *testing.T) { - // this also tests checkIss trustedList := []config.TrustedISS([]config.TrustedISS{{ISS: "https://demo.example", JKU: "https://mockauth:8000/idp/profile/oidc/keyset"}, {ISS: "https://demo1.example", JKU: "https://mockauth:8000/idp/profile/oidc/keyset"}}) @@ -364,7 +344,6 @@ func TestValidateTrustedIss(t *testing.T) { } func TestValidateTrustedIssNoConfig(t *testing.T) { - ok := validateTrustedIss(nil, "https://demo.example", "https://mockauth:8000/idp/profile/oidc/keyset") assert.True(t, ok, "this should be true") diff --git a/sda-download/pkg/request/request_test.go b/sda-download/pkg/request/request_test.go index 9de6e7d060..d34c26d606 100644 --- a/sda-download/pkg/request/request_test.go +++ b/sda-download/pkg/request/request_test.go @@ -41,7 +41,6 @@ func TestInitialiseClient(t *testing.T) { } func TestMakeRequest_Fail_HTTPNewRequest(t *testing.T) { - // Save original to-be-mocked functions originalHTTPMakeRequest := HTTPNewRequest @@ -68,11 +67,9 @@ func TestMakeRequest_Fail_HTTPNewRequest(t *testing.T) { // Return mock functions to originals HTTPNewRequest = originalHTTPMakeRequest - } func TestMakeRequest_Fail_StatusCode(t *testing.T) { - // Create mock client client := newTestClient(func(_ *http.Request) *http.Response { return &http.Response{ @@ -116,11 +113,9 @@ func TestMakeRequest_Fail_StatusCode(t *testing.T) { // Return mock functions to originals HTTPNewRequest = originalHTTPMakeRequest - } func TestMakeRequest_Success(t *testing.T) { - // Create mock client client := newTestClient(func(_ *http.Request) *http.Response { return &http.Response{ @@ -167,5 +162,4 @@ func TestMakeRequest_Success(t *testing.T) { // Return mock functions to originals HTTPNewRequest = originalHTTPMakeRequest - } diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 555cf5b590..8f0f1b5f93 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -83,7 +83,6 @@ func main() { log.Fatalln(err) } } - } func setup(config *config.Config) *http.Server { diff --git a/sda/cmd/auth/info_test.go b/sda/cmd/auth/info_test.go index a39fb1f109..b0fe1f8eea 100644 --- a/sda/cmd/auth/info_test.go +++ b/sda/cmd/auth/info_test.go @@ -44,7 +44,6 @@ func (suite *InfoTests) SetupTest() { if err != nil { suite.T().FailNow() } - } func (suite *InfoTests) TestReadPublicKeyFile() { diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index d0719f03d0..9efad24338 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -45,7 +45,6 @@ type AuthHandler struct { // downloadable s3cmd file with the specified fileName. Redirects to home if // config is missing. func (auth AuthHandler) getS3Config(ctx iris.Context, authType string, fileName string) { - log.Infoln(ctx.Request().URL.Path) s := sessions.Get(ctx) @@ -75,7 +74,6 @@ func (auth AuthHandler) getS3Config(ctx iris.Context, authType string, fileName // getMain returns the index.html page func (auth AuthHandler) getMain(ctx iris.Context) { - ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) err := ctx.View("index.html") @@ -88,7 +86,6 @@ func (auth AuthHandler) getMain(ctx iris.Context) { // getLoginOptions returns the available login providers as JSON func (auth AuthHandler) getLoginOptions(ctx iris.Context) { - var response []LoginOption // Only add the OIDC option if it has both id and secret if auth.Config.OIDC.ID != "" && auth.Config.OIDC.Secret != "" { @@ -109,7 +106,6 @@ func (auth AuthHandler) getLoginOptions(ctx iris.Context) { // postEGA handles post requests for logging in using EGA func (auth AuthHandler) postEGA(ctx iris.Context) { - s := sessions.Get(ctx) userform := ctx.FormValues() @@ -181,7 +177,6 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { err = ctx.View("loginform.html") log.Error("Failed to create backup view: ", err) } - } else { log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Invalid password entered by user") s.SetFlash("message", "Provided credentials are not valid") @@ -206,7 +201,6 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { // getEGALogin returns the EGA login form func (auth AuthHandler) getEGALogin(ctx iris.Context) { - s := sessions.Get(ctx) ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) @@ -301,7 +295,6 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { // getOIDCLogin renders the `oidc.html` template to the given iris context func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { - oidcData := auth.elixirLogin(ctx) if oidcData == nil { return @@ -331,7 +324,6 @@ func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { // getOIDCCORSLogin returns the oidc data as JSON to the given iris context func (auth AuthHandler) getOIDCCORSLogin(ctx iris.Context) { - oidcData := auth.elixirLogin(ctx) if oidcData == nil { return @@ -357,14 +349,12 @@ func (auth AuthHandler) getOIDCConfDownload(ctx iris.Context) { // globalHeaders presets common response headers func globalHeaders(ctx iris.Context) { - ctx.ResponseWriter().Header().Set("X-Content-Type-Options", "nosniff") ctx.Next() } // addCSPheaders implements CSP and recommended complementary policies func addCSPheaders(ctx iris.Context) { - ctx.ResponseWriter().Header().Set("Content-Security-Policy", "default-src 'self';"+ "script-src-elem 'self';"+ "img-src 'self' data:;"+ @@ -377,7 +367,6 @@ func addCSPheaders(ctx iris.Context) { } func main() { - // Initialise config config, err := config.NewConfig("auth") if err != nil { @@ -462,11 +451,9 @@ func main() { app.UseGlobal(globalHeaders) if config.Server.Cert != "" && config.Server.Key != "" { - log.Infoln("Serving content using https") err = app.Run(iris.TLS("0.0.0.0:8080", config.Server.Cert, config.Server.Key)) } else { - log.Infoln("Serving content using http") server := &http.Server{ Addr: "0.0.0.0:8080", diff --git a/sda/cmd/auth/oidc_test.go b/sda/cmd/auth/oidc_test.go index 7042447649..029d811144 100644 --- a/sda/cmd/auth/oidc_test.go +++ b/sda/cmd/auth/oidc_test.go @@ -53,7 +53,6 @@ func (suite *OIDCTests) TearDownTest() { } func (suite *OIDCTests) TestGetOidcClient() { - expectedEndpoint := oauth2.Endpoint{ AuthURL: suite.mockServer.AuthorizationEndpoint(), TokenURL: suite.mockServer.TokenEndpoint(), diff --git a/sda/cmd/auth/s3conf_test.go b/sda/cmd/auth/s3conf_test.go index 31a6e3de6a..71030b5149 100644 --- a/sda/cmd/auth/s3conf_test.go +++ b/sda/cmd/auth/s3conf_test.go @@ -25,7 +25,6 @@ func (suite *S3ConfTests) TearDownTest() {} //nolint:goconst func (suite *S3ConfTests) TestGetS3ConfigMap() { - // variable values token := "tokenvaluestring" inboxHost := "s3://inboxHost" @@ -59,5 +58,4 @@ func (suite *S3ConfTests) TestGetS3ConfigMap() { assert.Equal(suite.T(), fmt.Sprintf("%v", chunkSize), s3conf["multipart_chunk_size_mb"], fmt.Sprintf("multipart_chunk_size_mb should be %v", chunkSize)) assert.Equal(suite.T(), useHTTPS, s3conf["use_https"], fmt.Sprintf("use_https should be '%v'", useHTTPS)) assert.Equal(suite.T(), fmt.Sprintf("%v", socketTimeout), s3conf["socket_timeout"], fmt.Sprintf("socket_timeout should be %v", socketTimeout)) - } diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 0c436ee6d5..fa82ac9520 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -595,7 +595,6 @@ func tryDecrypt(key *[32]byte, buf []byte) ([]byte, error) { log.Error(err) return nil, err - } _, err = b.ReadByte() if err != nil { diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 215e52cedb..b51a3479f0 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -180,5 +180,4 @@ func schemaFromDatasetOperation(body []byte) (string, error) { default: return "", fmt.Errorf("could not recognize mapping operation") } - } diff --git a/sda/cmd/notify/notify_test.go b/sda/cmd/notify/notify_test.go index 4026b44188..5795d905ec 100644 --- a/sda/cmd/notify/notify_test.go +++ b/sda/cmd/notify/notify_test.go @@ -27,7 +27,6 @@ func (suite *TestSuite) SetupTest() { } func TestGetUser(t *testing.T) { - archivedMsg := schema.IngestionVerification{ User: "JohnDoe", FilePath: "path/to file", @@ -54,7 +53,6 @@ func TestGetUser(t *testing.T) { orgUser := getUser("error", infoErrorBytes) assert.Equal(t, "JohnDoe", orgUser) - } func TestSetSubject(t *testing.T) { diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index be94da4458..ce78a456e9 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -201,7 +201,6 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * continue } } - } } @@ -258,7 +257,6 @@ func schemaFromInboxOperation(body []byte) (string, error) { default: return "", errors.New("could not recognize inbox operation") } - } // schemaFromDatasetOperation returns the operation done with dataset @@ -290,7 +288,6 @@ func schemaFromDatasetOperation(body []byte) (string, error) { default: return "", errors.New("could not recognize inbox operation") } - } func ingestMessage(body []byte) ([]byte, any) { diff --git a/sda/cmd/reencrypt/reencrypt.go b/sda/cmd/reencrypt/reencrypt.go index 789a699aa1..5c25e721a9 100644 --- a/sda/cmd/reencrypt/reencrypt.go +++ b/sda/cmd/reencrypt/reencrypt.go @@ -68,7 +68,6 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r dataEditList := in.GetDataeditlist() if len(dataEditList) > 0 { // linter doesn't like checking for nil before len - // Check that G115: integer overflow conversion int -> uint32 is satisfied if len(dataEditList) > int(math.MaxUint32) { return nil, status.Error(400, "data edit list too long") @@ -106,7 +105,6 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r // server status. The corresponding grpc health server serves as a proxy to the internal health // service of the reencrypt server so that k8s grpc probes can be used when TLS is enabled. func (p *hServer) Check(ctx context.Context, in *healthgrpc.HealthCheckRequest) (*healthgrpc.HealthCheckResponse, error) { - rpcCtx, rpcCancel := context.WithTimeout(ctx, time.Second*2) defer rpcCancel() diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 134ce77c63..861e1ea85a 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -82,7 +82,6 @@ func (suite *ReEncryptTests) SetupTest() { suite.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") suite.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") - } func (suite *ReEncryptTests) TearDownTest() { @@ -203,7 +202,6 @@ func (suite *ReEncryptTests) TestReencryptHeader_DataEditList() { assert.NoError(suite.T(), err) packet = header.GetDataEditListHeaderPacket() assert.Nilf(suite.T(), packet, "DataEditList HeaderPacket found when not expected") - } func (suite *ReEncryptTests) TestReencryptHeader_BadPubKey() { diff --git a/sda/cmd/s3inbox/healthchecks.go b/sda/cmd/s3inbox/healthchecks.go index cf1881a70c..1fc432f2d3 100644 --- a/sda/cmd/s3inbox/healthchecks.go +++ b/sda/cmd/s3inbox/healthchecks.go @@ -14,7 +14,6 @@ import ( // CheckHealth checks and tries to repair the connections to MQ, DB and S3 func (p *Proxy) CheckHealth(w http.ResponseWriter, _ *http.Request) { - // try to connect to mq, check connection and channel var err error if p.messenger == nil { @@ -88,7 +87,6 @@ func (p *Proxy) httpsGetCheck(url string) error { } func (p *Proxy) getS3ReadyPath() (string, error) { - s3URL, err := url.Parse(p.s3.URL) if err != nil { return "", err diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index fe7448aba8..1502c90bb9 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -108,7 +108,6 @@ func (suite *HealthcheckTestSuite) TestHealthchecks() { resp := w.Result() defer resp.Body.Close() assert.Equal(suite.T(), 200, resp.StatusCode) - } func (suite *HealthcheckTestSuite) TestClosedDBHealthchecks() { diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index ee717be774..64e463c01c 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -177,7 +177,6 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } - } log.Debug("Forwarding to backend") @@ -304,21 +303,18 @@ func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { func (p *Proxy) uploadFinishedSuccessfully(req *http.Request, response *http.Response) bool { if response.StatusCode != 200 { - return false } switch req.Method { case http.MethodPut: if !strings.Contains(req.URL.String(), "partNumber") { - return true } return false case http.MethodPost: if strings.Contains(req.URL.String(), "uploadId") { - return true } @@ -530,7 +526,6 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { } return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.ReplaceAll(*result.Contents[0].ETag, "\"", "")))), *result.Contents[0].Size, nil - } // checkFileExists makes a request to the S3 to check whether the file already diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 3aa5ed932c..e8ce587a4e 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -49,7 +49,6 @@ func TestProxyTestSuite(t *testing.T) { } func (suite *ProxyTests) SetupTest() { - // Create fake server suite.fakeServer = startFakeServer("9024") diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index ebdd7977e3..18a10c7477 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -137,7 +137,6 @@ func main() { log.Errorf("GetHeader failed for file with ID: %v, readon: %v", message.FileID, err.Error()) if err := delivered.Ack(false); err != nil { log.Errorf("Failed to nack following getheader error message") - } // store full message info in case we want to fix the db entry and retry infoErrorMessage := broker.InfoError{ diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index 37ffe08411..4ad1ca8287 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -287,7 +287,6 @@ func (suite *BrokerTestSuite) TestCreateNewChannel() { assert.NoError(suite.T(), b.CreateNewChannel()) assert.False(suite.T(), b.Channel.IsClosed()) - } // Helper functions below this line diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 0320778f81..3e5afecab3 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -1047,7 +1047,6 @@ func (c *Config) configSyncAPI() { if viper.IsSet("sync.api.MappingRouting") { c.SyncAPI.MappingRouting = viper.GetString("sync.api.MappingRouting") } - } // GetC4GHKey reads and decrypts and returns the c4gh key diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 8e016f27f8..4295dce4fa 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -85,7 +85,6 @@ var RetryTimes = 5 // NewSDAdb creates a new DB connection from the given DBConf variables. // Currently, only postgresql connections are supported. func NewSDAdb(config DBConf) (*SDAdb, error) { - dbs := SDAdb{DB: nil, Version: -1, Config: config} err := dbs.Connect() @@ -173,7 +172,6 @@ func (config *DBConf) PgDataSource() (string, string) { // getVersion fetches the database schema version. This function return -1 when // the version could not be fetched. func (dbs *SDAdb) getVersion() (int, error) { - dbs.checkAndReconnectIfNeeded() log.Debug("Fetching database schema version") diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index d7264fb18c..9f8481112e 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -20,7 +20,6 @@ import ( // event. If the file already exists in the database, the entry is updated, but // a new file event is always inserted. func (dbs *SDAdb) RegisterFile(uploadPath, uploadUser string) (string, error) { - dbs.checkAndReconnectIfNeeded() if dbs.Version < 4 { @@ -551,7 +550,6 @@ func (dbs *SDAdb) updateDatasetEvent(datasetID, status, message string) error { } return nil - } // GetFileInfo returns info on a ingested file @@ -907,7 +905,6 @@ func (dbs *SDAdb) SetKeyHash(keyHash, fileID string) error { query := "UPDATE sda.files SET key_hash = $1 WHERE id = $2;" result, err := db.Exec(query, keyHash, fileID) if err != nil { - return err } if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index efca0f43fb..369220aea3 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -14,7 +14,6 @@ import ( // TestRegisterFile tests that RegisterFile() behaves as intended func (suite *DatabaseTests) TestRegisterFile() { - // create database connection db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) @@ -38,7 +37,6 @@ func (suite *DatabaseTests) TestRegisterFile() { err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.file_event_log WHERE file_id=$1 AND event='registered')", fileID).Scan(&exists) assert.NoError(suite.T(), err, "Failed to check if registered file event exists") assert.True(suite.T(), exists, "RegisterFile() did not insert a row into sda.file_event_log with id: "+fileID) - } func (suite *DatabaseTests) TestGetFileID() { @@ -539,7 +537,6 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { corrID, err := db.GetCorrID(user, filePath, "") assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") assert.Equal(suite.T(), fileID2, corrID) - } func (suite *DatabaseTests) TestGetCorrID_wrongFilePath() { @@ -954,7 +951,6 @@ func (suite *DatabaseTests) TestListUserDatasets() { if err != nil { suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } - } if err := db.MapFilesToDataset("test-user-dataset-01", []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}); err != nil { suite.FailNow("failed to map files to dataset") diff --git a/sda/internal/database/db_test.go b/sda/internal/database/db_test.go index 554a2385b2..fe8e3ba0f3 100644 --- a/sda/internal/database/db_test.go +++ b/sda/internal/database/db_test.go @@ -129,7 +129,6 @@ func (suite *DatabaseTests) TearDownTest() {} // TestNewSDAdb tests creation of new database connections, as well as fetching // of the database schema version. func (suite *DatabaseTests) TestNewSDAdb() { - // test working database connection db, err := NewSDAdb(suite.dbConf) assert.Nil(suite.T(), err, "got %v when creating new connection", err) @@ -151,12 +150,10 @@ func (suite *DatabaseTests) TestNewSDAdb() { _, err = NewSDAdb(wrongConf) assert.NotNil(suite.T(), err, "connection allowed with wrong credentials") - } // TestConnect tests creation of new database connections func (suite *DatabaseTests) TestConnect() { - // test connecting to a database db := SDAdb{DB: nil, Version: -1, Config: suite.dbConf} @@ -197,12 +194,10 @@ func (suite *DatabaseTests) TestConnect() { db.Config = wrongConf err = db.Connect() assert.NotNil(suite.T(), err, "connection allowed with wrong credentials") - } // TestClose tests that the connection is properly closed func (suite *DatabaseTests) TestClose() { - // test working database connection db, err := NewSDAdb(suite.dbConf) assert.Nil(suite.T(), err, "got %v when creating new connection", err) diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index 1dcacb72b1..f09c435928 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -262,7 +262,6 @@ func CreateHSToken(key []byte, tokenClaims map[string]any) (string, error) { if err := token.Set(key, value); err != nil { return "failed to set claim", err } - } jwtKey, err := jwk.FromRaw(key) diff --git a/sda/internal/jsonadapter/jsonadapter_test.go b/sda/internal/jsonadapter/jsonadapter_test.go index 8aa655cbcf..6fff5ba7ca 100644 --- a/sda/internal/jsonadapter/jsonadapter_test.go +++ b/sda/internal/jsonadapter/jsonadapter_test.go @@ -49,7 +49,6 @@ func (suite *AdapterTestSuite) SetupSuite() { if err != nil { suite.FailNow("failed to write policy file to disk") } - } func (suite *AdapterTestSuite) TearDownSuite() { diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index 4a1fd41b43..f65c1b29de 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -314,7 +314,6 @@ func (sb *s3Backend) GetFileSize(filePath string, expectDelay bool) (int64, erro Bucket: &sb.Bucket, Key: &filePath, }) - } if err != nil { diff --git a/sda/internal/userauth/userauth.go b/sda/internal/userauth/userauth.go index ba9b8c0dff..375e98eff7 100644 --- a/sda/internal/userauth/userauth.go +++ b/sda/internal/userauth/userauth.go @@ -42,7 +42,6 @@ func (u *ValidateFromToken) Authenticate(r *http.Request) (jwt.Token, error) { return nil, fmt.Errorf("error validating token keyset") } switch { - case r.Header.Get("X-Amz-Security-Token") != "": tokenStr := r.Header.Get("X-Amz-Security-Token") if tokenStr == "" { @@ -80,7 +79,6 @@ func (u *ValidateFromToken) Authenticate(r *http.Request) (jwt.Token, error) { default: return nil, fmt.Errorf("no access token supplied") - } } From e5a86d4973866f4b5a964c9b9ab07cfd0cc8526e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 10:10:21 +0200 Subject: [PATCH 205/250] [revive] don't use variable names that match library names --- sda-admin/c4ghkeyhash/c4ghKeyHash_test.go | 48 +- sda-download/api/s3/s3_test.go | 62 +- sda-download/internal/config/config.go | 6 +- sda-download/internal/config/config_test.go | 122 +-- sda-download/internal/database/database.go | 24 +- sda-download/pkg/auth/auth.go | 8 +- sda/cmd/api/api.go | 14 +- sda/cmd/api/api_test.go | 1012 +++++++++--------- sda/cmd/auth/cega_test.go | 10 +- sda/cmd/auth/info_test.go | 28 +- sda/cmd/auth/jwt_test.go | 60 +- sda/cmd/auth/main.go | 22 +- sda/cmd/auth/oidc_test.go | 142 +-- sda/cmd/auth/s3conf_test.go | 34 +- sda/cmd/finalize/finalize_test.go | 2 +- sda/cmd/ingest/ingest_test.go | 350 +++--- sda/cmd/intercept/intercept_test.go | 32 +- sda/cmd/mapper/mapper_test.go | 2 +- sda/cmd/notify/notify_test.go | 2 +- sda/cmd/reencrypt/reencrypt_test.go | 206 ++-- sda/cmd/s3inbox/healthchecks.go | 4 +- sda/cmd/s3inbox/healthchecks_test.go | 90 +- sda/cmd/s3inbox/proxy.go | 6 +- sda/cmd/s3inbox/proxy_test.go | 478 ++++----- sda/cmd/s3inbox/s3inbox.go | 10 +- sda/cmd/sync/sync.go | 14 +- sda/cmd/sync/sync_test.go | 42 +- sda/cmd/syncapi/syncapi.go | 4 +- sda/cmd/syncapi/syncapi_test.go | 98 +- sda/cmd/verify/verify_test.go | 2 +- sda/internal/broker/broker_test.go | 86 +- sda/internal/config/config.go | 42 +- sda/internal/config/config_test.go | 370 +++---- sda/internal/database/db_test.go | 44 +- sda/internal/helper/helper_test.go | 80 +- sda/internal/jsonadapter/jsonadapter.go | 16 +- sda/internal/jsonadapter/jsonadapter_test.go | 98 +- sda/internal/storage/storage.go | 34 +- sda/internal/storage/storage_test.go | 226 ++-- sda/internal/userauth/userauth_test.go | 122 +-- 40 files changed, 2026 insertions(+), 2026 deletions(-) diff --git a/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go b/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go index c3a6f61cb1..591e8accc3 100644 --- a/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go +++ b/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go @@ -40,41 +40,41 @@ func TestC4gh(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupSuite() { - suite.tempFolder = "/tmp/keys/" - err := os.MkdirAll(suite.tempFolder, 0750) +func (ts *TestSuite) SetupSuite() { + ts.tempFolder = "/tmp/keys/" + err := os.MkdirAll(ts.tempFolder, 0750) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } pub, _, err := keys.GenerateKeyPair() if err != nil { - suite.T().FailNow() + ts.T().FailNow() } buf := new(bytes.Buffer) if err := keys.WriteCrypt4GHX25519PublicKey(buf, pub); err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.b64String = base64.StdEncoding.EncodeToString(buf.Bytes()) + ts.b64String = base64.StdEncoding.EncodeToString(buf.Bytes()) - pubKeyFile, err := os.Create(suite.tempFolder + "/pub.key") + pubKeyFile, err := os.Create(ts.tempFolder + "/pub.key") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.pubkeyPath = suite.tempFolder + "/pub.key" + ts.pubkeyPath = ts.tempFolder + "/pub.key" _, err = pubKeyFile.Write(buf.Bytes()) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } } -func (suite *TestSuite) TearDownSuite() { - os.RemoveAll(suite.tempFolder) +func (ts *TestSuite) TearDownSuite() { + os.RemoveAll(ts.tempFolder) } -func (suite *TestSuite) TestAdd() { +func (ts *TestSuite) TestAdd() { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest @@ -84,21 +84,21 @@ func (suite *TestSuite) TestAdd() { token := "test-token" payload := C4ghPubKey{ - PubKey: suite.b64String, + PubKey: ts.b64String, Description: "test description", } jsonBody, err := json.Marshal(payload) if err != nil { - suite.T().Fail() + ts.T().Fail() } mockHelpers.On("PostRequest", expectedURL, token, jsonBody).Return([]byte(`{}`), nil) - assert.NoError(suite.T(), Add("http://example.com", token, suite.pubkeyPath, "test description")) - mockHelpers.AssertExpectations(suite.T()) + assert.NoError(ts.T(), Add("http://example.com", token, ts.pubkeyPath, "test description")) + mockHelpers.AssertExpectations(ts.T()) } -func (suite *TestSuite) TestDeprecate() { +func (ts *TestSuite) TestDeprecate() { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest @@ -108,11 +108,11 @@ func (suite *TestSuite) TestDeprecate() { token := "test-token" mockHelpers.On("PostRequest", expectedURL, token, []byte(`{}`)).Return([]byte(`{}`), nil) - assert.NoError(suite.T(), Deprecate("http://example.com", token, "6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc507")) - mockHelpers.AssertExpectations(suite.T()) + assert.NoError(ts.T(), Deprecate("http://example.com", token, "6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc507")) + mockHelpers.AssertExpectations(ts.T()) } -func (suite *TestSuite) TestList() { +func (ts *TestSuite) TestList() { mockHelpers := new(MockHelpers) mockHelpers.On("GetResponseBody", "http://example.com/c4gh-keys/list", "test-token").Return([]byte(`[{"hash":"cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23","description":"this is a test key","created_at":"2009-11-10 23:00:00","deprecated_at":""}]`), nil) originalFunc := helpers.GetResponseBody @@ -120,6 +120,6 @@ func (suite *TestSuite) TestList() { helpers.GetResponseBody = mockHelpers.GetResponseBody err := List("http://example.com", "test-token") - assert.NoError(suite.T(), err) - mockHelpers.AssertExpectations(suite.T()) + assert.NoError(ts.T(), err) + mockHelpers.AssertExpectations(ts.T()) } diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go index bf08274211..4e19159ac2 100644 --- a/sda-download/api/s3/s3_test.go +++ b/sda-download/api/s3/s3_test.go @@ -23,16 +23,16 @@ type S3TestSuite struct { Mock sqlmock.Sqlmock } -func (suite *S3TestSuite) SetupTest() { +func (ts *S3TestSuite) SetupTest() { var err error var db *sql.DB // create mock database testConnInfo := "host=localhost port=5432 user=user password=pass dbname=db sslmode=disable" - db, suite.Mock, err = sqlmock.New() + db, ts.Mock, err = sqlmock.New() if err != nil { - suite.T().Fatalf("error '%s' when creating mock database connection", err) + ts.T().Fatalf("error '%s' when creating mock database connection", err) } database.DB = &database.SQLdb{DB: db, ConnInfo: testConnInfo} @@ -52,14 +52,14 @@ func (suite *S3TestSuite) SetupTest() { } } -func (suite *S3TestSuite) TearDownTest() { +func (ts *S3TestSuite) TearDownTest() { } func TestS3TestSuite(t *testing.T) { suite.Run(t, new(S3TestSuite)) } -func (suite *S3TestSuite) TestGetBucketLocation() { +func (ts *S3TestSuite) TestGetBucketLocation() { // Send a request through the middleware to get datasets w := httptest.NewRecorder() @@ -69,23 +69,23 @@ func (suite *S3TestSuite) TestGetBucketLocation() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := "\nus-west-2" - assert.Equal(suite.T(), expected, string(body), "Wrong location from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong location from S3") } -func (suite *S3TestSuite) TestListBuckets() { +func (ts *S3TestSuite) TestListBuckets() { // Setup a mock database to handle queries query := `SELECT stable_id, created_at FROM sda.datasets WHERE stable_id = \$1` - suite.Mock.ExpectQuery(query).WithArgs("dataset1"). + ts.Mock.ExpectQuery(query).WithArgs("dataset1"). WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "nyss")) - suite.Mock.ExpectQuery(query).WithArgs("dataset10"). + ts.Mock.ExpectQuery(query).WithArgs("dataset10"). WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "nyligen")) - suite.Mock.ExpectQuery(query).WithArgs("https://url/dataset"). + ts.Mock.ExpectQuery(query).WithArgs("https://url/dataset"). WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "snart")) // Send a request through the middleware to get datasets @@ -97,7 +97,7 @@ func (suite *S3TestSuite) TestListBuckets() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := xml.Header + @@ -107,13 +107,13 @@ func (suite *S3TestSuite) TestListBuckets() { "snartdataset1" + "" - assert.Equal(suite.T(), expected, string(body), "Wrong bucket list from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong bucket list from S3") - err = suite.Mock.ExpectationsWereMet() - assert.Nilf(suite.T(), err, "there were unfulfilled expectations: %s", err) + err = ts.Mock.ExpectationsWereMet() + assert.Nilf(ts.T(), err, "there were unfulfilled expectations: %s", err) } -func (suite *S3TestSuite) TestListByPrefix() { +func (ts *S3TestSuite) TestListByPrefix() { // Setup a mock database to handle queries fileInfo := &database.FileInfo{ FileID: "file1", @@ -150,7 +150,7 @@ func (suite *S3TestSuite) TestListByPrefix() { LEFT JOIN \(SELECT file_id, checksum, type FROM sda.checksums WHERE source = 'UNENCRYPTED'\) sha ON files.id = sha.file_id WHERE datasets.stable_id = \$1; ` - suite.Mock.ExpectQuery(query). + ts.Mock.ExpectQuery(query). WithArgs("dataset1"). WillReturnRows(sqlmock.NewRows([]string{"file_id", "dataset_id", "display_file_name", "user_id", "file_path", "file_size", @@ -172,7 +172,7 @@ func (suite *S3TestSuite) TestListByPrefix() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := xml.Header + @@ -184,13 +184,13 @@ func (suite *S3TestSuite) TestListByPrefix() { "dataset1" + "" - assert.Equal(suite.T(), expected, string(body), "Wrong object list from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong object list from S3") - err = suite.Mock.ExpectationsWereMet() - assert.Nilf(suite.T(), err, "there were unfulfilled expectations: %s", err) + err = ts.Mock.ExpectationsWereMet() + assert.Nilf(ts.T(), err, "there were unfulfilled expectations: %s", err) } -func (suite *S3TestSuite) TestListObjects() { +func (ts *S3TestSuite) TestListObjects() { // Setup a mock database to handlequeries fileInfo := &database.FileInfo{ FileID: "file1", @@ -227,7 +227,7 @@ func (suite *S3TestSuite) TestListObjects() { LEFT JOIN \(SELECT file_id, checksum, type FROM sda.checksums WHERE source = 'UNENCRYPTED'\) sha ON files.id = sha.file_id WHERE datasets.stable_id = \$1; ` - suite.Mock.ExpectQuery(query). + ts.Mock.ExpectQuery(query). WithArgs("dataset1"). WillReturnRows(sqlmock.NewRows([]string{"file_id", "dataset_id", "display_file_name", "user_id", "file_path", "file_size", @@ -248,7 +248,7 @@ func (suite *S3TestSuite) TestListObjects() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := xml.Header + @@ -260,13 +260,13 @@ func (suite *S3TestSuite) TestListObjects() { "dataset1" + "" - assert.Equal(suite.T(), expected, string(body), "Wrong object list from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong object list from S3") - err = suite.Mock.ExpectationsWereMet() - assert.Nilf(suite.T(), err, "there were unfulfilled expectations: %s", err) + err = ts.Mock.ExpectationsWereMet() + assert.Nilf(ts.T(), err, "there were unfulfilled expectations: %s", err) } -func (suite *S3TestSuite) TestParseParams() { +func (ts *S3TestSuite) TestParseParams() { type paramTest struct { Path string Dataset string @@ -289,8 +289,8 @@ func (suite *S3TestSuite) TestParseParams() { testParseParams := func(c *gin.Context) { parseParams(c) - assert.Equal(suite.T(), params.Dataset, c.Param("dataset"), "Failed to parse dataset name") - assert.Equal(suite.T(), params.Filename, c.Param("filename"), "Failed to parse file name") + assert.Equal(ts.T(), params.Dataset, c.Param("dataset"), "Failed to parse dataset name") + assert.Equal(ts.T(), params.Filename, c.Param("filename"), "Failed to parse file name") c.AbortWithStatus(http.StatusAccepted) } @@ -305,6 +305,6 @@ func (suite *S3TestSuite) TestParseParams() { response := w.Result() defer response.Body.Close() - assert.Equal(suite.T(), http.StatusAccepted, response.StatusCode, "Request failed") + assert.Equal(ts.T(), http.StatusAccepted, response.StatusCode, "Request failed") } } diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 515af2ef3c..102c436e04 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -474,12 +474,12 @@ func readTrustedIssuers(filePath string) ([]TrustedISS, error) { } func constructWhitelist(obj []TrustedISS) *jwk.MapWhitelist { - keys := make(map[string]bool) + keyMap := make(map[string]bool) wl := jwk.NewMapWhitelist() for _, value := range obj { - if _, ok := keys[value.JKU]; !ok { - keys[value.JKU] = true + if _, ok := keyMap[value.JKU]; !ok { + keyMap[value.JKU] = true wl.Add(value.JKU) } } diff --git a/sda-download/internal/config/config_test.go b/sda-download/internal/config/config_test.go index 7f4c473c14..e656301ded 100644 --- a/sda-download/internal/config/config_test.go +++ b/sda-download/internal/config/config_test.go @@ -23,7 +23,7 @@ type TestSuite struct { suite.Suite } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("db.host", "test") viper.Set("db.user", "test") viper.Set("db.password", "test") @@ -33,7 +33,7 @@ func (suite *TestSuite) SetupTest() { viper.Set("oidc.configuration.url", "test") } -func (suite *TestSuite) TearDownTest() { +func (ts *TestSuite) TearDownTest() { viper.Reset() } @@ -41,38 +41,38 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) TestConfigFile() { +func (ts *TestSuite) TestConfigFile() { viper.Set("configFile", "test") config, err := NewConfig() - assert.Nil(suite.T(), config) - assert.Error(suite.T(), err) - assert.Equal(suite.T(), "test", viper.ConfigFileUsed()) + assert.Nil(ts.T(), config) + assert.Error(ts.T(), err) + assert.Equal(ts.T(), "test", viper.ConfigFileUsed()) } -func (suite *TestSuite) TestMissingRequiredConfVar() { +func (ts *TestSuite) TestMissingRequiredConfVar() { for _, requiredConfVar := range requiredConfVars { requiredConfVarValue := viper.Get(requiredConfVar) viper.Set(requiredConfVar, nil) expectedError := fmt.Errorf("%s not set", requiredConfVar) config, err := NewConfig() - assert.Nil(suite.T(), config) - if assert.Error(suite.T(), err) { - assert.Equal(suite.T(), expectedError, err) + assert.Nil(ts.T(), config) + if assert.Error(ts.T(), err) { + assert.Equal(ts.T(), expectedError, err) } viper.Set(requiredConfVar, requiredConfVarValue) } } -func (suite *TestSuite) TestAppConfig() { +func (ts *TestSuite) TestAppConfig() { // Generate a crypth4gh private key file _, privateKey, err := keys.GenerateKeyPair() - assert.NoError(suite.T(), err) - tempDir := suite.T().TempDir() + assert.NoError(ts.T(), err) + tempDir := ts.T().TempDir() defer os.RemoveAll(tempDir) privateKeyFile, err := os.Create(fmt.Sprintf("%s/c4fg.key", tempDir)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = keys.WriteCrypt4GHX25519PrivateKey(privateKeyFile, privateKey, []byte("password")) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Test fail on missing middleware viper.Set("app.host", "test") @@ -86,7 +86,7 @@ func (suite *TestSuite) TestAppConfig() { c := &Map{} err = c.appConfig() - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") viper.Reset() viper.Set("app.host", "test") @@ -100,42 +100,42 @@ func (suite *TestSuite) TestAppConfig() { c = &Map{} err = c.appConfig() - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "test", c.App.Host) - assert.Equal(suite.T(), 1234, c.App.Port) - assert.Equal(suite.T(), "test", c.App.ServerCert) - assert.Equal(suite.T(), "test", c.App.ServerKey) - assert.NotEmpty(suite.T(), c.C4GH.PrivateKey) - assert.NotEmpty(suite.T(), c.C4GH.PublicKeyB64) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "test", c.App.Host) + assert.Equal(ts.T(), 1234, c.App.Port) + assert.Equal(ts.T(), "test", c.App.ServerCert) + assert.Equal(ts.T(), "test", c.App.ServerKey) + assert.NotEmpty(ts.T(), c.C4GH.PrivateKey) + assert.NotEmpty(ts.T(), c.C4GH.PublicKeyB64) // Check the private key that was loaded by checking the derived public key publicKey, err := base64.StdEncoding.DecodeString(c.C4GH.PublicKeyB64) - assert.Nilf(suite.T(), err, "Incorrect public c4gh key generated (error in base64 encoding)") + assert.Nilf(ts.T(), err, "Incorrect public c4gh key generated (error in base64 encoding)") _, err = keys.ReadPublicKey(bytes.NewReader(publicKey)) - assert.Nilf(suite.T(), err, "Incorrect public c4gh key generated (bad key)") + assert.Nilf(ts.T(), err, "Incorrect public c4gh key generated (bad key)") // Check false c4gh key viper.Set("c4gh.transientKeyPath", "some/nonexistent.key") err = c.appConfig() - assert.ErrorContains(suite.T(), err, "no such file or directory") + assert.ErrorContains(ts.T(), err, "no such file or directory") // Check false c4gh key viper.Set("c4gh.transientKeyPath", privateKeyFile.Name()) viper.Set("c4gh.transientPassphrase", "blablabla") err = c.appConfig() - assert.ErrorContains(suite.T(), err, "chacha20poly1305: message authentication failed") + assert.ErrorContains(ts.T(), err, "chacha20poly1305: message authentication failed") } -func (suite *TestSuite) TestArchiveConfig() { +func (ts *TestSuite) TestArchiveConfig() { viper.Set("archive.type", POSIX) viper.Set("archive.location", "/test") c := &Map{} c.configArchive() - assert.Equal(suite.T(), "/test", c.Archive.Posix.Location) + assert.Equal(ts.T(), "/test", c.Archive.Posix.Location) } -func (suite *TestSuite) TestSessionConfig() { +func (ts *TestSuite) TestSessionConfig() { viper.Set("session.expiration", 3600) viper.Set("session.domain", "test") viper.Set("session.secure", false) @@ -145,24 +145,24 @@ func (suite *TestSuite) TestSessionConfig() { c := &Map{} c.sessionConfig() - assert.Equal(suite.T(), time.Duration(3600*time.Second), c.Session.Expiration) - assert.Equal(suite.T(), "test", c.Session.Domain) - assert.Equal(suite.T(), false, c.Session.Secure) - assert.Equal(suite.T(), false, c.Session.HTTPOnly) + assert.Equal(ts.T(), time.Duration(3600*time.Second), c.Session.Expiration) + assert.Equal(ts.T(), "test", c.Session.Domain) + assert.Equal(ts.T(), false, c.Session.Secure) + assert.Equal(ts.T(), false, c.Session.HTTPOnly) } -func (suite *TestSuite) TestDatabaseConfig() { +func (ts *TestSuite) TestDatabaseConfig() { // Test error on missing SSL vars viper.Set("db.sslmode", "verify-full") c := &Map{} err := c.configDatabase() - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") // Test no error on SSL disabled viper.Set("db.sslmode", "disable") c = &Map{} err = c.configDatabase() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Test pass on SSL vars set viper.Set("db.host", "test") @@ -177,28 +177,28 @@ func (suite *TestSuite) TestDatabaseConfig() { c = &Map{} err = c.configDatabase() - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "test", c.DB.Host) - assert.Equal(suite.T(), 1234, c.DB.Port) - assert.Equal(suite.T(), "test", c.DB.User) - assert.Equal(suite.T(), "test", c.DB.Password) - assert.Equal(suite.T(), "test", c.DB.Database) - assert.Equal(suite.T(), "test", c.DB.CACert) - assert.Equal(suite.T(), "test", c.DB.ClientCert) - assert.Equal(suite.T(), "test", c.DB.ClientKey) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "test", c.DB.Host) + assert.Equal(ts.T(), 1234, c.DB.Port) + assert.Equal(ts.T(), "test", c.DB.User) + assert.Equal(ts.T(), "test", c.DB.Password) + assert.Equal(ts.T(), "test", c.DB.Database) + assert.Equal(ts.T(), "test", c.DB.CACert) + assert.Equal(ts.T(), "test", c.DB.ClientCert) + assert.Equal(ts.T(), "test", c.DB.ClientKey) } -func (suite *TestSuite) TestOIDC() { +func (ts *TestSuite) TestOIDC() { // Test wrong file viper.Set("oidc.trusted.iss", "../../iss.json") c := &Map{} err := c.configureOIDC() - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") viper.Set("oidc.trusted.iss", "../../dev_utils/iss.json") c = &Map{} err = c.configureOIDC() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Test pass OIDC config viper.Set("oidc.trusted.iss", "../../dev_utils/iss.json") @@ -210,29 +210,29 @@ func (suite *TestSuite) TestOIDC() { whitelist := jwk.NewMapWhitelist().Add("https://mockauth:8000/idp/profile/oidc/keyset") c = &Map{} err = c.configureOIDC() - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "test", c.OIDC.ConfigurationURL) - assert.Equal(suite.T(), "test", c.OIDC.CACert) - assert.Equal(suite.T(), trustedList, c.OIDC.TrustedList) - assert.Equal(suite.T(), whitelist, c.OIDC.Whitelist) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "test", c.OIDC.ConfigurationURL) + assert.Equal(ts.T(), "test", c.OIDC.CACert) + assert.Equal(ts.T(), trustedList, c.OIDC.TrustedList) + assert.Equal(ts.T(), whitelist, c.OIDC.Whitelist) } -func (suite *TestSuite) TestConfigReencrypt() { - tempDir := suite.T().TempDir() +func (ts *TestSuite) TestConfigReencrypt() { + tempDir := ts.T().TempDir() c := &Map{} viper.Set("grpc.host", "localhost") - assert.NoError(suite.T(), c.configReencrypt()) - assert.Equal(suite.T(), 50051, c.Reencrypt.Port) + assert.NoError(ts.T(), c.configReencrypt()) + assert.Equal(ts.T(), 50051, c.Reencrypt.Port) // fail if set file doesn't exists viper.Set("grpc.clientcert", "/tmp/abracadabra") - assert.ErrorContains(suite.T(), c.configReencrypt(), "no such file or directory") + assert.ErrorContains(ts.T(), c.configReencrypt(), "no such file or directory") // any existing flle will make it pass viper.Set("grpc.clientcert", "config_test.go") - assert.NoError(suite.T(), c.configReencrypt()) + assert.NoError(ts.T(), c.configReencrypt()) // it will fail if certificate is set to a folder viper.Set("grpc.clientcert", tempDir) - assert.ErrorContains(suite.T(), c.configReencrypt(), "is a folder") + assert.ErrorContains(ts.T(), c.configReencrypt(), "is a folder") } diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index e7c8a58ae7..b6e7bf09f3 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -65,10 +65,10 @@ func sanitizeString(str string) string { } // NewDB creates a new DB connection -func NewDB(config config.DatabaseConfig) (*SQLdb, error) { - connInfo := buildConnInfo(config) +func NewDB(conf config.DatabaseConfig) (*SQLdb, error) { + connInfo := buildConnInfo(conf) - log.Debugf("Connecting to DB %s:%d on database: %s with user: %s", config.Host, config.Port, config.Database, config.User) + log.Debugf("Connecting to DB %s:%d on database: %s with user: %s", conf.Host, conf.Port, conf.Database, conf.User) db, err := sqlOpen("postgres", connInfo) if err != nil { log.Errorf("failed to connect to database, %s", err) @@ -88,24 +88,24 @@ func NewDB(config config.DatabaseConfig) (*SQLdb, error) { } // buildConnInfo builds a connection string for the database -func buildConnInfo(config config.DatabaseConfig) string { +func buildConnInfo(conf config.DatabaseConfig) string { connInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", - config.Host, config.Port, config.User, config.Password, config.Database, config.SslMode) + conf.Host, conf.Port, conf.User, conf.Password, conf.Database, conf.SslMode) - if config.SslMode == "disable" { + if conf.SslMode == "disable" { return connInfo } - if config.CACert != "" { - connInfo += fmt.Sprintf(" sslrootcert=%s", config.CACert) + if conf.CACert != "" { + connInfo += fmt.Sprintf(" sslrootcert=%s", conf.CACert) } - if config.ClientCert != "" { - connInfo += fmt.Sprintf(" sslcert=%s", config.ClientCert) + if conf.ClientCert != "" { + connInfo += fmt.Sprintf(" sslcert=%s", conf.ClientCert) } - if config.ClientKey != "" { - connInfo += fmt.Sprintf(" sslkey=%s", config.ClientKey) + if conf.ClientKey != "" { + connInfo += fmt.Sprintf(" sslkey=%s", conf.ClientKey) } return connInfo diff --git a/sda-download/pkg/auth/auth.go b/sda-download/pkg/auth/auth.go index 89ca8a3654..7133ecfd9a 100644 --- a/sda-download/pkg/auth/auth.go +++ b/sda-download/pkg/auth/auth.go @@ -30,12 +30,12 @@ type OIDCDetails struct { } // GetOIDCDetails requests OIDC configuration information -func GetOIDCDetails(url string) (OIDCDetails, error) { - log.Debugf("requesting OIDC config from %s", url) +func GetOIDCDetails(uri string) (OIDCDetails, error) { + log.Debugf("requesting OIDC config from %s", uri) // Prepare response body struct var u OIDCDetails // Do request - response, err := request.MakeRequest("GET", url, nil, nil) + response, err := request.MakeRequest("GET", uri, nil, nil) if err != nil { log.Errorf("request failed, %s", err) @@ -49,7 +49,7 @@ func GetOIDCDetails(url string) (OIDCDetails, error) { return u, err } defer response.Body.Close() - log.Debugf("received OIDC config %s from %s", u, url) + log.Debugf("received OIDC config %s from %s", u, uri) return u, nil } diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 8f0f1b5f93..dcb6327df5 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -85,9 +85,9 @@ func main() { } } -func setup(config *config.Config) *http.Server { - model, _ := model.NewModelFromString(jsonadapter.Model) - e, err := casbin.NewEnforcer(model, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) +func setup(conf *config.Config) *http.Server { + m, _ := model.NewModelFromString(jsonadapter.Model) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) if err != nil { shutdown() log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) // nolint # FIXME Fatal should only be called from main @@ -116,7 +116,7 @@ func setup(config *config.Config) *http.Server { cfg := &tls.Config{MinVersion: tls.VersionTLS12} srv := &http.Server{ - Addr: config.API.Host + ":" + fmt.Sprint(config.API.Port), + Addr: conf.API.Host + ":" + fmt.Sprint(conf.API.Port), Handler: r, TLSConfig: cfg, TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), @@ -183,14 +183,14 @@ func readinessResponse(c *gin.Context) { c.JSON(statusCode, "") } -func checkDB(database *database.SDAdb, timeout time.Duration) error { +func checkDB(db *database.SDAdb, timeout time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - if database.DB == nil { + if db.DB == nil { return fmt.Errorf("database is nil") } - return database.DB.PingContext(ctx) + return db.DB.PingContext(ctx) } func rbac(e *casbin.Enforcer) gin.HandlerFunc { diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 606ff59c6d..1c9e02d3c4 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -240,7 +240,7 @@ type TestSuite struct { User string } -func (suite *TestSuite) TestShutdown() { +func (s *TestSuite) TestShutdown() { Conf = &config.Config{} Conf.Broker = broker.MQConf{ Host: "localhost", @@ -251,7 +251,7 @@ func (suite *TestSuite) TestShutdown() { Vhost: "/sda", } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Database = database.DBConf{ Host: "localhost", @@ -262,20 +262,20 @@ func (suite *TestSuite) TestShutdown() { SslMode: "disable", } Conf.API.DB, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // make sure all conections are alive - assert.Equal(suite.T(), false, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), false, Conf.API.MQ.Connection.IsClosed()) - assert.Equal(suite.T(), nil, Conf.API.DB.DB.Ping()) + assert.Equal(s.T(), false, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), false, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), nil, Conf.API.DB.DB.Ping()) shutdown() - assert.Equal(suite.T(), true, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), true, Conf.API.MQ.Connection.IsClosed()) - assert.Equal(suite.T(), "sql: database is closed", Conf.API.DB.DB.Ping().Error()) + assert.Equal(s.T(), true, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), true, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), "sql: database is closed", Conf.API.DB.DB.Ping().Error()) } -func (suite *TestSuite) TestReadinessResponse() { +func (s *TestSuite) TestReadinessResponse() { gin.SetMode(gin.ReleaseMode) r := gin.Default() r.GET("/ready", readinessResponse) @@ -283,77 +283,77 @@ func (suite *TestSuite) TestReadinessResponse() { defer ts.Close() res, err := http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the connection to force a reconnection Conf.API.MQ.Connection.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the channel to force a reconneciton Conf.API.MQ.Channel.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close DB connection to force a reconnection Conf.API.DB.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() } // Initialise configuration and create jwt keys -func (suite *TestSuite) SetupSuite() { +func (s *TestSuite) SetupSuite() { log.SetLevel(log.DebugLevel) - suite.Path = "/tmp/keys/" - suite.KeyName = "example.demo" + s.Path = "/tmp/keys/" + s.KeyName = "example.demo" log.Print("Creating JWT keys for testing") - privpath, pubpath, err := helper.MakeFolder(suite.Path) - assert.NoError(suite.T(), err) - suite.PrivatePath = privpath - suite.PublicPath = pubpath + privpath, pubpath, err := helper.MakeFolder(s.Path) + assert.NoError(s.T(), err) + s.PrivatePath = privpath + s.PublicPath = pubpath err = helper.CreateRSAkeys(privpath, pubpath) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // Create a valid token for queries to the API - prKeyParsed, err := helper.ParsePrivateRSAKey(suite.PrivatePath, "/rsa") - assert.NoError(suite.T(), err) + prKeyParsed, err := helper.ParsePrivateRSAKey(s.PrivatePath, "/rsa") + assert.NoError(s.T(), err) token, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) - suite.Token = token + assert.NoError(s.T(), err) + s.Token = token user, ok := helper.DefaultTokenClaims["sub"].(string) - assert.True(suite.T(), ok) - suite.User = user + assert.True(s.T(), ok) + s.User = user c := &config.Config{} ServerConf := config.ServerConfig{} - ServerConf.Jwtpubkeypath = suite.PublicPath + ServerConf.Jwtpubkeypath = s.PublicPath c.Server = ServerConf Conf = c @@ -368,7 +368,7 @@ func (suite *TestSuite) SetupSuite() { SslMode: "disable", } Conf.API.DB, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Broker = broker.MQConf{ Host: "localhost", @@ -379,9 +379,9 @@ func (suite *TestSuite) SetupSuite() { Vhost: "/sda", } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) - suite.RBAC = []byte(`{"policy":[{"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, + s.RBAC = []byte(`{"policy":[{"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, {"role":"submission","path":"/dataset/release/*dataset","action":"POST"}, {"role":"submission","path":"/file/ingest","action":"POST"}, @@ -392,10 +392,10 @@ func (suite *TestSuite) SetupSuite() { "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"admin"}]}`) } -func (suite *TestSuite) TearDownSuite() { - assert.NoError(suite.T(), os.RemoveAll(suite.Path)) +func (s *TestSuite) TearDownSuite() { + assert.NoError(s.T(), os.RemoveAll(s.Path)) } -func (suite *TestSuite) SetupTest() { +func (s *TestSuite) SetupTest() { Conf.Database = database.DBConf{ Host: "localhost", Port: dbPort, @@ -405,10 +405,10 @@ func (suite *TestSuite) SetupTest() { SslMode: "disable", } Conf.API.DB, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) _, err = Conf.API.DB.DB.Exec("TRUNCATE sda.files CASCADE") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Broker = broker.MQConf{ Host: "localhost", @@ -420,41 +420,41 @@ func (suite *TestSuite) SetupTest() { SchemasPath: "../../schemas/isolated", } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // purge the queue so that the test passes when all tests are run as well as when run standalone. client := http.Client{Timeout: 30 * time.Second} for _, queue := range []string{"accession", "archived", "ingest", "mappings", "verified"} { req, err := http.NewRequest(http.MethodDelete, "http://"+BrokerAPI+"/api/queues/sda/"+queue+"/contents", http.NoBody) - assert.NoError(suite.T(), err, "failed to generate query") + assert.NoError(s.T(), err, "failed to generate query") req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") res.Body.Close() } } -func (suite *TestSuite) TestDatabasePingCheck() { +func (s *TestSuite) TestDatabasePingCheck() { emptyDB := database.SDAdb{} - assert.Error(suite.T(), checkDB(&emptyDB, 1*time.Second), "nil DB should fail") + assert.Error(s.T(), checkDB(&emptyDB, 1*time.Second), "nil DB should fail") db, err := database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), checkDB(db, 1*time.Second), "ping should succeed") + assert.NoError(s.T(), err) + assert.NoError(s.T(), checkDB(db, 1*time.Second), "ping should succeed") } -func (suite *TestSuite) TestAPIGetFiles() { +func (s *TestSuite) TestAPIGetFiles() { gin.SetMode(gin.ReleaseMode) r := gin.Default() m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r.GET("/files", rbac(e), func(c *gin.Context) { @@ -465,91 +465,91 @@ func (suite *TestSuite) TestAPIGetFiles() { filesURL := ts.URL + "/files" client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("GET", filesURL, nil) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) // Test query when no files is in db resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() filesData := []database.SubmissionFileInfo{} err = json.NewDecoder(resp.Body).Decode(&filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 0) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 0) + assert.NoError(s.T(), err) // Insert a file and make sure it is listed - file1 := fmt.Sprintf("/%v/TestAPIGetFiles.c4gh", suite.User) - fileID, err := Conf.API.DB.RegisterFile(file1, suite.User) - assert.NoError(suite.T(), err, "failed to register file in database") + file1 := fmt.Sprintf("/%v/TestAPIGetFiles.c4gh", s.User) + fileID, err := Conf.API.DB.RegisterFile(file1, s.User) + assert.NoError(s.T(), err, "failed to register file in database") corrID := uuid.New().String() latestStatus := "uploaded" - err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, suite.User, "{}", "{}") - assert.NoError(suite.T(), err, "got (%v) when trying to update file status") + err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "got (%v) when trying to update file status") resp, err = client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() err = json.NewDecoder(resp.Body).Decode(&filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 1) - assert.Equal(suite.T(), filesData[0].Status, latestStatus) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 1) + assert.Equal(s.T(), filesData[0].Status, latestStatus) + assert.NoError(s.T(), err) // Update the file's status and make sure only the lastest status is listed err = Conf.API.DB.SetAccessionID("stableID", fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) } latestStatus = "ready" - err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, suite.User, "{}", "{}") - assert.NoError(suite.T(), err, "got (%v) when trying to update file status") + err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "got (%v) when trying to update file status") resp, err = client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() data, err := io.ReadAll(resp.Body) - assert.NoError(suite.T(), err) - assert.NotContains(suite.T(), string(data), "accessionID") + assert.NoError(s.T(), err) + assert.NotContains(s.T(), string(data), "accessionID") err = json.Unmarshal(data, &filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 1) - assert.Equal(suite.T(), filesData[0].Status, latestStatus) - assert.Empty(suite.T(), filesData[0].AccessionID) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 1) + assert.Equal(s.T(), filesData[0].Status, latestStatus) + assert.Empty(s.T(), filesData[0].AccessionID) // Insert a second file and make sure it is listed - file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", suite.User) - _, err = Conf.API.DB.RegisterFile(file2, suite.User) - assert.NoError(suite.T(), err, "failed to register file in database") + file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", s.User) + _, err = Conf.API.DB.RegisterFile(file2, s.User) + assert.NoError(s.T(), err, "failed to register file in database") resp, err = client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() err = json.NewDecoder(resp.Body).Decode(&filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 2) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 2) for _, fileInfo := range filesData { switch fileInfo.InboxPath { case file1: - assert.Equal(suite.T(), fileInfo.Status, latestStatus) + assert.Equal(s.T(), fileInfo.Status, latestStatus) case file2: - assert.Equal(suite.T(), fileInfo.Status, "registered") + assert.Equal(s.T(), fileInfo.Status, "registered") } } - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) } func TestApiTestSuite(t *testing.T) { @@ -560,23 +560,23 @@ func testEndpoint(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) } -func (suite *TestSuite) TestRBAC() { +func (s *TestSuite) TestRBAC() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/c4gh-keys/list", nil) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/c4gh-keys/list", rbac(e), testEndpoint) @@ -585,30 +585,30 @@ func (suite *TestSuite) TestRBAC() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "ok") + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "ok") } -func (suite *TestSuite) TestRBAC_badUser() { +func (s *TestSuite) TestRBAC_badUser() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.API.RBACpolicy = []byte(`{"policy":[{"role":"admin","path":"/admin/*","action":"(GET)|(POST)|(PUT)"}], "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"submission"}]}`) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/admin/list-users", nil) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/admin/list-users", rbac(e), testEndpoint) @@ -616,21 +616,21 @@ func (suite *TestSuite) TestRBAC_badUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusUnauthorized, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) } -func (suite *TestSuite) TestRBAC_noToken() { +func (s *TestSuite) TestRBAC_noToken() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&[]byte{})) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() @@ -643,27 +643,27 @@ func (suite *TestSuite) TestRBAC_noToken() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusUnauthorized, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "no access token supplied") + assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "no access token supplied") } -func (suite *TestSuite) TestRBAC_emptyPolicy() { +func (s *TestSuite) TestRBAC_emptyPolicy() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&[]byte{})) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/files", nil) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/files", rbac(e), testEndpoint) @@ -672,30 +672,30 @@ func (suite *TestSuite) TestRBAC_emptyPolicy() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusUnauthorized, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "not authorized") + assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "not authorized") } -func (suite *TestSuite) TestIngestFile() { +func (s *TestSuite) TestIngestFile() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" type ingest struct { @@ -706,7 +706,7 @@ func (suite *TestSuite) TestIngestFile() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/ingest", rbac(e), ingestFile) @@ -714,7 +714,7 @@ func (suite *TestSuite) TestIngestFile() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -722,29 +722,29 @@ func (suite *TestSuite) TestIngestFile() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/ingest", http.NoBody) req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestIngestFile_NoUser() { +func (s *TestSuite) TestIngestFile_NoUser() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -763,19 +763,19 @@ func (suite *TestSuite) TestIngestFile_NoUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (suite *TestSuite) TestIngestFile_WrongUser() { +func (s *TestSuite) TestIngestFile_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -795,21 +795,21 @@ func (suite *TestSuite) TestIngestFile_WrongUser() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "sql: no rows in result set") + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") } -func (suite *TestSuite) TestIngestFile_WrongFilePath() { +func (s *TestSuite) TestIngestFile_WrongFilePath() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -821,7 +821,7 @@ func (suite *TestSuite) TestIngestFile_WrongFilePath() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("POST", "/file/ingest", bytes.NewBuffer(ingestMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/ingest", ingestFile) @@ -830,26 +830,26 @@ func (suite *TestSuite) TestIngestFile_WrongFilePath() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "sql: no rows in result set") + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") } -func (suite *TestSuite) TestSetAccession() { +func (s *TestSuite) TestSetAccession() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), @@ -859,23 +859,23 @@ func (suite *TestSuite) TestSetAccession() { DecryptedSize: 948, } err = Conf.API.DB.SetArchived(fileInfo, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } type accession struct { @@ -888,7 +888,7 @@ func (suite *TestSuite) TestSetAccession() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) @@ -896,7 +896,7 @@ func (suite *TestSuite) TestSetAccession() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -904,31 +904,31 @@ func (suite *TestSuite) TestSetAccession() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody) req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestSetAccession_WrongUser() { +func (s *TestSuite) TestSetAccession_WrongUser() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } type accession struct { @@ -941,7 +941,7 @@ func (suite *TestSuite) TestSetAccession_WrongUser() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) @@ -949,22 +949,22 @@ func (suite *TestSuite) TestSetAccession_WrongUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (suite *TestSuite) TestSetAccession_WrongFormat() { +func (s *TestSuite) TestSetAccession_WrongFormat() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/federated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } type accession struct { @@ -977,7 +977,7 @@ func (suite *TestSuite) TestSetAccession_WrongFormat() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) @@ -985,25 +985,25 @@ func (suite *TestSuite) TestSetAccession_WrongFormat() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (suite *TestSuite) TestCreateDataset() { +func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), @@ -1013,34 +1013,34 @@ func (suite *TestSuite) TestCreateDataset() { DecryptedSize: 948, } err = Conf.API.DB.SetArchived(fileInfo, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "dummy"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", rbac(e), createDataset) @@ -1048,7 +1048,7 @@ func (suite *TestSuite) TestCreateDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -1056,33 +1056,33 @@ func (suite *TestSuite) TestCreateDataset() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/mappings", http.NoBody) req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") - assert.NoError(suite.T(), json.Unmarshal(body, &data), "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to read response from broker") + assert.NoError(s.T(), json.Unmarshal(body, &data), "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestCreateDataset_BadFormat() { +func (s *TestSuite) TestCreateDataset_BadFormat() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), @@ -1092,40 +1092,40 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() { DecryptedSize: 948, } err = Conf.API.DB.SetArchived(fileInfo, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}") - assert.NoError(suite.T(), err, "got (%v) when marking file as ready", err) + assert.NoError(s.T(), err, "got (%v) when marking file as ready", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/federated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "dummy"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", rbac(e), createDataset) @@ -1133,16 +1133,16 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "does not match pattern") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "does not match pattern") } -func (suite *TestSuite) TestCreateDataset_MissingAccessionIDs() { +func (s *TestSuite) TestCreateDataset_MissingAccessionIDs() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -1150,7 +1150,7 @@ func (suite *TestSuite) TestCreateDataset_MissingAccessionIDs() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", createDataset) @@ -1158,23 +1158,23 @@ func (suite *TestSuite) TestCreateDataset_MissingAccessionIDs() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "at least one accessionID is reqired") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "at least one accessionID is reqired") } -func (suite *TestSuite) TestCreateDataset_WrongIDs() { +func (s *TestSuite) TestCreateDataset_WrongIDs() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "dummy"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", createDataset) @@ -1182,29 +1182,29 @@ func (suite *TestSuite) TestCreateDataset_WrongIDs() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "accession ID not found: ") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "accession ID not found: ") } -func (suite *TestSuite) TestCreateDataset_WrongUser() { +func (s *TestSuite) TestCreateDataset_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), @@ -1214,26 +1214,26 @@ func (suite *TestSuite) TestCreateDataset_WrongUser() { DecryptedSize: 948, } err = Conf.API.DB.SetArchived(fileInfo, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + assert.NoError(s.T(), err, "failed to mark file as Archived") err = Conf.API.DB.SetVerified(fileInfo, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "tester"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", createDataset) @@ -1241,58 +1241,58 @@ func (suite *TestSuite) TestCreateDataset_WrongUser() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "accession ID owned by other user") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "accession ID owned by other user") } -func (suite *TestSuite) TestReleaseDataset() { +func (s *TestSuite) TestReleaseDataset() { user := "TestReleaseDataset" for i := 0; i < 3; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } if err := Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_TestReleaseDataset_00", "accession_TestReleaseDataset_01", "accession_TestReleaseDataset_02"}); err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/API:dataset-01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1300,7 +1300,7 @@ func (suite *TestSuite) TestReleaseDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -1308,38 +1308,38 @@ func (suite *TestSuite) TestReleaseDataset() { req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestReleaseDataset_NoDataset() { +func (s *TestSuite) TestReleaseDataset_NoDataset() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1347,29 +1347,29 @@ func (suite *TestSuite) TestReleaseDataset_NoDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } -func (suite *TestSuite) TestReleaseDataset_BadDataset() { +func (s *TestSuite) TestReleaseDataset_BadDataset() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/non-existing", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1377,58 +1377,58 @@ func (suite *TestSuite) TestReleaseDataset_BadDataset() { router.ServeHTTP(w, r) response := w.Result() defer response.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, response.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, response.StatusCode) } -func (suite *TestSuite) TestReleaseDataset_DeprecatedDataset() { +func (s *TestSuite) TestReleaseDataset_DeprecatedDataset() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } } if err := Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}); err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("test-dataset-01", "deprecated", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/test-dataset-01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1436,53 +1436,53 @@ func (suite *TestSuite) TestReleaseDataset_DeprecatedDataset() { router.ServeHTTP(w, r) response := w.Result() defer response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) } -func (suite *TestSuite) TestListActiveUsers() { +func (s *TestSuite) TestListActiveUsers() { testUsers := []string{"User-A", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 3; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } } err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_User-A_00", "accession_User-A_01", "accession_User-A_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/users", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/users", rbac(e), listActiveUsers) @@ -1490,58 +1490,58 @@ func (suite *TestSuite) TestListActiveUsers() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) var users []string err = json.NewDecoder(okResponse.Body).Decode(&users) - assert.NoError(suite.T(), err, "failed to list users from DB") - assert.Equal(suite.T(), []string{"User-B", "User-C"}, users) + assert.NoError(s.T(), err, "failed to list users from DB") + assert.Equal(s.T(), []string{"User-B", "User-C"}, users) } -func (suite *TestSuite) TestListUserFiles() { +func (s *TestSuite) TestListUserFiles() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } } err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/users/user@example.org/files", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/users/:username/files", rbac(e), listUserFiles) @@ -1549,27 +1549,27 @@ func (suite *TestSuite) TestListUserFiles() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) files := []database.SubmissionFileInfo{} err = json.NewDecoder(okResponse.Body).Decode(&files) - assert.NoError(suite.T(), err, "failed to list users from DB") - assert.Equal(suite.T(), 2, len(files)) - assert.Contains(suite.T(), files[0].AccessionID, "accession_user_example.org_0") + assert.NoError(s.T(), err, "failed to list users from DB") + assert.Equal(s.T(), 2, len(files)) + assert.Contains(s.T(), files[0].AccessionID, "accession_user_example.org_0") } -func (suite *TestSuite) TestAddC4ghHash() { +func (s *TestSuite) TestAddC4ghHash() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r := gin.Default() @@ -1578,7 +1578,7 @@ func (suite *TestSuite) TestAddC4ghHash() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Create a valid request body keyhash := schema.C4ghPubKey{ @@ -1586,38 +1586,38 @@ func (suite *TestSuite) TestAddC4ghHash() { Description: "Test key description", } body, err := json.Marshal(keyhash) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() // Isert pubkey again and expect error resp2, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusConflict, resp2.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusConflict, resp2.StatusCode) defer resp2.Body.Close() } -func (suite *TestSuite) TestAddC4ghHash_emptyJson() { +func (s *TestSuite) TestAddC4ghHash_emptyJson() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r := gin.Default() r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) @@ -1625,34 +1625,34 @@ func (suite *TestSuite) TestAddC4ghHash_emptyJson() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Create an invalid request body body := []byte(`{"invalid_json"}`) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() } -func (suite *TestSuite) TestAddC4ghHash_notBase64() { +func (s *TestSuite) TestAddC4ghHash_notBase64() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r := gin.Default() r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) @@ -1660,24 +1660,24 @@ func (suite *TestSuite) TestAddC4ghHash_notBase64() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Create an invalid request body body := []byte(`{"pubkey": "asdkjsahfd=", "decription": ""}`) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() } -func (suite *TestSuite) TestListC4ghHashes() { - assert.NoError(suite.T(), Conf.API.DB.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", "this is a test key"), "failed to register key in database") +func (s *TestSuite) TestListC4ghHashes() { + assert.NoError(s.T(), Conf.API.DB.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", "this is a test key"), "failed to register key in database") expectedResponse := database.C4ghKeyHash{ Hash: "cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", @@ -1687,7 +1687,7 @@ func (suite *TestSuite) TestListC4ghHashes() { } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) r := gin.Default() r.GET("/c4gh-keys/list", listC4ghHashes) @@ -1695,34 +1695,34 @@ func (suite *TestSuite) TestListC4ghHashes() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("GET", ts.URL+"/c4gh-keys/list", nil) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() hashes := []database.C4ghKeyHash{} err = json.NewDecoder(resp.Body).Decode(&hashes) - assert.NoError(suite.T(), err, "failed to list users from DB") + assert.NoError(s.T(), err, "failed to list users from DB") for n, h := range hashes { if h.Hash == "cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23" { - assert.Equal(suite.T(), expectedResponse, hashes[n]) + assert.Equal(s.T(), expectedResponse, hashes[n]) break } } } -func (suite *TestSuite) TestDeprecateC4ghHash() { - assert.NoError(suite.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd9991453839581fc69ee26e0daefde6a5d2660fc23", "this is a deprecation test key"), "failed to register key in database") +func (s *TestSuite) TestDeprecateC4ghHash() { + assert.NoError(s.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd9991453839581fc69ee26e0daefde6a5d2660fc23", "this is a deprecation test key"), "failed to register key in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) r := gin.Default() r.POST("/c4gh-keys/deprecate/*keyHash", deprecateC4ghHash) @@ -1730,29 +1730,29 @@ func (suite *TestSuite) TestDeprecateC4ghHash() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/deprecate/abc8f5cc8d936ce437a52cd9991453839581fc69ee26e0daefde6a5d2660fc23", http.NoBody) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() // a second time gives an error since the key is alreadu deprecated resp2, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp2.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp2.StatusCode) defer resp2.Body.Close() } -func (suite *TestSuite) TestDeprecateC4ghHash_wrongHash() { - assert.NoError(suite.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc99", "this is a deprecation test key"), "failed to register key in database") +func (s *TestSuite) TestDeprecateC4ghHash_wrongHash() { + assert.NoError(s.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc99", "this is a deprecation test key"), "failed to register key in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) r := gin.Default() r.POST("/c4gh-keys/deprecate/*keyHash", deprecateC4ghHash) @@ -1760,58 +1760,58 @@ func (suite *TestSuite) TestDeprecateC4ghHash_wrongHash() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/deprecate/xyz8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", http.NoBody) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() } -func (suite *TestSuite) TestListDatasets() { +func (s *TestSuite) TestListDatasets() { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } stableID := fmt.Sprintf("accession_%s_0%d", "dummy", i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_dummy_00", "accession_dummy_01", "accession_dummy_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_dummy_03", "accession_dummy_04"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/datasets/list", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/datasets/list", listAllDatasets) @@ -1820,56 +1820,56 @@ func (suite *TestSuite) TestListDatasets() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) datasets := []database.DatasetInfo{} err = json.NewDecoder(okResponse.Body).Decode(&datasets) - assert.NoError(suite.T(), err, "failed to list datasets from DB") - assert.Equal(suite.T(), 2, len(datasets)) - assert.Equal(suite.T(), "released", datasets[1].Status) - assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) + assert.NoError(s.T(), err, "failed to list datasets from DB") + assert.Equal(s.T(), 2, len(datasets)) + assert.Equal(s.T(), "released", datasets[1].Status) + assert.Equal(s.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) } -func (suite *TestSuite) TestListUserDatasets() { +func (s *TestSuite) TestListUserDatasets() { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } stableID := fmt.Sprintf("accession_%s_0%d", "user_example.org", i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/datasets/list/user@example.org", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/datasets/list/:username", listUserDatasets) @@ -1877,56 +1877,56 @@ func (suite *TestSuite) TestListUserDatasets() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) datasets := []database.DatasetInfo{} err = json.NewDecoder(okResponse.Body).Decode(&datasets) - assert.NoError(suite.T(), err, "failed to list datasets from DB") - assert.Equal(suite.T(), 2, len(datasets)) - assert.Equal(suite.T(), "released", datasets[1].Status) - assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) + assert.NoError(s.T(), err, "failed to list datasets from DB") + assert.Equal(s.T(), 2, len(datasets)) + assert.Equal(s.T(), "released", datasets[1].Status) + assert.Equal(s.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) } -func (suite *TestSuite) TestListDatasetsAsUser() { +func (s *TestSuite) TestListDatasetsAsUser() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), suite.User) + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } stableID := fmt.Sprintf("accession_user_example.org_0%d", i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/datasets", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/datasets", listDatasets) @@ -1934,38 +1934,38 @@ func (suite *TestSuite) TestListDatasetsAsUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) datasets := []database.DatasetInfo{} err = json.NewDecoder(okResponse.Body).Decode(&datasets) - assert.NoError(suite.T(), err, "failed to list datasets from DB") - assert.Equal(suite.T(), 2, len(datasets)) - assert.Equal(suite.T(), "released", datasets[1].Status) - assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) + assert.NoError(s.T(), err, "failed to list datasets from DB") + assert.Equal(s.T(), 2, len(datasets)) + assert.Equal(s.T(), "released", datasets[1].Status) + assert.Equal(s.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) } -func (suite *TestSuite) TestReVerifyFile() { +func (s *TestSuite) TestReVerifyFile() { user := "TestReVerify" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerify-00%d.c4gh", user, i) fileID, err := Conf.API.DB.RegisterFile(filePath, user) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}"); err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) if err != nil { - suite.FailNow("failed to calculate Checksum") + s.FailNow("failed to calculate Checksum") } decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) if err != nil { - suite.FailNow("failed to calculate DecryptedChecksum") + s.FailNow("failed to calculate DecryptedChecksum") } fileInfo := database.FileInfo{ @@ -1977,29 +1977,29 @@ func (suite *TestSuite) TestReVerifyFile() { UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { - suite.FailNow("failed to mark file as Archived") + s.FailNow("failed to mark file as Archived") } if err := Conf.API.DB.SetVerified(fileInfo, fileID); err != nil { - suite.FailNow("failed to mark file as Verified") + s.FailNow("failed to mark file as Verified") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) if err := Conf.API.DB.SetAccessionID(stableID, fileID); err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}"); err != nil { - suite.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) + s.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) } } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/file/verify/accession_TestReVerify_01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.PUT("/file/verify/:accession", reVerifyFile) @@ -2007,7 +2007,7 @@ func (suite *TestSuite) TestReVerifyFile() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -2015,27 +2015,27 @@ func (suite *TestSuite) TestReVerifyFile() { req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestReVerifyFile_wrongAccession() { +func (s *TestSuite) TestReVerifyFile_wrongAccession() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/file/verify/accession_TestReVerify_99", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/verify/:accession", reVerifyFile) @@ -2043,31 +2043,31 @@ func (suite *TestSuite) TestReVerifyFile_wrongAccession() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } -func (suite *TestSuite) TestReVerifyDataset() { +func (s *TestSuite) TestReVerifyDataset() { user := "TestReVerifyDataset" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerifyDataset-00%d.c4gh", user, i) fileID, err := Conf.API.DB.RegisterFile(filePath, user) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}"); err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) if err != nil { - suite.FailNow("failed to calculate Checksum") + s.FailNow("failed to calculate Checksum") } decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) if err != nil { - suite.FailNow("failed to calculate DecryptedChecksum") + s.FailNow("failed to calculate DecryptedChecksum") } fileInfo := database.FileInfo{ @@ -2079,34 +2079,34 @@ func (suite *TestSuite) TestReVerifyDataset() { UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { - suite.FailNow("failed to mark file as Archived") + s.FailNow("failed to mark file as Archived") } if err := Conf.API.DB.SetVerified(fileInfo, fileID); err != nil { - suite.FailNow("failed to mark file as Verified") + s.FailNow("failed to mark file as Verified") } stableID := fmt.Sprintf("%s_0%d", user, i) if err := Conf.API.DB.SetAccessionID(stableID, fileID); err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}"); err != nil { - suite.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) + s.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) } } if err := Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"TestReVerifyDataset_00", "TestReVerifyDataset_01", "TestReVerifyDataset_02"}); err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/dataset/verify/test-dataset-01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.PUT("/dataset/verify/*dataset", reVerifyDataset) @@ -2114,7 +2114,7 @@ func (suite *TestSuite) TestReVerifyDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the messages shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -2122,27 +2122,27 @@ func (suite *TestSuite) TestReVerifyDataset() { req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 3, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 3, data.MessagesReady) } -func (suite *TestSuite) TestReVerifyDataset_wrongDatasetName() { +func (s *TestSuite) TestReVerifyDataset_wrongDatasetName() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/dataset/verify/wrong_dataset", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.PUT("/dataset/verify/*dataset", reVerifyDataset) @@ -2150,5 +2150,5 @@ func (suite *TestSuite) TestReVerifyDataset_wrongDatasetName() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } diff --git a/sda/cmd/auth/cega_test.go b/sda/cmd/auth/cega_test.go index 647296400a..cf22a36d6a 100644 --- a/sda/cmd/auth/cega_test.go +++ b/sda/cmd/auth/cega_test.go @@ -21,22 +21,22 @@ func TestCegaTestSuite(t *testing.T) { suite.Run(t, new(CegaTests)) } -func (suite *CegaTests) TestGetb64Credentials() { +func (ts *CegaTests) TestGetb64Credentials() { user := "testUser" password := "password" expected := base64.StdEncoding.EncodeToString([]byte(user + ":" + password)) - assert.Equal(suite.T(), expected, getb64Credentials(user, password), "base64 encoding of credentials failing") + assert.Equal(ts.T(), expected, getb64Credentials(user, password), "base64 encoding of credentials failing") } -func (suite *CegaTests) TestVerifyPassword() { +func (ts *CegaTests) TestVerifyPassword() { password := "password" hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { log.Error(err) } - assert.Equal(suite.T(), true, verifyPassword(password, string(hash)), "password hash verification failing on correct hash") - assert.Equal(suite.T(), false, verifyPassword(password, "wronghash"), "password hash verification returning true for wrong hash") + assert.Equal(ts.T(), true, verifyPassword(password, string(hash)), "password hash verification failing on correct hash") + assert.Equal(ts.T(), false, verifyPassword(password, "wronghash"), "password hash verification returning true for wrong hash") } diff --git a/sda/cmd/auth/info_test.go b/sda/cmd/auth/info_test.go index b0fe1f8eea..ae46fc5ad2 100644 --- a/sda/cmd/auth/info_test.go +++ b/sda/cmd/auth/info_test.go @@ -21,37 +21,37 @@ func TestInfoTestSuite(t *testing.T) { suite.Run(t, new(InfoTests)) } -func (suite *InfoTests) SetupTest() { - suite.TempDir, _ = os.MkdirTemp("", "key") +func (ts *InfoTests) SetupTest() { + ts.TempDir, _ = os.MkdirTemp("", "key") pub, _, err := keys.GenerateKeyPair() if err != nil { - suite.FailNowf("Filed to generate crypt4gh keypair", err.Error()) + ts.FailNowf("Filed to generate crypt4gh keypair", err.Error()) } buf := new(bytes.Buffer) if err := keys.WriteCrypt4GHX25519PublicKey(buf, pub); err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.pubKeyb64 = base64.StdEncoding.EncodeToString(buf.Bytes()) + ts.pubKeyb64 = base64.StdEncoding.EncodeToString(buf.Bytes()) - pubKeyFile, err := os.Create(suite.TempDir + "/pub.key") + pubKeyFile, err := os.Create(ts.TempDir + "/pub.key") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } _, err = pubKeyFile.Write(buf.Bytes()) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } } -func (suite *InfoTests) TestReadPublicKeyFile() { - pubKey, err := readPublicKeyFile(suite.TempDir + "/pub.key") - assert.NoError(suite.T(), err, "Reading public key from disk failed") - assert.Equal(suite.T(), suite.pubKeyb64, pubKey) +func (ts *InfoTests) TestReadPublicKeyFile() { + pubKey, err := readPublicKeyFile(ts.TempDir + "/pub.key") + assert.NoError(ts.T(), err, "Reading public key from disk failed") + assert.Equal(ts.T(), ts.pubKeyb64, pubKey) } -func (suite *InfoTests) TearDownTest() { - os.RemoveAll(suite.TempDir) +func (ts *InfoTests) TearDownTest() { + os.RemoveAll(ts.TempDir) } diff --git a/sda/cmd/auth/jwt_test.go b/sda/cmd/auth/jwt_test.go index 7b923a29ec..055df14b32 100644 --- a/sda/cmd/auth/jwt_test.go +++ b/sda/cmd/auth/jwt_test.go @@ -25,38 +25,38 @@ func TestJWTTestSuite(t *testing.T) { suite.Run(t, new(JWTTests)) } -func (suite *JWTTests) SetupTest() { +func (ts *JWTTests) SetupTest() { var err error - suite.TempDir, err = os.MkdirTemp(os.TempDir(), "jwt-test") - assert.NoError(suite.T(), err) + ts.TempDir, err = os.MkdirTemp(os.TempDir(), "jwt-test") + assert.NoError(ts.T(), err) ecKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) ecKeyBytes, err := jwk.EncodePEM(ecKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/ec", ecKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/ec", ecKeyBytes, 0600)) ecPubKeyBytes, err := jwk.EncodePEM(&ecKey.PublicKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/ec.pub", ecPubKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/ec.pub", ecPubKeyBytes, 0600)) rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) rsaKeyBytes, err := jwk.EncodePEM(rsaKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/rsa", rsaKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/rsa", rsaKeyBytes, 0600)) rsaPubKeyBytes, err := jwk.EncodePEM(&rsaKey.PublicKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/rsa.pub", rsaPubKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/rsa.pub", rsaPubKeyBytes, 0600)) } -func (suite *JWTTests) TearDownTest() { - os.RemoveAll(suite.TempDir) +func (ts *JWTTests) TearDownTest() { + os.RemoveAll(ts.TempDir) } -func (suite *JWTTests) TestGenerateJwtToken() { +func (ts *JWTTests) TestGenerateJwtToken() { type KeyAlgo struct { Algorithm string Keyfile string @@ -64,8 +64,8 @@ func (suite *JWTTests) TestGenerateJwtToken() { } algorithms := []KeyAlgo{ - {Algorithm: "RS256", Keyfile: suite.TempDir + "/rsa", Pubfile: suite.TempDir + "/rsa.pub"}, - {Algorithm: "ES256", Keyfile: suite.TempDir + "/ec", Pubfile: suite.TempDir + "/ec.pub"}, + {Algorithm: "RS256", Keyfile: ts.TempDir + "/rsa", Pubfile: ts.TempDir + "/rsa.pub"}, + {Algorithm: "ES256", Keyfile: ts.TempDir + "/ec", Pubfile: ts.TempDir + "/ec.pub"}, } claims := map[string]any{ @@ -76,25 +76,25 @@ func (suite *JWTTests) TestGenerateJwtToken() { } for _, test := range algorithms { - ts, expiration, err := generateJwtToken(claims, test.Keyfile, test.Algorithm) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), ts) + t, expiration, err := generateJwtToken(claims, test.Keyfile, test.Algorithm) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), t) keyData, err := os.ReadFile(test.Pubfile) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) key, err := jwk.ParseKey(keyData, jwk.WithPEM(true)) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), jwk.AssignKeyID(key)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), jwk.AssignKeyID(key)) keySet := jwk.NewSet() - assert.NoError(suite.T(), keySet.AddKey(key)) + assert.NoError(ts.T(), keySet.AddKey(key)) - token, err := jwt.Parse([]byte(ts), jwt.WithKeySet(keySet, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "http://local.issuer", token.Issuer()) - assert.Equal(suite.T(), "test@foo.bar", token.Subject()) + token, err := jwt.Parse([]byte(t), jwt.WithKeySet(keySet, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "http://local.issuer", token.Issuer()) + assert.Equal(ts.T(), "test@foo.bar", token.Subject()) // check that the expiration string is a date _, err = time.Parse("2006-01-02 15:04:05", expiration) - assert.Nil(suite.T(), err, "Couldn't parse expiration date for jwt") + assert.Nil(ts.T(), err, "Couldn't parse expiration date for jwt") } } diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index 9efad24338..eb8fa6448c 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -368,7 +368,7 @@ func addCSPheaders(ctx iris.Context) { func main() { // Initialise config - config, err := config.NewConfig("auth") + conf, err := config.NewConfig("auth") if err != nil { log.Errorf("Failed to generate config, reason: %v", err) os.Exit(1) @@ -377,14 +377,14 @@ func main() { var oauth2Config oauth2.Config var provider *oidc.Provider - if config.Auth.OIDC.ID != "" && config.Auth.OIDC.Secret != "" { + if conf.Auth.OIDC.ID != "" && conf.Auth.OIDC.Secret != "" { // Initialise OIDC client - oauth2Config, provider = getOidcClient(config.Auth.OIDC) + oauth2Config, provider = getOidcClient(conf.Auth.OIDC) } // Create handler struct for the web server authHandler := AuthHandler{ - Config: config.Auth, + Config: conf.Auth, OAuth2Config: oauth2Config, OIDCProvider: provider, htmlDir: "./frontend/templates", @@ -398,12 +398,12 @@ func main() { // Start sessions handler in order to send flash messages sess := sessions.New(sessions.Config{Cookie: "_session_id", AllowReclaim: true}) - if config.Server.CORS.AllowOrigin != "" { + if conf.Server.CORS.AllowOrigin != "" { // Set CORS context corsContext := cors.New(cors.Options{ - AllowedOrigins: strings.Split(config.Server.CORS.AllowOrigin, ","), - AllowedMethods: strings.Split(config.Server.CORS.AllowMethods, ","), - AllowCredentials: config.Server.CORS.AllowCredentials, + AllowedOrigins: strings.Split(conf.Server.CORS.AllowOrigin, ","), + AllowedMethods: strings.Split(conf.Server.CORS.AllowMethods, ","), + AllowCredentials: conf.Server.CORS.AllowCredentials, }) app.Use(corsContext) } @@ -411,7 +411,7 @@ func main() { app.Use(sess.Handler()) // Connect to DB - authHandler.Config.DB, err = database.NewSDAdb(config.Database) + authHandler.Config.DB, err = database.NewSDAdb(conf.Database) if err != nil { log.Error(err) panic(err) @@ -450,9 +450,9 @@ func main() { app.UseGlobal(globalHeaders) - if config.Server.Cert != "" && config.Server.Key != "" { + if conf.Server.Cert != "" && conf.Server.Key != "" { log.Infoln("Serving content using https") - err = app.Run(iris.TLS("0.0.0.0:8080", config.Server.Cert, config.Server.Key)) + err = app.Run(iris.TLS("0.0.0.0:8080", conf.Server.Cert, conf.Server.Key)) } else { log.Infoln("Serving content using http") server := &http.Server{ diff --git a/sda/cmd/auth/oidc_test.go b/sda/cmd/auth/oidc_test.go index 029d811144..500aa5d591 100644 --- a/sda/cmd/auth/oidc_test.go +++ b/sda/cmd/auth/oidc_test.go @@ -33,64 +33,64 @@ func TestOIDCTestSuite(t *testing.T) { suite.Run(t, new(OIDCTests)) } -func (suite *OIDCTests) SetupTest() { +func (ts *OIDCTests) SetupTest() { var err error - suite.mockServer, err = mockoidc.Run() - assert.NoError(suite.T(), err) + ts.mockServer, err = mockoidc.Run() + assert.NoError(ts.T(), err) // create an elixir config that has the needed endpoints set - suite.OIDCConfig = config.OIDCConfig{ - ID: suite.mockServer.ClientID, - Provider: suite.mockServer.Issuer(), + ts.OIDCConfig = config.OIDCConfig{ + ID: ts.mockServer.ClientID, + Provider: ts.mockServer.Issuer(), RedirectURL: "http://redirect", - Secret: suite.mockServer.ClientSecret, + Secret: ts.mockServer.ClientSecret, } } -func (suite *OIDCTests) TearDownTest() { - err := suite.mockServer.Shutdown() - assert.NoError(suite.T(), err) +func (ts *OIDCTests) TearDownTest() { + err := ts.mockServer.Shutdown() + assert.NoError(ts.T(), err) } -func (suite *OIDCTests) TestGetOidcClient() { +func (ts *OIDCTests) TestGetOidcClient() { expectedEndpoint := oauth2.Endpoint{ - AuthURL: suite.mockServer.AuthorizationEndpoint(), - TokenURL: suite.mockServer.TokenEndpoint(), + AuthURL: ts.mockServer.AuthorizationEndpoint(), + TokenURL: ts.mockServer.TokenEndpoint(), AuthStyle: 0} - oauth2Config, provider := getOidcClient(suite.OIDCConfig) - assert.Equal(suite.T(), suite.mockServer.ClientID, oauth2Config.ClientID, "ClientID was modified when creating the oauth2Config") - assert.Equal(suite.T(), suite.mockServer.ClientSecret, oauth2Config.ClientSecret, "ClientSecret was modified when creating the oauth2Config") - assert.Equal(suite.T(), suite.OIDCConfig.RedirectURL, oauth2Config.RedirectURL, "RedirectURL was modified when creating the oauth2Config") - assert.Equal(suite.T(), expectedEndpoint, oauth2Config.Endpoint, "Issuer was modified when creating the oauth2Config") - assert.Equal(suite.T(), expectedEndpoint, provider.Endpoint(), "provider has the wrong endpoint") - assert.Equal(suite.T(), []string{"openid", "ga4gh_passport_v1 profile email eduperson_entitlement"}, oauth2Config.Scopes, "oauth2Config has the wrong scopes") + oauth2Config, provider := getOidcClient(ts.OIDCConfig) + assert.Equal(ts.T(), ts.mockServer.ClientID, oauth2Config.ClientID, "ClientID was modified when creating the oauth2Config") + assert.Equal(ts.T(), ts.mockServer.ClientSecret, oauth2Config.ClientSecret, "ClientSecret was modified when creating the oauth2Config") + assert.Equal(ts.T(), ts.OIDCConfig.RedirectURL, oauth2Config.RedirectURL, "RedirectURL was modified when creating the oauth2Config") + assert.Equal(ts.T(), expectedEndpoint, oauth2Config.Endpoint, "Issuer was modified when creating the oauth2Config") + assert.Equal(ts.T(), expectedEndpoint, provider.Endpoint(), "provider has the wrong endpoint") + assert.Equal(ts.T(), []string{"openid", "ga4gh_passport_v1 profile email eduperson_entitlement"}, oauth2Config.Scopes, "oauth2Config has the wrong scopes") } -func (suite *OIDCTests) TestAuthenticateWithOidc() { +func (ts *OIDCTests) TestAuthenticateWithOidc() { // Create a code to authenticate - session, err := suite.mockServer.SessionStore.NewSession( + session, err := ts.mockServer.SessionStore.NewSession( "openid email profile", "nonce", mockoidc.DefaultUser(), "", "") if err != nil { log.Error(err) } code := session.SessionID - jwkURL := suite.mockServer.JWKSEndpoint() + jwkURL := ts.mockServer.JWKSEndpoint() - oauth2Config, provider := getOidcClient(suite.OIDCConfig) + oauth2Config, provider := getOidcClient(ts.OIDCConfig) elixirIdentity, err := authenticateWithOidc(oauth2Config, provider, code, jwkURL) - assert.Nil(suite.T(), err, "Failed to authenticate with OIDC") + assert.Nil(ts.T(), err, "Failed to authenticate with OIDC") // Ensure both RawToken and ResignedToken are not empty - assert.NotEqual(suite.T(), "", elixirIdentity.RawToken, "Empty RawToken returned from OIDC authentication") - assert.NotEqual(suite.T(), "", elixirIdentity.ResignedToken, "Empty ResignedToken returned from OIDC authentication") + assert.NotEqual(ts.T(), "", elixirIdentity.RawToken, "Empty RawToken returned from OIDC authentication") + assert.NotEqual(ts.T(), "", elixirIdentity.ResignedToken, "Empty ResignedToken returned from OIDC authentication") } -func (suite *OIDCTests) TestValidateJwt() { - session, err := suite.mockServer.SessionStore.NewSession("openid email profile", "nonce", mockoidc.DefaultUser(), "", "") - assert.NoError(suite.T(), err) - oauth2Config, provider := getOidcClient(suite.OIDCConfig) - jwkURL := suite.mockServer.JWKSEndpoint() +func (ts *OIDCTests) TestValidateJwt() { + session, err := ts.mockServer.SessionStore.NewSession("openid email profile", "nonce", mockoidc.DefaultUser(), "", "") + assert.NoError(ts.T(), err) + oauth2Config, provider := getOidcClient(ts.OIDCConfig) + jwkURL := ts.mockServer.JWKSEndpoint() elixirIdentity, _ := authenticateWithOidc(oauth2Config, provider, session.SessionID, jwkURL) elixirJWT := elixirIdentity.RawToken @@ -102,88 +102,88 @@ func (suite *OIDCTests) TestValidateJwt() { } token := jwt.New() for key, value := range claims { - assert.NoError(suite.T(), token.Set(key, value)) + assert.NoError(ts.T(), token.Set(key, value)) } // key from mock server - suiteKey, err := jwk.FromRaw(suite.mockServer.Keypair.PrivateKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), suiteKey.Set(jwk.KeyIDKey, suite.mockServer.Keypair.Kid)) + suiteKey, err := jwk.FromRaw(ts.mockServer.Keypair.PrivateKey) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), suiteKey.Set(jwk.KeyIDKey, ts.mockServer.Keypair.Kid)) // Create HS256 test token mySigningKey := []byte("AllYourBase") testTokenHS256, err := jwt.Sign(token, jwt.WithKey(jwa.HS256, mySigningKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create RSA test token rsaRawKey, err := rsa.GenerateKey(rand.Reader, 2048) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) rsaKey, err := jwk.FromRaw(rsaRawKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), jwk.AssignKeyID(rsaKey)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), jwk.AssignKeyID(rsaKey)) testTokenRSA, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, rsaKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create ECDSA test token ecRawKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) ecKey, err := jwk.FromRaw(ecRawKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), jwk.AssignKeyID(ecKey)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), jwk.AssignKeyID(ecKey)) testTokenEC, err := jwt.Sign(token, jwt.WithKey(jwa.ES256, ecKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create expired RSA test token expiredTokenRSA := jwt.New() for key, value := range claims { - assert.NoError(suite.T(), token.Set(key, value)) + assert.NoError(ts.T(), token.Set(key, value)) } - assert.NoError(suite.T(), expiredTokenRSA.Set(jwt.ExpirationKey, time.Now().UTC().Add(-4*time.Hour))) + assert.NoError(ts.T(), expiredTokenRSA.Set(jwt.ExpirationKey, time.Now().UTC().Add(-4*time.Hour))) testExpiredTokenRSA, err := jwt.Sign(expiredTokenRSA, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // sanity check - _, expDate, err := validateToken(elixirJWT, suite.mockServer.JWKSEndpoint()) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), expDate, elixirIdentity.ExpDateRaw, "Returned wrong exp date but without returning errors") + _, expDate, err := validateToken(elixirJWT, ts.mockServer.JWKSEndpoint()) + assert.Nil(ts.T(), err) + assert.Equal(ts.T(), expDate, elixirIdentity.ExpDateRaw, "Returned wrong exp date but without returning errors") // Not a jwk url _, _, err = validateToken(elixirJWT, "http://some/jwk/endpoint") - assert.ErrorContains(suite.T(), err, "failed to fetch \"http://some/jwk/endpoint") + assert.ErrorContains(ts.T(), err, "failed to fetch \"http://some/jwk/endpoint") // correct private key, RSA oidcTokenRSA, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) - _, _, err = validateToken(string(oidcTokenRSA), suite.mockServer.JWKSEndpoint()) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) + _, _, err = validateToken(string(oidcTokenRSA), ts.mockServer.JWKSEndpoint()) + assert.NoError(ts.T(), err) // wrong signing method - _, _, err = validateToken(string(testTokenHS256), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: key provider 0 failed:") + _, _, err = validateToken(string(testTokenHS256), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: key provider 0 failed:") // wrong private key, RSA - _, _, err = validateToken(string(testTokenRSA), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: key provider 0 failed:") + _, _, err = validateToken(string(testTokenRSA), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: key provider 0 failed:") // wrong private key, ECDSA - _, _, err = validateToken(string(testTokenEC), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: key provider 0 failed:") + _, _, err = validateToken(string(testTokenEC), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: key provider 0 failed:") // expired token - _, _, err = validateToken(string(testExpiredTokenRSA), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: \"exp\" not satisfied") + _, _, err = validateToken(string(testExpiredTokenRSA), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: \"exp\" not satisfied") // check that we handle the case where token has no expiration date - assert.NoError(suite.T(), token.Set(jwt.ExpirationKey, 0)) + assert.NoError(ts.T(), token.Set(jwt.ExpirationKey, 0)) noExpiryToken, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) - _, _, err = validateToken(string(noExpiryToken), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: iitr between exp and iat is less than") + assert.NoError(ts.T(), err) + _, _, err = validateToken(string(noExpiryToken), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: iitr between exp and iat is less than") // check that we handle the case where token has no exp key - assert.NoError(suite.T(), token.Remove(jwt.ExpirationKey)) + assert.NoError(ts.T(), token.Remove(jwt.ExpirationKey)) noExpiryToken, err = jwt.Sign(token, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) - _, _, err = validateToken(string(noExpiryToken), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: \"exp\" not satisfied: required claim not found") + assert.NoError(ts.T(), err) + _, _, err = validateToken(string(noExpiryToken), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: \"exp\" not satisfied: required claim not found") } diff --git a/sda/cmd/auth/s3conf_test.go b/sda/cmd/auth/s3conf_test.go index 71030b5149..040118617f 100644 --- a/sda/cmd/auth/s3conf_test.go +++ b/sda/cmd/auth/s3conf_test.go @@ -19,12 +19,12 @@ func TestS3ConfTestSuite(t *testing.T) { suite.Run(t, new(S3ConfTests)) } -func (suite *S3ConfTests) SetupTest() {} +func (ts *S3ConfTests) SetupTest() {} -func (suite *S3ConfTests) TearDownTest() {} +func (ts *S3ConfTests) TearDownTest() {} //nolint:goconst -func (suite *S3ConfTests) TestGetS3ConfigMap() { +func (ts *S3ConfTests) TestGetS3ConfigMap() { // variable values token := "tokenvaluestring" inboxHost := "s3://inboxHost" @@ -44,18 +44,18 @@ func (suite *S3ConfTests) TestGetS3ConfigMap() { s3conf := getS3ConfigMap(token, inboxHost, user) - assert.Equal(suite.T(), user, s3conf["access_key"], fmt.Sprintf("access_key should be %v", user)) - assert.Equal(suite.T(), user, s3conf["secret_key"], fmt.Sprintf("secret_key should be %v", user)) - assert.Equal(suite.T(), token, s3conf["access_token"], fmt.Sprintf("access_token should be %v", token)) - assert.Equal(suite.T(), checkSslCertificate, s3conf["check_ssl_certificate"], fmt.Sprintf("check_ssl_certificate should be %v", checkSslCertificate)) - assert.Equal(suite.T(), checkSslHostname, s3conf["check_ssl_hostname"], fmt.Sprintf("check_ssl_hostname should be %v", checkSslHostname)) - assert.Equal(suite.T(), encoding, s3conf["encoding"], fmt.Sprintf("encoding should be %v", encoding)) - assert.Equal(suite.T(), encrypt, s3conf["encrypt"], fmt.Sprintf("encrypt should be %v", encrypt)) - assert.Equal(suite.T(), guessMimeType, s3conf["guess_mime_type"], fmt.Sprintf("guess_mime_type should be %v", guessMimeType)) - assert.Equal(suite.T(), inboxHost, s3conf["host_base"], fmt.Sprintf("host_base should be %v", inboxHost)) - assert.Equal(suite.T(), inboxHost, s3conf["host_bucket"], fmt.Sprintf("host_bucket should be %v", inboxHost)) - assert.Equal(suite.T(), humanReadableSizes, s3conf["human_readable_sizes"], fmt.Sprintf("human_readable_sizes should be %v", humanReadableSizes)) - assert.Equal(suite.T(), fmt.Sprintf("%v", chunkSize), s3conf["multipart_chunk_size_mb"], fmt.Sprintf("multipart_chunk_size_mb should be %v", chunkSize)) - assert.Equal(suite.T(), useHTTPS, s3conf["use_https"], fmt.Sprintf("use_https should be '%v'", useHTTPS)) - assert.Equal(suite.T(), fmt.Sprintf("%v", socketTimeout), s3conf["socket_timeout"], fmt.Sprintf("socket_timeout should be %v", socketTimeout)) + assert.Equal(ts.T(), user, s3conf["access_key"], fmt.Sprintf("access_key should be %v", user)) + assert.Equal(ts.T(), user, s3conf["secret_key"], fmt.Sprintf("secret_key should be %v", user)) + assert.Equal(ts.T(), token, s3conf["access_token"], fmt.Sprintf("access_token should be %v", token)) + assert.Equal(ts.T(), checkSslCertificate, s3conf["check_ssl_certificate"], fmt.Sprintf("check_ssl_certificate should be %v", checkSslCertificate)) + assert.Equal(ts.T(), checkSslHostname, s3conf["check_ssl_hostname"], fmt.Sprintf("check_ssl_hostname should be %v", checkSslHostname)) + assert.Equal(ts.T(), encoding, s3conf["encoding"], fmt.Sprintf("encoding should be %v", encoding)) + assert.Equal(ts.T(), encrypt, s3conf["encrypt"], fmt.Sprintf("encrypt should be %v", encrypt)) + assert.Equal(ts.T(), guessMimeType, s3conf["guess_mime_type"], fmt.Sprintf("guess_mime_type should be %v", guessMimeType)) + assert.Equal(ts.T(), inboxHost, s3conf["host_base"], fmt.Sprintf("host_base should be %v", inboxHost)) + assert.Equal(ts.T(), inboxHost, s3conf["host_bucket"], fmt.Sprintf("host_bucket should be %v", inboxHost)) + assert.Equal(ts.T(), humanReadableSizes, s3conf["human_readable_sizes"], fmt.Sprintf("human_readable_sizes should be %v", humanReadableSizes)) + assert.Equal(ts.T(), fmt.Sprintf("%v", chunkSize), s3conf["multipart_chunk_size_mb"], fmt.Sprintf("multipart_chunk_size_mb should be %v", chunkSize)) + assert.Equal(ts.T(), useHTTPS, s3conf["use_https"], fmt.Sprintf("use_https should be '%v'", useHTTPS)) + assert.Equal(ts.T(), fmt.Sprintf("%v", socketTimeout), s3conf["socket_timeout"], fmt.Sprintf("socket_timeout should be %v", socketTimeout)) } diff --git a/sda/cmd/finalize/finalize_test.go b/sda/cmd/finalize/finalize_test.go index 0f8a2819d2..aea1d90847 100644 --- a/sda/cmd/finalize/finalize_test.go +++ b/sda/cmd/finalize/finalize_test.go @@ -15,6 +15,6 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index af1dbc8c75..c76d2dd1ab 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -171,22 +171,22 @@ type TestSuite struct { ingest Ingest } -func (suite *TestSuite) SetupSuite() { +func (ts *TestSuite) SetupSuite() { viper.Set("log.level", "debug") - tempDir := suite.T().TempDir() + tempDir := ts.T().TempDir() keyFile1 := fmt.Sprintf("%s/c4gh1.key", tempDir) keyFile2 := fmt.Sprintf("%s/c4gh2.key", tempDir) publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test") if err != nil { - suite.FailNow("Failed to create c4gh key") + ts.FailNow("Failed to create c4gh key") } // Add only the first public key to the list - suite.pubKeyList = append(suite.pubKeyList, publicKey) + ts.pubKeyList = append(ts.pubKeyList, publicKey) _, err = helper.CreatePrivateKeyFile(keyFile2, "test") if err != nil { - suite.FailNow("Failed to create c4gh key") + ts.FailNow("Failed to create c4gh key") } viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ @@ -212,164 +212,164 @@ func (suite *TestSuite) SetupSuite() { viper.Set("inbox.location", "/tmp/") viper.Set("schema.path", "../../schemas/isolated/") - suite.ingest.Conf, err = config.NewConfig("ingest") + ts.ingest.Conf, err = config.NewConfig("ingest") if err != nil { - suite.FailNowf("failed to init config: %s", err.Error()) + ts.FailNowf("failed to init config: %s", err.Error()) } - suite.ingest.DB, err = database.NewSDAdb(suite.ingest.Conf.Database) + ts.ingest.DB, err = database.NewSDAdb(ts.ingest.Conf.Database) if err != nil { - suite.FailNowf("failed to setup database connection: %s", err.Error()) + ts.FailNowf("failed to setup database connection: %s", err.Error()) } - suite.ingest.MQ, err = broker.NewMQ(suite.ingest.Conf.Broker) + ts.ingest.MQ, err = broker.NewMQ(ts.ingest.Conf.Broker) if err != nil { - suite.FailNowf("failed to setup rabbitMQ connection: %s", err.Error()) + ts.FailNowf("failed to setup rabbitMQ connection: %s", err.Error()) } - suite.ingest.ArchiveKeyList, err = config.GetC4GHprivateKeys() + ts.ingest.ArchiveKeyList, err = config.GetC4GHprivateKeys() if err != nil { - suite.FailNow("no private keys configured") + ts.FailNow("no private keys configured") } - if err := suite.ingest.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), "the test key"); err != nil { - suite.FailNow("failed to register the public key") + if err := ts.ingest.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), "the test key"); err != nil { + ts.FailNow("failed to register the public key") } } -func (suite *TestSuite) TearDownSuite() { - _ = os.RemoveAll(suite.ingest.Conf.Archive.Posix.Location) - _ = os.RemoveAll(suite.ingest.Conf.Inbox.Posix.Location) +func (ts *TestSuite) TearDownSuite() { + _ = os.RemoveAll(ts.ingest.Conf.Archive.Posix.Location) + _ = os.RemoveAll(ts.ingest.Conf.Inbox.Posix.Location) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { var err error - suite.ingest.Conf.Archive.Posix.Location, err = os.MkdirTemp("", "archive") + ts.ingest.Conf.Archive.Posix.Location, err = os.MkdirTemp("", "archive") if err != nil { - suite.FailNow("failed to create temp folder") + ts.FailNow("failed to create temp folder") } - suite.ingest.Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") + ts.ingest.Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") if err != nil { - suite.FailNow("failed to create temp folder") + ts.FailNow("failed to create temp folder") } - f, err := os.CreateTemp(suite.ingest.Conf.Inbox.Posix.Location, "") + f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") if err != nil { - suite.FailNow("failed to create test file") + ts.FailNow("failed to create test file") } defer f.Close() _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) if err != nil { - suite.FailNow("failed to write data to test file") + ts.FailNow("failed to write data to test file") } outFileName := f.Name() + ".c4gh" outFile, err := os.Create(outFileName) if err != nil { - suite.FailNow("failed to create encrypted test file") + ts.FailNow("failed to create encrypted test file") } defer outFile.Close() _, privateKey, err := keys.GenerateKeyPair() if err != nil { - suite.FailNow("failed to create private c4gh key") + ts.FailNow("failed to create private c4gh key") } - crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, suite.pubKeyList, nil) + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, ts.pubKeyList, nil) if err != nil { - suite.FailNow("failed to create c4gh writer") + ts.FailNow("failed to create c4gh writer") } _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) if err != nil { - suite.FailNow("failed to write data to encrypted test file") + ts.FailNow("failed to write data to encrypted test file") } crypt4GHWriter.Close() - suite.filePath = filepath.Base(outFileName) + ts.filePath = filepath.Base(outFileName) - suite.ingest.Archive, err = storage.NewBackend(suite.ingest.Conf.Archive) + ts.ingest.Archive, err = storage.NewBackend(ts.ingest.Conf.Archive) if err != nil { - suite.FailNow("failed to setup archive backend") + ts.FailNow("failed to setup archive backend") } - suite.ingest.Inbox, err = storage.NewBackend(suite.ingest.Conf.Inbox) + ts.ingest.Inbox, err = storage.NewBackend(ts.ingest.Conf.Inbox) if err != nil { - suite.FailNow("failed to setup inbox backend") + ts.FailNow("failed to setup inbox backend") } } -func (suite *TestSuite) TestTryDecrypt_wrongFile() { - tempDir := suite.T().TempDir() +func (ts *TestSuite) TestTryDecrypt_wrongFile() { + tempDir := ts.T().TempDir() err := os.WriteFile(fmt.Sprintf("%s/dummy.file", tempDir), []byte("hello\ngo\n"), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) file, err := os.Open(fmt.Sprintf("%s/dummy.file", tempDir)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer file.Close() buf, err := io.ReadAll(file) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) privateKeys, err := config.GetC4GHprivateKeys() - assert.NoError(suite.T(), err) - assert.Len(suite.T(), privateKeys, 2) + assert.NoError(ts.T(), err) + assert.Len(ts.T(), privateKeys, 2) header, err := tryDecrypt(privateKeys[0], buf) - assert.Nil(suite.T(), header) - assert.EqualError(suite.T(), err, "not a Crypt4GH file") + assert.Nil(ts.T(), header) + assert.EqualError(ts.T(), err, "not a Crypt4GH file") } -func (suite *TestSuite) TestTryDecrypt() { +func (ts *TestSuite) TestTryDecrypt() { _, signingKey, err := keys.GenerateKeyPair() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // encrypt test file - tempDir := suite.T().TempDir() + tempDir := ts.T().TempDir() unencryptedFile, err := os.CreateTemp(tempDir, "unencryptedFile-") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = os.WriteFile(unencryptedFile.Name(), []byte("content"), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) encryptedFile, err := os.CreateTemp(tempDir, "encryptedFile-") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) - crypt4GHWriter, err := streaming.NewCrypt4GHWriter(encryptedFile, signingKey, suite.pubKeyList, nil) - assert.NoError(suite.T(), err) + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(encryptedFile, signingKey, ts.pubKeyList, nil) + assert.NoError(ts.T(), err) _, err = io.Copy(crypt4GHWriter, unencryptedFile) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) crypt4GHWriter.Close() file, err := os.Open(encryptedFile.Name()) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer file.Close() buf, err := io.ReadAll(file) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) privateKeys, err := config.GetC4GHprivateKeys() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) for i, key := range privateKeys { header, err := tryDecrypt(key, buf) switch { case i == 0: - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), header) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), header) default: - assert.Contains(suite.T(), err.Error(), "could not find matching public key heade") - assert.Nil(suite.T(), header) + assert.Contains(ts.T(), err.Error(), "could not find matching public key heade") + assert.Nil(ts.T(), header) } } } // messages of type `cancel` -func (suite *TestSuite) TestCancelFile() { +func (ts *TestSuite) TestCancelFile() { // prepare the DB entries UserName := "test-cancel" file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", UserName) - fileID, err := suite.ingest.DB.RegisterFile(file1, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(file1, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ @@ -378,18 +378,18 @@ func (suite *TestSuite) TestCancelFile() { User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.cancelFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(corrID, message)) } -func (suite *TestSuite) TestCancelFile_wrongCorrelationID() { +func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // prepare the DB entries UserName := "test-cancel" file1 := fmt.Sprintf("/%v/TestCancelMessage_wrongCorrelationID.c4gh", UserName) - fileID, err := suite.ingest.DB.RegisterFile(file1, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(file1, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ @@ -398,176 +398,176 @@ func (suite *TestSuite) TestCancelFile_wrongCorrelationID() { User: UserName, } - assert.Equal(suite.T(), "reject", suite.ingest.cancelFile(uuid.New().String(), message)) + assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(uuid.New().String(), message)) } // messages of type `ingest` -func (suite *TestSuite) TestIngestFile() { +func (ts *TestSuite) TestIngestFile() { // prepare the DB entries UserName := "test-ingest" - fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } -func (suite *TestSuite) TestIngestFile_secondTime() { +func (ts *TestSuite) TestIngestFile_secondTime() { // prepare the DB entries UserName := "test-ingest" - fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) // file is already in `archived` state - assert.Equal(suite.T(), "reject", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) } -func (suite *TestSuite) TestIngestFile_unknownInboxType() { +func (ts *TestSuite) TestIngestFile_unknownInboxType() { UserName := "test-ingest-unknown" message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(uuid.New().String(), message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(uuid.New().String(), message)) } -func (suite *TestSuite) TestIngestFile_reingestCancelledFile() { +func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { // prepare the DB entries UserName := "test-ingest" - fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } -func (suite *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { +func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { // prepare the DB entries UserName := "test-ingest" - fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } // over write the encrypted file to generate new checksum - f, err := os.CreateTemp(suite.ingest.Conf.Inbox.Posix.Location, "") + f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") if err != nil { - suite.FailNow("failed to create test file") + ts.FailNow("failed to create test file") } defer f.Close() _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) if err != nil { - suite.FailNow("failed to write data to test file") + ts.FailNow("failed to write data to test file") } _, privateKey, err := keys.GenerateKeyPair() if err != nil { - suite.FailNow("failed to create private c4gh key") + ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(suite.ingest.Conf.Inbox.Posix.Location, suite.filePath)) + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) if err != nil { - suite.FailNowf("failed to create encrypted test file: %s", err.Error()) + ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } defer outFile.Close() sha256hash := sha256.New() mr := io.MultiWriter(outFile, sha256hash) - crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, suite.pubKeyList, nil) + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, ts.pubKeyList, nil) if err != nil { - suite.FailNowf("failed to create c4gh writer: %s", err.Error()) + ts.FailNowf("failed to create c4gh writer: %s", err.Error()) } _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) if err != nil { - suite.FailNow("failed to write data to encrypted test file") + ts.FailNow("failed to write data to encrypted test file") } crypt4GHWriter.Close() // reingestion should work - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) // DB should have the new checksum var dbChecksum string const q = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" - if err := suite.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { - suite.FailNow("failed to get checksum from database") + if err := ts.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { + ts.FailNow("failed to get checksum from database") } - assert.Equal(suite.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) + assert.Equal(ts.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) } -func (suite *TestSuite) TestIngestFile_reingestVerifiedFile() { +func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { // prepare the DB entries UserName := "test-ingest" - fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) // fake file verification sha256hash := sha256.New() @@ -576,30 +576,30 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedFile() { fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 - if err := suite.ingest.DB.SetVerified(fi, fileID); err != nil { - suite.Fail("failed to mark file as verified") + if err := ts.ingest.DB.SetVerified(fi, fileID); err != nil { + ts.Fail("failed to mark file as verified") } - assert.Equal(suite.T(), "reject", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) } -func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { +func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // prepare the DB entries UserName := "test-ingest" - fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) // fake file verification sha256hash := sha256.New() @@ -608,39 +608,39 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 - if err := suite.ingest.DB.SetVerified(fi, fileID); err != nil { - suite.Fail("failed to mark file as verified") + if err := ts.ingest.DB.SetVerified(fi, fileID); err != nil { + ts.Fail("failed to mark file as verified") } - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } -func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { +func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { // prepare the DB entries UserName := "test-ingest" - fileID, err := suite.ingest.DB.RegisterFile(suite.filePath, UserName) - assert.NoError(suite.T(), err, "failed to register file in database") + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", - FilePath: suite.filePath, + FilePath: ts.filePath, User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) var firstDbChecksum string const q1 = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" - if err := suite.ingest.DB.DB.QueryRow(q1, fileID).Scan(&firstDbChecksum); err != nil { - suite.FailNow("failed to get checksum from database") + if err := ts.ingest.DB.DB.QueryRow(q1, fileID).Scan(&firstDbChecksum); err != nil { + ts.FailNow("failed to get checksum from database") } // fake file verification @@ -650,70 +650,70 @@ func (suite *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum( fi.DecryptedChecksum = hex.EncodeToString(verifiedSha256.Sum(nil)) fi.DecryptedSize = 10 * 1024 * 1024 fi.Size = (10 * 1024 * 1024) + 456 - if err := suite.ingest.DB.SetVerified(fi, fileID); err != nil { - suite.Fail("failed to mark file as verified") + if err := ts.ingest.DB.SetVerified(fi, fileID); err != nil { + ts.Fail("failed to mark file as verified") } - if err = suite.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { - suite.Fail("failed to update file event log") + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") } // over write the encrypted file to generate new checksum - f, err := os.CreateTemp(suite.ingest.Conf.Inbox.Posix.Location, "") + f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") if err != nil { - suite.FailNow("failed to create test file") + ts.FailNow("failed to create test file") } defer f.Close() _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) if err != nil { - suite.FailNow("failed to write data to test file") + ts.FailNow("failed to write data to test file") } _, privateKey, err := keys.GenerateKeyPair() if err != nil { - suite.FailNow("failed to create private c4gh key") + ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(suite.ingest.Conf.Inbox.Posix.Location, suite.filePath)) + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) if err != nil { - suite.FailNowf("failed to create encrypted test file: %s", err.Error()) + ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } defer outFile.Close() sha256hash := sha256.New() mr := io.MultiWriter(outFile, sha256hash) - crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, suite.pubKeyList, nil) + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, ts.pubKeyList, nil) if err != nil { - suite.FailNowf("failed to create c4gh writer: %s", err.Error()) + ts.FailNowf("failed to create c4gh writer: %s", err.Error()) } _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) if err != nil { - suite.FailNow("failed to write data to encrypted test file") + ts.FailNow("failed to write data to encrypted test file") } crypt4GHWriter.Close() // reingestion should work - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) // DB should have the new checksum var dbChecksum string const q = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" - if err := suite.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { - suite.FailNow("failed to get checksum from database") + if err := ts.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { + ts.FailNow("failed to get checksum from database") } - assert.Equal(suite.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) + assert.Equal(ts.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) - assert.NotEqual(suite.T(), dbChecksum, firstDbChecksum) + assert.NotEqual(ts.T(), dbChecksum, firstDbChecksum) } -func (suite *TestSuite) TestIngestFile_missingFile() { +func (ts *TestSuite) TestIngestFile_missingFile() { // prepare the DB entries UserName := "test-ingest" corrID := uuid.New().String() - basepath := filepath.Dir(suite.filePath) + basepath := filepath.Dir(ts.filePath) message := schema.IngestionTrigger{ Type: "ingest", @@ -721,5 +721,5 @@ func (suite *TestSuite) TestIngestFile_missingFile() { User: UserName, } - assert.Equal(suite.T(), "ack", suite.ingest.ingestFile(corrID, message)) + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } diff --git a/sda/cmd/intercept/intercept_test.go b/sda/cmd/intercept/intercept_test.go index a591cea4fe..01e5cb8eef 100644 --- a/sda/cmd/intercept/intercept_test.go +++ b/sda/cmd/intercept/intercept_test.go @@ -17,7 +17,7 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } @@ -51,7 +51,7 @@ type missing struct { FilePath string `json:"filepath"` } -func (suite *TestSuite) TestMessageSelection_Accession() { +func (ts *TestSuite) TestMessageSelection_Accession() { msg := accession{ Type: "accession", User: "foo", @@ -65,11 +65,11 @@ func (suite *TestSuite) TestMessageSelection_Accession() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgType, msgAccession, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgType, msgAccession, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Cancel() { +func (ts *TestSuite) TestMessageSelection_Cancel() { msg := ingest{ Type: "cancel", User: "foo", @@ -79,11 +79,11 @@ func (suite *TestSuite) TestMessageSelection_Cancel() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgType, msgCancel, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgType, msgCancel, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Ingest() { +func (ts *TestSuite) TestMessageSelection_Ingest() { msg := ingest{ Type: "ingest", User: "foo", @@ -93,11 +93,11 @@ func (suite *TestSuite) TestMessageSelection_Ingest() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgIngest, msgType, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgIngest, msgType, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Mapping() { +func (ts *TestSuite) TestMessageSelection_Mapping() { msg := mapping{ Type: "mapping", DatasetID: "EGAD12345678900", @@ -109,11 +109,11 @@ func (suite *TestSuite) TestMessageSelection_Mapping() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgMapping, msgType, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgMapping, msgType, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Notype() { +func (ts *TestSuite) TestMessageSelection_Notype() { msg := missing{ User: "foo", FilePath: "/tmp/foo", @@ -122,6 +122,6 @@ func (suite *TestSuite) TestMessageSelection_Notype() { msgType, err := typeFromMessage(message) - assert.Error(suite.T(), err, "Unexpected lack of error from typeFromMessage") - assert.Equal(suite.T(), "", msgType, "message type from message does not match expected") + assert.Error(ts.T(), err, "Unexpected lack of error from typeFromMessage") + assert.Equal(ts.T(), "", msgType, "message type from message does not match expected") } diff --git a/sda/cmd/mapper/mapper_test.go b/sda/cmd/mapper/mapper_test.go index 0f8a2819d2..aea1d90847 100644 --- a/sda/cmd/mapper/mapper_test.go +++ b/sda/cmd/mapper/mapper_test.go @@ -15,6 +15,6 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } diff --git a/sda/cmd/notify/notify_test.go b/sda/cmd/notify/notify_test.go index 5795d905ec..2bb29d1b08 100644 --- a/sda/cmd/notify/notify_test.go +++ b/sda/cmd/notify/notify_test.go @@ -22,7 +22,7 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 861e1ea85a..71da4b588f 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -43,63 +43,63 @@ func TestReEncryptTests(t *testing.T) { suite.Run(t, new(ReEncryptTests)) } -func (suite *ReEncryptTests) SetupTest() { +func (ts *ReEncryptTests) SetupTest() { var err error log.SetLevel(log.InfoLevel) repKey := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" - suite.KeyPath, _ = os.MkdirTemp("", "key") - if err := os.WriteFile(suite.KeyPath+"/c4gh.key", []byte(repKey), 0600); err != nil { - suite.T().FailNow() + ts.KeyPath, _ = os.MkdirTemp("", "key") + if err := os.WriteFile(ts.KeyPath+"/c4gh.key", []byte(repKey), 0600); err != nil { + ts.T().FailNow() } - suite.UserPublicKey, suite.UserPrivateKey, err = keys.GenerateKeyPair() + ts.UserPublicKey, ts.UserPrivateKey, err = keys.GenerateKeyPair() if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - key, err := os.Create(suite.KeyPath + "/new.key") + key, err := os.Create(ts.KeyPath + "/new.key") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - if err := keys.WriteCrypt4GHX25519PrivateKey(key, suite.UserPrivateKey, []byte("password")); err != nil { - suite.T().FailNow() + if err := keys.WriteCrypt4GHX25519PrivateKey(key, ts.UserPrivateKey, []byte("password")); err != nil { + ts.T().FailNow() } buf := new(bytes.Buffer) - if err := keys.WriteCrypt4GHX25519PublicKey(buf, suite.UserPublicKey); err != nil { - suite.T().FailNow() + if err := keys.WriteCrypt4GHX25519PublicKey(buf, ts.UserPublicKey); err != nil { + ts.T().FailNow() } - suite.UserPubKeyString = base64.StdEncoding.EncodeToString(buf.Bytes()) + ts.UserPubKeyString = base64.StdEncoding.EncodeToString(buf.Bytes()) - viper.Set("c4gh.filepath", suite.KeyPath+"/c4gh.key") + viper.Set("c4gh.filepath", ts.KeyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") - suite.PrivateKey, err = config.GetC4GHKey() + ts.PrivateKey, err = config.GetC4GHKey() if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") - suite.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") + ts.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") + ts.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") } -func (suite *ReEncryptTests) TearDownTest() { - os.RemoveAll(suite.KeyPath) +func (ts *ReEncryptTests) TearDownTest() { + os.RemoveAll(ts.KeyPath) } -func (suite *ReEncryptTests) TestReencryptHeader() { +func (ts *ReEncryptTests) TestReencryptHeader() { lis, err := net.Listen("tcp", "localhost:50051") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) if err := s.Serve(lis); err != nil { - suite.T().Fail() + ts.T().Fail() } }() @@ -107,7 +107,7 @@ func (suite *ReEncryptTests) TestReencryptHeader() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50051", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -115,33 +115,33 @@ func (suite *ReEncryptTests) TestReencryptHeader() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString}) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString}) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr := bytes.NewReader(res.Header) - fileStream := io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err := streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err := io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "content", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) } -func (suite *ReEncryptTests) TestReencryptHeader_DataEditList() { +func (ts *ReEncryptTests) TestReencryptHeader_DataEditList() { lis, err := net.Listen("tcp", "localhost:50054") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) if err := s.Serve(lis); err != nil { - suite.T().Fail() + ts.T().Fail() } }() @@ -149,7 +149,7 @@ func (suite *ReEncryptTests) TestReencryptHeader_DataEditList() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50054", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -157,63 +157,63 @@ func (suite *ReEncryptTests) TestReencryptHeader_DataEditList() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString, Dataeditlist: []uint64{1, 2, 1, 2}}) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString, Dataeditlist: []uint64{1, 2, 1, 2}}) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr := bytes.NewReader(res.Header) - fileStream := io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err := streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err := io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "onen", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "onen", string(data)) hr = bytes.NewReader(res.Header) - header, err := headers.NewHeader(hr, suite.UserPrivateKey) - assert.NoError(suite.T(), err) + header, err := headers.NewHeader(hr, ts.UserPrivateKey) + assert.NoError(ts.T(), err) packet := header.GetDataEditListHeaderPacket() - assert.NotNilf(suite.T(), packet, "DataEditList HeaderPacket not found") - assert.Equal(suite.T(), uint32(4), packet.NumberLengths) - assert.Equal(suite.T(), uint64(1), packet.Lengths[0]) - assert.Equal(suite.T(), uint64(2), packet.Lengths[1]) - assert.Equal(suite.T(), uint64(1), packet.Lengths[2]) - assert.Equal(suite.T(), uint64(2), packet.Lengths[3]) - assert.Equal(suite.T(), int(4), len(packet.Lengths)) - - res, err = c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString, Dataeditlist: []uint64{}}) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + assert.NotNilf(ts.T(), packet, "DataEditList HeaderPacket not found") + assert.Equal(ts.T(), uint32(4), packet.NumberLengths) + assert.Equal(ts.T(), uint64(1), packet.Lengths[0]) + assert.Equal(ts.T(), uint64(2), packet.Lengths[1]) + assert.Equal(ts.T(), uint64(1), packet.Lengths[2]) + assert.Equal(ts.T(), uint64(2), packet.Lengths[3]) + assert.Equal(ts.T(), int(4), len(packet.Lengths)) + + res, err = c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString, Dataeditlist: []uint64{}}) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr = bytes.NewReader(res.Header) - fileStream = io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream = io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err = streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err = streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err = io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "content", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) hr = bytes.NewReader(res.Header) - header, err = headers.NewHeader(hr, suite.UserPrivateKey) - assert.NoError(suite.T(), err) + header, err = headers.NewHeader(hr, ts.UserPrivateKey) + assert.NoError(ts.T(), err) packet = header.GetDataEditListHeaderPacket() - assert.Nilf(suite.T(), packet, "DataEditList HeaderPacket found when not expected") + assert.Nilf(ts.T(), packet, "DataEditList HeaderPacket found when not expected") } -func (suite *ReEncryptTests) TestReencryptHeader_BadPubKey() { +func (ts *ReEncryptTests) TestReencryptHeader_BadPubKey() { lis, err := net.Listen("tcp", "localhost:50052") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) _ = s.Serve(lis) }() @@ -221,7 +221,7 @@ func (suite *ReEncryptTests) TestReencryptHeader_BadPubKey() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50052", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -229,21 +229,21 @@ func (suite *ReEncryptTests) TestReencryptHeader_BadPubKey() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: "BadKey"}) - assert.Contains(suite.T(), err.Error(), "illegal base64 data") - assert.Nil(suite.T(), res) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: "BadKey"}) + assert.Contains(ts.T(), err.Error(), "illegal base64 data") + assert.Nil(ts.T(), res) } -func (suite *ReEncryptTests) TestReencryptHeader_NoHeader() { +func (ts *ReEncryptTests) TestReencryptHeader_NoHeader() { lis, err := net.Listen("tcp", "localhost:50053") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) _ = s.Serve(lis) }() @@ -251,7 +251,7 @@ func (suite *ReEncryptTests) TestReencryptHeader_NoHeader() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50053", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -259,32 +259,32 @@ func (suite *ReEncryptTests) TestReencryptHeader_NoHeader() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: make([]byte, 0), Publickey: suite.UserPubKeyString}) - assert.Contains(suite.T(), err.Error(), "no header received") - assert.Nil(suite.T(), res) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: make([]byte, 0), Publickey: ts.UserPubKeyString}) + assert.Contains(ts.T(), err.Error(), "no header received") + assert.Nil(ts.T(), res) } -func (suite *ReEncryptTests) TestReencryptHeader_TLS() { - certPath := suite.T().TempDir() +func (ts *ReEncryptTests) TestReencryptHeader_TLS() { + certPath := ts.T().TempDir() helper.MakeCerts(certPath) rootCAs := x509.NewCertPool() cacertFile, err := os.ReadFile(certPath + "/ca.crt") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } ok := rootCAs.AppendCertsFromPEM(cacertFile) if !ok { - suite.T().FailNow() + ts.T().FailNow() } certs, err := tls.LoadX509KeyPair(certPath+"/tls.crt", certPath+"/tls.key") if err != nil { - suite.T().Log(err.Error()) - suite.T().FailNow() + ts.T().Log(err.Error()) + ts.T().FailNow() } lis, err := net.Listen("tcp", "localhost:50443") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { @@ -298,9 +298,9 @@ func (suite *ReEncryptTests) TestReencryptHeader_TLS() { ) opts := []grpc.ServerOption{grpc.Creds(serverCreds)} s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) if err := s.Serve(lis); err != nil { - suite.T().Fail() + ts.T().Fail() } }() @@ -313,8 +313,8 @@ func (suite *ReEncryptTests) TestReencryptHeader_TLS() { ) conn, err := grpc.NewClient("localhost:50443", grpc.WithTransportCredentials(clientCreds)) if err != nil { - suite.T().Log(err.Error()) - suite.T().FailNow() + ts.T().Log(err.Error()) + ts.T().FailNow() } defer conn.Close() @@ -322,22 +322,22 @@ func (suite *ReEncryptTests) TestReencryptHeader_TLS() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString}) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString}) if err != nil { - suite.T().Log(err.Error()) - suite.T().FailNow() + ts.T().Log(err.Error()) + ts.T().FailNow() } - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), res) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), res) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr := bytes.NewReader(res.Header) - fileStream := io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err := streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err := io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "content", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) } diff --git a/sda/cmd/s3inbox/healthchecks.go b/sda/cmd/s3inbox/healthchecks.go index 1fc432f2d3..9cba134a4d 100644 --- a/sda/cmd/s3inbox/healthchecks.go +++ b/sda/cmd/s3inbox/healthchecks.go @@ -73,8 +73,8 @@ func (p *Proxy) CheckHealth(w http.ResponseWriter, _ *http.Request) { } // httpsGetCheck sends a request to the S3 backend and makes sure it is healthy -func (p *Proxy) httpsGetCheck(url string) error { - resp, e := p.client.Get(url) +func (p *Proxy) httpsGetCheck(uri string) error { + resp, e := p.client.Get(uri) if e != nil { return e } diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index 1502c90bb9..870a3fdf9d 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -30,11 +30,11 @@ func TestHealthTestSuite(t *testing.T) { suite.Run(t, new(HealthcheckTestSuite)) } -func (suite *HealthcheckTestSuite) SetupTest() { - suite.fakeServer = startFakeServer("9024") +func (ts *HealthcheckTestSuite) SetupTest() { + ts.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - suite.S3Fakeconf = storage.S3Conf{ + ts.S3Fakeconf = storage.S3Conf{ URL: "http://127.0.0.1", Port: 9024, AccessKey: "someAccess", @@ -44,7 +44,7 @@ func (suite *HealthcheckTestSuite) SetupTest() { } // Create a configuration for the fake MQ - suite.MQConf = broker.MQConf{ + ts.MQConf = broker.MQConf{ Host: "127.0.0.1", Port: MQport, User: "guest", @@ -53,10 +53,10 @@ func (suite *HealthcheckTestSuite) SetupTest() { Exchange: "", } - suite.messenger = &broker.AMQPBroker{} + ts.messenger = &broker.AMQPBroker{} // Create a database configuration for the fake database - suite.DBConf = database.DBConf{ + ts.DBConf = database.DBConf{ Host: "127.0.0.1", Port: DBport, User: "postgres", @@ -68,61 +68,61 @@ func (suite *HealthcheckTestSuite) SetupTest() { ClientKey: "", } - suite.database = &database.SDAdb{} + ts.database = &database.SDAdb{} } -func (suite *HealthcheckTestSuite) TearDownTest() { - suite.fakeServer.Close() - suite.database.Close() +func (ts *HealthcheckTestSuite) TearDownTest() { + ts.fakeServer.Close() + ts.database.Close() } -func (suite *HealthcheckTestSuite) TestHttpsGetCheck() { - p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) +func (ts *HealthcheckTestSuite) TestHttpsGetCheck() { + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) url, _ := p.getS3ReadyPath() - assert.NoError(suite.T(), p.httpsGetCheck(url)) - assert.Error(suite.T(), p.httpsGetCheck("http://127.0.0.1:8888/nonexistent"), "404 should fail") + assert.NoError(ts.T(), p.httpsGetCheck(url)) + assert.Error(ts.T(), p.httpsGetCheck("http://127.0.0.1:8888/nonexistent"), "404 should fail") } -func (suite *HealthcheckTestSuite) TestS3URL() { - p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) +func (ts *HealthcheckTestSuite) TestS3URL() { + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) _, err := p.getS3ReadyPath() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) p.s3.URL = "://badurl" url, err := p.getS3ReadyPath() - assert.Empty(suite.T(), url) - assert.Error(suite.T(), err) + assert.Empty(ts.T(), url) + assert.Error(ts.T(), err) } -func (suite *HealthcheckTestSuite) TestHealthchecks() { +func (ts *HealthcheckTestSuite) TestHealthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 200, resp.StatusCode) + assert.Equal(ts.T(), 200, resp.StatusCode) } -func (suite *HealthcheckTestSuite) TestClosedDBHealthchecks() { +func (ts *HealthcheckTestSuite) TestClosedDBHealthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Check that 200 is reported w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 200, resp.StatusCode) + assert.Equal(ts.T(), 200, resp.StatusCode) // Close connection to DB, check that connection is restored and 200 returned w = httptest.NewRecorder() @@ -130,15 +130,15 @@ func (suite *HealthcheckTestSuite) TestClosedDBHealthchecks() { p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp = w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 200, resp.StatusCode) + assert.Equal(ts.T(), 200, resp.StatusCode) } -func (suite *HealthcheckTestSuite) TestNoS3Healthchecks() { +func (ts *HealthcheckTestSuite) TestNoS3Healthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // S3 unavailable, check that 503 is reported w := httptest.NewRecorder() @@ -146,23 +146,23 @@ func (suite *HealthcheckTestSuite) TestNoS3Healthchecks() { p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 503, resp.StatusCode) + assert.Equal(ts.T(), 503, resp.StatusCode) } -func (suite *HealthcheckTestSuite) TestNoMQHealthchecks() { +func (ts *HealthcheckTestSuite) TestNoMQHealthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Messenger unavailable, check that 503 is reported p.messenger.Conf.Port = 123456 p.messenger.Connection.Close() - assert.True(suite.T(), p.messenger.Connection.IsClosed()) + assert.True(ts.T(), p.messenger.Connection.IsClosed()) w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 503, resp.StatusCode) + assert.Equal(ts.T(), 503, resp.StatusCode) } diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 64e463c01c..64b9d1e526 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -76,11 +76,11 @@ const ( ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. -func NewProxy(s3conf storage.S3Conf, auth userauth.Authenticator, messenger *broker.AMQPBroker, database *database.SDAdb, tls *tls.Config) *Proxy { - tr := &http.Transport{TLSClientConfig: tls} +func NewProxy(s3conf storage.S3Conf, auth userauth.Authenticator, messenger *broker.AMQPBroker, db *database.SDAdb, tlsConf *tls.Config) *Proxy { + tr := &http.Transport{TLSClientConfig: tlsConf} client := &http.Client{Transport: tr, Timeout: 30 * time.Second} - return &Proxy{s3conf, auth, messenger, database, client, make(map[string]string)} + return &Proxy{s3conf, auth, messenger, db, client, make(map[string]string)} } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index e8ce587a4e..187e95d28c 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -48,12 +48,12 @@ func TestProxyTestSuite(t *testing.T) { suite.Run(t, new(ProxyTests)) } -func (suite *ProxyTests) SetupTest() { +func (s *ProxyTests) SetupTest() { // Create fake server - suite.fakeServer = startFakeServer("9024") + s.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - suite.S3Fakeconf = storage.S3Conf{ + s.S3Fakeconf = storage.S3Conf{ URL: "http://127.0.0.1", Port: 9024, AccessKey: "someAccess", @@ -62,7 +62,7 @@ func (suite *ProxyTests) SetupTest() { Region: "us-east-1", } - suite.S3conf = storage.S3Conf{ + s.S3conf = storage.S3Conf{ URL: "http://127.0.0.1", Port: s3Port, AccessKey: "access", @@ -72,7 +72,7 @@ func (suite *ProxyTests) SetupTest() { } // Create a configuration for the fake MQ - suite.MQConf = broker.MQConf{ + s.MQConf = broker.MQConf{ Host: "127.0.0.1", Port: MQport, User: "guest", @@ -81,10 +81,10 @@ func (suite *ProxyTests) SetupTest() { Exchange: "", } - suite.messenger = &broker.AMQPBroker{} + s.messenger = &broker.AMQPBroker{} // Create a database configuration for the fake database - suite.DBConf = database.DBConf{ + s.DBConf = database.DBConf{ Host: "127.0.0.1", Port: DBport, User: "postgres", @@ -96,65 +96,65 @@ func (suite *ProxyTests) SetupTest() { ClientKey: "", } - suite.database = &database.SDAdb{} + s.database = &database.SDAdb{} // Create temp demo rsa key pair demoKeysPath := "demo-rsa-keys" defer os.RemoveAll(demoKeysPath) prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // Parse demo private key prKeyParsed, err := helper.ParsePrivateRSAKey(prKeyPath, "/rsa") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // Create token and set up request defaults defaultToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" - suite.auth = userauth.NewValidateFromToken(jwk.NewSet()) - _ = suite.auth.ReadJwtPubKeyPath(jwtpubkeypath) + s.auth = userauth.NewValidateFromToken(jwk.NewSet()) + _ = s.auth.ReadJwtPubKeyPath(jwtpubkeypath) - suite.token, err = jwt.Parse([]byte(defaultToken), jwt.WithKeySet(suite.auth.Keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) + s.token, err = jwt.Parse([]byte(defaultToken), jwt.WithKeySet(s.auth.Keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) if err != nil { - suite.T().FailNow() + s.T().FailNow() } - s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(suite.S3conf.AccessKey, suite.S3conf.SecretKey, ""))) + s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) if err != nil { - suite.FailNow("bad") + s.FailNow("bad") } s3Client := s3.NewFromConfig( s3cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", suite.S3conf.URL, suite.S3conf.Port)) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(suite.S3conf.URL, "http:") - o.Region = suite.S3conf.Region + o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", s.S3conf.URL, s.S3conf.Port)) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.URL, "http:") + o.Region = s.S3conf.Region o.UsePathStyle = true }, ) - _, _ = s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(suite.S3conf.Bucket)}) + _, _ = s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(s.S3conf.Bucket)}) if err != nil { fmt.Println(err.Error()) } output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ Body: strings.NewReader("This is a test"), - Bucket: aws.String(suite.S3conf.Bucket), + Bucket: aws.String(s.S3conf.Bucket), Key: aws.String("/dummy/file"), ContentEncoding: aws.String("application/octet-stream"), }) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), output, output) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), output, output) } -func (suite *ProxyTests) TearDownTest() { - suite.fakeServer.Close() - suite.database.Close() +func (s *ProxyTests) TearDownTest() { + s.fakeServer.Close() + s.database.Close() } type FakeServer struct { @@ -220,26 +220,26 @@ func (m *MockMessenger) SendMessage(uuid string, body []byte) error { } // nolint:bodyclose -func (suite *ProxyTests) TestServeHTTP_disallowed() { - proxy := NewProxy(suite.S3Fakeconf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) +func (s *ProxyTests) TestServeHTTP_disallowed() { + proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() - log.Warnf("using proxy on port %d", suite.S3Fakeconf.Port) + log.Warnf("using proxy on port %d", s.S3Fakeconf.Port) // Remove bucket disallowed r.Method = "DELETE" r.URL, _ = url.Parse("/asdf/") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Deletion of files are disallowed r.Method = "DELETE" r.URL, _ = url.Parse("/asdf/asdf") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) log.Warnf("getting to not allowed stuff") // Policy methods are not allowed @@ -247,44 +247,44 @@ func (suite *ProxyTests) TestServeHTTP_disallowed() { r.Method = "GET" r.URL, _ = url.Parse("/asdf?acl=rw") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Normal get is dissallowed w = httptest.NewRecorder() r.Method = "GET" r.URL, _ = url.Parse("/asdf/") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Put policy is disallowed w = httptest.NewRecorder() r.Method = "PUT" r.URL, _ = url.Parse("/asdf?policy=rw") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Create bucket disallowed w = httptest.NewRecorder() r.Method = "PUT" r.URL, _ = url.Parse("/asdf/") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Not authorized user get 401 response - proxy = NewProxy(suite.S3Fakeconf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) + proxy = NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) w = httptest.NewRecorder() r.Method = "GET" r.URL, _ = url.Parse("/username/file") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 401, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 401, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) } -func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { +func (s *ProxyTests) TestServeHTTPS3Unresponsive() { s3conf := storage.S3Conf{ URL: "http://localhost:40211", AccessKey: "someAccess", @@ -292,7 +292,7 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { Bucket: "buckbuck", Region: "us-east-1", } - proxy := NewProxy(s3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) + proxy := NewProxy(s3conf, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() @@ -300,119 +300,119 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { // Just try to list the files r.Method = "GET" r.URL, _ = url.Parse("/dummy/asdf") - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 500, w.Result().StatusCode) // nolint:bodyclose + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 500, w.Result().StatusCode) // nolint:bodyclose } -func (suite *ProxyTests) TestServeHTTP_MQConnectionClosed() { +func (s *ProxyTests) TestServeHTTP_MQConnectionClosed() { // Set up - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the mq connection will be restored when needed proxy.messenger.Connection.Close() - assert.True(suite.T(), proxy.messenger.Connection.IsClosed()) + assert.True(s.T(), proxy.messenger.Connection.IsClosed()) r, _ := http.NewRequest("PUT", "/dummy/connectionclosed-file", nil) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) // nolint:bodyclose - assert.False(suite.T(), proxy.messenger.Connection.IsClosed()) + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose + assert.False(s.T(), proxy.messenger.Connection.IsClosed()) } -func (suite *ProxyTests) TestServeHTTP_MQChannelClosed() { +func (s *ProxyTests) TestServeHTTP_MQChannelClosed() { // Set up - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the mq channel will be restored when needed proxy.messenger.Channel.Close() - assert.True(suite.T(), proxy.messenger.Channel.IsClosed()) + assert.True(s.T(), proxy.messenger.Channel.IsClosed()) r, _ := http.NewRequest("PUT", "/dummy/channelclosed-file", nil) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) // nolint:bodyclose - assert.False(suite.T(), proxy.messenger.Channel.IsClosed()) + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose + assert.False(s.T(), proxy.messenger.Channel.IsClosed()) } -func (suite *ProxyTests) TestServeHTTP_MQ_Unavailable() { +func (s *ProxyTests) TestServeHTTP_MQ_Unavailable() { // Set up - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the correct status code is returned when mq connection can't be created proxy.messenger.Conf.Port = 123456 proxy.messenger.Connection.Close() - assert.True(suite.T(), proxy.messenger.Connection.IsClosed()) + assert.True(s.T(), proxy.messenger.Connection.IsClosed()) r, _ := http.NewRequest("PUT", "/dummy/mqunavailable-file", nil) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 500, w.Result().StatusCode) // nolint:bodyclose + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 500, w.Result().StatusCode) // nolint:bodyclose } // nolint:bodyclose -func (suite *ProxyTests) TestServeHTTP_allowed() { - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) +func (s *ProxyTests) TestServeHTTP_allowed() { + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // List files works r, err := http.NewRequest("GET", "/dummy/file", nil) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) w := httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) // Testing the pinged interface + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Testing the pinged interface // Put file works w = httptest.NewRecorder() r.Method = "PUT" - suite.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + s.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Put with partnumber sends no message w = httptest.NewRecorder() r.Method = "PUT" r.URL, _ = url.Parse("/dummy/file?partNumber=5") - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Post with uploadId sends message r.Method = "POST" r.URL, _ = url.Parse("/dummy/file?uploadId=5") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Post without uploadId sends no message r.Method = "POST" r.URL, _ = url.Parse("/dummy/file") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Abort multipart works r.Method = "DELETE" r.URL, _ = url.Parse("/dummy/asdf?uploadId=123") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Going through the different extra stuff that can be in the get request // that trigger different code paths in the code. @@ -420,45 +420,45 @@ func (suite *ProxyTests) TestServeHTTP_allowed() { r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?delimiter=puppe") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Show multiparts uploads r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?uploads") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Delimiter alone together with prefix r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?delimiter=puppe&prefix=asdf") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Location parameter r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?location=fnuffe") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Filenames with platform incompatible characters are disallowed // not checked in TestServeHTTP_allowed() because we look for a non 403 response r.Method = "PUT" r.URL, _ = url.Parse("/dummy/fi|le") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 406, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 406, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) } -func (suite *ProxyTests) TestMessageFormatting() { +func (s *ProxyTests) TestMessageFormatting() { // Set up basic request for multipart upload r, _ := http.NewRequest("POST", "/buckbuck/user/new_file.txt", nil) r.Host = "localhost" @@ -466,42 +466,42 @@ func (suite *ProxyTests) TestMessageFormatting() { r.Header.Set("x-amz-content-sha256", "checksum") claims := jwt.New() - assert.NoError(suite.T(), claims.Set("sub", "user@host.domain")) + assert.NoError(s.T(), claims.Set("sub", "user@host.domain")) // start proxy that denies everything - proxy := NewProxy(suite.S3Fakeconf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) - suite.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" + proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) + s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" msg, err := proxy.CreateMessageFromRequest(r, claims) - assert.Nil(suite.T(), err) - assert.IsType(suite.T(), Event{}, msg) + assert.Nil(s.T(), err) + assert.IsType(s.T(), Event{}, msg) - assert.Equal(suite.T(), int64(1234), msg.Filesize) - assert.Equal(suite.T(), "user/new_file.txt", msg.Filepath) - assert.Equal(suite.T(), "user@host.domain", msg.Username) + assert.Equal(s.T(), int64(1234), msg.Filesize) + assert.Equal(s.T(), "user/new_file.txt", msg.Filepath) + assert.Equal(s.T(), "user@host.domain", msg.Username) c, _ := json.Marshal(msg.Checksum[0]) checksum := Checksum{} _ = json.Unmarshal(c, &checksum) - assert.Equal(suite.T(), "sha256", checksum.Type) - assert.Equal(suite.T(), "5b233b981dc12e7ccf4c242b99c042b7842b73b956ad662e4fe0f8354151538b", checksum.Value) + assert.Equal(s.T(), "sha256", checksum.Type) + assert.Equal(s.T(), "5b233b981dc12e7ccf4c242b99c042b7842b73b956ad662e4fe0f8354151538b", checksum.Value) // Test single shot upload r.Method = "PUT" msg, err = proxy.CreateMessageFromRequest(r, jwt.New()) - assert.Nil(suite.T(), err) - assert.IsType(suite.T(), Event{}, msg) - assert.Equal(suite.T(), "upload", msg.Operation) + assert.Nil(s.T(), err) + assert.IsType(s.T(), Event{}, msg) + assert.Equal(s.T(), "upload", msg.Operation) } -func (suite *ProxyTests) TestDatabaseConnection() { - database, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) - defer database.Close() - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) +func (s *ProxyTests) TestDatabaseConnection() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer messenger.Connection.Close() // Start proxy that allows everything - proxy := NewProxy(suite.S3Fakeconf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // PUT a file into the system filename := "/dummy/db-test-file" @@ -509,224 +509,224 @@ func (suite *ProxyTests) TestDatabaseConnection() { stringReader := strings.NewReader("a brand new string") r, _ := http.NewRequest("PUT", filename, stringReader) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) res := w.Result() defer res.Body.Close() - assert.Equal(suite.T(), 200, res.StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 200, res.StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Check that the file is registered and uploaded in the database // connect to the database - db, err := sql.Open(suite.DBConf.PgDataSource()) - assert.Nil(suite.T(), err, "Failed to connect to database") + db.DB, err = sql.Open(s.DBConf.PgDataSource()) + assert.Nil(s.T(), err, "Failed to connect to database") // Check that the file is in the database var fileID string query := "SELECT id FROM sda.files WHERE submission_file_path = $1;" - err = db.QueryRow(query, filename[1:]).Scan(&fileID) - assert.Nil(suite.T(), err, "Failed to query database") - assert.NotNil(suite.T(), fileID, "File not found in database") + err = db.DB.QueryRow(query, filename[1:]).Scan(&fileID) + assert.Nil(s.T(), err, "Failed to query database") + assert.NotNil(s.T(), fileID, "File not found in database") // Check that the "registered" status is in the database for this file for _, status := range []string{"registered", "uploaded"} { var exists int query = "SELECT 1 FROM sda.file_event_log WHERE event = $1 AND file_id = $2;" - err = db.QueryRow(query, status, fileID).Scan(&exists) - assert.Nil(suite.T(), err, "Failed to find '%v' event in database", status) - assert.Equal(suite.T(), exists, 1, "File '%v' event does not exist", status) + err = db.DB.QueryRow(query, status, fileID).Scan(&exists) + assert.Nil(s.T(), err, "Failed to find '%v' event in database", status) + assert.Equal(s.T(), exists, 1, "File '%v' event does not exist", status) } } -func (suite *ProxyTests) TestFormatUploadFilePath() { +func (s *ProxyTests) TestFormatUploadFilePath() { unixPath := "a/b/c.c4gh" testPath := "a\\b\\c.c4gh" uploadPath, err := formatUploadFilePath(testPath) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), unixPath, uploadPath) + assert.NoError(s.T(), err) + assert.Equal(s.T(), unixPath, uploadPath) // mixed "\" and "/" weirdPath := `dq\sw:*?"<>|\t\s/df.c4gh` _, err = formatUploadFilePath(weirdPath) - assert.EqualError(suite.T(), err, "filepath contains mixed '\\' and '/' characters") + assert.EqualError(s.T(), err, "filepath contains mixed '\\' and '/' characters") // no mixed "\" and "/" but not allowed weirdPath = `dq\sw:*?"<>|\t\sdf!s'(a);w@4&f=+e$,g#[]d%.c4gh` _, err = formatUploadFilePath(weirdPath) - assert.EqualError(suite.T(), err, "filepath contains disallowed characters: :, *, ?, \", <, >, |, !, ', (, ), ;, @, &, =, +, $, ,, #, [, ], %") + assert.EqualError(s.T(), err, "filepath contains disallowed characters: :, *, ?, \", <, >, |, !, ', (, ), ;, @, &, =, +, $, ,, #, [, ], %") } -func (suite *ProxyTests) TestCheckFileExists() { - database, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) - defer database.Close() - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) +func (s *ProxyTests) TestCheckFileExists() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer messenger.Connection.Close() - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) res, err := proxy.checkFileExists("/dummy/file") - assert.True(suite.T(), res) - assert.Nil(suite.T(), err) + assert.True(s.T(), res) + assert.Nil(s.T(), err) } -func (suite *ProxyTests) TestCheckFileExists_nonExistingFile() { +func (s *ProxyTests) TestCheckFileExists_nonExistingFile() { // Check that looking for a non-existing file gives (false, nil) // from checkFileExists - database, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) - defer database.Close() - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer messenger.Connection.Close() - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) res, err := proxy.checkFileExists("nonexistingfilepath") - assert.False(suite.T(), res) - assert.Nil(suite.T(), err) + assert.False(s.T(), res) + assert.Nil(s.T(), err) } -func (suite *ProxyTests) TestCheckFileExists_unresponsive() { +func (s *ProxyTests) TestCheckFileExists_unresponsive() { // Check that errors when connecting to S3 are forwarded // and that checkFileExists return (false, someError) - database, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) - defer database.Close() - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer messenger.Connection.Close() // Unaccessible S3 (wrong port) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) proxy.s3.Port = 1111 res, err := proxy.checkFileExists("nonexistingfilepath") - assert.False(suite.T(), res) - assert.NotNil(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "S3: HeadObject") + assert.False(s.T(), res) + assert.NotNil(s.T(), err) + assert.Contains(s.T(), err.Error(), "S3: HeadObject") // Bad access key gives 403 - proxy.s3.Port = suite.S3conf.Port + proxy.s3.Port = s.S3conf.Port proxy.s3.AccessKey = "invaild" res, err = proxy.checkFileExists("nonexistingfilepath") - assert.False(suite.T(), res) - assert.NotNil(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "StatusCode: 403") + assert.False(s.T(), res) + assert.NotNil(s.T(), err) + assert.Contains(s.T(), err.Error(), "StatusCode: 403") } -func (suite *ProxyTests) TestStoreObjectSizeInDB() { - db, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) +func (s *ProxyTests) TestStoreObjectSizeInDB() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) defer db.Close() - mq, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer mq.Connection.Close() - p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db fileID, err := db.RegisterFile("/dummy/file", "test-user") - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), fileID) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) - assert.NoError(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + assert.NoError(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" var objectSize int64 - assert.NoError(suite.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) - assert.Equal(suite.T(), int64(14), objectSize) + assert.NoError(s.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) + assert.Equal(s.T(), int64(14), objectSize) } -func (suite *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { - db, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) +func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) - mq, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer mq.Connection.Close() - p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db fileID, err := db.RegisterFile("/dummy/file", "test-user") - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), fileID) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) db.Close() - assert.NoError(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + assert.NoError(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) } -func (suite *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { - db, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) +func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) defer db.Close() - mq, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer mq.Connection.Close() - p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db fileID, err := db.RegisterFile("/dummy/file", "test-user") - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), fileID) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) // Detect autentication failure p.s3.AccessKey = "badKey" - assert.Error(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) // Detect unresponsive backend service p.s3.Port = 1234 - assert.Error(suite.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) } // This test is intended to try to catch some issues we sometimes see when a query to the S3 backend // happens to fast so that it is not ready and returns a false 404. -func (suite *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { - db, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) +func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) defer db.Close() - mq, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer mq.Connection.Close() - p := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), suite.messenger, suite.database, new(tls.Config)) + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) p.database = db fileID, err := db.RegisterFile("/test/new_file", "test-user") - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), fileID) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) - s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(suite.S3conf.AccessKey, suite.S3conf.SecretKey, ""))) + s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) if err != nil { - suite.FailNow("bad") + s.FailNow("bad") } s3Client := s3.NewFromConfig( s3cfg, func(o *s3.Options) { - o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", suite.S3conf.URL, suite.S3conf.Port)) - o.EndpointOptions.DisableHTTPS = strings.HasPrefix(suite.S3conf.URL, "http:") - o.Region = suite.S3conf.Region + o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", s.S3conf.URL, s.S3conf.Port)) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.URL, "http:") + o.Region = s.S3conf.Region o.UsePathStyle = true }, ) output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ Body: strings.NewReader(strings.Repeat("A", 10*1024*1024)), - Bucket: aws.String(suite.S3conf.Bucket), + Bucket: aws.String(s.S3conf.Bucket), Key: aws.String("/test/new_file"), ContentEncoding: aws.String("application/octet-stream"), }) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), output, output) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), output, output) - assert.NoError(suite.T(), p.storeObjectSizeInDB("/test/new_file", fileID)) + assert.NoError(s.T(), p.storeObjectSizeInDB("/test/new_file", fileID)) const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" var objectSize int64 // If the S3 backend haven't had the time to process the request above correctly this will generate an error. - assert.NoError(suite.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) - assert.Equal(suite.T(), int64(10*1024*1024), objectSize) + assert.NoError(s.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) + assert.Equal(s.T(), int64(10*1024*1024), objectSize) } diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index ade30d9078..292d690325 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -102,11 +102,11 @@ func main() { log.Panicf("Error while getting key %s: %v", Conf.Server.Jwtpubkeypath, err) } } - mux := mux.NewRouter() + router := mux.NewRouter() proxy := NewProxy(Conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy) - mux.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") - mux.HandleFunc("/health", proxy.CheckHealth) - mux.PathPrefix("/").Handler(proxy) + router.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") + router.HandleFunc("/health", proxy.CheckHealth) + router.PathPrefix("/").Handler(proxy) server := &http.Server{ Addr: ":8000", @@ -114,7 +114,7 @@ func main() { WriteTimeout: 30 * time.Second, IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 30 * time.Second, - Handler: mux, + Handler: router, } if Conf.Server.Cert != "" && Conf.Server.Key != "" { diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 618b7944ee..93c16aa427 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -242,12 +242,12 @@ func buildSyncDatasetJSON(b []byte) ([]byte, error) { dataset.User = data.User } - json, err := json.Marshal(dataset) + datasetJSON, err := json.Marshal(dataset) if err != nil { return nil, err } - return json, nil + return datasetJSON, nil } func sendPOST(payload []byte) error { @@ -279,14 +279,14 @@ func sendPOST(payload []byte) error { } func createHostURL(host string, port int) (string, error) { - url, err := url.ParseRequestURI(host) + uri, err := url.ParseRequestURI(host) if err != nil { return "", err } - if url.Port() == "" && port != 0 { - url.Host += fmt.Sprintf(":%d", port) + if uri.Port() == "" && port != 0 { + uri.Host += fmt.Sprintf(":%d", port) } - url.Path = "/dataset" + uri.Path = "/dataset" - return url.String(), nil + return uri.String(), nil } diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index ef2ba65735..0f582bff5f 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -111,7 +111,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *SyncTest) SetupTest() { +func (s *SyncTest) SetupTest() { viper.Set("log.level", "debug") viper.Set("archive.type", "posix") viper.Set("archive.location", "../../dev_utils") @@ -137,63 +137,63 @@ func (suite *SyncTest) SetupTest() { key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" keyPath, _ := os.MkdirTemp("", "key") err := os.WriteFile(keyPath+"/c4gh.key", []byte(key), 0600) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) viper.Set("c4gh.filepath", keyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" err = os.WriteFile(keyPath+"/c4gh.pub", []byte(pubKey), 0600) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) viper.Set("c4gh.syncPubKeyPath", keyPath+"/c4gh.pub") defer os.RemoveAll(keyPath) } -func (suite *SyncTest) TestBuildSyncDatasetJSON() { - suite.SetupTest() +func (s *SyncTest) TestBuildSyncDatasetJSON() { + s.SetupTest() Conf, err := config.NewConfig("sync") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) db, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileID, err := db.RegisterFile("dummy.user/test/file1.c4gh", "dummy.user") - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = db.SetAccessionID("ed6af454-d910-49e3-8cda-488a6f246e67", fileID) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) fileInfo := database.FileInfo{ArchiveChecksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} err = db.SetArchived(fileInfo, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + assert.NoError(s.T(), err, "failed to mark file as Archived") err = db.SetVerified(fileInfo, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Verified") + assert.NoError(s.T(), err, "failed to mark file as Verified") accessions := []string{"ed6af454-d910-49e3-8cda-488a6f246e67"} - assert.NoError(suite.T(), db.MapFilesToDataset("cd532362-e06e-4461-8490-b9ce64b8d9e7", accessions), "failed to map file to dataset") + assert.NoError(s.T(), db.MapFilesToDataset("cd532362-e06e-4461-8490-b9ce64b8d9e7", accessions), "failed to map file to dataset") m := []byte(`{"type":"mapping", "dataset_id": "cd532362-e06e-4461-8490-b9ce64b8d9e7", "accession_ids": ["ed6af454-d910-49e3-8cda-488a6f246e67"]}`) jsonData, err := buildSyncDatasetJSON(m) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) dataset := []byte(`{"dataset_id":"cd532362-e06e-4461-8490-b9ce64b8d9e7","dataset_files":[{"filepath":"dummy.user/test/file1.c4gh","file_id":"ed6af454-d910-49e3-8cda-488a6f246e67","sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}],"user":"dummy.user"}`) - assert.Equal(suite.T(), string(dataset), string(jsonData)) + assert.Equal(s.T(), string(dataset), string(jsonData)) } -func (suite *SyncTest) TestCreateHostURL() { +func (s *SyncTest) TestCreateHostURL() { conf = &config.Config{} conf.Sync = config.Sync{ RemoteHost: "http://localhost", RemotePort: 443, } - s, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "http://localhost:443/dataset", s) + h, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) + assert.NoError(s.T(), err) + assert.Equal(s.T(), "http://localhost:443/dataset", h) } -func (suite *SyncTest) TestSendPOST() { +func (s *SyncTest) TestSendPOST() { r := http.NewServeMux() r.HandleFunc("/dataset", func(w http.ResponseWriter, r *http.Request) { username, _, ok := r.BasicAuth() @@ -214,12 +214,12 @@ func (suite *SyncTest) TestSendPOST() { } syncJSON := []byte(`{"user":"test.user@example.com", "dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "dataset_files": [{"filepath": "inbox/user/file1.c4gh","file_id": "5fe7b660-afea-4c3a-88a9-3daabf055ebb", "sha256": "82E4e60e7beb3db2e06A00a079788F7d71f75b61a4b75f28c4c942703dabb6d6"}, {"filepath": "inbox/user/file2.c4gh","file_id": "ed6af454-d910-49e3-8cda-488a6f246e76", "sha256": "c967d96e56dec0f0cfee8f661846238b7f15771796ee1c345cae73cd812acc2b"}]}`) err := sendPOST(syncJSON) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) conf.Sync = config.Sync{ RemoteHost: ts.URL, RemoteUser: "foo", RemotePassword: "bar", } - assert.EqualError(suite.T(), sendPOST(syncJSON), "401 Unauthorized") + assert.EqualError(s.T(), sendPOST(syncJSON), "401 Unauthorized") } diff --git a/sda/cmd/syncapi/syncapi.go b/sda/cmd/syncapi/syncapi.go index 1a17789e6d..5fe07f229f 100644 --- a/sda/cmd/syncapi/syncapi.go +++ b/sda/cmd/syncapi/syncapi.go @@ -72,7 +72,7 @@ func main() { } } -func setup(config *config.Config) *http.Server { +func setup(conf *config.Config) *http.Server { r := mux.NewRouter().SkipClean(true) r.HandleFunc("/ready", readinessResponse).Methods("GET") @@ -82,7 +82,7 @@ func setup(config *config.Config) *http.Server { cfg := &tls.Config{MinVersion: tls.VersionTLS12} srv := &http.Server{ - Addr: config.API.Host + ":" + fmt.Sprint(config.API.Port), + Addr: conf.API.Host + ":" + fmt.Sprint(conf.API.Port), Handler: r, TLSConfig: cfg, TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), diff --git a/sda/cmd/syncapi/syncapi_test.go b/sda/cmd/syncapi/syncapi_test.go index 0a0012969a..da21dbab48 100644 --- a/sda/cmd/syncapi/syncapi_test.go +++ b/sda/cmd/syncapi/syncapi_test.go @@ -105,7 +105,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *SyncAPITest) SetupTest() { +func (s *SyncAPITest) SetupTest() { viper.Set("log.level", "debug") viper.Set("log.format", "json") @@ -125,43 +125,43 @@ func (suite *SyncAPITest) SetupTest() { viper.Set("sync.api.password", "admin") } -func (suite *SyncAPITest) TestSetup() { - suite.SetupTest() +func (s *SyncAPITest) TestSetup() { + s.SetupTest() conf, err := config.NewConfig("sync-api") - assert.NoError(suite.T(), err, "Failed to setup config") - assert.Equal(suite.T(), mqPort, conf.Broker.Port) - assert.Equal(suite.T(), mqPort, viper.GetInt("broker.port")) + assert.NoError(s.T(), err, "Failed to setup config") + assert.Equal(s.T(), mqPort, conf.Broker.Port) + assert.Equal(s.T(), mqPort, viper.GetInt("broker.port")) server := setup(conf) - assert.Equal(suite.T(), "0.0.0.0:8080", server.Addr) + assert.Equal(s.T(), "0.0.0.0:8080", server.Addr) } -func (suite *SyncAPITest) TestShutdown() { - suite.SetupTest() +func (s *SyncAPITest) TestShutdown() { + s.SetupTest() Conf, err = config.NewConfig("sync-api") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "127.0.0.1", Conf.API.MQ.Conf.Host) + assert.NoError(s.T(), err) + assert.Equal(s.T(), "127.0.0.1", Conf.API.MQ.Conf.Host) // make sure all conections are alive - assert.Equal(suite.T(), false, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), false, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), false, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), false, Conf.API.MQ.Connection.IsClosed()) shutdown() - assert.Equal(suite.T(), true, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), true, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), true, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), true, Conf.API.MQ.Connection.IsClosed()) } -func (suite *SyncAPITest) TestReadinessResponse() { - suite.SetupTest() +func (s *SyncAPITest) TestReadinessResponse() { + s.SetupTest() Conf, err = config.NewConfig("sync-api") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) r := mux.NewRouter() r.HandleFunc("/ready", readinessResponse) @@ -169,44 +169,44 @@ func (suite *SyncAPITest) TestReadinessResponse() { defer ts.Close() res, err := http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the connection to force a reconneciton Conf.API.MQ.Connection.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the channel to force a reconneciton Conf.API.MQ.Channel.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() } -func (suite *SyncAPITest) TestDatasetRoute() { - suite.SetupTest() +func (s *SyncAPITest) TestDatasetRoute() { + s.SetupTest() Conf, err = config.NewConfig("sync-api") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Broker.SchemasPath = "../../schemas/isolated/" @@ -217,18 +217,18 @@ func (suite *SyncAPITest) TestDatasetRoute() { goodJSON := []byte(`{"user": "test.user@example.com", "dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e6", "dataset_files": [{"filepath": "inbox/user/file-1.c4gh","file_id": "5fe7b660-afea-4c3a-88a9-3daabf055ebb", "sha256": "82E4e60e7beb3db2e06A00a079788F7d71f75b61a4b75f28c4c942703dabb6d6"}, {"filepath": "inbox/user/file2.c4gh","file_id": "ed6af454-d910-49e3-8cda-488a6f246e76", "sha256": "c967d96e56dec0f0cfee8f661846238b7f15771796ee1c345cae73cd812acc2b"}]}`) good, err := http.Post(ts.URL+"/dataset", "application/json", bytes.NewBuffer(goodJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, good.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, good.StatusCode) defer good.Body.Close() badJSON := []byte(`{"dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "dataset_files": []}`) bad, err := http.Post(ts.URL+"/dataset", "application/json", bytes.NewBuffer(badJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, bad.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, bad.StatusCode) defer bad.Body.Close() } -func (suite *SyncAPITest) TestMetadataRoute() { +func (s *SyncAPITest) TestMetadataRoute() { Conf = &config.Config{} Conf.Broker.SchemasPath = "../../schemas" @@ -239,18 +239,18 @@ func (suite *SyncAPITest) TestMetadataRoute() { goodJSON := []byte(`{"dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "metadata": {"dummy":"data"}}`) good, err := http.Post(ts.URL+"/metadata", "application/json", bytes.NewBuffer(goodJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, good.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, good.StatusCode) defer good.Body.Close() badJSON := []byte(`{"dataset_id": "phail", "metadata": {}}`) bad, err := http.Post(ts.URL+"/metadata", "application/json", bytes.NewBuffer(badJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, bad.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, bad.StatusCode) defer bad.Body.Close() } -func (suite *SyncAPITest) TestBasicAuth() { +func (s *SyncAPITest) TestBasicAuth() { Conf = &config.Config{} Conf.Broker.SchemasPath = "../../schemas" Conf.SyncAPI = config.SyncAPIConf{ @@ -265,16 +265,16 @@ func (suite *SyncAPITest) TestBasicAuth() { goodJSON := []byte(`{"dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "metadata": {"dummy":"data"}}`) req, err := http.NewRequest("POST", ts.URL+"/metadata", bytes.NewBuffer(goodJSON)) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) req.SetBasicAuth(Conf.SyncAPI.APIUser, Conf.SyncAPI.APIPassword) good, err := ts.Client().Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, good.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, good.StatusCode) defer good.Body.Close() req.SetBasicAuth(Conf.SyncAPI.APIUser, "wrongpass") bad, err := ts.Client().Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusUnauthorized, bad.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusUnauthorized, bad.StatusCode) defer bad.Body.Close() } diff --git a/sda/cmd/verify/verify_test.go b/sda/cmd/verify/verify_test.go index 0f8a2819d2..aea1d90847 100644 --- a/sda/cmd/verify/verify_test.go +++ b/sda/cmd/verify/verify_test.go @@ -15,6 +15,6 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index 4ad1ca8287..dc0795d5aa 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -107,7 +107,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *BrokerTestSuite) SetupTest() { +func (ts *BrokerTestSuite) SetupTest() { tMqconf = MQConf{ "127.0.0.1", mqPort, @@ -132,14 +132,14 @@ func TestBrokerTestSuite(t *testing.T) { suite.Run(t, new(BrokerTestSuite)) } -func (suite *BrokerTestSuite) TestBuildMqURI() { - amqps := buildMQURI("localhost", "user", "pass", "/vhost", 5555, true) - assert.Equal(suite.T(), "amqps://user:pass@localhost:5555/vhost", amqps) - amqp := buildMQURI("localhost", "user", "pass", "/vhost", 5555, false) - assert.Equal(suite.T(), "amqp://user:pass@localhost:5555/vhost", amqp) +func (ts *BrokerTestSuite) TestBuildMqURI() { + amqpsURL := buildMQURI("localhost", "user", "pass", "/vhost", 5555, true) + assert.Equal(ts.T(), "amqps://user:pass@localhost:5555/vhost", amqpsURL) + amqpURL := buildMQURI("localhost", "user", "pass", "/vhost", 5555, false) + assert.Equal(ts.T(), "amqp://user:pass@localhost:5555/vhost", amqpURL) } -func (suite *BrokerTestSuite) TestTLSConfigBroker() { +func (ts *BrokerTestSuite) TestTLSConfigBroker() { confOK := tMqconf confOK.Ssl = true confOK.VerifyPeer = true @@ -148,15 +148,15 @@ func (suite *BrokerTestSuite) TestTLSConfigBroker() { confOK.ClientKey = certPath + "/tls.key" tlsConfig, err := TLSConfigBroker(confOK) - assert.NoError(suite.T(), err, "Unexpected error") - assert.NotZero(suite.T(), tlsConfig.Certificates, "Expected warnings were missing") - assert.NotZero(suite.T(), tlsConfig.RootCAs, "Expected warnings were missing") - assert.EqualValues(suite.T(), tlsConfig.ServerName, "mq") + assert.NoError(ts.T(), err, "Unexpected error") + assert.NotZero(ts.T(), tlsConfig.Certificates, "Expected warnings were missing") + assert.NotZero(ts.T(), tlsConfig.RootCAs, "Expected warnings were missing") + assert.EqualValues(ts.T(), tlsConfig.ServerName, "mq") noCa := confOK noCa.CACert = "" _, err = TLSConfigBroker(noCa) - assert.NoError(suite.T(), err, "Unexpected error") + assert.NoError(ts.T(), err, "Unexpected error") var buf bytes.Buffer log.SetOutput(&buf) @@ -165,23 +165,23 @@ func (suite *BrokerTestSuite) TestTLSConfigBroker() { }() noCa.CACert = certPath + "/tls.key" _, err = TLSConfigBroker(noCa) - assert.NoError(suite.T(), err, "Unexpected error") - assert.Contains(suite.T(), buf.String(), "No certs appended, using system certs only") + assert.NoError(ts.T(), err, "Unexpected error") + assert.Contains(ts.T(), buf.String(), "No certs appended, using system certs only") badCertConf := confOK badCertConf.ClientCert = certPath + "/bar" _, err = CatchTLSConfigBrokerPanic(badCertConf) - assert.EqualError(suite.T(), err, "open "+certPath+"/bar: no such file or directory") + assert.EqualError(ts.T(), err, "open "+certPath+"/bar: no such file or directory") badKeyConf := confOK badKeyConf.ClientKey = certPath + "/foo" _, err = CatchTLSConfigBrokerPanic(badKeyConf) - assert.EqualError(suite.T(), err, "open "+certPath+"/foo: no such file or directory") + assert.EqualError(ts.T(), err, "open "+certPath+"/foo: no such file or directory") noPemFile := confOK noPemFile.ClientKey = "broker.go" _, err = CatchTLSConfigBrokerPanic(noPemFile) - assert.EqualError(suite.T(), err, "tls: failed to find any PEM data in key input") + assert.EqualError(ts.T(), err, "tls: failed to find any PEM data in key input") } func CatchTLSConfigBrokerPanic(c MQConf) (cfg *tls.Config, err error) { @@ -196,49 +196,49 @@ func CatchTLSConfigBrokerPanic(c MQConf) (cfg *tls.Config, err error) { return cfg, err } -func (suite *BrokerTestSuite) TestNewMQNoTLS() { +func (ts *BrokerTestSuite) TestNewMQNoTLS() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) b.Channel.Close() b.Connection.Close() } -func (suite *BrokerTestSuite) TestNewMQTLS() { +func (ts *BrokerTestSuite) TestNewMQTLS() { SslConf := tMqconf SslConf.Port = tlsPort SslConf.Ssl = true SslConf.VerifyPeer = true b, err := NewMQ(SslConf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) b.Channel.Close() b.Connection.Close() } -func (suite *BrokerTestSuite) TestSendMessage() { +func (ts *BrokerTestSuite) TestSendMessage() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) err = b.SendMessage("1", "", "ingest", []byte("test message")) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) b.Channel.Close() b.Connection.Close() } -func (suite *BrokerTestSuite) TestGetMessages() { +func (ts *BrokerTestSuite) TestGetMessages() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -259,15 +259,15 @@ func (suite *BrokerTestSuite) TestGetMessages() { Body: []byte("test message"), }, ) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) d, err := b.GetMessages("ingest") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) for message := range d { if string(message.Body) == "test message" { err := message.Ack(false) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) break } @@ -277,16 +277,16 @@ func (suite *BrokerTestSuite) TestGetMessages() { b.Connection.Close() } -func (suite *BrokerTestSuite) TestCreateNewChannel() { +func (ts *BrokerTestSuite) TestCreateNewChannel() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.False(suite.T(), b.Channel.IsClosed()) + assert.NoError(ts.T(), err) + assert.False(ts.T(), b.Channel.IsClosed()) b.Channel.Close() - assert.True(suite.T(), b.Channel.IsClosed()) + assert.True(ts.T(), b.Channel.IsClosed()) - assert.NoError(suite.T(), b.CreateNewChannel()) - assert.False(suite.T(), b.Channel.IsClosed()) + assert.NoError(ts.T(), b.CreateNewChannel()) + assert.False(ts.T(), b.Channel.IsClosed()) } // Helper functions below this line diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 3e5afecab3..e5c20063ad 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -744,62 +744,62 @@ func (c *Config) configBackup() { // configBroker provides configuration for the message broker func (c *Config) configBroker() error { // Setup broker - broker := broker.MQConf{} + mq := broker.MQConf{} - broker.Host = viper.GetString("broker.host") - broker.Port = viper.GetInt("broker.port") - broker.User = viper.GetString("broker.user") - broker.Password = viper.GetString("broker.password") + mq.Host = viper.GetString("broker.host") + mq.Port = viper.GetInt("broker.port") + mq.User = viper.GetString("broker.user") + mq.Password = viper.GetString("broker.password") - broker.Queue = viper.GetString("broker.queue") + mq.Queue = viper.GetString("broker.queue") if viper.IsSet("broker.serverName") { - broker.ServerName = viper.GetString("broker.serverName") + mq.ServerName = viper.GetString("broker.serverName") } if viper.IsSet("broker.routingkey") { - broker.RoutingKey = viper.GetString("broker.routingkey") + mq.RoutingKey = viper.GetString("broker.routingkey") } if viper.IsSet("broker.exchange") { - broker.Exchange = viper.GetString("broker.exchange") + mq.Exchange = viper.GetString("broker.exchange") } if viper.IsSet("broker.vhost") { if strings.HasPrefix(viper.GetString("broker.vhost"), "/") { - broker.Vhost = viper.GetString("broker.vhost") + mq.Vhost = viper.GetString("broker.vhost") } else { - broker.Vhost = "/" + viper.GetString("broker.vhost") + mq.Vhost = "/" + viper.GetString("broker.vhost") } } else { - broker.Vhost = "/" + mq.Vhost = "/" } if viper.IsSet("broker.ssl") { - broker.Ssl = viper.GetBool("broker.ssl") + mq.Ssl = viper.GetBool("broker.ssl") } if viper.IsSet("broker.verifyPeer") { - broker.VerifyPeer = viper.GetBool("broker.verifyPeer") - if broker.VerifyPeer { + mq.VerifyPeer = viper.GetBool("broker.verifyPeer") + if mq.VerifyPeer { // Since verifyPeer is specified, these are required. if !(viper.IsSet("broker.clientCert") && viper.IsSet("broker.clientKey")) { return errors.New("when broker.verifyPeer is set both broker.clientCert and broker.clientKey is needed") } - broker.ClientCert = viper.GetString("broker.clientCert") - broker.ClientKey = viper.GetString("broker.clientKey") + mq.ClientCert = viper.GetString("broker.clientCert") + mq.ClientKey = viper.GetString("broker.clientKey") } } if viper.IsSet("broker.cacert") { - broker.CACert = viper.GetString("broker.cacert") + mq.CACert = viper.GetString("broker.cacert") } - broker.PrefetchCount = 2 + mq.PrefetchCount = 2 if viper.IsSet("broker.prefetchCount") { - broker.PrefetchCount = viper.GetInt("broker.prefetchCount") + mq.PrefetchCount = viper.GetInt("broker.prefetchCount") } - c.Broker = broker + c.Broker = mq return nil } diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index ea3d7a32e6..c9d534e6a5 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -24,7 +24,7 @@ type ConfigTestSuite struct { var certPath, rootDir string -func (suite *ConfigTestSuite) SetupTest() { +func (ts *ConfigTestSuite) SetupTest() { _, b, _, _ := runtime.Caller(0) rootDir = path.Join(path.Dir(b), "../../../") // pwd, _ = os.Getwd() @@ -32,7 +32,7 @@ func (suite *ConfigTestSuite) SetupTest() { helper.MakeCerts(certPath) rbacFile, err := os.CreateTemp(certPath, "admins") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) rbac := []byte(`{"policy":[ {"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, @@ -43,7 +43,7 @@ func (suite *ConfigTestSuite) SetupTest() { {"role":"dummy@example.org","rolebinding":"admin"}, {"role":"foo@example.org","rolebinding":"submission"}]}`) _, err = rbacFile.Write(rbac) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("api.rbacFile", rbacFile.Name()) viper.Set("broker.host", "testhost") @@ -68,7 +68,7 @@ func (suite *ConfigTestSuite) SetupTest() { viper.Set("log.level", "debug") } -func (suite *ConfigTestSuite) TearDownTest() { +func (ts *ConfigTestSuite) TearDownTest() { viper.Reset() defer os.RemoveAll(certPath) } @@ -77,172 +77,172 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(ConfigTestSuite)) } -func (suite *ConfigTestSuite) TestNonExistingApplication() { +func (ts *ConfigTestSuite) TestNonExistingApplication() { expectedError := errors.New("application 'test' doesn't exist") config, err := NewConfig("test") - assert.Nil(suite.T(), config) - if assert.Error(suite.T(), err) { - assert.Equal(suite.T(), expectedError, err) + assert.Nil(ts.T(), config) + if assert.Error(ts.T(), err) { + assert.Equal(ts.T(), expectedError, err) } } -func (suite *ConfigTestSuite) TestConfigFile() { +func (ts *ConfigTestSuite) TestConfigFile() { viper.Set("configFile", rootDir+"/.github/integration/sda/config.yaml") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/sda/config.yaml") - assert.Equal(suite.T(), absPath, viper.ConfigFileUsed()) + assert.Equal(ts.T(), absPath, viper.ConfigFileUsed()) } -func (suite *ConfigTestSuite) TestWrongConfigFile() { +func (ts *ConfigTestSuite) TestWrongConfigFile() { viper.Set("configFile", rootDir+"/.github/integration/rabbitmq/cega.conf") config, err := NewConfig("s3inbox") - assert.Nil(suite.T(), config) - assert.Error(suite.T(), err) + assert.Nil(ts.T(), config) + assert.Error(ts.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/rabbitmq/cega.conf") - assert.Equal(suite.T(), absPath, viper.ConfigFileUsed()) + assert.Equal(ts.T(), absPath, viper.ConfigFileUsed()) } -func (suite *ConfigTestSuite) TestConfigPath() { +func (ts *ConfigTestSuite) TestConfigPath() { viper.Reset() viper.Set("configPath", rootDir+"/.github/integration/sda/") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/sda/config.yaml") - assert.Equal(suite.T(), absPath, viper.ConfigFileUsed()) + assert.Equal(ts.T(), absPath, viper.ConfigFileUsed()) } -func (suite *ConfigTestSuite) TestNoConfig() { +func (ts *ConfigTestSuite) TestNoConfig() { viper.Reset() config, err := NewConfig("s3inbox") - assert.Nil(suite.T(), config) - assert.Error(suite.T(), err) + assert.Nil(ts.T(), config) + assert.Error(ts.T(), err) } -func (suite *ConfigTestSuite) TestMissingRequiredConfVar() { +func (ts *ConfigTestSuite) TestMissingRequiredConfVar() { for _, requiredConfVar := range requiredConfVars { requiredConfVarValue := viper.Get(requiredConfVar) viper.Set(requiredConfVar, nil) expectedError := fmt.Errorf("%s not set", requiredConfVar) config, err := NewConfig("s3inbox") - assert.Nil(suite.T(), config) - if assert.Error(suite.T(), err) { - assert.Equal(suite.T(), expectedError, err) + assert.Nil(ts.T(), config) + if assert.Error(ts.T(), err) { + assert.Equal(ts.T(), expectedError, err) } viper.Set(requiredConfVar, requiredConfVarValue) } } -func (suite *ConfigTestSuite) TestConfigS3Storage() { +func (ts *ConfigTestSuite) TestConfigS3Storage() { config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.Inbox.S3) - assert.Equal(suite.T(), "testurl", config.Inbox.S3.URL) - assert.Equal(suite.T(), "testaccess", config.Inbox.S3.AccessKey) - assert.Equal(suite.T(), "testsecret", config.Inbox.S3.SecretKey) - assert.Equal(suite.T(), "testbucket", config.Inbox.S3.Bucket) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.Inbox.S3) + assert.Equal(ts.T(), "testurl", config.Inbox.S3.URL) + assert.Equal(ts.T(), "testaccess", config.Inbox.S3.AccessKey) + assert.Equal(ts.T(), "testsecret", config.Inbox.S3.SecretKey) + assert.Equal(ts.T(), "testbucket", config.Inbox.S3.Bucket) } -func (suite *ConfigTestSuite) TestConfigBroker() { +func (ts *ConfigTestSuite) TestConfigBroker() { config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.Inbox.S3) - assert.Equal(suite.T(), "/testvhost", config.Broker.Vhost) - assert.Equal(suite.T(), false, config.Broker.Ssl) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.Inbox.S3) + assert.Equal(ts.T(), "/testvhost", config.Broker.Vhost) + assert.Equal(ts.T(), false, config.Broker.Ssl) viper.Set("broker.ssl", true) viper.Set("broker.verifyPeer", true) _, err = NewConfig("s3inbox") - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") viper.Set("broker.clientCert", "dummy-value") viper.Set("broker.clientKey", "dummy-value") _, err = NewConfig("s3inbox") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("broker.vhost", nil) config, err = NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "/", config.Broker.Vhost) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "/", config.Broker.Vhost) } -func (suite *ConfigTestSuite) TestTLSConfigBroker() { +func (ts *ConfigTestSuite) TestTLSConfigBroker() { viper.Set("broker.serverName", "broker") viper.Set("broker.ssl", true) viper.Set("broker.cacert", certPath+"/ca.crt") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsBroker, err := TLSConfigBroker(config) - assert.NotNil(suite.T(), tlsBroker) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), tlsBroker) + assert.NoError(ts.T(), err) viper.Set("broker.verifyPeer", true) viper.Set("broker.clientCert", certPath+"/tls.crt") viper.Set("broker.clientKey", certPath+"/tls.key") config, err = NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsBroker, err = TLSConfigBroker(config) - assert.NotNil(suite.T(), tlsBroker) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), tlsBroker) + assert.NoError(ts.T(), err) viper.Set("broker.clientCert", certPath+"tls.crt") viper.Set("broker.clientKey", certPath+"/tls.key") config, err = NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsBroker, err = TLSConfigBroker(config) - assert.Nil(suite.T(), tlsBroker) - assert.Error(suite.T(), err) + assert.Nil(ts.T(), tlsBroker) + assert.Error(ts.T(), err) } -func (suite *ConfigTestSuite) TestTLSConfigProxy() { +func (ts *ConfigTestSuite) TestTLSConfigProxy() { viper.Set("inbox.cacert", certPath+"/ca.crt") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsProxy, err := TLSConfigProxy(config) - assert.NotNil(suite.T(), tlsProxy) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), tlsProxy) + assert.NoError(ts.T(), err) } -func (suite *ConfigTestSuite) TestDefaultLogLevel() { +func (ts *ConfigTestSuite) TestDefaultLogLevel() { viper.Set("log.level", "test") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), log.TraceLevel, log.GetLevel()) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), log.TraceLevel, log.GetLevel()) } -func (suite *ConfigTestSuite) TestAPIConfiguration() { +func (ts *ConfigTestSuite) TestAPIConfiguration() { // At this point we should fail because we lack configuration viper.Reset() config, err := NewConfig("api") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), config) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), config) // testing deafult values - suite.SetupTest() + ts.SetupTest() config, err = NewConfig("api") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.API) - assert.Equal(suite.T(), "0.0.0.0", config.API.Host) - assert.Equal(suite.T(), 8080, config.API.Port) - assert.Equal(suite.T(), true, config.API.Session.Secure) - assert.Equal(suite.T(), true, config.API.Session.HTTPOnly) - assert.Equal(suite.T(), "api_session_key", config.API.Session.Name) - assert.Equal(suite.T(), -1*time.Second, config.API.Session.Expiration) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.API) + assert.Equal(ts.T(), "0.0.0.0", config.API.Host) + assert.Equal(ts.T(), 8080, config.API.Port) + assert.Equal(ts.T(), true, config.API.Session.Secure) + assert.Equal(ts.T(), true, config.API.Session.HTTPOnly) + assert.Equal(ts.T(), "api_session_key", config.API.Session.Name) + assert.Equal(ts.T(), -1*time.Second, config.API.Session.Expiration) rbac, _ := os.ReadFile(viper.GetString("api.rbacFile")) - assert.Equal(suite.T(), rbac, config.API.RBACpolicy) + assert.Equal(ts.T(), rbac, config.API.RBACpolicy) viper.Reset() - suite.SetupTest() + ts.SetupTest() // over write defaults viper.Set("api.port", 8443) viper.Set("api.session.secure", false) @@ -250,21 +250,21 @@ func (suite *ConfigTestSuite) TestAPIConfiguration() { viper.Set("api.session.expiration", 60) config, err = NewConfig("api") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.API) - assert.Equal(suite.T(), "0.0.0.0", config.API.Host) - assert.Equal(suite.T(), 8443, config.API.Port) - assert.Equal(suite.T(), false, config.API.Session.Secure) - assert.Equal(suite.T(), "test", config.API.Session.Domain) - assert.Equal(suite.T(), 60*time.Second, config.API.Session.Expiration) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.API) + assert.Equal(ts.T(), "0.0.0.0", config.API.Host) + assert.Equal(ts.T(), 8443, config.API.Port) + assert.Equal(ts.T(), false, config.API.Session.Secure) + assert.Equal(ts.T(), "test", config.API.Session.Domain) + assert.Equal(ts.T(), 60*time.Second, config.API.Session.Expiration) } -func (suite *ConfigTestSuite) TestNotifyConfiguration() { +func (ts *ConfigTestSuite) TestNotifyConfiguration() { // At this point we should fail because we lack configuration config, err := NewConfig("notify") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), config) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), config) viper.Set("broker.host", "test") viper.Set("broker.port", 123) @@ -280,16 +280,16 @@ func (suite *ConfigTestSuite) TestNotifyConfiguration() { viper.Set("smtp.from", "noreply") config, err = NewConfig("notify") - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config) } -func (suite *ConfigTestSuite) TestSyncConfig() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestSyncConfig() { + ts.SetupTest() // At this point we should fail because we lack configuration config, err := NewConfig("backup") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), config) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), config) viper.Set("archive.type", "posix") viper.Set("archive.location", "test") @@ -303,32 +303,32 @@ func (suite *ConfigTestSuite) TestSyncConfig() { viper.Set("c4gh.passphrase", "pass") viper.Set("c4gh.syncPubKeyPath", "/keys/recipient") config, err = NewConfig("sync") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.Broker) - assert.Equal(suite.T(), "testhost", config.Broker.Host) - assert.Equal(suite.T(), 123, config.Broker.Port) - assert.Equal(suite.T(), "testuser", config.Broker.User) - assert.Equal(suite.T(), "testpassword", config.Broker.Password) - assert.Equal(suite.T(), "testqueue", config.Broker.Queue) - assert.NotNil(suite.T(), config.Database) - assert.Equal(suite.T(), "test", config.Database.Host) - assert.Equal(suite.T(), 123, config.Database.Port) - assert.Equal(suite.T(), "test", config.Database.User) - assert.Equal(suite.T(), "test", config.Database.Password) - assert.Equal(suite.T(), "test", config.Database.Database) - assert.NotNil(suite.T(), config.Archive) - assert.NotNil(suite.T(), config.Archive.Posix) - assert.Equal(suite.T(), "test", config.Archive.Posix.Location) - assert.NotNil(suite.T(), config.Sync) - assert.NotNil(suite.T(), config.Sync.Destination.Posix) - assert.Equal(suite.T(), "test", config.Sync.Destination.Posix.Location) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.Broker) + assert.Equal(ts.T(), "testhost", config.Broker.Host) + assert.Equal(ts.T(), 123, config.Broker.Port) + assert.Equal(ts.T(), "testuser", config.Broker.User) + assert.Equal(ts.T(), "testpassword", config.Broker.Password) + assert.Equal(ts.T(), "testqueue", config.Broker.Queue) + assert.NotNil(ts.T(), config.Database) + assert.Equal(ts.T(), "test", config.Database.Host) + assert.Equal(ts.T(), 123, config.Database.Port) + assert.Equal(ts.T(), "test", config.Database.User) + assert.Equal(ts.T(), "test", config.Database.Password) + assert.Equal(ts.T(), "test", config.Database.Database) + assert.NotNil(ts.T(), config.Archive) + assert.NotNil(ts.T(), config.Archive.Posix) + assert.Equal(ts.T(), "test", config.Archive.Posix.Location) + assert.NotNil(ts.T(), config.Sync) + assert.NotNil(ts.T(), config.Sync.Destination.Posix) + assert.Equal(ts.T(), "test", config.Sync.Destination.Posix.Location) } -func (suite *ConfigTestSuite) TestGetC4GHPublicKey() { +func (ts *ConfigTestSuite) TestGetC4GHPublicKey() { pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" pubKeyPath, _ := os.MkdirTemp("", "pubkey") err := os.WriteFile(pubKeyPath+"/c4gh.pub", []byte(pubKey), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) var kb [32]byte k, _ := base64.StdEncoding.DecodeString("uQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=") @@ -336,41 +336,41 @@ func (suite *ConfigTestSuite) TestGetC4GHPublicKey() { viper.Set("c4gh.syncPubKeyPath", pubKeyPath+"/c4gh.pub") pkBytes, err := GetC4GHPublicKey() - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), pkBytes) - assert.Equal(suite.T(), pkBytes, &kb, "GetC4GHPublicKey didn't return correct pubKey") + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), pkBytes) + assert.Equal(ts.T(), pkBytes, &kb, "GetC4GHPublicKey didn't return correct pubKey") defer os.RemoveAll(pubKeyPath) } -func (suite *ConfigTestSuite) TestGetC4GHKey() { +func (ts *ConfigTestSuite) TestGetC4GHKey() { key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" keyPath, _ := os.MkdirTemp("", "key") err := os.WriteFile(keyPath+"/c4gh.key", []byte(key), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.filepath", keyPath+"/c4gh.key") pkBytes, err := GetC4GHKey() - assert.EqualError(suite.T(), err, "chacha20poly1305: message authentication failed") - assert.Nil(suite.T(), pkBytes) + assert.EqualError(ts.T(), err, "chacha20poly1305: message authentication failed") + assert.Nil(ts.T(), pkBytes) viper.Set("c4gh.filepath", keyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") pkBytes, err = GetC4GHKey() - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), pkBytes) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), pkBytes) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_AllOK() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_AllOK() { keyPath, _ := os.MkdirTemp("", "key") keyFile1 := keyPath + "/c4gh1.key" keyFile2 := keyPath + "/c4gh2.key" _, err := helper.CreatePrivateKeyFile(keyFile1, "test") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) _, err = helper.CreatePrivateKeyFile(keyFile2, "test") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: keyFile1, Passphrase: "test"}, @@ -378,121 +378,121 @@ func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_AllOK() { }) privateKeys, err := GetC4GHprivateKeys() - assert.NoError(suite.T(), err) - assert.Len(suite.T(), privateKeys, 2) + assert.NoError(ts.T(), err) + assert.Len(ts.T(), privateKeys, 2) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_MissingKeyPath() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_MissingKeyPath() { viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: "/non/existent/path1", Passphrase: "test"}, }) privateKeys, err := GetC4GHprivateKeys() - assert.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "failed to open key file") - assert.Nil(suite.T(), privateKeys) + assert.Error(ts.T(), err) + assert.Contains(ts.T(), err.Error(), "failed to open key file") + assert.Nil(ts.T(), privateKeys) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_WrongPassphrase() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_WrongPassphrase() { keyPath, _ := os.MkdirTemp("", "key") keyFile := keyPath + "/c4gh1.key" _, err := helper.CreatePrivateKeyFile(keyFile, "test") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: keyFile, Passphrase: "wrong"}, }) privateKeys, err := GetC4GHprivateKeys() - assert.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "chacha20poly1305: message authentication faile") - assert.Nil(suite.T(), privateKeys) + assert.Error(ts.T(), err) + assert.Contains(ts.T(), err.Error(), "chacha20poly1305: message authentication faile") + assert.Nil(ts.T(), privateKeys) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_InvalidKey() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_InvalidKey() { key := "not a valid key" keyPath, _ := os.MkdirTemp("", "key") keyFile := keyPath + "/c4gh1.key" err := os.WriteFile(keyFile, []byte(key), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: keyFile, Passphrase: "wrong"}, }) privateKeys, err := GetC4GHprivateKeys() - assert.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "read of unrecognized private key format") - assert.Nil(suite.T(), privateKeys) + assert.Error(ts.T(), err) + assert.Contains(ts.T(), err.Error(), "read of unrecognized private key format") + assert.Nil(ts.T(), privateKeys) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestConfigSyncAPI() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigSyncAPI() { + ts.SetupTest() noConfig, err := NewConfig("sync-api") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) viper.Set("sync.api.user", "user") viper.Set("sync.api.password", "password") config, err := NewConfig("sync-api") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "user", config.SyncAPI.APIUser) - assert.Equal(suite.T(), "password", config.SyncAPI.APIPassword) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "user", config.SyncAPI.APIUser) + assert.Equal(ts.T(), "password", config.SyncAPI.APIPassword) viper.Set("sync.api.AccessionRouting", "wrong") config, err = NewConfig("sync-api") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "wrong", config.SyncAPI.AccessionRouting) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "wrong", config.SyncAPI.AccessionRouting) } -func (suite *ConfigTestSuite) TestConfigReEncryptServer() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigReEncryptServer() { + ts.SetupTest() noConfig, err := NewConfig("reencrypt") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" keyPath, _ := os.MkdirTemp("", "key") defer os.RemoveAll(keyPath) if err := os.WriteFile(keyPath+"/c4gh.key", []byte(key), 0600); err != nil { - suite.T().FailNow() + ts.T().FailNow() } viper.Set("c4gh.filepath", keyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") config, err := NewConfig("reencrypt") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), 50051, config.ReEncrypt.Port) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 50051, config.ReEncrypt.Port) viper.Set("grpc.CACert", certPath+"/ca.crt") viper.Set("grpc.serverCert", certPath+"/tls.crt") viper.Set("grpc.serverKey", certPath+"/tls.key") config, err = NewConfig("reencrypt") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), certPath+"/ca.crt", config.ReEncrypt.CACert) - assert.Equal(suite.T(), certPath+"/tls.crt", config.ReEncrypt.ServerCert) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), certPath+"/ca.crt", config.ReEncrypt.CACert) + assert.Equal(ts.T(), certPath+"/tls.crt", config.ReEncrypt.ServerCert) } -func (suite *ConfigTestSuite) TestConfigAuth_CEGA() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigAuth_CEGA() { + ts.SetupTest() ECPath, _ := os.MkdirTemp("", "EC") if err := helper.CreateECkeys(ECPath, ECPath); err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer os.RemoveAll(ECPath) noConfig, err := NewConfig("auth") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) viper.Set("auth.s3Inbox", "http://inbox:8000") viper.Set("auth.publicFile", "no-file") @@ -504,38 +504,38 @@ func (suite *ConfigTestSuite) TestConfigAuth_CEGA() { viper.Set("auth.Jwt.signatureAlg", "ES256") viper.Set("auth.Jwt.tokenTTL", 168) _, err = NewConfig("auth") - assert.ErrorContains(suite.T(), err, "no such file or directory") + assert.ErrorContains(ts.T(), err, "no such file or directory") viper.Set("auth.publicFile", ECPath+"/ec.pub") viper.Set("auth.Jwt.privateKey", ECPath+"/ec") c, err := NewConfig("auth") - assert.Equal(suite.T(), c.Auth.JwtPrivateKey, fmt.Sprintf("%s/ec", ECPath)) - assert.Equal(suite.T(), c.Auth.JwtTTL, 168) - assert.NoError(suite.T(), err, "unexpected failure") + assert.Equal(ts.T(), c.Auth.JwtPrivateKey, fmt.Sprintf("%s/ec", ECPath)) + assert.Equal(ts.T(), c.Auth.JwtTTL, 168) + assert.NoError(ts.T(), err, "unexpected failure") } -func (suite *ConfigTestSuite) TestConfigAuth_OIDC() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigAuth_OIDC() { + ts.SetupTest() ECPath, _ := os.MkdirTemp("", "EC") if err := helper.CreateECkeys(ECPath, ECPath); err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer os.RemoveAll(ECPath) noConfig, err := NewConfig("auth") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) viper.Set("auth.s3Inbox", "http://inbox:8000") viper.Set("auth.publicFile", ECPath+"/ec.pub") viper.Set("oidc.id", "oidcTestID") viper.Set("oidc.secret", "oidcTestIssuer") _, err = NewConfig("auth") - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) viper.Set("oidc.provider", "http://provider:9000") viper.Set("oidc.redirectUrl", "http://auth/oidc/login") _, err = NewConfig("auth") - assert.NoError(suite.T(), err, "unexpected failure") + assert.NoError(ts.T(), err, "unexpected failure") } diff --git a/sda/internal/database/db_test.go b/sda/internal/database/db_test.go index fe8e3ba0f3..6027aaa6d5 100644 --- a/sda/internal/database/db_test.go +++ b/sda/internal/database/db_test.go @@ -104,9 +104,9 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *DatabaseTests) SetupTest() { +func (ts *DatabaseTests) SetupTest() { // Database connection variables - suite.dbConf = DBConf{ + ts.dbConf = DBConf{ Host: "127.0.0.1", Port: dbPort, User: "postgres", @@ -118,20 +118,20 @@ func (suite *DatabaseTests) SetupTest() { ClientKey: "", } - db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + db, err := NewSDAdb(ts.dbConf) + assert.Nil(ts.T(), err, "got %v when creating new connection", err) _, err = db.DB.Exec("TRUNCATE sda.files, sda.encryption_keys CASCADE") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) } -func (suite *DatabaseTests) TearDownTest() {} +func (ts *DatabaseTests) TearDownTest() {} // TestNewSDAdb tests creation of new database connections, as well as fetching // of the database schema version. -func (suite *DatabaseTests) TestNewSDAdb() { +func (ts *DatabaseTests) TestNewSDAdb() { // test working database connection - db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + db, err := NewSDAdb(ts.dbConf) + assert.Nil(ts.T(), err, "got %v when creating new connection", err) db.Close() @@ -149,20 +149,20 @@ func (suite *DatabaseTests) TestNewSDAdb() { } _, err = NewSDAdb(wrongConf) - assert.NotNil(suite.T(), err, "connection allowed with wrong credentials") + assert.NotNil(ts.T(), err, "connection allowed with wrong credentials") } // TestConnect tests creation of new database connections -func (suite *DatabaseTests) TestConnect() { +func (ts *DatabaseTests) TestConnect() { // test connecting to a database - db := SDAdb{DB: nil, Version: -1, Config: suite.dbConf} + db := SDAdb{DB: nil, Version: -1, Config: ts.dbConf} err := db.Connect() - assert.Nil(suite.T(), err, "failed connecting: %s", err) + assert.Nil(ts.T(), err, "failed connecting: %s", err) // test that nothing happens if you connect when already connected err = db.Connect() - assert.Nil(suite.T(), err, "Connect() should return nil when called on an"+ + assert.Nil(ts.T(), err, "Connect() should return nil when called on an"+ " already open connection: %s", err) // test querying a closed connection @@ -170,11 +170,11 @@ func (suite *DatabaseTests) TestConnect() { query := "SELECT MAX(version) FROM sda.dbschema_version" var dbVersion = -1 err = db.DB.QueryRow(query).Scan(&dbVersion) - assert.NotNil(suite.T(), err, "query possible on closed connection") + assert.NotNil(ts.T(), err, "query possible on closed connection") // test reconnection by using getVersion() _, err = db.getVersion() - assert.Nil(suite.T(), err, "failed reconnecting: %s", err) + assert.Nil(ts.T(), err, "failed reconnecting: %s", err) db.Close() @@ -193,14 +193,14 @@ func (suite *DatabaseTests) TestConnect() { db.Config = wrongConf err = db.Connect() - assert.NotNil(suite.T(), err, "connection allowed with wrong credentials") + assert.NotNil(ts.T(), err, "connection allowed with wrong credentials") } // TestClose tests that the connection is properly closed -func (suite *DatabaseTests) TestClose() { +func (ts *DatabaseTests) TestClose() { // test working database connection - db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + db, err := NewSDAdb(ts.dbConf) + assert.Nil(ts.T(), err, "got %v when creating new connection", err) db.Close() @@ -208,10 +208,10 @@ func (suite *DatabaseTests) TestClose() { query := "SELECT MAX(version) FROM sda.dbschema_version" var dbVersion = -1 err = db.DB.QueryRow(query).Scan(&dbVersion) - assert.NotNil(suite.T(), err, "query possible on closed connection") + assert.NotNil(ts.T(), err, "query possible on closed connection") // check that nothing happens if Close is called on a closed connection db.Close() - assert.NotPanics(suite.T(), db.Close, + assert.NotPanics(ts.T(), db.Close, "Close paniced when called on closed connection") } diff --git a/sda/internal/helper/helper_test.go b/sda/internal/helper/helper_test.go index 384cc0df2a..814872fc79 100644 --- a/sda/internal/helper/helper_test.go +++ b/sda/internal/helper/helper_test.go @@ -21,114 +21,114 @@ func TestUserAuthTestSuite(t *testing.T) { suite.Run(t, new(HelperTest)) } -func (suite *HelperTest) SetupTest() { +func (ts *HelperTest) SetupTest() { } -func (suite *HelperTest) TestMakeFolder() { +func (ts *HelperTest) TestMakeFolder() { privateK, publicK, _ := MakeFolder("dummy-folder") - assert.Equal(suite.T(), "dummy-folder/private-key", privateK) - assert.Equal(suite.T(), "dummy-folder/public-key", publicK) + assert.Equal(ts.T(), "dummy-folder/private-key", privateK) + assert.Equal(ts.T(), "dummy-folder/public-key", publicK) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateRSAkeys() { +func (ts *HelperTest) TestCreateRSAkeys() { privateK, publicK, _ := MakeFolder("dummy-folder") - assert.Nil(suite.T(), CreateRSAkeys(privateK, publicK)) + assert.Nil(ts.T(), CreateRSAkeys(privateK, publicK)) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestParsePrivateRSAKey() { +func (ts *HelperTest) TestParsePrivateRSAKey() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateRSAkeys(privateK, publicK) - assert.Nil(suite.T(), e) + assert.Nil(ts.T(), e) _, err := ParsePrivateRSAKey(privateK, "/rsa") - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateRSAToken() { +func (ts *HelperTest) TestCreateRSAToken() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateRSAkeys(privateK, publicK) - assert.Nil(suite.T(), e) + assert.Nil(ts.T(), e) ParsedPrKey, _ := ParsePrivateRSAKey(privateK, "/rsa") tok, err := CreateRSAToken(ParsedPrKey, "RS256", DefaultTokenClaims) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) set := jwk.NewSet() keyData, err := os.ReadFile(filepath.Join(filepath.Clean(publicK), "/rsa.pub")) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) key, err := jwk.ParseKey(keyData, jwk.WithPEM(true)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = jwk.AssignKeyID(key) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), set.AddKey(key)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), set.AddKey(key)) fmt.Println(tok) _, err = jwt.Parse([]byte(tok), jwt.WithKeySet(set, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateECkeys() { +func (ts *HelperTest) TestCreateECkeys() { privateK, publicK, _ := MakeFolder("dummy-folder") - assert.Nil(suite.T(), CreateECkeys(privateK, publicK)) + assert.Nil(ts.T(), CreateECkeys(privateK, publicK)) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestParsePrivateECKey() { +func (ts *HelperTest) TestParsePrivateECKey() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateECkeys(privateK, publicK) - assert.Nil(suite.T(), e) + assert.Nil(ts.T(), e) k, err := ParsePrivateECKey(privateK, "/ec") - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), "EC", fmt.Sprintf("%v", k.KeyType())) + assert.Nil(ts.T(), err) + assert.Equal(ts.T(), "EC", fmt.Sprintf("%v", k.KeyType())) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateECToken() { +func (ts *HelperTest) TestCreateECToken() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateECkeys(privateK, publicK) - assert.Nil(suite.T(), e) + assert.Nil(ts.T(), e) ParsedPrKey, err := ParsePrivateECKey(privateK, "/ec") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) _, err = CreateECToken(ParsedPrKey, "ES256", DefaultTokenClaims) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateHSToken() { +func (ts *HelperTest) TestCreateHSToken() { key := make([]byte, 256) tok, err := CreateHSToken(key, DefaultTokenClaims) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) set := jwk.NewSet() jwtKey, err := jwk.FromRaw(key) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = jwk.AssignKeyID(jwtKey) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) - assert.NoError(suite.T(), set.AddKey(jwtKey)) + assert.NoError(ts.T(), set.AddKey(jwtKey)) fmt.Println(tok) _, err = jwt.Parse([]byte(tok), jwt.WithKeySet(set, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreatePrivateKeyFile() { +func (ts *HelperTest) TestCreatePrivateKeyFile() { tempDir, err := os.MkdirTemp("", "key") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer os.RemoveAll(tempDir) // Define the key file path and passphrase @@ -137,17 +137,17 @@ func (suite *HelperTest) TestCreatePrivateKeyFile() { // Call the function under test publicKey, err := CreatePrivateKeyFile(keyFile, passphrase) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Verify the file was created _, err = os.Stat(keyFile) - assert.NoError(suite.T(), err, "Private key file should exist") + assert.NoError(ts.T(), err, "Private key file should exist") // Read and verify the file contents fileContent, err := os.ReadFile(keyFile) - assert.NoError(suite.T(), err, "Should be able to read the private key file") - assert.Contains(suite.T(), string(fileContent), "BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY", "File should contain a valid private key") + assert.NoError(ts.T(), err, "Should be able to read the private key file") + assert.Contains(ts.T(), string(fileContent), "BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY", "File should contain a valid private key") // Verify the public key is not empty - assert.NotEqual(suite.T(), [32]byte{}, publicKey, "Public key should not be empty") + assert.NotEqual(ts.T(), [32]byte{}, publicKey, "Public key should not be empty") } diff --git a/sda/internal/jsonadapter/jsonadapter.go b/sda/internal/jsonadapter/jsonadapter.go index 970fbc9e9b..74ee928876 100644 --- a/sda/internal/jsonadapter/jsonadapter.go +++ b/sda/internal/jsonadapter/jsonadapter.go @@ -54,8 +54,8 @@ func NewAdapter(source *[]byte) *Adapter { return &Adapter{policy: []CasbinRule{}, source: source} } -func (a *Adapter) LoadPolicy(model model.Model) error { - err := a.loadFromBuffer(model) +func (a *Adapter) LoadPolicy(m model.Model) error { + err := a.loadFromBuffer(m) if err != nil { return err } @@ -63,7 +63,7 @@ func (a *Adapter) LoadPolicy(model model.Model) error { return nil } -func (a *Adapter) loadFromBuffer(model model.Model) error { +func (a *Adapter) loadFromBuffer(m model.Model) error { if len(*a.source) == 0 { return nil } @@ -75,13 +75,13 @@ func (a *Adapter) loadFromBuffer(model model.Model) error { } for _, p := range input.Policy { - if err := persist.LoadPolicyLine(fmt.Sprintf("p,%s,%s,%s", p.Role, p.Path, p.Action), model); err != nil { + if err := persist.LoadPolicyLine(fmt.Sprintf("p,%s,%s,%s", p.Role, p.Path, p.Action), m); err != nil { return err } } for _, r := range input.Roles { - if err := persist.LoadPolicyLine(fmt.Sprintf("g,%s,%s", r.Role, r.RoleBinding), model); err != nil { + if err := persist.LoadPolicyLine(fmt.Sprintf("g,%s,%s", r.Role, r.RoleBinding), m); err != nil { return err } } @@ -105,16 +105,16 @@ func (a *Adapter) RemoveFilteredPolicy(_ string, _ string, _ int, _ ...string) e } // SavePolicy saves policy -func (a *Adapter) SavePolicy(model model.Model) error { +func (a *Adapter) SavePolicy(m model.Model) error { a.policy = []CasbinRule{} var rules []CasbinRule - for ptype, ast := range model["p"] { + for ptype, ast := range m["p"] { for _, rule := range ast.Policy { rules = append(rules, CasbinRule{PType: ptype, V0: rule[0], V1: rule[1], V2: rule[2]}) } } - for ptype, ast := range model["g"] { + for ptype, ast := range m["g"] { for _, rule := range ast.Policy { rules = append(rules, CasbinRule{PType: ptype, V0: rule[0], V1: rule[1]}) } diff --git a/sda/internal/jsonadapter/jsonadapter_test.go b/sda/internal/jsonadapter/jsonadapter_test.go index 6fff5ba7ca..406a02c527 100644 --- a/sda/internal/jsonadapter/jsonadapter_test.go +++ b/sda/internal/jsonadapter/jsonadapter_test.go @@ -26,10 +26,10 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(AdapterTestSuite)) } -func (suite *AdapterTestSuite) SetupSuite() { - suite.Model, _ = model.NewModelFromString(Model) - suite.EmptyPolicy = []byte(`{"policy":[],"roles":[]}`) - suite.DefaultPolicy = []byte(`{"policy":[{"role":"admin","path":"/keys/*","action":"(GET)|(POST)|(PUT)"}, +func (ts *AdapterTestSuite) SetupSuite() { + ts.Model, _ = model.NewModelFromString(Model) + ts.EmptyPolicy = []byte(`{"policy":[],"roles":[]}`) + ts.DefaultPolicy = []byte(`{"policy":[{"role":"admin","path":"/keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, {"role":"submission","path":"/dataset/release/*dataset","action":"POST"}, {"role":"submission","path":"/file/ingest","action":"POST"}, @@ -37,80 +37,80 @@ func (suite *AdapterTestSuite) SetupSuite() { "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy@example.org","rolebinding":"admin"}, {"role":"foo@example.org","rolebinding":"submission"}]}`) - suite.ExpectedPolicy = [][]string{{"admin", "/keys/*", "(GET)|(POST)|(PUT)"}, + ts.ExpectedPolicy = [][]string{{"admin", "/keys/*", "(GET)|(POST)|(PUT)"}, {"submission", "/dataset/create", "POST"}, {"submission", "/dataset/release/*dataset", "POST"}, {"submission", "/file/ingest", "POST"}, {"submission", "/file/accession", "POST"}} - suite.ExpectedGroups = [][]string{{"admin", "submission"}, {"dummy@example.org", "admin"}, {"foo@example.org", "submission"}} + ts.ExpectedGroups = [][]string{{"admin", "submission"}, {"dummy@example.org", "admin"}, {"foo@example.org", "submission"}} - suite.File, _ = os.CreateTemp("", "policy") - _, err := suite.File.Write(suite.DefaultPolicy) + ts.File, _ = os.CreateTemp("", "policy") + _, err := ts.File.Write(ts.DefaultPolicy) if err != nil { - suite.FailNow("failed to write policy file to disk") + ts.FailNow("failed to write policy file to disk") } } -func (suite *AdapterTestSuite) TearDownSuite() { - os.RemoveAll(suite.File.Name()) +func (ts *AdapterTestSuite) TearDownSuite() { + os.RemoveAll(ts.File.Name()) } -func (suite *AdapterTestSuite) TestAdapter_empty() { - a := NewAdapter(&suite.EmptyPolicy) - e, err := casbin.NewEnforcer(suite.Model, a) - assert.NoError(suite.T(), err, "New enforcer failed on empty policy") +func (ts *AdapterTestSuite) TestAdapter_empty() { + a := NewAdapter(&ts.EmptyPolicy) + e, err := casbin.NewEnforcer(ts.Model, a) + assert.NoError(ts.T(), err, "New enforcer failed on empty policy") p, err := e.GetPolicy() - assert.NoError(suite.T(), err, "failed to get policy") - assert.Equal(suite.T(), [][]string(nil), p) + assert.NoError(ts.T(), err, "failed to get policy") + assert.Equal(ts.T(), [][]string(nil), p) } -func (suite *AdapterTestSuite) TestAdapter() { - a := NewAdapter(&suite.DefaultPolicy) - e, err := casbin.NewEnforcer(suite.Model, a) - assert.NoError(suite.T(), err, "New enforcer failed withpolicy") +func (ts *AdapterTestSuite) TestAdapter() { + a := NewAdapter(&ts.DefaultPolicy) + e, err := casbin.NewEnforcer(ts.Model, a) + assert.NoError(ts.T(), err, "New enforcer failed withpolicy") p, err := e.GetPolicy() - assert.Equal(suite.T(), len(suite.ExpectedPolicy), len(p)) - assert.NoError(suite.T(), err, "failed to get policy") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, suite.ExpectedPolicy)) + assert.Equal(ts.T(), len(ts.ExpectedPolicy), len(p)) + assert.NoError(ts.T(), err, "failed to get policy") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, ts.ExpectedPolicy)) g, err := e.GetGroupingPolicy() - assert.NoError(suite.T(), err, "failed to get groups") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, suite.ExpectedGroups)) + assert.NoError(ts.T(), err, "failed to get groups") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, ts.ExpectedGroups)) } -func (suite *AdapterTestSuite) TestAdapter_fromFile() { - b, err := os.ReadFile(suite.File.Name()) - assert.NoError(suite.T(), err, "failed to read json file") - e, err := casbin.NewEnforcer(suite.Model, NewAdapter(&b)) - assert.NoError(suite.T(), err, "New enforcer failed with policy from file: %s", suite.File.Name()) +func (ts *AdapterTestSuite) TestAdapter_fromFile() { + b, err := os.ReadFile(ts.File.Name()) + assert.NoError(ts.T(), err, "failed to read json file") + e, err := casbin.NewEnforcer(ts.Model, NewAdapter(&b)) + assert.NoError(ts.T(), err, "New enforcer failed with policy from file: %s", ts.File.Name()) p, err := e.GetPolicy() - assert.Equal(suite.T(), len(suite.ExpectedPolicy), len(p)) - assert.NoError(suite.T(), err, "failed to get policy") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, suite.ExpectedPolicy)) + assert.Equal(ts.T(), len(ts.ExpectedPolicy), len(p)) + assert.NoError(ts.T(), err, "failed to get policy") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, ts.ExpectedPolicy)) g, err := e.GetGroupingPolicy() - assert.NoError(suite.T(), err, "failed to get groups") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, suite.ExpectedGroups)) + assert.NoError(ts.T(), err, "failed to get groups") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, ts.ExpectedGroups)) } -func (suite *AdapterTestSuite) TestAdapter_save() { - a := NewAdapter(&suite.EmptyPolicy) - e, err := casbin.NewEnforcer(suite.Model, a) - assert.NoError(suite.T(), err, "New enforcer failed without policy") +func (ts *AdapterTestSuite) TestAdapter_save() { + a := NewAdapter(&ts.EmptyPolicy) + e, err := casbin.NewEnforcer(ts.Model, a) + assert.NoError(ts.T(), err, "New enforcer failed without policy") _, err = e.AddPolicy("foo", "/data/*", "GET") - assert.NoError(suite.T(), err, "failed to add policy") + assert.NoError(ts.T(), err, "failed to add policy") _, err = e.AddPolicy("foo", "/data/:value", "POST") - assert.NoError(suite.T(), err, "failed to add policy") + assert.NoError(ts.T(), err, "failed to add policy") - assert.NoError(suite.T(), e.SavePolicy(), "failed to save policy") - assert.Equal(suite.T(), 2, len(a.policy)) + assert.NoError(ts.T(), e.SavePolicy(), "failed to save policy") + assert.Equal(ts.T(), 2, len(a.policy)) } -func (suite *AdapterTestSuite) TestAdapter_notImplemented() { - a := NewAdapter(&suite.EmptyPolicy) - assert.Error(suite.T(), a.AddPolicy("", "", []string{""})) - assert.Error(suite.T(), a.RemovePolicy("", "", []string{""})) - assert.Error(suite.T(), a.RemoveFilteredPolicy("", "", 0, "")) +func (ts *AdapterTestSuite) TestAdapter_notImplemented() { + a := NewAdapter(&ts.EmptyPolicy) + assert.Error(ts.T(), a.AddPolicy("", "", []string{""})) + assert.Error(ts.T(), a.RemovePolicy("", "", []string{""})) + assert.Error(ts.T(), a.RemoveFilteredPolicy("", "", 0, "")) } diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index f65c1b29de..66d5ff3407 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -56,29 +56,29 @@ type posixConf struct { } // NewBackend initiates a storage backend -func NewBackend(config Conf) (Backend, error) { - switch config.Type { +func NewBackend(conf Conf) (Backend, error) { + switch conf.Type { case "s3": - return newS3Backend(config.S3) + return newS3Backend(conf.S3) case "sftp": - return newSftpBackend(config.SFTP) + return newSftpBackend(conf.SFTP) default: - return newPosixBackend(config.Posix) + return newPosixBackend(conf.Posix) } } -func newPosixBackend(config posixConf) (*posixBackend, error) { - fileInfo, err := os.Stat(config.Location) +func newPosixBackend(conf posixConf) (*posixBackend, error) { + fileInfo, err := os.Stat(conf.Location) if err != nil { return nil, err } if !fileInfo.IsDir() { - return nil, fmt.Errorf("%s is not a directory", config.Location) + return nil, fmt.Errorf("%s is not a directory", conf.Location) } - return &posixBackend{Location: config.Location}, nil + return &posixBackend{Location: conf.Location}, nil } // NewFileReader returns an io.Reader instance @@ -378,29 +378,29 @@ type SftpConf struct { HostKey string } -func newSftpBackend(config SftpConf) (*sftpBackend, error) { +func newSftpBackend(conf SftpConf) (*sftpBackend, error) { // read in and parse pem key - key, err := os.ReadFile(config.PemKeyPath) + key, err := os.ReadFile(conf.PemKeyPath) if err != nil { return nil, fmt.Errorf("failed to read from key file, %v", err) } var signer ssh.Signer - if config.PemKeyPass == "" { + if conf.PemKeyPass == "" { signer, err = ssh.ParsePrivateKey(key) } else { - signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(config.PemKeyPass)) + signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(conf.PemKeyPass)) } if err != nil { return nil, fmt.Errorf("failed to parse private key, %v", err) } // connect - conn, err := ssh.Dial("tcp", config.Host+":"+config.Port, + conn, err := ssh.Dial("tcp", conf.Host+":"+conf.Port, &ssh.ClientConfig{ - User: config.UserName, + User: conf.UserName, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: TrustedHostKeyCallback(config.HostKey), + HostKeyCallback: TrustedHostKeyCallback(conf.HostKey), }, ) if err != nil { @@ -416,7 +416,7 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { sfb := &sftpBackend{ Connection: conn, Client: client, - Conf: &config, + Conf: &conf, } _, err = client.ReadDir("./") diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index a4c87402a3..a97819b191 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -140,7 +140,7 @@ func TestStorageTestSuite(t *testing.T) { suite.Run(t, new(StorageTestSuite)) } -func (suite *StorageTestSuite) SetupTest() { +func (ts *StorageTestSuite) SetupTest() { testS3Conf := S3Conf{ "http://127.0.0.1", s3Port, @@ -171,300 +171,300 @@ func (suite *StorageTestSuite) SetupTest() { testConf = Conf{posixType, testS3Conf, testPosixConf, testSftpConf} } -func (suite *StorageTestSuite) TestNewBackend() { +func (ts *StorageTestSuite) TestNewBackend() { testConf.Type = posixType p, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend posix failed") - assert.IsType(suite.T(), p, &posixBackend{}, "Wrong type from NewBackend with posix") + assert.NoError(ts.T(), err, "Backend posix failed") + assert.IsType(ts.T(), p, &posixBackend{}, "Wrong type from NewBackend with posix") var buf bytes.Buffer log.SetOutput(&buf) testConf.Type = sftpType sf, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend sftp failed") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") - assert.IsType(suite.T(), sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") + assert.NoError(ts.T(), err, "Backend sftp failed") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") + assert.IsType(ts.T(), sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") buf.Reset() testConf.Type = s3Type s, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend s3 failed") - assert.IsType(suite.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") + assert.NoError(ts.T(), err, "Backend s3 failed") + assert.IsType(ts.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") // test some extra ssl handling testConf.S3.CAcert = "/dev/null" s, err = NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend s3 failed") - assert.IsType(suite.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") + assert.NoError(ts.T(), err, "Backend s3 failed") + assert.IsType(ts.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") } -func (suite *StorageTestSuite) TestNewS3Client() { +func (ts *StorageTestSuite) TestNewS3Client() { c, err := NewS3Client(testConf.S3) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), c) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), c) } -func (suite *StorageTestSuite) TestCheckS3Bucket() { +func (ts *StorageTestSuite) TestCheckS3Bucket() { s3, err := newS3Backend(testConf.S3) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), CheckS3Bucket(testConf.S3.Bucket, s3.Client)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), CheckS3Bucket(testConf.S3.Bucket, s3.Client)) testConf.S3.URL = "file://tmp/" bad, err := newS3Backend(testConf.S3) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) err = CheckS3Bucket(testConf.S3.Bucket, bad.Client) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) } -func (suite *StorageTestSuite) TestPosixBackend() { +func (ts *StorageTestSuite) TestPosixBackend() { posixPath, _ := os.MkdirTemp("", "posix") defer os.RemoveAll(posixPath) testConf.Type = posixType testConf.Posix = posixConf{posixPath} backend, err := NewBackend(testConf) - assert.Nil(suite.T(), err, "POSIX backend failed unexpectedly") + assert.Nil(ts.T(), err, "POSIX backend failed unexpectedly") log.SetOutput(os.Stdout) writer, err := backend.NewFileWriter("testFile") - assert.NotNil(suite.T(), writer, "Got a nil reader for writer from posix") - assert.NoError(suite.T(), err, "posix NewFileWriter failed when it shouldn't") + assert.NotNil(ts.T(), writer, "Got a nil reader for writer from posix") + assert.NoError(ts.T(), err, "posix NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - assert.NoError(suite.T(), err, "Failure when writing to posix writer") - assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") + assert.NoError(ts.T(), err, "Failure when writing to posix writer") + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") writer.Close() reader, err := backend.NewFileReader("testFile") - assert.Nil(suite.T(), err, "posix NewFileReader failed when it should work") - assert.NotNil(suite.T(), reader, "Reader that should be usable is nosuite.T(), bailing out") + assert.Nil(ts.T(), err, "posix NewFileReader failed when it should work") + assert.NotNil(ts.T(), reader, "Reader that should be usable is nosuite.T(), bailing out") var buf bytes.Buffer log.SetOutput(&buf) writer, err = backend.NewFileWriter("posix/Not/Creatable") - assert.Nil(suite.T(), writer, "Got a non-nil reader for writer from posix") - assert.Error(suite.T(), err, "posix NewFileWriter worked when it shouldn't") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + assert.Nil(ts.T(), writer, "Got a non-nil reader for writer from posix") + assert.Error(ts.T(), err, "posix NewFileWriter worked when it shouldn't") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") buf.Reset() log.SetOutput(os.Stdout) var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") - assert.Nil(suite.T(), err, "unexpected error when reading back data") + assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Nil(ts.T(), err, "unexpected error when reading back data") size, err := backend.GetFileSize("testFile", false) - assert.Nil(suite.T(), err, "posix NewFileReader failed when it should work") - assert.NotNil(suite.T(), size, "Got a nil size for posix") + assert.Nil(ts.T(), err, "posix NewFileReader failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for posix") err = backend.RemoveFile("testFile") - assert.Nil(suite.T(), err, "posix RemoveFile failed when it should work") + assert.Nil(ts.T(), err, "posix RemoveFile failed when it should work") log.SetOutput(&buf) reader, err = backend.NewFileReader("posixDoesNotExist") - assert.Error(suite.T(), err, "posix NewFileReader worked when it should not") - assert.Nil(suite.T(), reader, "Got a non-nil reader for posix") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + assert.Error(ts.T(), err, "posix NewFileReader worked when it should not") + assert.Nil(ts.T(), reader, "Got a non-nil reader for posix") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") buf.Reset() _, err = backend.GetFileSize("posixDoesNotExist", false) - assert.Error(suite.T(), err, "posix GetFileSize worked when it should not") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + assert.Error(ts.T(), err, "posix GetFileSize worked when it should not") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") log.SetOutput(os.Stdout) testConf.Posix.Location = "/thisdoesnotexist" backEnd, err := NewBackend(testConf) - assert.NotNil(suite.T(), err, "Backend worked when it should not") - assert.Nil(suite.T(), backEnd, "Got a backend when expected not to") + assert.NotNil(ts.T(), err, "Backend worked when it should not") + assert.Nil(ts.T(), backEnd, "Got a backend when expected not to") testConf.Posix.Location = "/etc/passwd" backEnd, err = NewBackend(testConf) - assert.NotNil(suite.T(), err, "Backend worked when it should not") - assert.Nil(suite.T(), backEnd, "Got a backend when expected not to") + assert.NotNil(ts.T(), err, "Backend worked when it should not") + assert.Nil(ts.T(), backEnd, "Got a backend when expected not to") var dummyBackend *posixBackend failReader, err := dummyBackend.NewFileReader("/") - assert.NotNil(suite.T(), err, "NewFileReader worked when it should not") - assert.Nil(suite.T(), failReader, "Got a Reader when expected not to") + assert.NotNil(ts.T(), err, "NewFileReader worked when it should not") + assert.Nil(ts.T(), failReader, "Got a Reader when expected not to") failWriter, err := dummyBackend.NewFileWriter("/") - assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") - assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") + assert.NotNil(ts.T(), err, "NewFileWriter worked when it should not") + assert.Nil(ts.T(), failWriter, "Got a Writer when expected not to") _, err = dummyBackend.GetFileSize("/", false) - assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") + assert.NotNil(ts.T(), err, "GetFileSize worked when it should not") err = dummyBackend.RemoveFile("/") - assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") + assert.NotNil(ts.T(), err, "RemoveFile worked when it should not") } -func (suite *StorageTestSuite) TestS3Backend() { +func (ts *StorageTestSuite) TestS3Backend() { testConf.Type = s3Type s3back, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend failed") + assert.NoError(ts.T(), err, "Backend failed") writer, err := s3back.NewFileWriter("s3Creatable") - assert.NotNil(suite.T(), writer, "Got a nil reader for writer from s3") - assert.Nil(suite.T(), err, "s3 NewFileWriter failed when it shouldn't") + assert.NotNil(ts.T(), writer, "Got a nil reader for writer from s3") + assert.Nil(ts.T(), err, "s3 NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - assert.Nil(suite.T(), err, "Failure when writing to s3 writer") - assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") + assert.Nil(ts.T(), err, "Failure when writing to s3 writer") + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") writer.Close() // sleep to allow the write to complete, otherwise the next step will fail due to timing issues. time.Sleep(1 * time.Second) reader, err := s3back.NewFileReader("s3Creatable") - assert.NoError(suite.T(), err, "s3 NewFileReader failed when it should work") - assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") + assert.NoError(ts.T(), err, "s3 NewFileReader failed when it should work") + assert.NotNil(ts.T(), reader, "Reader that should be usable is not, bailing out") size, err := s3back.GetFileSize("s3Creatable", false) - assert.Nil(suite.T(), err, "s3 GetFileSize failed when it should work") - assert.NotNil(suite.T(), size, "Got a nil size for s3") - assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") + assert.Nil(ts.T(), err, "s3 GetFileSize failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for s3") + assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") // make sure expectDelay=true works // delete file and make sure the file size can not be retrieved anymore err = s3back.RemoveFile("s3Creatable") - assert.Nil(suite.T(), err, "s3 RemoveFile failed when it should work") + assert.Nil(ts.T(), err, "s3 RemoveFile failed when it should work") _, err = s3back.GetFileSize("s3Creatable", true) - assert.NotNil(suite.T(), err, "s3 GetFileSize worked when it should not") - assert.Error(suite.T(), err) + assert.NotNil(ts.T(), err, "s3 GetFileSize worked when it should not") + assert.Error(ts.T(), err) // rewrite file, do not wait before retrieving file size writer, err = s3back.NewFileWriter("s3Creatable") - assert.Nil(suite.T(), err, "s3 NewFileWriter failed when it shouldn't") + assert.Nil(ts.T(), err, "s3 NewFileWriter failed when it shouldn't") written, err = writer.Write(writeData) - assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") - assert.Nil(suite.T(), err, "Failure when writing to s3 writer") + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") + assert.Nil(ts.T(), err, "Failure when writing to s3 writer") writer.Close() size, err = s3back.GetFileSize("s3Creatable", true) - assert.Nil(suite.T(), err, "s3 GetFileSize with expected delay failed when it should work") - assert.NotNil(suite.T(), size, "Got a nil size for s3") - assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") + assert.Nil(ts.T(), err, "s3 GetFileSize with expected delay failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for s3") + assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") if err != nil && err != io.EOF { - assert.Nil(suite.T(), err, "unexpected error when reading back data") + assert.Nil(ts.T(), err, "unexpected error when reading back data") } err = s3back.RemoveFile("s3Creatable") - assert.Nil(suite.T(), err, "s3 RemoveFile failed when it should work") + assert.Nil(ts.T(), err, "s3 RemoveFile failed when it should work") _, err = s3back.GetFileSize("s3DoesNotExist", false) - assert.NotNil(suite.T(), err, "s3 GetFileSize worked when it should not") - assert.Error(suite.T(), err) + assert.NotNil(ts.T(), err, "s3 GetFileSize worked when it should not") + assert.Error(ts.T(), err) reader, err = s3back.NewFileReader("s3DoesNotExist") - assert.NotNil(suite.T(), err, "s3 NewFileReader worked when it should not") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), reader, "Got a non-nil reader for s3") + assert.NotNil(ts.T(), err, "s3 NewFileReader worked when it should not") + assert.Error(ts.T(), err) + assert.Nil(ts.T(), reader, "Got a non-nil reader for s3") testConf.S3.URL = "file://tmp/" _, err = NewBackend(testConf) - assert.Error(suite.T(), err, "Backend worked when it should not") + assert.Error(ts.T(), err, "Backend worked when it should not") } -func (suite *StorageTestSuite) TestSftpBackend() { +func (ts *StorageTestSuite) TestSftpBackend() { var buf bytes.Buffer log.SetOutput(&buf) testConf.Type = sftpType sftpBack, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend failed") + assert.NoError(ts.T(), err, "Backend failed") buf.Reset() var sftpDoesNotExist = "nonexistent/file" var sftpCreatable = "/share/file/exists" writer, err := sftpBack.NewFileWriter(sftpCreatable) - assert.NotNil(suite.T(), writer, "Got a nil reader for writer from sftp") - assert.Nil(suite.T(), err, "sftp NewFileWriter failed when it shouldn't") + assert.NotNil(ts.T(), writer, "Got a nil reader for writer from sftp") + assert.Nil(ts.T(), err, "sftp NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - assert.Nil(suite.T(), err, "Failure when writing to sftp writer") - assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") + assert.Nil(ts.T(), err, "Failure when writing to sftp writer") + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") writer.Close() reader, err := sftpBack.NewFileReader(sftpCreatable) - assert.Nil(suite.T(), err, "sftp NewFileReader failed when it should work") - assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") + assert.Nil(ts.T(), err, "sftp NewFileReader failed when it should work") + assert.NotNil(ts.T(), reader, "Reader that should be usable is not, bailing out") size, err := sftpBack.GetFileSize(sftpCreatable, false) - assert.Nil(suite.T(), err, "sftp GetFileSize failed when it should work") - assert.NotNil(suite.T(), size, "Got a nil size for sftp") - assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") + assert.Nil(ts.T(), err, "sftp GetFileSize failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for sftp") + assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") err = sftpBack.RemoveFile(sftpCreatable) - assert.Nil(suite.T(), err, "sftp RemoveFile failed when it should work") + assert.Nil(ts.T(), err, "sftp RemoveFile failed when it should work") err = sftpBack.RemoveFile(sftpDoesNotExist) - assert.ErrorContains(suite.T(), err, "file does not exist") + assert.ErrorContains(ts.T(), err, "file does not exist") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") if err != nil && err != io.EOF { - assert.Nil(suite.T(), err, "unexpected error when reading back data") + assert.Nil(ts.T(), err, "unexpected error when reading back data") } _, err = sftpBack.GetFileSize(sftpDoesNotExist, false) - assert.EqualError(suite.T(), err, "failed to get file size with sftp, file does not exist") + assert.EqualError(ts.T(), err, "failed to get file size with sftp, file does not exist") reader, err = sftpBack.NewFileReader(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "failed to open file with sftp, file does not exist") - assert.Nil(suite.T(), reader, "Got a non-nil reader for sftp") + assert.EqualError(ts.T(), err, "failed to open file with sftp, file does not exist") + assert.Nil(ts.T(), reader, "Got a non-nil reader for sftp") // wrong host key testConf.SFTP.HostKey = "wronghostkey" _, err = NewBackend(testConf) - assert.ErrorContains(suite.T(), err, "failed to start ssh connection, ssh: handshake failed: host key verification expected") + assert.ErrorContains(ts.T(), err, "failed to start ssh connection, ssh: handshake failed: host key verification expected") // wrong key password testConf.SFTP.PemKeyPass = "wrongkey" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to parse private key, x509: decryption password incorrect") + assert.EqualError(ts.T(), err, "failed to parse private key, x509: decryption password incorrect") // missing key password testConf.SFTP.PemKeyPass = "" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to parse private key, ssh: this private key is passphrase protected") + assert.EqualError(ts.T(), err, "failed to parse private key, ssh: this private key is passphrase protected") // wrong key testConf.SFTP.PemKeyPath = "nonexistentkey" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to read from key file, open nonexistentkey: no such file or directory") + assert.EqualError(ts.T(), err, "failed to read from key file, open nonexistentkey: no such file or directory") f, _ := os.CreateTemp(sshPath, "dummy") testConf.SFTP.PemKeyPath = f.Name() _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to parse private key, ssh: no key found") + assert.EqualError(ts.T(), err, "failed to parse private key, ssh: no key found") testConf.SFTP.Host = "nonexistenthost" _, err = NewBackend(testConf) - assert.NotNil(suite.T(), err, "Backend worked when it should not") + assert.NotNil(ts.T(), err, "Backend worked when it should not") var dummyBackend *sftpBackend failReader, err := dummyBackend.NewFileReader("/") - assert.NotNil(suite.T(), err, "NewFileReader worked when it should not") - assert.Nil(suite.T(), failReader, "Got a Reader when expected not to") + assert.NotNil(ts.T(), err, "NewFileReader worked when it should not") + assert.Nil(ts.T(), failReader, "Got a Reader when expected not to") failWriter, err := dummyBackend.NewFileWriter("/") - assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") - assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") + assert.NotNil(ts.T(), err, "NewFileWriter worked when it should not") + assert.Nil(ts.T(), failWriter, "Got a Writer when expected not to") _, err = dummyBackend.GetFileSize("/", false) - assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") + assert.NotNil(ts.T(), err, "GetFileSize worked when it should not") err = dummyBackend.RemoveFile("/") - assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") - assert.EqualError(suite.T(), err, "invalid sftpBackend") + assert.NotNil(ts.T(), err, "RemoveFile worked when it should not") + assert.EqualError(ts.T(), err, "invalid sftpBackend") } diff --git a/sda/internal/userauth/userauth_test.go b/sda/internal/userauth/userauth_test.go index f181f783a9..25770c27af 100644 --- a/sda/internal/userauth/userauth_test.go +++ b/sda/internal/userauth/userauth_test.go @@ -33,7 +33,7 @@ func TestUserAuthTestSuite(t *testing.T) { suite.Run(t, new(UserAuthTest)) } -func (suite *UserAuthTest) SetupTest() { +func (ts *UserAuthTest) SetupTest() { } func TestMain(m *testing.M) { @@ -115,75 +115,75 @@ func TestMain(m *testing.M) { os.Exit(0) } -func (suite *UserAuthTest) TestAlwaysAuthenticator() { +func (ts *UserAuthTest) TestAlwaysAuthenticator() { a := helper.NewAlwaysAllow() r, _ := http.NewRequest("Get", "/", nil) _, err := a.Authenticate(r) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_NoFile() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_NoFile() { a := NewValidateFromToken(jwk.NewSet()) err := a.ReadJwtPubKeyPath("") - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_GetFile() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_GetFile() { demoKeysPath := "temp-keys" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateECkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" a := NewValidateFromToken(jwk.NewSet()) err = a.ReadJwtPubKeyPath(jwtpubkeypath) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), 2, a.Keyset.Len()) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, a.Keyset.Len()) defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_WrongURL() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_WrongURL() { a := NewValidateFromToken(jwk.NewSet()) jwtpubkeyurl := "/dummy/" err := a.FetchJwtPubKeyURL(jwtpubkeyurl) - assert.Equal(suite.T(), "jwtpubkeyurl is not a proper URL (/dummy/)", err.Error()) + assert.Equal(ts.T(), "jwtpubkeyurl is not a proper URL (/dummy/)", err.Error()) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_BadURL() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_BadURL() { a := NewValidateFromToken(jwk.NewSet()) jwtpubkeyurl := "dummy.com/jwk" err := a.FetchJwtPubKeyURL(jwtpubkeyurl) - assert.Equal(suite.T(), "parse \"dummy.com/jwk\": invalid URI for request", err.Error()) + assert.Equal(ts.T(), "parse \"dummy.com/jwk\": invalid URI for request", err.Error()) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_GoodURL() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_GoodURL() { a := NewValidateFromToken(jwk.NewSet()) jwtpubkeyurl := fmt.Sprintf("http://localhost:%d/jwk", OIDCport) err := a.FetchJwtPubKeyURL(jwtpubkeyurl) - assert.NoError(suite.T(), err, "failed to fetch remote JWK") - assert.Equal(suite.T(), 3, a.Keyset.Len()) + assert.NoError(ts.T(), err, "failed to fetch remote JWK") + assert.Equal(ts.T(), 3, a.Keyset.Len()) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { // These tests should be possible to reuse with all correct authenticators somehow // Create temp demo rsa key pair demoKeysPath := "demo-rsa-keys" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" a := NewValidateFromToken(jwk.NewSet()) @@ -191,11 +191,11 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { // Parse demo private key prKeyParsed, err := helper.ParsePrivateRSAKey(prKeyPath, "/rsa") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create token and set up request defaults defaultToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ := http.NewRequest("", "/", nil) r.Host = "localhost" @@ -204,7 +204,7 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.Header.Set("X-Amz-Security-Token", "notJWT") r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) r.Header.Set("X-Amz-Security-Token", defaultToken) @@ -212,24 +212,24 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.URL.Path = "/dummy/" signer.SignV4(*r, "username", "testpass", "", "us-east-1") token, err := a.Authenticate(r) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) privateClaims := token.PrivateClaims() - assert.Equal(suite.T(), privateClaims["pilot"], helper.DefaultTokenClaims["pilot"]) + assert.Equal(ts.T(), privateClaims["pilot"], helper.DefaultTokenClaims["pilot"]) // Create and test expired Elixir token expiredToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.ExpiredClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", expiredToken) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) // Elixir token is not valid (e.g. issued in a future time) nonValidToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.NonValidClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" @@ -237,7 +237,7 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.URL.Path = "/username/" _, nonvalidToken := a.Authenticate(r) // The error output is huge so a smaller part is compared - assert.Equal(suite.T(), "\"iat\" not satisfied", nonvalidToken.Error()) + assert.Equal(ts.T(), "\"iat\" not satisfied", nonvalidToken.Error()) // Elixir tokens broken r, _ = http.NewRequest("", "/", nil) @@ -245,40 +245,40 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.Header.Set("X-Amz-Security-Token", defaultToken[3:]) r.URL.Path = "/username/" _, brokenToken := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", "random"+defaultToken) r.URL.Path = "/username/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) _, brokenToken2 := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") // Bad issuer basIss, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.WrongTokenAlgClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", basIss) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Contains(suite.T(), err.Error(), "failed to get issuer from token") + assert.Contains(ts.T(), err.Error(), "failed to get issuer from token") // Delete the keys when testing is done or failed defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { // Create temp demo ec key pair demoKeysPath := "demo-ec-keys" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateECkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" @@ -287,11 +287,11 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { // Parse demo private key prKeyParsed, err := helper.ParsePrivateECKey(prKeyPath, "/ec") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create token and set up request defaults defaultToken, err := helper.CreateECToken(prKeyParsed, "ES256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ := http.NewRequest("", "/", nil) r.Host = "localhost" @@ -301,22 +301,22 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { r.URL.Path = "/dummy/" signer.SignV4(*r, "username", "testpass", "", "us-east-1") _, err = a.Authenticate(r) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) // Create and test expired Elixir token expiredToken, err := helper.CreateECToken(prKeyParsed, "ES256", helper.ExpiredClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", expiredToken) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) // Elixir token is not valid nonValidToken, err := helper.CreateECToken(prKeyParsed, "ES256", helper.NonValidClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" @@ -324,7 +324,7 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { r.URL.Path = "/username/" _, nonvalidToken := a.Authenticate(r) // The error output is huge so a smaller part is compared - assert.Equal(suite.T(), "\"iat\" not satisfied", nonvalidToken.Error()) + assert.Equal(ts.T(), "\"iat\" not satisfied", nonvalidToken.Error()) // Elixir tokens broken r, _ = http.NewRequest("", "/", nil) @@ -332,38 +332,38 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { r.Header.Set("X-Amz-Security-Token", defaultToken[3:]) r.URL.Path = "/username/" _, brokenToken := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", "random"+defaultToken) r.URL.Path = "/username/" _, brokenToken2 := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") // Bad issuer basIss, err := helper.CreateECToken(prKeyParsed, "ES256", helper.WrongTokenAlgClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", basIss) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Contains(suite.T(), err.Error(), "failed to get issuer from token") + assert.Contains(ts.T(), err.Error(), "failed to get issuer from token") defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestWrongKeyType_RSA() { +func (ts *UserAuthTest) TestWrongKeyType_RSA() { // Create temp demo ec key pair demoKeysPath := "demo-ec-keys" demoPrKeyName := "/ec" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateECkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" @@ -372,20 +372,20 @@ func (suite *UserAuthTest) TestWrongKeyType_RSA() { // Parse demo private key _, err = helper.ParsePrivateRSAKey(prKeyPath, demoPrKeyName) - assert.Equal(suite.T(), "bad key format, expected RSA got EC", err.Error()) + assert.Equal(ts.T(), "bad key format, expected RSA got EC", err.Error()) defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestWrongKeyType_EC() { +func (ts *UserAuthTest) TestWrongKeyType_EC() { // Create temp demo ec key pair demoKeysPath := "demo-rsa-keys" demoPrKeyName := "/rsa" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" @@ -394,20 +394,20 @@ func (suite *UserAuthTest) TestWrongKeyType_EC() { // Parse demo private key _, err = helper.ParsePrivateECKey(prKeyPath, demoPrKeyName) - assert.Equal(suite.T(), "bad key format, expected EC got RSA", err.Error()) + assert.Equal(ts.T(), "bad key format, expected EC got RSA", err.Error()) defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_HS() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_HS() { // Create random secret key := make([]byte, 256) _, err := rand.Read(key) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create HS256 token wrongAlgToken, err := helper.CreateHSToken(key, helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) a := NewValidateFromToken(jwk.NewSet()) @@ -416,7 +416,7 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_HS() { r.Header.Set("X-Amz-Security-Token", wrongAlgToken) r.URL.Path = "/username/" _, WrongAlg := a.Authenticate(r) - assert.Contains(suite.T(), WrongAlg.Error(), "failed to find key with key ID") + assert.Contains(ts.T(), WrongAlg.Error(), "failed to find key with key ID") } func TestGetBearerToken(t *testing.T) { From 176e5295a00bbe125a90b429361662b65014506e Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 10:27:24 +0200 Subject: [PATCH 206/250] [revive] Don't capitalize local variables --- sda/cmd/api/api.go | 8 +-- sda/cmd/api/api_test.go | 22 +++---- sda/cmd/ingest/ingest_test.go | 84 +++++++++++++------------- sda/cmd/s3inbox/s3inbox.go | 33 +++++----- sda/cmd/s3inbox/s3inbox_test.go | 12 ++-- sda/cmd/sync/sync.go | 4 +- sda/cmd/sync/sync_test.go | 4 +- sda/internal/broker/broker_test.go | 10 +-- sda/internal/config/config_test.go | 20 +++--- sda/internal/helper/helper_test.go | 8 +-- sda/internal/userauth/userauth_test.go | 8 +-- 11 files changed, 105 insertions(+), 108 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index dcb6327df5..3967bf818d 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -174,8 +174,8 @@ func readinessResponse(c *gin.Context) { } } - if DBRes := checkDB(Conf.API.DB, 5*time.Millisecond); DBRes != nil { - log.Debugf("DB connection error :%v", DBRes) + if dbRes := checkDB(Conf.API.DB, 5*time.Millisecond); dbRes != nil { + log.Debugf("DB connection error :%v", dbRes) Conf.API.DB.Reconnect() statusCode = http.StatusServiceUnavailable } @@ -319,8 +319,8 @@ func deleteFile(c *gin.Context) { return } - var RetryTimes = 5 - for count := 1; count <= RetryTimes; count++ { + var retryTimes = 5 + for count := 1; count <= retryTimes; count++ { err = inbox.RemoveFile(filePath) if err == nil { break diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 1c9e02d3c4..c8a38b4387 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -145,12 +145,12 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to rabbitmq: %s", err) } - RSAPath, _ := os.MkdirTemp("", "RSA") - if err := helper.CreateRSAkeys(RSAPath, RSAPath); err != nil { + rsaPath, _ := os.MkdirTemp("", "RSA") + if err := helper.CreateRSAkeys(rsaPath, rsaPath); err != nil { log.Panic("Failed to create RSA keys") } - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { log.Panic("Failed to create EC keys") } @@ -179,10 +179,10 @@ func TestMain(m *testing.M) { } OIDCport, _ = strconv.Atoi(oidc.GetPort("8080/tcp")) - OIDCHostAndPort := oidc.GetHostPort("8080/tcp") + oidcHostAndPort := oidc.GetHostPort("8080/tcp") client = http.Client{Timeout: 5 * time.Second} - req, err = http.NewRequest(http.MethodGet, "http://"+OIDCHostAndPort+"/jwk", http.NoBody) + req, err = http.NewRequest(http.MethodGet, "http://"+oidcHostAndPort+"/jwk", http.NoBody) if err != nil { log.Panic(err) } @@ -223,8 +223,8 @@ func TestMain(m *testing.M) { log.Fatalf("Could not purge resource: %s", err) } // cleanup temp files - _ = os.RemoveAll(ECPath) - _ = os.RemoveAll(RSAPath) + _ = os.RemoveAll(ecPath) + _ = os.RemoveAll(rsaPath) os.Exit(code) } @@ -352,9 +352,9 @@ func (s *TestSuite) SetupSuite() { s.User = user c := &config.Config{} - ServerConf := config.ServerConfig{} - ServerConf.Jwtpubkeypath = s.PublicPath - c.Server = ServerConf + serverConf := config.ServerConfig{} + serverConf.Jwtpubkeypath = s.PublicPath + c.Server = serverConf Conf = c diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index c76d2dd1ab..3a1e347bc5 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -362,40 +362,40 @@ func (ts *TestSuite) TestTryDecrypt() { // messages of type `cancel` func (ts *TestSuite) TestCancelFile() { // prepare the DB entries - UserName := "test-cancel" - file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", UserName) - fileID, err := ts.ingest.DB.RegisterFile(file1, UserName) + userName := "test-cancel" + file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", userName) + fileID, err := ts.ingest.DB.RegisterFile(file1, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "cancel", FilePath: file1, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(corrID, message)) } func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // prepare the DB entries - UserName := "test-cancel" - file1 := fmt.Sprintf("/%v/TestCancelMessage_wrongCorrelationID.c4gh", UserName) - fileID, err := ts.ingest.DB.RegisterFile(file1, UserName) + userName := "test-cancel" + file1 := fmt.Sprintf("/%v/TestCancelMessage_wrongCorrelationID.c4gh", userName) + fileID, err := ts.ingest.DB.RegisterFile(file1, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "cancel", FilePath: file1, - User: UserName, + User: userName, } assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(uuid.New().String(), message)) @@ -404,38 +404,38 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // messages of type `ingest` func (ts *TestSuite) TestIngestFile() { // prepare the DB entries - UserName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } func (ts *TestSuite) TestIngestFile_secondTime() { // prepare the DB entries - UserName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -444,30 +444,30 @@ func (ts *TestSuite) TestIngestFile_secondTime() { assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) } func (ts *TestSuite) TestIngestFile_unknownInboxType() { - UserName := "test-ingest-unknown" + userName := "test-ingest-unknown" message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(uuid.New().String(), message)) } func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { // prepare the DB entries - UserName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -480,19 +480,19 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { // prepare the DB entries - UserName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -552,19 +552,19 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { // prepare the DB entries - UserName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -584,19 +584,19 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // prepare the DB entries - UserName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -620,19 +620,19 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { // prepare the DB entries - UserName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, UserName) + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, UserName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -711,14 +711,14 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_missingFile() { // prepare the DB entries - UserName := "test-ingest" + userName := "test-ingest" corrID := uuid.New().String() basepath := filepath.Dir(ts.filePath) message := schema.IngestionTrigger{ Type: "ingest", FilePath: fmt.Sprintf("%s/missing.file.c4gh", basepath), - User: UserName, + User: userName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index 292d690325..ae7898bd96 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -19,9 +19,6 @@ import ( log "github.com/sirupsen/logrus" ) -// Export Conf so we can access it in the other modules -var Conf *config.Config - func main() { sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) @@ -34,21 +31,21 @@ func main() { } }() - Conf, err := config.NewConfig("s3inbox") + conf, err := config.NewConfig("s3inbox") if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - tlsProxy, err := config.TLSConfigProxy(Conf) + tlsProxy, err := config.TLSConfigProxy(conf) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - sdaDB, err := database.NewSDAdb(Conf.Database) + sdaDB, err := database.NewSDAdb(conf.Database) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -62,21 +59,21 @@ func main() { log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - s3, err := storage.NewS3Client(Conf.Inbox.S3) + s3, err := storage.NewS3Client(conf.Inbox.S3) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - err = storage.CheckS3Bucket(Conf.Inbox.S3.Bucket, s3) + err = storage.CheckS3Bucket(conf.Inbox.S3.Bucket, s3) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - messenger, err := broker.NewMQ(Conf.Broker) + messenger, err := broker.NewMQ(conf.Broker) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -92,18 +89,18 @@ func main() { }() auth := userauth.NewValidateFromToken(jwk.NewSet()) // Load keys for JWT verification - if Conf.Server.Jwtpubkeyurl != "" { - if err := auth.FetchJwtPubKeyURL(Conf.Server.Jwtpubkeyurl); err != nil { - log.Panicf("Error while getting key %s: %v", Conf.Server.Jwtpubkeyurl, err) + if conf.Server.Jwtpubkeyurl != "" { + if err := auth.FetchJwtPubKeyURL(conf.Server.Jwtpubkeyurl); err != nil { + log.Panicf("Error while getting key %s: %v", conf.Server.Jwtpubkeyurl, err) } } - if Conf.Server.Jwtpubkeypath != "" { - if err := auth.ReadJwtPubKeyPath(Conf.Server.Jwtpubkeypath); err != nil { - log.Panicf("Error while getting key %s: %v", Conf.Server.Jwtpubkeypath, err) + if conf.Server.Jwtpubkeypath != "" { + if err := auth.ReadJwtPubKeyPath(conf.Server.Jwtpubkeypath); err != nil { + log.Panicf("Error while getting key %s: %v", conf.Server.Jwtpubkeypath, err) } } router := mux.NewRouter() - proxy := NewProxy(Conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy) + proxy := NewProxy(conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy) router.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") router.HandleFunc("/health", proxy.CheckHealth) router.PathPrefix("/").Handler(proxy) @@ -117,8 +114,8 @@ func main() { Handler: router, } - if Conf.Server.Cert != "" && Conf.Server.Key != "" { - if err := server.ListenAndServeTLS(Conf.Server.Cert, Conf.Server.Key); err != nil { + if conf.Server.Cert != "" && conf.Server.Key != "" { + if err := server.ListenAndServeTLS(conf.Server.Cert, conf.Server.Key); err != nil { panic(err) } } else { diff --git a/sda/cmd/s3inbox/s3inbox_test.go b/sda/cmd/s3inbox/s3inbox_test.go index 3837fa5bde..5f2c7eae2e 100644 --- a/sda/cmd/s3inbox/s3inbox_test.go +++ b/sda/cmd/s3inbox/s3inbox_test.go @@ -171,12 +171,12 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to rabbitmq: %s", err) } - RSAPath, _ := os.MkdirTemp("", "RSA") - if err := helper.CreateRSAkeys(RSAPath, RSAPath); err != nil { + rsaPath, _ := os.MkdirTemp("", "RSA") + if err := helper.CreateRSAkeys(rsaPath, rsaPath); err != nil { log.Panic("Failed to create RSA keys") } - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { log.Panic("Failed to create EC keys") } // pulls an image, creates a container based on it and runs it @@ -204,10 +204,10 @@ func TestMain(m *testing.M) { } OIDCport, _ = strconv.Atoi(oidc.GetPort("8080/tcp")) - OIDCHostAndPort := oidc.GetHostPort("8080/tcp") + oidcHostAndPort := oidc.GetHostPort("8080/tcp") client = http.Client{Timeout: 5 * time.Second} - req, err = http.NewRequest(http.MethodGet, "http://"+OIDCHostAndPort+"/jwk", http.NoBody) + req, err = http.NewRequest(http.MethodGet, "http://"+oidcHostAndPort+"/jwk", http.NoBody) if err != nil { log.Panic(err) } diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 93c16aa427..1f78cc6132 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -255,12 +255,12 @@ func sendPOST(payload []byte) error { Timeout: 30 * time.Second, } - URL, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) + uri, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, URL, bytes.NewBuffer(payload)) + req, err := http.NewRequest(http.MethodPost, uri, bytes.NewBuffer(payload)) if err != nil { return err } diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index 0f582bff5f..f1c24ae87e 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -152,10 +152,10 @@ func (s *SyncTest) SetupTest() { func (s *SyncTest) TestBuildSyncDatasetJSON() { s.SetupTest() - Conf, err := config.NewConfig("sync") + conf, err := config.NewConfig("sync") assert.NoError(s.T(), err) - db, err = database.NewSDAdb(Conf.Database) + db, err = database.NewSDAdb(conf.Database) assert.NoError(s.T(), err) fileID, err := db.RegisterFile("dummy.user/test/file1.c4gh", "dummy.user") diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index dc0795d5aa..99c61420d9 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -207,12 +207,12 @@ func (ts *BrokerTestSuite) TestNewMQNoTLS() { } func (ts *BrokerTestSuite) TestNewMQTLS() { - SslConf := tMqconf - SslConf.Port = tlsPort - SslConf.Ssl = true - SslConf.VerifyPeer = true + sslConf := tMqconf + sslConf.Port = tlsPort + sslConf.Ssl = true + sslConf.VerifyPeer = true - b, err := NewMQ(SslConf) + b, err := NewMQ(sslConf) assert.NoError(ts.T(), err) assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") assert.False(ts.T(), b.Connection.IsClosed()) diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index c9d534e6a5..4c33844ff2 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -484,11 +484,11 @@ func (ts *ConfigTestSuite) TestConfigReEncryptServer() { func (ts *ConfigTestSuite) TestConfigAuth_CEGA() { ts.SetupTest() - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { ts.T().FailNow() } - defer os.RemoveAll(ECPath) + defer os.RemoveAll(ecPath) noConfig, err := NewConfig("auth") assert.Error(ts.T(), err) @@ -506,10 +506,10 @@ func (ts *ConfigTestSuite) TestConfigAuth_CEGA() { _, err = NewConfig("auth") assert.ErrorContains(ts.T(), err, "no such file or directory") - viper.Set("auth.publicFile", ECPath+"/ec.pub") - viper.Set("auth.Jwt.privateKey", ECPath+"/ec") + viper.Set("auth.publicFile", ecPath+"/ec.pub") + viper.Set("auth.Jwt.privateKey", ecPath+"/ec") c, err := NewConfig("auth") - assert.Equal(ts.T(), c.Auth.JwtPrivateKey, fmt.Sprintf("%s/ec", ECPath)) + assert.Equal(ts.T(), c.Auth.JwtPrivateKey, fmt.Sprintf("%s/ec", ecPath)) assert.Equal(ts.T(), c.Auth.JwtTTL, 168) assert.NoError(ts.T(), err, "unexpected failure") } @@ -517,18 +517,18 @@ func (ts *ConfigTestSuite) TestConfigAuth_CEGA() { func (ts *ConfigTestSuite) TestConfigAuth_OIDC() { ts.SetupTest() - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { ts.T().FailNow() } - defer os.RemoveAll(ECPath) + defer os.RemoveAll(ecPath) noConfig, err := NewConfig("auth") assert.Error(ts.T(), err) assert.Nil(ts.T(), noConfig) viper.Set("auth.s3Inbox", "http://inbox:8000") - viper.Set("auth.publicFile", ECPath+"/ec.pub") + viper.Set("auth.publicFile", ecPath+"/ec.pub") viper.Set("oidc.id", "oidcTestID") viper.Set("oidc.secret", "oidcTestIssuer") _, err = NewConfig("auth") diff --git a/sda/internal/helper/helper_test.go b/sda/internal/helper/helper_test.go index 814872fc79..42f538817c 100644 --- a/sda/internal/helper/helper_test.go +++ b/sda/internal/helper/helper_test.go @@ -54,8 +54,8 @@ func (ts *HelperTest) TestCreateRSAToken() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateRSAkeys(privateK, publicK) assert.Nil(ts.T(), e) - ParsedPrKey, _ := ParsePrivateRSAKey(privateK, "/rsa") - tok, err := CreateRSAToken(ParsedPrKey, "RS256", DefaultTokenClaims) + parsedPrKey, _ := ParsePrivateRSAKey(privateK, "/rsa") + tok, err := CreateRSAToken(parsedPrKey, "RS256", DefaultTokenClaims) assert.Nil(ts.T(), err) set := jwk.NewSet() @@ -97,9 +97,9 @@ func (ts *HelperTest) TestCreateECToken() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateECkeys(privateK, publicK) assert.Nil(ts.T(), e) - ParsedPrKey, err := ParsePrivateECKey(privateK, "/ec") + parsedPrKey, err := ParsePrivateECKey(privateK, "/ec") assert.NoError(ts.T(), err) - _, err = CreateECToken(ParsedPrKey, "ES256", DefaultTokenClaims) + _, err = CreateECToken(parsedPrKey, "ES256", DefaultTokenClaims) assert.Nil(ts.T(), err) defer os.RemoveAll("dummy-folder") diff --git a/sda/internal/userauth/userauth_test.go b/sda/internal/userauth/userauth_test.go index 25770c27af..1a984172c1 100644 --- a/sda/internal/userauth/userauth_test.go +++ b/sda/internal/userauth/userauth_test.go @@ -80,10 +80,10 @@ func TestMain(m *testing.M) { } OIDCport, _ = strconv.Atoi(oidc.GetPort("8080/tcp")) - OIDCHostAndPort := oidc.GetHostPort("8080/tcp") + oidcHostAndPort := oidc.GetHostPort("8080/tcp") client := http.Client{Timeout: 5 * time.Second} - req, err := http.NewRequest(http.MethodGet, "http://"+OIDCHostAndPort+"/jwk", http.NoBody) + req, err := http.NewRequest(http.MethodGet, "http://"+oidcHostAndPort+"/jwk", http.NoBody) if err != nil { log.Panic(err) } @@ -415,8 +415,8 @@ func (ts *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_HS() { r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", wrongAlgToken) r.URL.Path = "/username/" - _, WrongAlg := a.Authenticate(r) - assert.Contains(ts.T(), WrongAlg.Error(), "failed to find key with key ID") + _, wrongAlg := a.Authenticate(r) + assert.Contains(ts.T(), wrongAlg.Error(), "failed to find key with key ID") } func TestGetBearerToken(t *testing.T) { From 62660e492149d0847981c5168a8c8f4a4dc53b31 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 10:43:26 +0200 Subject: [PATCH 207/250] [revive] variable naming --- sda/cmd/s3inbox/proxy.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 64b9d1e526..1f4a074776 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -34,7 +34,7 @@ type Proxy struct { messenger *broker.AMQPBroker database *database.SDAdb client *http.Client - fileIds map[string]string + fileIDs map[string]string } // The Event struct @@ -158,11 +158,11 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw } // if this is an upload request - if p.detectRequestType(r) == Put && p.fileIds[r.URL.Path] == "" { + if p.detectRequestType(r) == Put && p.fileIDs[r.URL.Path] == "" { // register file in database log.Debugf("registering file %v in the database", r.URL.Path) - p.fileIds[r.URL.Path], err = p.database.RegisterFile(filepath, username) - log.Debugf("fileId: %v", p.fileIds[r.URL.Path]) + p.fileIDs[r.URL.Path], err = p.database.RegisterFile(filepath, username) + log.Debugf("fileId: %v", p.fileIDs[r.URL.Path]) if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to register file in database: %v", err)) @@ -213,33 +213,33 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // The following block is for treating the case when the client loses connection to the server and then it reconnects to a // different instance of s3inbox. For more details see #1358. - if p.fileIds[r.URL.Path] == "" { - p.fileIds[r.URL.Path], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") + if p.fileIDs[r.URL.Path] == "" { + p.fileIDs[r.URL.Path], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) return } - log.Debugf("resuming work on file with fileId: %v", p.fileIds[r.URL.Path]) + log.Debugf("resuming work on file with fileId: %v", p.fileIDs[r.URL.Path]) } - if err := p.storeObjectSizeInDB(rawFilepath, p.fileIds[r.URL.Path]); err != nil { + if err := p.storeObjectSizeInDB(rawFilepath, p.fileIDs[r.URL.Path]); err != nil { log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) p.internalServerError(w, r, "storeObjectSizeInDB failed") return } - log.Debugf("marking file %v as 'uploaded' in database", p.fileIds[r.URL.Path]) - err = p.database.UpdateFileEventLog(p.fileIds[r.URL.Path], "uploaded", p.fileIds[r.URL.Path], "inbox", "{}", string(jsonMessage)) + log.Debugf("marking file %v as 'uploaded' in database", p.fileIDs[r.URL.Path]) + err = p.database.UpdateFileEventLog(p.fileIDs[r.URL.Path], "uploaded", p.fileIDs[r.URL.Path], "inbox", "{}", string(jsonMessage)) if err != nil { p.internalServerError(w, r, fmt.Sprintf("could not connect to db: %v", err)) return } - delete(p.fileIds, r.URL.Path) + delete(p.fileIDs, r.URL.Path) } // Writing non-200 to the response before the headers propagate the error @@ -294,7 +294,7 @@ func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { } } - if err := p.messenger.SendMessage(p.fileIds[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { + if err := p.messenger.SendMessage(p.fileIDs[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { return fmt.Errorf("error when sending message to broker: %v", err) } From 644259e7029305cbdad3858de19825f923ee18a8 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 10:56:14 +0200 Subject: [PATCH 208/250] [revive] Use errors.New where applicable --- sda-admin/main.go | 4 ++-- sda-download/api/sda/sda.go | 2 +- sda-download/internal/database/database_test.go | 8 ++++---- sda-download/internal/storage/seekable.go | 9 +++++---- sda-download/internal/storage/storage.go | 13 +++++++------ sda/cmd/api/api.go | 3 ++- sda/cmd/finalize/finalize.go | 3 ++- sda/cmd/mapper/mapper.go | 7 ++++--- sda/cmd/notify/notify.go | 3 ++- sda/cmd/s3inbox/proxy.go | 5 +++-- sda/cmd/s3inbox/proxy_test.go | 3 ++- sda/cmd/sync/sync.go | 3 ++- sda/internal/config/config.go | 14 +++++++------- sda/internal/database/database.go | 3 ++- sda/internal/database/db_functions.go | 2 +- sda/internal/helper/helper.go | 3 ++- sda/internal/schema/schema.go | 3 ++- sda/internal/storage/storage.go | 16 ++++++++-------- sda/internal/userauth/userauth.go | 11 ++++++----- 19 files changed, 64 insertions(+), 51 deletions(-) diff --git a/sda-admin/main.go b/sda-admin/main.go index 560874f3ba..c6499ae8bb 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -182,14 +182,14 @@ func parseFlagsAndEnv() error { if apiURI == "" { apiURI = os.Getenv("API_HOST") if apiURI == "" { - return fmt.Errorf("error: either -uri must be provided or API_HOST environment variable must be set") + return errors.New("error: either -uri must be provided or API_HOST environment variable must be set") } } if token == "" { token = os.Getenv("ACCESS_TOKEN") if token == "" { - return fmt.Errorf("error: either -token must be provided or ACCESS_TOKEN environment variable must be set") + return errors.New("error: either -token must be provided or ACCESS_TOKEN environment variable must be set") } } diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index 4998efa6bc..e0354eafc7 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -539,7 +539,7 @@ var calculateCoords = func(start, end int64, htsget_range string, fileDetails *d return 0, 0, err } if a > b { - return 0, 0, fmt.Errorf("endCoordinate must be greater than startCoordinate") + return 0, 0, errors.New("endCoordinate must be greater than startCoordinate") } // Byte ranges are inclusive; +1 so that the last byte is included diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index 5b45385c40..6f18c2ca18 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -65,7 +65,7 @@ func testLogFatalf(f string, args ...any) { func TestCheckAndReconnect(t *testing.T) { db, mock, _ := sqlmock.New(sqlmock.MonitorPingsOption(true)) - mock.ExpectPing().WillReturnError(fmt.Errorf("ping fail for testing bad conn")) + mock.ExpectPing().WillReturnError(errors.New("ping fail for testing bad conn")) err := CatchPanicCheckAndReconnect(SQLdb{db, ""}) assert.Error(t, err, "Should have received error from checkAndReconnectOnNeeded fataling") @@ -75,7 +75,7 @@ func CatchPanicCheckAndReconnect(db SQLdb) (err error) { defer func() { r := recover() if r != nil { - err = fmt.Errorf("Caught panic") + err = errors.New("Caught panic") } }() @@ -92,7 +92,7 @@ func CatchNewDBPanic() (err error) { defer func() { r := recover() if r != nil { - err = fmt.Errorf("Caught panic") + err = errors.New("Caught panic") } }() @@ -136,7 +136,7 @@ func TestNewDB(t *testing.T) { return db, nil } - mock.ExpectPing().WillReturnError(fmt.Errorf("ping fail for testing")) + mock.ExpectPing().WillReturnError(errors.New("ping fail for testing")) err = CatchNewDBPanic() diff --git a/sda-download/internal/storage/seekable.go b/sda-download/internal/storage/seekable.go index a6ba50da4b..383a81db68 100644 --- a/sda-download/internal/storage/seekable.go +++ b/sda-download/internal/storage/seekable.go @@ -3,6 +3,7 @@ package storage import ( "bytes" + "errors" "fmt" "io" "strings" @@ -21,7 +22,7 @@ func (pb *posixBackend) NewFileReadSeeker(filePath string) (io.ReadSeekCloser, e seeker, ok := reader.(io.ReadSeekCloser) if !ok { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, errors.New("Invalid posixBackend") } return seeker, nil @@ -213,7 +214,7 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { return r.currentOffset, nil } - return r.currentOffset, fmt.Errorf("Bad whence") + return r.currentOffset, errors.New("Bad whence") } // removeFromOutstanding removes a prefetch from the list of outstanding prefetches once it's no longer active @@ -414,7 +415,7 @@ func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { case io.SeekEnd: r.currentOffset = r.totalSize + offset default: - return 0, fmt.Errorf("Unsupported whence") + return 0, errors.New("Unsupported whence") } return r.currentOffset, nil @@ -442,7 +443,7 @@ func (r *seekableMultiReader) Read(dst []byte) (int, error) { seekable, ok := reader.(io.ReadSeeker) if !ok { - return 0, fmt.Errorf("Expected seekable reader but changed") + return 0, errors.New("Expected seekable reader but changed") } _, err := seekable.Seek(r.currentOffset-int64(readerStartAt), 0) diff --git a/sda-download/internal/storage/storage.go b/sda-download/internal/storage/storage.go index 9a6c3dd159..98c37aa088 100644 --- a/sda-download/internal/storage/storage.go +++ b/sda-download/internal/storage/storage.go @@ -4,6 +4,7 @@ package storage import ( "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net/http" @@ -74,7 +75,7 @@ func newPosixBackend(config posixConf) (*posixBackend, error) { // NewFileReader returns an io.Reader instance func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if pb == nil { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, errors.New("Invalid posixBackend") } file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -90,7 +91,7 @@ func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter returns an io.Writer instance func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if pb == nil { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, errors.New("Invalid posixBackend") } file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) @@ -106,7 +107,7 @@ func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of the file func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { if pb == nil { - return 0, fmt.Errorf("Invalid posixBackend") + return 0, errors.New("Invalid posixBackend") } stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -191,7 +192,7 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { // NewFileReader returns an io.Reader instance func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { if sb == nil { - return nil, fmt.Errorf("Invalid s3Backend") + return nil, errors.New("Invalid s3Backend") } r, err := sb.Client.GetObject(&s3.GetObjectInput{ @@ -225,7 +226,7 @@ func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter uploads the contents of an io.Reader to a S3 bucket func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { if sb == nil { - return nil, fmt.Errorf("Invalid s3Backend") + return nil, errors.New("Invalid s3Backend") } reader, writer := io.Pipe() @@ -248,7 +249,7 @@ func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of a specific object func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { if sb == nil { - return 0, fmt.Errorf("Invalid s3Backend") + return 0, errors.New("Invalid s3Backend") } r, err := sb.Client.HeadObject(&s3.HeadObjectInput{ diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 3967bf818d..cb873b7add 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "math" "net/http" @@ -187,7 +188,7 @@ func checkDB(db *database.SDAdb, timeout time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() if db.DB == nil { - return fmt.Errorf("database is nil") + return errors.New("database is nil") } return db.DB.PingContext(ctx) diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 48fd1a4f9b..df2096cac9 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "errors" "fmt" "io" @@ -250,7 +251,7 @@ func backupFile(delivered amqp.Delivery) error { } if diskFileSize != int64(fileSize) { - return fmt.Errorf("file size in archive does not match database for archive file") + return errors.New("file size in archive does not match database for archive file") } file, err := archive.NewFileReader(filePath) diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index b51a3479f0..a0c11693a6 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "errors" "fmt" "github.com/neicnordic/sensitive-data-archive/internal/broker" @@ -162,12 +163,12 @@ func schemaFromDatasetOperation(body []byte) (string, error) { datasetMessageType, ok := message["type"] if !ok { - return "", fmt.Errorf("malformed message, dataset message type is missing") + return "", errors.New("malformed message, dataset message type is missing") } datasetOpsType, ok := datasetMessageType.(string) if !ok { - return "", fmt.Errorf("could not cast operation attribute to string") + return "", errors.New("could not cast operation attribute to string") } switch datasetOpsType { @@ -178,6 +179,6 @@ func schemaFromDatasetOperation(body []byte) (string, error) { case "deprecate": return "dataset-deprecate", nil default: - return "", fmt.Errorf("could not recognize mapping operation") + return "", errors.New("could not recognize mapping operation") } } diff --git a/sda/cmd/notify/notify.go b/sda/cmd/notify/notify.go index 8e6a923524..8e85ca3ac2 100644 --- a/sda/cmd/notify/notify.go +++ b/sda/cmd/notify/notify.go @@ -4,6 +4,7 @@ package main import ( "encoding/base64" "encoding/json" + "errors" "fmt" "net/smtp" "strconv" @@ -157,5 +158,5 @@ func validator(queue, schemaPath string, delivery amqp091.Delivery) error { return nil } - return fmt.Errorf("Error") + return errors.New("Error") } diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 1f4a074776..ebf18b8e4b 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "encoding/json" "encoding/xml" + "errors" "fmt" "io" "net/http" @@ -276,7 +277,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { var err error if p.messenger == nil { - return fmt.Errorf("messenger is down") + return errors.New("messenger is down") } if p.messenger.IsConnClosed() { log.Warning("connection is closed, reconnecting...") @@ -587,7 +588,7 @@ func formatUploadFilePath(filePath string) (string, error) { // Check for mixed "\" and "/" in filepath. Stop and throw an error if true so that // we do not end up with unintended folder structure when applying ReplaceAll below if strings.Contains(filePath, "\\") && strings.Contains(filePath, "/") { - return filePath, fmt.Errorf("filepath contains mixed '\\' and '/' characters") + return filePath, errors.New("filepath contains mixed '\\' and '/' characters") } // make any windows path separators linux compatible diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 187e95d28c..313e389661 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "database/sql" "encoding/json" + "errors" "fmt" "net" "net/http" @@ -213,7 +214,7 @@ func NewMockMessenger() *MockMessenger { func (m *MockMessenger) SendMessage(uuid string, body []byte) error { if uuid == "" || body == nil { - return fmt.Errorf("bad message") + return errors.New("bad message") } return nil diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 1f78cc6132..8dcf42e3e8 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -5,6 +5,7 @@ package main import ( "bytes" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -211,7 +212,7 @@ func syncFiles(stableID string) error { if err != nil || copiedSize != int64(fileSize) { switch { case copiedSize != int64(fileSize): - return fmt.Errorf("copied size does not match file size") + return errors.New("copied size does not match file size") default: return err } diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index e5c20063ad..54afbbb52b 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -227,7 +227,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) default: - return nil, fmt.Errorf("inbox.type not set") + return nil, errors.New("inbox.type not set") } case "auth": requiredConfVars = []string{ @@ -273,7 +273,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) default: - return nil, fmt.Errorf("archive.type not set") + return nil, errors.New("archive.type not set") } switch viper.GetString("inbox.type") { @@ -282,7 +282,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) default: - return nil, fmt.Errorf("inbox.type not set") + return nil, errors.New("inbox.type not set") } case "finalize": requiredConfVars = []string{ @@ -408,7 +408,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) default: - return nil, fmt.Errorf("archive.type not set") + return nil, errors.New("archive.type not set") } switch viper.GetString("sync.destination.type") { @@ -419,7 +419,7 @@ func NewConfig(app string) (*Config, error) { case SFTP: requiredConfVars = append(requiredConfVars, []string{"sync.destination.sftp.host", "sync.destination.sftp.port", "sync.destination.sftp.userName", "sync.destination.sftp.pemKeyPath", "sync.destination.sftp.pemKeyPass"}...) default: - return nil, fmt.Errorf("sync.destination.type not set") + return nil, errors.New("sync.destination.type not set") } case "sync-api": requiredConfVars = []string{ @@ -452,7 +452,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) default: - return nil, fmt.Errorf("archive.type not set") + return nil, errors.New("archive.type not set") } default: @@ -508,7 +508,7 @@ func NewConfig(app string) (*Config, error) { } if (c.Auth.OIDC.ID == "" || c.Auth.OIDC.Secret == "") && (c.Auth.Cega.ID == "" || c.Auth.Cega.Secret == "") { - return nil, fmt.Errorf("neither cega or oidc login configured") + return nil, errors.New("neither cega or oidc login configured") } c.Auth.InfoURL = viper.GetString("auth.infoUrl") diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index 4295dce4fa..bb00a755f9 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -4,6 +4,7 @@ package database import ( "database/sql" + "errors" "fmt" "time" @@ -117,7 +118,7 @@ func (dbs *SDAdb) Connect() error { } // default error - err := fmt.Errorf("failed to connect within reconnect time") + err := errors.New("failed to connect within reconnect time") log.Infoln("Connecting to database") log.Debugf("host: %s:%d, database: %s, user: %s", dbs.Config.Host, dbs.Config.Port, dbs.Config.Database, dbs.Config.User) diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 9f8481112e..66be75d5c6 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -806,7 +806,7 @@ func (dbs *SDAdb) getCorrID(user, path, accession string) (string, error) { return "", rows.Err() } - return "", fmt.Errorf("sql: no rows in result set") + return "", errors.New("sql: no rows in result set") } // list all users with files not yet assigned to a dataset diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index f09c435928..e337768d83 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "log" "math/big" @@ -96,7 +97,7 @@ type AlwaysDeny struct{} // Authenticate does not authenticate anyone. func (u *AlwaysDeny) Authenticate(_ *http.Request) (jwt.Token, error) { - return nil, fmt.Errorf("denied") + return nil, errors.New("denied") } // MakeFolder creates a folder and subfolders for the keys pair diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go index 6f2d771422..f249ed52f7 100644 --- a/sda/internal/schema/schema.go +++ b/sda/internal/schema/schema.go @@ -2,6 +2,7 @@ package schema import ( "encoding/json" + "errors" "fmt" "path/filepath" "strings" @@ -12,7 +13,7 @@ import ( func ValidateJSON(reference string, body []byte) error { dest := getStructName(reference) if dest == "" { - return fmt.Errorf("unknown reference schema") + return errors.New("unknown reference schema") } compiler := jsonschema.NewCompiler() compiler.Draft = jsonschema.Draft7 diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index 66d5ff3407..3788301770 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -84,7 +84,7 @@ func newPosixBackend(conf posixConf) (*posixBackend, error) { // NewFileReader returns an io.Reader instance func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if pb == nil { - return nil, fmt.Errorf("invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -100,7 +100,7 @@ func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter returns an io.Writer instance func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if pb == nil { - return nil, fmt.Errorf("invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) @@ -116,7 +116,7 @@ func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of the file func (pb *posixBackend) GetFileSize(filePath string, _ bool) (int64, error) { if pb == nil { - return 0, fmt.Errorf("invalid posixBackend") + return 0, errors.New("invalid posixBackend") } stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -132,7 +132,7 @@ func (pb *posixBackend) GetFileSize(filePath string, _ bool) (int64, error) { // RemoveFile removes a file from a given path func (pb *posixBackend) RemoveFile(filePath string) error { if pb == nil { - return fmt.Errorf("invalid posixBackend") + return errors.New("invalid posixBackend") } err := os.Remove(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -431,7 +431,7 @@ func newSftpBackend(conf SftpConf) (*sftpBackend, error) { // NewFileWriter returns an io.Writer instance for the sftp remote func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if sfb == nil { - return nil, fmt.Errorf("invalid sftpBackend") + return nil, errors.New("invalid sftpBackend") } // Make remote directories parent := filepath.Dir(filePath) @@ -451,7 +451,7 @@ func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of the file func (sfb *sftpBackend) GetFileSize(filePath string, _ bool) (int64, error) { if sfb == nil { - return 0, fmt.Errorf("invalid sftpBackend") + return 0, errors.New("invalid sftpBackend") } stat, err := sfb.Client.Lstat(filePath) @@ -465,7 +465,7 @@ func (sfb *sftpBackend) GetFileSize(filePath string, _ bool) (int64, error) { // NewFileReader returns an io.Reader instance func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if sfb == nil { - return nil, fmt.Errorf("invalid sftpBackend") + return nil, errors.New("invalid sftpBackend") } file, err := sfb.Client.Open(filePath) @@ -479,7 +479,7 @@ func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // RemoveFile removes a file or an empty directory. func (sfb *sftpBackend) RemoveFile(filePath string) error { if sfb == nil { - return fmt.Errorf("invalid sftpBackend") + return errors.New("invalid sftpBackend") } err := sfb.Client.Remove(filePath) diff --git a/sda/internal/userauth/userauth.go b/sda/internal/userauth/userauth.go index 375e98eff7..13c73005e5 100644 --- a/sda/internal/userauth/userauth.go +++ b/sda/internal/userauth/userauth.go @@ -2,6 +2,7 @@ package userauth import ( "context" + "errors" "fmt" "net/http" "net/url" @@ -39,13 +40,13 @@ func NewValidateFromToken(keyset jwk.Set) *ValidateFromToken { func (u *ValidateFromToken) Authenticate(r *http.Request) (jwt.Token, error) { // Verify signature by parsing the token with the given key if u == nil { - return nil, fmt.Errorf("error validating token keyset") + return nil, errors.New("error validating token keyset") } switch { case r.Header.Get("X-Amz-Security-Token") != "": tokenStr := r.Header.Get("X-Amz-Security-Token") if tokenStr == "" { - return nil, fmt.Errorf("no access token supplied") + return nil, errors.New("no access token supplied") } token, err := jwt.Parse([]byte(tokenStr), jwt.WithKeySet(u.Keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) if err != nil { @@ -78,7 +79,7 @@ func (u *ValidateFromToken) Authenticate(r *http.Request) (jwt.Token, error) { return token, nil default: - return nil, fmt.Errorf("no access token supplied") + return nil, errors.New("no access token supplied") } } @@ -153,10 +154,10 @@ func (u *ValidateFromToken) FetchJwtPubKeyURL(jwtpubkeyurl string) error { func readTokenFromHeader(authStr string) (string, error) { headerParts := strings.Split(authStr, " ") if headerParts[0] != "Bearer" { - return "", fmt.Errorf("authorization scheme must be bearer") + return "", errors.New("authorization scheme must be bearer") } if len(headerParts) != 2 || headerParts[1] == "" { - return "", fmt.Errorf("token string is missing from authorization header") + return "", errors.New("token string is missing from authorization header") } return headerParts[1], nil From 7537ebdc06d9e2b6cc55e165c9cc7ddf8811be88 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 10:59:45 +0200 Subject: [PATCH 209/250] [revive] remove redundant aliases --- sda/cmd/auth/cega.go | 2 +- sda/cmd/orchestrate/orchestrate.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/auth/cega.go b/sda/cmd/auth/cega.go index 8f99aa9977..2cd199bdbd 100644 --- a/sda/cmd/auth/cega.go +++ b/sda/cmd/auth/cega.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/neicnordic/sensitive-data-archive/internal/config" - bcrypt "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/bcrypt" ) // CegaUserResponse captures the response list diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index ce78a456e9..5c847b6542 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -9,7 +9,7 @@ import ( "os" "time" - uuid "github.com/google/uuid" + "github.com/google/uuid" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/schema" From 4b74de4b5e43df648afa7f2a91a58a1bb27fbdab Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 11:08:29 +0200 Subject: [PATCH 210/250] [revive] return early --- sda-download/internal/config/config.go | 5 ++--- sda-download/pkg/auth/auth.go | 5 ++--- sda/internal/config/config.go | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 102c436e04..70965ec444 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -171,11 +171,10 @@ func NewConfig() (*Map, error) { } if err := viper.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - log.Infoln("No config file found, using ENVs only") - } else { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { return nil, err } + log.Infoln("No config file found, using ENVs only") } requiredConfVars := []string{ "db.host", "db.user", "db.password", "db.database", "oidc.configuration.url", "grpc.host", diff --git a/sda-download/pkg/auth/auth.go b/sda-download/pkg/auth/auth.go index 7133ecfd9a..2c403deb15 100644 --- a/sda-download/pkg/auth/auth.go +++ b/sda-download/pkg/auth/auth.go @@ -123,13 +123,12 @@ var GetToken = func(headers http.Header) (string, int, error) { // Check that header contains a token string var token string - if len(headerParts) == 2 { - token = headerParts[1] - } else { + if len(headerParts) != 2 { log.Debug("authorization check failed") return "", 400, errors.New("token string is missing from authorization header") } + token = headerParts[1] log.Debug("access token found") return token, 0, nil diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 54afbbb52b..3d11ffa711 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -180,13 +180,12 @@ func NewConfig(app string) (*Config, error) { log.Infoln("reading config") if err := viper.ReadInConfig(); err != nil { log.Infoln(err.Error()) - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - log.Infoln("No config file found, using ENVs only") - } else { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { log.Infoln("ReadInConfig Error") return nil, err } + log.Infoln("No config file found, using ENVs only") } if viper.IsSet("log.format") { From 2410fa8e267d8bbb4323213fc4cfb9db4266f07d Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 11:23:19 +0200 Subject: [PATCH 211/250] [revive] don't use defer inside of loops --- sda-download/api/s3/s3_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go index 4e19159ac2..5c05a05af9 100644 --- a/sda-download/api/s3/s3_test.go +++ b/sda-download/api/s3/s3_test.go @@ -303,7 +303,7 @@ func (ts *S3TestSuite) TestParseParams() { router.ServeHTTP(w, httptest.NewRequest("GET", params.Path, nil)) response := w.Result() - defer response.Body.Close() + response.Body.Close() assert.Equal(ts.T(), http.StatusAccepted, response.StatusCode, "Request failed") } From 36cee5b7c767ae491292e31c8375b79d3a138cf6 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 11:26:44 +0200 Subject: [PATCH 212/250] [sda-download] ignore unnecessary-stmt error --- sda-download/cmd/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index 2c4b2c347b..11734022df 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -26,7 +26,7 @@ func init() { // Set middleware // nolint:gocritic // this nolint can be removed, if you have more than one middlewares available - switch conf.App.Middleware { + switch conf.App.Middleware { //nolint:revive default: api.SelectedMiddleware = middleware.TokenMiddleware } From d2768c714c1aec18772084c6569fbd4fcfc552f0 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 11:34:40 +0200 Subject: [PATCH 213/250] [static check] sda --- sda/cmd/api/api.go | 10 ++++------ sda/cmd/ingest/ingest_test.go | 4 ++-- sda/internal/config/config.go | 6 +++--- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index cb873b7add..9d9d94dddb 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -268,10 +268,9 @@ func ingestFile(c *gin.Context) { corrID, err := Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath, "") if err != nil { - switch { - case corrID == "": + if corrID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - default: + } else { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) } @@ -361,10 +360,9 @@ func setAccession(c *gin.Context) { corrID, err := Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") if err != nil { - switch { - case corrID == "": + if corrID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - default: + } else { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) } diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 3a1e347bc5..bd647a2e93 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -348,8 +348,8 @@ func (ts *TestSuite) TestTryDecrypt() { for i, key := range privateKeys { header, err := tryDecrypt(key, buf) - switch { - case i == 0: + switch i { + case 0: assert.NoError(ts.T(), err) assert.NotNil(ts.T(), header) default: diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 3d11ffa711..27de641f69 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -782,7 +782,7 @@ func (c *Config) configBroker() error { mq.VerifyPeer = viper.GetBool("broker.verifyPeer") if mq.VerifyPeer { // Since verifyPeer is specified, these are required. - if !(viper.IsSet("broker.clientCert") && viper.IsSet("broker.clientKey")) { + if !viper.IsSet("broker.clientCert") && !viper.IsSet("broker.clientKey") { return errors.New("when broker.verifyPeer is set both broker.clientCert and broker.clientKey is needed") } mq.ClientCert = viper.GetString("broker.clientCert") @@ -818,7 +818,7 @@ func (c *Config) configDatabase() error { // Optional settings if db.SslMode == "verify-full" { // Since verify-full is specified, these are required. - if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { + if !viper.IsSet("db.clientCert") && !viper.IsSet("db.clientKey") { return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") } } @@ -1119,7 +1119,7 @@ func GetC4GHPublicKey() (*[32]byte, error) { func (c *Config) configServer() error { s := ServerConfig{} - if !(viper.IsSet("server.jwtpubkeypath") || viper.IsSet("server.jwtpubkeyurl")) { + if !viper.IsSet("server.jwtpubkeypath") && !viper.IsSet("server.jwtpubkeyurl") { return errors.New("either server.pubkeypath or server.jwtpubkeyurl should be present to start the service") } From 477d6f258107bac88570ef8ebf93853cc8f21a21 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 11:34:54 +0200 Subject: [PATCH 214/250] [static check] sda-admin --- sda-admin/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sda-admin/main.go b/sda-admin/main.go index c6499ae8bb..62490f6aaf 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -21,7 +21,7 @@ var ( ) // Command-line usage -const usage = `Usage: sda-admin [-uri URI] [-token TOKEN] [options] +const usage = `usage: sda-admin [-uri URI] [-token TOKEN] [options] Commands: user list List all users. @@ -42,7 +42,7 @@ Global Options: Additional Commands: version Show the version of sda-admin. help Show this help message. - -h, -help Show this help message.` + -h, -help Show this help message` var userUsage = `List Users: Usage: sda-admin user list From 56990b83ef040c4506b534c9d75b35ac6478b4dd Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 11:44:18 +0200 Subject: [PATCH 215/250] [static-check] sda-download --- sda-download/api/sda/sda.go | 2 +- sda-download/internal/config/config.go | 2 +- sda-download/internal/database/database.go | 16 ++++++------- sda-download/internal/storage/seekable.go | 28 +++++++++++----------- sda-download/internal/storage/storage.go | 14 +++++------ sda-download/pkg/request/request.go | 2 +- 6 files changed, 32 insertions(+), 32 deletions(-) diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index e0354eafc7..53a08ae73d 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -287,7 +287,7 @@ func Download(c *gin.Context) { return } - wholeFile := true + wholeFile := true //nolint:staticcheck if start != 0 || end != 0 { wholeFile = false } diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index 70965ec444..2e9a7129c0 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -432,7 +432,7 @@ func (c *Map) configDatabase() error { } if db.SslMode == "verify-full" { // Since verify-full is specified, these are required. - if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { + if !viper.IsSet("db.clientCert") && !viper.IsSet("db.clientKey") { return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") } } diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index b6e7bf09f3..ec46a393db 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -134,7 +134,7 @@ var GetFiles = func(datasetID string) ([]*FileInfo, error) { var ( r []*FileInfo = nil err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { @@ -251,9 +251,9 @@ func (dbs *SQLdb) getFiles(datasetID string) ([]*FileInfo, error) { // CheckDataset checks if dataset name exists var CheckDataset = func(dataset string) (bool, error) { var ( - r bool = false + r = false err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { @@ -291,7 +291,7 @@ var GetDatasetInfo = func(datasetID string) (*DatasetInfo, error) { var ( d *DatasetInfo = nil err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { @@ -409,9 +409,9 @@ func (dbs *SQLdb) getDatasetFileInfo(datasetID, filePath string) (*FileInfo, err // CheckFilePermission checks if user has permissions to access the dataset the file is a part of var CheckFilePermission = func(fileID string) (string, error) { var ( - r string = "" - err error = nil - count int = 0 + r = "" + err error = nil + count = 0 ) for count < dbRetryTimes { @@ -467,7 +467,7 @@ var GetFile = func(fileID string) (*FileDownload, error) { var ( r *FileDownload = nil err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { r, err = DB.getFile(fileID) diff --git a/sda-download/internal/storage/seekable.go b/sda-download/internal/storage/seekable.go index 383a81db68..845d150a70 100644 --- a/sda-download/internal/storage/seekable.go +++ b/sda-download/internal/storage/seekable.go @@ -22,7 +22,7 @@ func (pb *posixBackend) NewFileReadSeeker(filePath string) (io.ReadSeekCloser, e seeker, ok := reader.(io.ReadSeekCloser) if !ok { - return nil, errors.New("Invalid posixBackend") + return nil, errors.New("invalid posixBackend") } return seeker, nil @@ -176,10 +176,10 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: if offset < 0 { - return r.currentOffset, fmt.Errorf("Invalid offset %v- can't be negative when seeking from start", offset) + return r.currentOffset, fmt.Errorf("invalid offset %v- can't be negative when seeking from start", offset) } if offset > r.objectSize { - return r.currentOffset, fmt.Errorf("Invalid offset %v - beyond end of object (size %v)", offset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset %v - beyond end of object (size %v)", offset, r.objectSize) } r.currentOffset = offset @@ -189,10 +189,10 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { case io.SeekCurrent: if r.currentOffset+offset < 0 { - return r.currentOffset, fmt.Errorf("Invalid offset %v from %v would be be before start", offset, r.currentOffset) + return r.currentOffset, fmt.Errorf("invalid offset %v from %v would be be before start", offset, r.currentOffset) } if offset > r.objectSize { - return r.currentOffset, fmt.Errorf("Invalid offset - %v from %v would end up beyond of object %v", offset, r.currentOffset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset - %v from %v would end up beyond of object %v", offset, r.currentOffset, r.objectSize) } r.currentOffset += offset @@ -202,10 +202,10 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { case io.SeekEnd: if r.objectSize+offset < 0 { - return r.currentOffset, fmt.Errorf("Invalid offset %v from end in %v bytes object, would be before file start", offset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset %v from end in %v bytes object, would be before file start", offset, r.objectSize) } if r.objectSize+offset > r.objectSize { - return r.currentOffset, fmt.Errorf("Invalid offset %v from end in %v bytes object", offset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset %v from end in %v bytes object", offset, r.objectSize) } r.currentOffset = r.objectSize + offset @@ -214,7 +214,7 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { return r.currentOffset, nil } - return r.currentOffset, errors.New("Bad whence") + return r.currentOffset, errors.New("bad whence") } // removeFromOutstanding removes a prefetch from the list of outstanding prefetches once it's no longer active @@ -353,7 +353,7 @@ func (r *s3Reader) Read(dst []byte) (n int, err error) { responseRange := fmt.Sprintf("bytes %d-", r.currentOffset) if object.ContentRange == nil || !strings.HasPrefix(*object.ContentRange, responseRange) { - return 0, fmt.Errorf("Unexpected content range %v - expected prefix %v", object.ContentRange, responseRange) + return 0, fmt.Errorf("unexpected content range %v - expected prefix %v", object.ContentRange, responseRange) } b := bytes.Buffer{} @@ -391,12 +391,12 @@ func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { for i, reader := range readers { seeker, ok := reader.(io.ReadSeeker) if !ok { - return nil, fmt.Errorf("Reader %d to SeekableMultiReader is not seekable", i) + return nil, fmt.Errorf("reader %d to SeekableMultiReader is not seekable", i) } size, err := seeker.Seek(0, io.SeekEnd) if err != nil { - return nil, fmt.Errorf("Size determination failed for reader %d to SeekableMultiReader: %v", i, err) + return nil, fmt.Errorf("size determination failed for reader %d to SeekableMultiReader: %v", i, err) } sizes[i] = size @@ -415,7 +415,7 @@ func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { case io.SeekEnd: r.currentOffset = r.totalSize + offset default: - return 0, errors.New("Unsupported whence") + return 0, errors.New("unsupported whence") } return r.currentOffset, nil @@ -443,12 +443,12 @@ func (r *seekableMultiReader) Read(dst []byte) (int, error) { seekable, ok := reader.(io.ReadSeeker) if !ok { - return 0, errors.New("Expected seekable reader but changed") + return 0, errors.New("expected seekable reader but changed") } _, err := seekable.Seek(r.currentOffset-int64(readerStartAt), 0) if err != nil { - return 0, fmt.Errorf("Unexpected error while seeking: %v", err) + return 0, fmt.Errorf("unexpected error while seeking: %v", err) } n, err := seekable.Read(dst) diff --git a/sda-download/internal/storage/storage.go b/sda-download/internal/storage/storage.go index 98c37aa088..9c499b81e0 100644 --- a/sda-download/internal/storage/storage.go +++ b/sda-download/internal/storage/storage.go @@ -75,7 +75,7 @@ func newPosixBackend(config posixConf) (*posixBackend, error) { // NewFileReader returns an io.Reader instance func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if pb == nil { - return nil, errors.New("Invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -91,7 +91,7 @@ func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter returns an io.Writer instance func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if pb == nil { - return nil, errors.New("Invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) @@ -107,7 +107,7 @@ func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of the file func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { if pb == nil { - return 0, errors.New("Invalid posixBackend") + return 0, errors.New("invalid posixBackend") } stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -165,7 +165,7 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { if aerr, ok := err.(awserr.Error); ok { if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && aerr.Code() != s3.ErrCodeBucketAlreadyExists { - log.Error("Unexpected issue while creating bucket", err) + log.Error("unexpected issue while creating bucket", err) } } } @@ -192,7 +192,7 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { // NewFileReader returns an io.Reader instance func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { if sb == nil { - return nil, errors.New("Invalid s3Backend") + return nil, errors.New("invalid s3Backend") } r, err := sb.Client.GetObject(&s3.GetObjectInput{ @@ -226,7 +226,7 @@ func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter uploads the contents of an io.Reader to a S3 bucket func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { if sb == nil { - return nil, errors.New("Invalid s3Backend") + return nil, errors.New("invalid s3Backend") } reader, writer := io.Pipe() @@ -249,7 +249,7 @@ func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of a specific object func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { if sb == nil { - return 0, errors.New("Invalid s3Backend") + return 0, errors.New("invalid s3Backend") } r, err := sb.Client.HeadObject(&s3.HeadObjectInput{ diff --git a/sda-download/pkg/request/request.go b/sda-download/pkg/request/request.go index 5aba81fb7c..49cca9c61f 100644 --- a/sda-download/pkg/request/request.go +++ b/sda-download/pkg/request/request.go @@ -56,7 +56,7 @@ var HTTPNewRequest = http.NewRequest var MakeRequest = func(method string, url string, headers map[string]string, body []byte) (*http.Response, error) { var ( response *http.Response - count int = 0 + count = 0 ) // Build HTTP request From 7e61315741cfaf52d85cea9b0b35dbd74ecea303 Mon Sep 17 00:00:00 2001 From: Joakim Bygdell Date: Tue, 15 Apr 2025 11:45:17 +0200 Subject: [PATCH 216/250] Final linter config file --- .golangci.yml | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/.golangci.yml b/.golangci.yml index bd04b6b0be..c05076ac27 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -12,7 +12,7 @@ linters: - govet - nestif - nlreturn - # - revive + - revive - rowserrcheck - staticcheck settings: @@ -23,3 +23,38 @@ linters: - G304 - G401 - G501 + revive: + enable-all-rules: true + rules: + - name: add-constant + disabled: true + - name: bare-return + disabled: true + - name: cognitive-complexity # disable for now + disabled: true + - name: confusing-naming # disable for now + disabled: true + - name: confusing-results + disabled: true + - name: cyclomatic # disable for now, requires refactoring + disabled: true + - name: exported + disabled: true + - name: flag-parameter # this one requires refactoring + disabled: true + - name: function-length # this one requires refactoring + disabled: true + - name: get-return # this triggers on function names + disabled: true + - name: line-length-limit + disabled: true + - name: max-public-structs + disabled: true + - name: package-comments + disabled: true + - name: unchecked-type-assertion # disable for now + disabled: true + - name: unused-parameter # disable for now + disabled: true + - name: unused-receiver # disable for now + disabled: true From 4bb962036619c9a82e48465366bbcf1f28ab3aec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 23:19:53 +0000 Subject: [PATCH 217/250] Bump golang.org/x/net in /sda-download in the go_modules group Bumps the go_modules group in /sda-download with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.36.0 to 0.38.0 - [Commits](https://github.com/golang/net/compare/v0.36.0...v0.38.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.38.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 09767f9169..328d21aa10 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -70,7 +70,7 @@ require ( github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect - golang.org/x/net v0.36.0 // indirect + golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect golang.org/x/tools v0.25.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index 3069d973ef..344d7d47b5 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -189,8 +189,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.36.0 h1:vWF2fRbw4qslQsQzgFqZff+BItCvGFQqKzKIzx1rmoA= -golang.org/x/net v0.36.0/go.mod h1:bFmbeoIPfrw4sMHNhb4J9f6+tPziuGjq7Jk/38fxi1I= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From d34d57f22f96a1026c7961a2679149e001a95350 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 16 Apr 2025 23:25:36 +0000 Subject: [PATCH 218/250] Bump golang.org/x/net in /sda in the go_modules group Bumps the go_modules group in /sda with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.37.0 to 0.38.0 - [Commits](https://github.com/golang/net/compare/v0.37.0...v0.38.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.38.0 dependency-type: indirect dependency-group: go_modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 2 +- sda/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 318bcd39e0..ef5b1e586d 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -156,7 +156,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.37.0 // indirect + golang.org/x/net v0.38.0 // indirect golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect golang.org/x/time v0.8.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index f7bb47b95f..3bf6f57a7b 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -435,8 +435,8 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= -golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 1fab6fe5456a1a6e14967bd383eccde25c8eb090 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Apr 2025 19:32:02 +0000 Subject: [PATCH 219/250] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/lestrrat-go/jwx/v2](https://github.com/lestrrat-go/jwx) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/lestrrat-go/jwx/v2` from 2.1.4 to 2.1.5 - [Release notes](https://github.com/lestrrat-go/jwx/releases) - [Changelog](https://github.com/lestrrat-go/jwx/blob/v2.1.5/Changes) - [Commits](https://github.com/lestrrat-go/jwx/compare/v2.1.4...v2.1.5) Updates `google.golang.org/grpc` from 1.71.1 to 1.72.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.71.1...v1.72.0) --- updated-dependencies: - dependency-name: github.com/lestrrat-go/jwx/v2 dependency-version: 2.1.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.72.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 8 ++++---- sda-download/go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 09767f9169..7e147c8731 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -11,7 +11,7 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 - github.com/lestrrat-go/jwx/v2 v2.1.4 + github.com/lestrrat-go/jwx/v2 v2.1.5 github.com/lib/pq v1.10.9 github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 @@ -19,7 +19,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.37.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - google.golang.org/grpc v1.71.1 + google.golang.org/grpc v1.72.0 google.golang.org/protobuf v1.36.6 ) @@ -46,7 +46,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lestrrat-go/blackmagic v1.0.2 // indirect + github.com/lestrrat-go/blackmagic v1.0.3 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect @@ -74,7 +74,7 @@ require ( golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect golang.org/x/tools v0.25.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index 3069d973ef..4a17997af3 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -84,16 +84,16 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= -github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= +github.com/lestrrat-go/blackmagic v1.0.3 h1:94HXkVLxkZO9vJI/w2u1T0DAoprShFd13xtnSINtDWs= +github.com/lestrrat-go/blackmagic v1.0.3/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.4 h1:uBCMmJX8oRZStmKuMMOFb0Yh9xmEMgNJLgjuKKt4/qc= -github.com/lestrrat-go/jwx/v2 v2.1.4/go.mod h1:nWRbDFR1ALG2Z6GJbBXzfQaYyvn751KuuyySN2yR6is= +github.com/lestrrat-go/jwx/v2 v2.1.5 h1:PQI5gzadLfJ22ckLrejPVX6eGXKM4M4eGi5fW2jjA3o= +github.com/lestrrat-go/jwx/v2 v2.1.5/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -231,10 +231,10 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= -google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= -google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From f6b0fa1d09d3550506ea3945a56b164a324e293b Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Wed, 9 Apr 2025 23:49:40 +0200 Subject: [PATCH 220/250] docs: update documentation for running the download service --- DEVELOPMENT.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index be6bdc9991..5969badb6a 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -27,14 +27,15 @@ If successful, the curl command should output the HTTP code `200`. You can further check the endpoint `/metadata/datasets` using: ```sh -token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets ``` -If successful, the curl command should output a JSON body containing: +where `token` can be obtained by logging to http://localhost:8801 with the dummy user. + +If successful, the curl command should output a JSON body with an empty list and with HTTP code `200`: ```json -["EGAD74900000101"] +[] ``` ## Running other SDA services with `go run` From a4e250dd2c4248a16657445e0860b31b247166b0 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Thu, 10 Apr 2025 00:16:10 +0200 Subject: [PATCH 221/250] docs: update documentation for running the ingest service --- DEVELOPMENT.md | 61 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5969badb6a..5e7d8da891 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -76,27 +76,52 @@ CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go - Check if the `ingest` service works as expected by following these steps -```sh -# create a test file -seq 10 > /tmp/t1.txt + - Create a test file -# update the s3cmd config file -sed -i '/host_/s/s3inbox:8000/localhost:18000/g' /tmp/shared/s3cfg + ```sh + seq 10 > /tmp/t1.txt + ``` -# upload /tmp/t1.txt to s3inbox by sda-cli -sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt + - Download the s3cmd config file (`s3cmd-inbox.conf`) by logging to http://localhost:8801 with the admin user. After that update the config file by running -# use sda-admin to check if t1.txt has been uploaded -export API_HOST=http://localhost:8090 -export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') -sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'uploaded' + ```sh + sed -i '/host_/s/inbox:8000/localhost:18000/g' s3cmd-inbox.conf + sed -i 's/use_https = True/use_https = False/g' s3cmd-inbox.conf + ``` -# register the Crypt4GH key -curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add + - Upload the test file to S3inbox by `sda-cli` -# use sda-admin to ingest the file t1.txt -sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org + ```sh + sda-cli -config s3cmd-inbox.conf upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt + ``` -# verify that t1.txt has been ingested using sda-admin -sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'verified' -``` + - Use `sda-admin` to check if the test file has been uploaded + + ```sh + export API_HOST=http://localhost:8090 + export ACCESS_TOKEN=$(grep access_token s3cmd-inbox.conf | awk '{print $3}') + sda-admin file list -user testu@lifescience-ri.eu + ``` + + The file `testu_lifescience-ri.eu/t1.txt.c4gh` should have fileStatus `uploaded` + + + - Register the Crypt4GH key + + ```sh + curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add + ``` + + - Ingest the file test file + + ```sh + sda-admin file ingest -filepath testu_lifescience-ri.eu/t1.txt.c4gh -user testu@lifescience-ri.eu + ``` + + - Verify that the test file has been ingested + + ```sh + sda-admin file list -user testu@lifescience-ri.eu + ``` + + The file `testu_lifescience-ri.eu/t1.txt.c4gh` should have fileStatus `verified` From d5fa707a7217e733d68878239f6cbc442e205d12 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Mon, 14 Apr 2025 15:10:21 +0200 Subject: [PATCH 222/250] [docs] add reviewers suggestion --- DEVELOPMENT.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 5e7d8da891..5a3b15ea6e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -112,6 +112,14 @@ CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add ``` + If the key is already registered in the database, you may see the following error: + + ``` + {"error":"key hash already exists or no rows were updated","status":409} + ``` + + This is expected and can be safely ignored. + - Ingest the file test file ```sh From 6170c18ff39739094d87335ac09d45b7d24f41eb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 22 Apr 2025 08:16:42 +0000 Subject: [PATCH 223/250] Bump the all-modules group in /sda with 3 updates Bumps the all-modules group in /sda with 3 updates: [github.com/casbin/casbin/v2](https://github.com/casbin/casbin), [github.com/lestrrat-go/jwx/v2](https://github.com/lestrrat-go/jwx) and [google.golang.org/grpc](https://github.com/grpc/grpc-go). Updates `github.com/casbin/casbin/v2` from 2.104.0 to 2.105.0 - [Release notes](https://github.com/casbin/casbin/releases) - [Changelog](https://github.com/casbin/casbin/blob/master/.releaserc.json) - [Commits](https://github.com/casbin/casbin/compare/v2.104.0...v2.105.0) Updates `github.com/lestrrat-go/jwx/v2` from 2.1.4 to 2.1.5 - [Release notes](https://github.com/lestrrat-go/jwx/releases) - [Changelog](https://github.com/lestrrat-go/jwx/blob/v2.1.5/Changes) - [Commits](https://github.com/lestrrat-go/jwx/compare/v2.1.4...v2.1.5) Updates `google.golang.org/grpc` from 1.71.1 to 1.72.0 - [Release notes](https://github.com/grpc/grpc-go/releases) - [Commits](https://github.com/grpc/grpc-go/compare/v1.71.1...v1.72.0) --- updated-dependencies: - dependency-name: github.com/casbin/casbin/v2 dependency-version: 2.105.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: github.com/lestrrat-go/jwx/v2 dependency-version: 2.1.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: google.golang.org/grpc dependency-version: 1.72.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 10 +++++----- sda/go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index ef5b1e586d..1ed0ca44ef 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -11,14 +11,14 @@ require ( github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 github.com/aws/smithy-go v1.22.3 - github.com/casbin/casbin/v2 v2.104.0 + github.com/casbin/casbin/v2 v2.105.0 github.com/coreos/go-oidc/v3 v3.14.1 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 github.com/kataras/iris/v12 v12.2.11 - github.com/lestrrat-go/jwx/v2 v2.1.4 + github.com/lestrrat-go/jwx/v2 v2.1.5 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 @@ -35,7 +35,7 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.37.0 golang.org/x/oauth2 v0.29.0 - google.golang.org/grpc v1.71.1 + google.golang.org/grpc v1.72.0 google.golang.org/protobuf v1.36.6 ) @@ -112,7 +112,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lestrrat-go/blackmagic v1.0.2 // indirect + github.com/lestrrat-go/blackmagic v1.0.3 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect @@ -160,7 +160,7 @@ require ( golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect golang.org/x/time v0.8.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 3bf6f57a7b..50872a5b44 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -71,8 +71,8 @@ github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKz github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/casbin/casbin/v2 v2.104.0 h1:qDakyBZ4jUg1VskF1+UzIwkg+uXWcp0u0M9PMm1RsTA= -github.com/casbin/casbin/v2 v2.104.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.105.0 h1:dLj5P6pLApBRat9SADGiLxLZjiDPvA1bsPkyV4PGx6I= +github.com/casbin/casbin/v2 v2.105.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -224,16 +224,16 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= -github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= +github.com/lestrrat-go/blackmagic v1.0.3 h1:94HXkVLxkZO9vJI/w2u1T0DAoprShFd13xtnSINtDWs= +github.com/lestrrat-go/blackmagic v1.0.3/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.4 h1:uBCMmJX8oRZStmKuMMOFb0Yh9xmEMgNJLgjuKKt4/qc= -github.com/lestrrat-go/jwx/v2 v2.1.4/go.mod h1:nWRbDFR1ALG2Z6GJbBXzfQaYyvn751KuuyySN2yR6is= +github.com/lestrrat-go/jwx/v2 v2.1.5 h1:PQI5gzadLfJ22ckLrejPVX6eGXKM4M4eGi5fW2jjA3o= +github.com/lestrrat-go/jwx/v2 v2.1.5/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -511,10 +511,10 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f h1:OxYkA3wjPsZyBylwymxSHa7ViiW1Sml4ToBrncvFehI= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f/go.mod h1:+2Yz8+CLJbIfL9z73EW45avw8Lmge3xVElCP9zEKi50= -google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= -google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From aa1486853af30b39bfb3b3b1bbd50661c6cf7f33 Mon Sep 17 00:00:00 2001 From: Emmi Rehn Date: Tue, 15 Apr 2025 16:48:40 +0300 Subject: [PATCH 224/250] [syncapi] Fix error handling when parsing dataset fails --- sda/cmd/ingest/ingest.go | 2 +- sda/cmd/sync/sync_test.go | 2 ++ sda/cmd/syncapi/syncapi.go | 7 +++++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index fa82ac9520..6c66889fb6 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -364,7 +364,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "ack" default: - log.Errorf("unexpected eror when opening file for reading: %s", err.Error()) + log.Errorf("unexpected error when opening file for reading: %s", err.Error()) return "nack" } diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index f1c24ae87e..95d35a9810 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -199,6 +199,8 @@ func (s *SyncTest) TestSendPOST() { username, _, ok := r.BasicAuth() if ok && username == "foo" { w.WriteHeader(http.StatusUnauthorized) + + return } w.WriteHeader(http.StatusOK) diff --git a/sda/cmd/syncapi/syncapi.go b/sda/cmd/syncapi/syncapi.go index 5fe07f229f..672df53f37 100644 --- a/sda/cmd/syncapi/syncapi.go +++ b/sda/cmd/syncapi/syncapi.go @@ -137,13 +137,16 @@ func dataset(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() if err := schema.ValidateJSON(fmt.Sprintf("%s/../bigpicture/file-sync.json", Conf.Broker.SchemasPath), b); err != nil { - respondWithError(w, http.StatusBadRequest, fmt.Sprintf("eror on JSON validation: %s", err.Error())) + respondWithError(w, http.StatusBadRequest, fmt.Sprintf("error on JSON validation: %s", err.Error())) return } if err := parseDatasetMessage(b); err != nil { - w.WriteHeader(http.StatusInternalServerError) + log.Errorf("error on parsing dataset message: %v", err) + respondWithError(w, http.StatusInternalServerError, "error while processing message") + + return } w.WriteHeader(http.StatusOK) From 99f8fa513a8751e52e881b2119c0eb83a33fedf8 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 18 Mar 2025 14:52:46 +0100 Subject: [PATCH 225/250] fix: update rbac for testing purposes --- .github/integration/sda/rbac.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 7d37530658..545e93fd1b 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -14,6 +14,11 @@ "role": "admin", "path": "/dataset/*", "action": "(POST)|(PUT)" + }, + { + "role": "submission", + "path": "/dataset/*", + "action": "POST" }, { "role": "submission", @@ -61,6 +66,10 @@ "role": "admin", "rolebinding": "submission" }, + { + "role": "test@dummy.org", + "rolebinding": "submission" + }, { "role": "testu@lifescience-ri.eu", "rolebinding": "admin" From 70d9265d18441ff4c1b1a62ff9fb2a8b7dbfa804 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 18 Mar 2025 14:53:08 +0100 Subject: [PATCH 226/250] fix typos and improve documentation --- sda/cmd/api/api.go | 2 +- sda/cmd/api/api.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 9d9d94dddb..b57981b1a7 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -216,7 +216,7 @@ func rbac(e *casbin.Enforcer) gin.HandlerFunc { return } - log.Debugln("authoriozed") + log.Debugln("authorized") } } diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index abccbb8c62..0501f3c583 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -121,7 +121,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th Example: ```bash - curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01"}' https://HOSTNAME/dataset/create + curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01", "user": "user@example.org"}' https://HOSTNAME/dataset/create ``` - `/dataset/release/*dataset` From f9dcaa954867f88e379c39c02f69ccde3d2d4e43 Mon Sep 17 00:00:00 2001 From: MalinAhlberg Date: Tue, 18 Mar 2025 14:56:47 +0100 Subject: [PATCH 227/250] [tests] failing(!!) test for double verifying of a file --- .../tests/sda/20_ingest-verify_test.sh | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/.github/integration/tests/sda/20_ingest-verify_test.sh b/.github/integration/tests/sda/20_ingest-verify_test.sh index f9443cfb31..7d55ba670b 100644 --- a/.github/integration/tests/sda/20_ingest-verify_test.sh +++ b/.github/integration/tests/sda/20_ingest-verify_test.sh @@ -76,4 +76,50 @@ if [ "$key_hashes" -eq 0 ]; then exit 1 fi +# Testing that verify does not loop when re-queueing already verified file +# NB currently will make verify loop be prepared to stop it before it fills out memory with endless logs +CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NA12878.bam.c4gh';") + +encrypted_checksums=$( + jq -c -n \ + --arg sha256 "$ENC_SHA" \ + '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))' +) + +properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg correlation_id "$CORRID" \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' +) + +verify_payload=$( + jq -r -c -n \ + --arg user test@dummy.com \ + --arg archive_path "$CORRID" \ + --arg file_id "$CORRID" \ + --arg filepath test_dummy.org/NA12878.bam.c4gh \ + --argjson encrypted_checksums "$encrypted_checksums" \ + --argjson re_verify false \ + '$ARGS.named|@base64' +) + +verify_body=$( + jq -c -n \ + --arg vhost sda \ + --arg name sda \ + --argjson properties "$properties" \ + --arg routing_key "archived" \ + --arg payload_encoding base64 \ + --arg payload "$verify_payload" \ + '$ARGS.named' +) + +curl -k -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ + -H 'Content-Type: application/json;charset=UTF-8' \ + -d "$verify_body" | jq + + echo "ingestion and verification test completed successfully" \ No newline at end of file From 7b9fa078041e6c8a5f1784c111a5af900039cb82 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 14 Apr 2025 08:50:08 +0200 Subject: [PATCH 228/250] handle already verified file better --- sda/cmd/verify/verify.go | 44 ++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 18a10c7477..d216b9489f 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -344,42 +344,42 @@ func main() { continue } - switch status { - case "disabled": + + if status == "disabled" { log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } continue - case "enabled": - fileInfo, err := db.GetFileInfo(message.FileID) - if err != nil { - log.Errorf("failed to get info for file: %s", message.FileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } + } - continue + fileInfo, err := db.GetFileInfo(message.FileID) + if err != nil { + log.Errorf("failed to get info for file: %s", message.FileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - if fileInfo.DecryptedChecksum != "" { - log.Debugln("file already verified") - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } + continue + } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + if fileInfo.DecryptedChecksum != "" { + log.Warnln("file already verified") + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue } + + if err := delivered.Ack(false); err != nil { + log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + } + + continue } if err := db.SetVerified(file, message.FileID); err != nil { From e97d5a4b8272ab3dd9a2a30fa684630dd80bf895 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 14 Apr 2025 22:55:32 +0200 Subject: [PATCH 229/250] set verified status for file already verified --- sda/cmd/verify/verify.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index d216b9489f..508afb1517 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -364,8 +364,18 @@ func main() { continue } - if fileInfo.DecryptedChecksum != "" { + if fileInfo.DecryptedChecksum != "" && fileInfo.DecryptedChecksum == fmt.Sprintf("%x", sha256hash.Sum(nil)) { log.Warnln("file already verified") + + if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { + log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + } + + continue + } + if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) if err := delivered.Nack(false, true); err != nil { From 8ffbbef0ac73434405ffde0cbfd1fdfcdb20e67c Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 14 Apr 2025 23:52:06 +0200 Subject: [PATCH 230/250] simplify logic for already verified files --- sda/cmd/verify/verify.go | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 508afb1517..ccadb86447 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -364,7 +364,8 @@ func main() { continue } - if fileInfo.DecryptedChecksum != "" && fileInfo.DecryptedChecksum == fmt.Sprintf("%x", sha256hash.Sum(nil)) { + switch { + case fileInfo.DecryptedChecksum != "" && fileInfo.DecryptedChecksum == fmt.Sprintf("%x", sha256hash.Sum(nil)): log.Warnln("file already verified") if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { @@ -375,9 +376,9 @@ func main() { continue } - - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + default: + if err := db.SetVerified(file, message.FileID); err != nil { + log.Errorf("SetVerified failed, reason: (%s)", err.Error()) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } @@ -385,29 +386,14 @@ func main() { continue } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } - - continue - } - - if err := db.SetVerified(file, message.FileID); err != nil { - log.Errorf("SetVerified failed, reason: (%s)", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } + if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { + log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + } - if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { - log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) + continue } - - continue } // Send message to verified queue From c49a32345a321790815ed7fdbc5f1b83cb6bc6dd Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Mon, 14 Apr 2025 23:52:40 +0200 Subject: [PATCH 231/250] remove explicit test for already verified file - case is covered by the cancel test script --- .../tests/sda/20_ingest-verify_test.sh | 46 ------------------- 1 file changed, 46 deletions(-) diff --git a/.github/integration/tests/sda/20_ingest-verify_test.sh b/.github/integration/tests/sda/20_ingest-verify_test.sh index 7d55ba670b..f9443cfb31 100644 --- a/.github/integration/tests/sda/20_ingest-verify_test.sh +++ b/.github/integration/tests/sda/20_ingest-verify_test.sh @@ -76,50 +76,4 @@ if [ "$key_hashes" -eq 0 ]; then exit 1 fi -# Testing that verify does not loop when re-queueing already verified file -# NB currently will make verify loop be prepared to stop it before it fills out memory with endless logs -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NA12878.bam.c4gh';") - -encrypted_checksums=$( - jq -c -n \ - --arg sha256 "$ENC_SHA" \ - '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))' -) - -properties=$( - jq -c -n \ - --argjson delivery_mode 2 \ - --arg correlation_id "$CORRID" \ - --arg content_encoding UTF-8 \ - --arg content_type application/json \ - '$ARGS.named' -) - -verify_payload=$( - jq -r -c -n \ - --arg user test@dummy.com \ - --arg archive_path "$CORRID" \ - --arg file_id "$CORRID" \ - --arg filepath test_dummy.org/NA12878.bam.c4gh \ - --argjson encrypted_checksums "$encrypted_checksums" \ - --argjson re_verify false \ - '$ARGS.named|@base64' -) - -verify_body=$( - jq -c -n \ - --arg vhost sda \ - --arg name sda \ - --argjson properties "$properties" \ - --arg routing_key "archived" \ - --arg payload_encoding base64 \ - --arg payload "$verify_payload" \ - '$ARGS.named' -) - -curl -k -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \ - -H 'Content-Type: application/json;charset=UTF-8' \ - -d "$verify_body" | jq - - echo "ingestion and verification test completed successfully" \ No newline at end of file From 1545a719b7a874b1004c14f8cf7a28e36a37af91 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 23 Apr 2025 11:52:18 +0200 Subject: [PATCH 232/250] further simplify logic for already verified files --- sda/cmd/verify/verify.go | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index ccadb86447..349461b27f 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -364,19 +364,7 @@ func main() { continue } - switch { - case fileInfo.DecryptedChecksum != "" && fileInfo.DecryptedChecksum == fmt.Sprintf("%x", sha256hash.Sum(nil)): - log.Warnln("file already verified") - - if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { - log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - default: + if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { if err := db.SetVerified(file, message.FileID); err != nil { log.Errorf("SetVerified failed, reason: (%s)", err.Error()) if err := delivered.Nack(false, true); err != nil { @@ -385,15 +373,17 @@ func main() { continue } + } else { + log.Warnln("file already verified") + } - if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { - log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue + if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { + log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } + + continue } // Send message to verified queue From 498b20fb8cc256e88ac7a26a0173157fa49350e8 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 23 Apr 2025 11:53:31 +0200 Subject: [PATCH 233/250] remove non-essential test permissions --- .github/integration/sda/rbac.json | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 545e93fd1b..7d37530658 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -14,11 +14,6 @@ "role": "admin", "path": "/dataset/*", "action": "(POST)|(PUT)" - }, - { - "role": "submission", - "path": "/dataset/*", - "action": "POST" }, { "role": "submission", @@ -66,10 +61,6 @@ "role": "admin", "rolebinding": "submission" }, - { - "role": "test@dummy.org", - "rolebinding": "submission" - }, { "role": "testu@lifescience-ri.eu", "rolebinding": "admin" From 8b5ea7c063f25d3ad461b97b330f554ac6129ffc Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Wed, 23 Apr 2025 16:28:48 +0200 Subject: [PATCH 234/250] add suggestion from review --- sda/cmd/verify/verify.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 349461b27f..908a2546a6 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -374,7 +374,7 @@ func main() { continue } } else { - log.Warnln("file already verified") + log.Infof("file with correlation id: %s is already verified", delivered.CorrelationId) } if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { From 46b5f8270b9598885086499c64d80c1a324713c5 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Thu, 24 Apr 2025 06:43:27 +0000 Subject: [PATCH 235/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index a3c81180ab..2b2d492156 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.81 -appVersion: v0.3.223 +version: 0.8.82 +appVersion: v0.5.7 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index b04f94b984..3c381b10bd 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.82 -appVersion: v0.3.223 +version: 0.7.83 +appVersion: v0.5.7 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index e7fbf0a37c..fd58be22c4 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.11 -appVersion: v0.3.223 +version: 0.30.12 +appVersion: v0.5.7 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 5c7ece1f5b51e818a9f614c6a78891aab02b689a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 21:02:46 +0000 Subject: [PATCH 236/250] Bump org.springframework.boot:spring-boot-starter-parent Bumps the all-modules group in /sda-sftp-inbox with 1 update: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.4 to 3.4.5 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.4...v3.4.5) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.4.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index b30137a36c..2798622cd9 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.4 + 3.4.5 From 59e3236269f2576ba27307195c1a98c71ad67684 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 21:06:31 +0000 Subject: [PATCH 237/250] Bump github.com/aws/aws-sdk-go in /sda-download in the all-modules group Bumps the all-modules group in /sda-download with 1 update: [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go). Updates `github.com/aws/aws-sdk-go` from 1.55.6 to 1.55.7 - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG_PENDING.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.55.6...v1.55.7) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-version: 1.55.7 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 2 +- sda-download/go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index a0423bf89c..88794debed 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -6,7 +6,7 @@ toolchain go1.24.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go v1.55.6 + github.com/aws/aws-sdk-go v1.55.7 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 diff --git a/sda-download/go.sum b/sda-download/go.sum index 287e34aa0e..c69f9253de 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -3,8 +3,8 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.55.6 h1:cSg4pvZ3m8dgYcgqB97MrcdjUmZ1BeMYKUxMMB89IPk= -github.com/aws/aws-sdk-go v1.55.6/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= +github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= From ab042c3971bbfbecc52025ce544beb8388dc5f3f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 21:13:47 +0000 Subject: [PATCH 238/250] Bump the all-modules group in /sda-doa with 4 updates Bumps the all-modules group in /sda-doa with 4 updates: [org.springframework.boot:spring-boot-starter-parent](https://github.com/spring-projects/spring-boot), [com.google.code.gson:gson](https://github.com/google/gson), [no.elixir:crypt4gh](https://github.com/ELIXIR-NO/FEGA-Norway) and [no.elixir:clearinghouse](https://github.com/ELIXIR-NO/FEGA-Norway). Updates `org.springframework.boot:spring-boot-starter-parent` from 3.4.4 to 3.4.5 - [Release notes](https://github.com/spring-projects/spring-boot/releases) - [Commits](https://github.com/spring-projects/spring-boot/compare/v3.4.4...v3.4.5) Updates `com.google.code.gson:gson` from 2.13.0 to 2.13.1 - [Release notes](https://github.com/google/gson/releases) - [Changelog](https://github.com/google/gson/blob/main/CHANGELOG.md) - [Commits](https://github.com/google/gson/compare/gson-parent-2.13.0...gson-parent-2.13.1) Updates `no.elixir:crypt4gh` from 3.0.22 to 3.0.24 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/crypt4gh-3.0.22...crypt4gh-3.0.24) Updates `no.elixir:clearinghouse` from 2.0.17 to 2.0.18 - [Release notes](https://github.com/ELIXIR-NO/FEGA-Norway/releases) - [Commits](https://github.com/ELIXIR-NO/FEGA-Norway/compare/clearinghouse-2.0.17...clearinghouse-2.0.18) --- updated-dependencies: - dependency-name: org.springframework.boot:spring-boot-starter-parent dependency-version: 3.4.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: com.google.code.gson:gson dependency-version: 2.13.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:crypt4gh dependency-version: 3.0.24 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: no.elixir:clearinghouse dependency-version: 2.0.18 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-doa/pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index b5cb4bddbf..5b42709df8 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.4 + 3.4.5 no.uio.ifi @@ -21,7 +21,7 @@ com.google.code.gson gson - 2.13.0 + 2.13.1 org.springframework.boot @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.22 + 3.0.24 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.17 + 2.0.18 org.slf4j From 993f54d1d52bff619fdab497ba167ec982fd44ba Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 28 Apr 2025 21:18:18 +0000 Subject: [PATCH 239/250] Bump the all-modules group in /sda with 2 updates Bumps the all-modules group in /sda with 2 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2) and [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.72 to 1.17.74 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.72...feature/s3/manager/v1.17.74) Updates `github.com/aws/aws-sdk-go-v2/service/s3` from 1.79.2 to 1.79.3 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.79.2...service/s3/v1.79.3) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.74 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-version: 1.79.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 6 +++--- sda/go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 1ed0ca44ef..754bc964cc 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -8,8 +8,8 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 - github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.105.0 github.com/coreos/go-oidc/v3 v3.14.1 @@ -58,7 +58,7 @@ require ( github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect diff --git a/sda/go.sum b/sda/go.sum index 50872a5b44..7c4789c039 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9 github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72 h1:PcKMOZfp+kNtJTw2HF2op6SjDvwPBYRvz0Y24PQLUR4= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.72/go.mod h1:vq7/m7dahFXcdzWVOvvjasDI9RcsD3RsTfHmDundJYg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 h1:+1lc5oMFFHlVBclPXQf/POqlvdpBzjLaN2c3ujDCcZw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74/go.mod h1:EiskBoFr4SpYnFIbw8UM7DP7CacQXDHEmJqLI1xpRFI= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -46,14 +46,14 @@ github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcu github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0 h1:lguz0bmOoGzozP9XfRJR1QIayEYo+2vP/No3OfLF0pU= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.0/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2 h1:tWUG+4wZqdMl/znThEk9tcCy8tTMxq8dW0JTgamohrY= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.2/go.mod h1:U5SNqwhXB3Xe6F47kXvWihPl/ilGaEDe8HD/50Z9wxc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= From ebe275418ffa7b9f95ae61e3f81b699d662b19e3 Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 29 Apr 2025 14:31:08 +0000 Subject: [PATCH 240/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 2b2d492156..88ec15cac4 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.82 -appVersion: v0.5.7 +version: 0.8.83 +appVersion: v0.5.10 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index 3c381b10bd..afdcef676b 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.83 -appVersion: v0.5.7 +version: 0.7.84 +appVersion: v0.5.10 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index fd58be22c4..730688db50 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.12 -appVersion: v0.5.7 +version: 0.30.13 +appVersion: v0.5.10 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 8577158d96756e0bebe5f060c76f916816100bbc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 19:19:40 +0000 Subject: [PATCH 241/250] Bump golangci/golangci-lint-action from 7.0.0 to 8.0.0 Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 7.0.0 to 8.0.0. - [Release notes](https://github.com/golangci/golangci-lint-action/releases) - [Commits](https://github.com/golangci/golangci-lint-action/compare/v7.0.0...v8.0.0) --- updated-dependencies: - dependency-name: golangci/golangci-lint-action dependency-version: 8.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/code-linter.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 26c3741630..56be2be57b 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v7.0.0 + uses: golangci/golangci-lint-action@v8.0.0 with: args: --timeout 5m working-directory: sda-download @@ -48,7 +48,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v7.0.0 + uses: golangci/golangci-lint-action@v8.0.0 with: args: --timeout 5m working-directory: sda @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v7.0.0 + uses: golangci/golangci-lint-action@v8.0.0 with: args: --timeout 5m working-directory: sda-admin From 96764d680127aedc79e8b2e723b2a82551bd6fad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 19:32:53 +0000 Subject: [PATCH 242/250] Bump the all-modules group in /sda-download with 2 updates Bumps the all-modules group in /sda-download with 2 updates: [github.com/lestrrat-go/jwx/v2](https://github.com/lestrrat-go/jwx) and [golang.org/x/crypto](https://github.com/golang/crypto). Updates `github.com/lestrrat-go/jwx/v2` from 2.1.5 to 2.1.6 - [Release notes](https://github.com/lestrrat-go/jwx/releases) - [Changelog](https://github.com/lestrrat-go/jwx/blob/v2.1.6/Changes) - [Commits](https://github.com/lestrrat-go/jwx/compare/v2.1.5...v2.1.6) Updates `golang.org/x/crypto` from 0.37.0 to 0.38.0 - [Commits](https://github.com/golang/crypto/compare/v0.37.0...v0.38.0) --- updated-dependencies: - dependency-name: github.com/lestrrat-go/jwx/v2 dependency-version: 2.1.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.38.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-download/go.mod | 8 ++++---- sda-download/go.sum | 20 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/sda-download/go.mod b/sda-download/go.mod index 88794debed..73eae28d65 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -11,13 +11,13 @@ require ( github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 - github.com/lestrrat-go/jwx/v2 v2.1.5 + github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.10.9 github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.37.0 + golang.org/x/crypto v0.38.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 google.golang.org/grpc v1.72.0 google.golang.org/protobuf v1.36.6 @@ -71,8 +71,8 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect golang.org/x/tools v0.25.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda-download/go.sum b/sda-download/go.sum index c69f9253de..e4383bafcb 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -92,8 +92,8 @@ github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCG github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.5 h1:PQI5gzadLfJ22ckLrejPVX6eGXKM4M4eGi5fW2jjA3o= -github.com/lestrrat-go/jwx/v2 v2.1.5/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= +github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVfecA= +github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -176,8 +176,8 @@ golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -205,23 +205,23 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From 2a4072f86e9de64dad0b0c3300c5b3e260e1767f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 19:36:21 +0000 Subject: [PATCH 243/250] Bump com.amazonaws:aws-java-sdk-s3 Bumps the all-modules group in /sda-sftp-inbox with 1 update: [com.amazonaws:aws-java-sdk-s3](https://github.com/aws/aws-sdk-java). Updates `com.amazonaws:aws-java-sdk-s3` from 1.12.782 to 1.12.783 - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.12.782...1.12.783) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-version: 1.12.783 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda-sftp-inbox/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index 2798622cd9..e52ee1ebbd 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -108,7 +108,7 @@ com.amazonaws aws-java-sdk-s3 - 1.12.782 + 1.12.783 com.google.guava From d2b78d8a39532aeba5f7f09f747eb83d09820b52 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 May 2025 20:08:01 +0000 Subject: [PATCH 244/250] Bump the all-modules group in /sda with 4 updates Bumps the all-modules group in /sda with 4 updates: [github.com/aws/aws-sdk-go-v2/feature/s3/manager](https://github.com/aws/aws-sdk-go-v2), [github.com/lestrrat-go/jwx/v2](https://github.com/lestrrat-go/jwx), [golang.org/x/crypto](https://github.com/golang/crypto) and [golang.org/x/oauth2](https://github.com/golang/oauth2). Updates `github.com/aws/aws-sdk-go-v2/feature/s3/manager` from 1.17.74 to 1.17.75 - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Changelog](https://github.com/aws/aws-sdk-go-v2/blob/main/changelog-template.json) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/feature/s3/manager/v1.17.74...feature/s3/manager/v1.17.75) Updates `github.com/lestrrat-go/jwx/v2` from 2.1.5 to 2.1.6 - [Release notes](https://github.com/lestrrat-go/jwx/releases) - [Changelog](https://github.com/lestrrat-go/jwx/blob/v2.1.6/Changes) - [Commits](https://github.com/lestrrat-go/jwx/compare/v2.1.5...v2.1.6) Updates `golang.org/x/crypto` from 0.37.0 to 0.38.0 - [Commits](https://github.com/golang/crypto/compare/v0.37.0...v0.38.0) Updates `golang.org/x/oauth2` from 0.29.0 to 0.30.0 - [Commits](https://github.com/golang/oauth2/compare/v0.29.0...v0.30.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/feature/s3/manager dependency-version: 1.17.75 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: github.com/lestrrat-go/jwx/v2 dependency-version: 2.1.6 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: all-modules - dependency-name: golang.org/x/crypto dependency-version: 0.38.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules - dependency-name: golang.org/x/oauth2 dependency-version: 0.30.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: all-modules ... Signed-off-by: dependabot[bot] --- sda/go.mod | 12 ++++++------ sda/go.sum | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/sda/go.mod b/sda/go.mod index 754bc964cc..d41bb2cefc 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.36.3 github.com/aws/aws-sdk-go-v2/config v1.29.14 github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 github.com/aws/smithy-go v1.22.3 github.com/casbin/casbin/v2 v2.105.0 @@ -18,7 +18,7 @@ require ( github.com/gorilla/mux v1.8.1 github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 github.com/kataras/iris/v12 v12.2.11 - github.com/lestrrat-go/jwx/v2 v2.1.5 + github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 @@ -33,8 +33,8 @@ require ( github.com/sirupsen/logrus v1.9.3 github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.37.0 - golang.org/x/oauth2 v0.29.0 + golang.org/x/crypto v0.38.0 + golang.org/x/oauth2 v0.30.0 google.golang.org/grpc v1.72.0 google.golang.org/protobuf v1.36.6 ) @@ -157,8 +157,8 @@ require ( golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.32.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect golang.org/x/time v0.8.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/sda/go.sum b/sda/go.sum index 7c4789c039..6933adbf9a 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -34,8 +34,8 @@ github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9 github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74 h1:+1lc5oMFFHlVBclPXQf/POqlvdpBzjLaN2c3ujDCcZw= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.74/go.mod h1:EiskBoFr4SpYnFIbw8UM7DP7CacQXDHEmJqLI1xpRFI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= @@ -232,8 +232,8 @@ github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCG github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.5 h1:PQI5gzadLfJ22ckLrejPVX6eGXKM4M4eGi5fW2jjA3o= -github.com/lestrrat-go/jwx/v2 v2.1.5/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= +github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVfecA= +github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= @@ -408,8 +408,8 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= -golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -437,8 +437,8 @@ golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= -golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -469,8 +469,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= -golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -480,8 +480,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= -golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -492,8 +492,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= -golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From dd570ff41d0db726c6d35e5a84696dc2997b317e Mon Sep 17 00:00:00 2001 From: neicnordic Date: Tue, 6 May 2025 07:32:40 +0000 Subject: [PATCH 245/250] Bump chart version --- charts/sda-db/Chart.yaml | 4 ++-- charts/sda-mq/Chart.yaml | 4 ++-- charts/sda-svc/Chart.yaml | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/charts/sda-db/Chart.yaml b/charts/sda-db/Chart.yaml index 88ec15cac4..e2cb47ecdf 100644 --- a/charts/sda-db/Chart.yaml +++ b/charts/sda-db/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-db -version: 0.8.83 -appVersion: v0.5.10 +version: 0.8.84 +appVersion: v0.5.14 kubeVersion: '>= 1.26.0' description: Database component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-mq/Chart.yaml b/charts/sda-mq/Chart.yaml index afdcef676b..15080c15d3 100644 --- a/charts/sda-mq/Chart.yaml +++ b/charts/sda-mq/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-mq -version: 0.7.84 -appVersion: v0.5.10 +version: 0.7.85 +appVersion: v0.5.14 kubeVersion: '>= 1.26.0' description: RabbitMQ component for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index 730688db50..f0237495ba 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: sda-svc -version: 0.30.13 -appVersion: v0.5.10 +version: 0.30.14 +appVersion: v0.5.14 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation home: https://neic-sda.readthedocs.io From 1cc947e4a245bf6586149f7c22dbe70e3cbce4b3 Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 9 May 2025 14:12:11 +0200 Subject: [PATCH 246/250] require a pubkey for s3inbox when resigning jwt --- charts/sda-svc/templates/s3-inbox-deploy.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index e71e077d38..bf97b7faa4 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -190,14 +190,16 @@ spec: - name: SERVER_KEY value: {{ include "tlsPath" . }}/tls.key {{- end }} - {{- if.Values.global.auth.jwtPub }} + {{- if .Values.global.auth.jwtPub }} - name: SERVER_JWTPUBKEYPATH value: {{ include "jwtPath" . }} - {{- end }} - {{- if not .Values.global.auth.resignJwt }} + {{- end }} + {{- if .Values.global.auth.resignJwt }} + {{- $_ := required "The signing pub key is required for s3inbox when auth resigns tokens" .Values.global.auth.jwtPub }} + {{- else }} - name: SERVER_JWTPUBKEYURL value: {{ .Values.global.oidc.provider }}{{ .Values.global.oidc.jwkPath }} - {{- end }} + {{- end }} {{- if .Values.global.log.format }} - name: LOG_FORMAT value: {{ .Values.global.log.format | quote }} From f41a6f9c7566062992dfeb6e3c9cd434d3747a8b Mon Sep 17 00:00:00 2001 From: Alex Aperis Date: Fri, 9 May 2025 15:30:42 +0200 Subject: [PATCH 247/250] bump chart version --- charts/sda-svc/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/sda-svc/Chart.yaml b/charts/sda-svc/Chart.yaml index f0237495ba..4103b3095f 100644 --- a/charts/sda-svc/Chart.yaml +++ b/charts/sda-svc/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: sda-svc -version: 0.30.14 +version: 0.30.15 appVersion: v0.5.14 kubeVersion: '>= 1.26.0' description: Components for Sensitive Data Archive (SDA) installation From 6ccea86dd8bb3e92943cd47749238bf112b07dc4 Mon Sep 17 00:00:00 2001 From: Nanjiang Shu Date: Fri, 25 Apr 2025 13:57:34 +0200 Subject: [PATCH 248/250] [lint] Update lint command in Makefile to match V2 [lint] remove unnecessary arguments --- Makefile | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 278d0bfa91..9f8282f92a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,3 @@ -# linter modules to be included/excluded -LINT_INCLUDE=-E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -LINT_EXCLUDE=-e G401,G501,G107 - help: @echo 'Welcome!' @echo '' @@ -152,13 +148,13 @@ integrationtest-sda-doa-s3-down: lint-all: lint-sda lint-sda-download lint-sda-admin lint-sda: @echo 'Running golangci-lint in the `sda` folder' - @cd sda && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) + @cd sda && golangci-lint run lint-sda-download: @echo 'Running golangci-lint in the `sda-download` folder' - @cd sda-download && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) + @cd sda-download && golangci-lint run lint-sda-admin: @echo 'Running golangci-lint in the `sda-admin` folder' - @cd sda-admin && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) + @cd sda-admin && golangci-lint run # run static code tests test-all: test-sda test-sda-download test-sda-sftp-inbox @@ -207,4 +203,4 @@ k3d-deploy-sda-s3: k3d-deploy-sda-posix: @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false posix k3d-cleanup-all-deployments: - @bash .github/integration/scripts/charts/cleanup.sh \ No newline at end of file + @bash .github/integration/scripts/charts/cleanup.sh From 9e27a1a13f7f1d336e01b3ba22f3bba07da107a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 12 May 2025 19:48:48 +0000 Subject: [PATCH 249/250] Bump anothrNick/github-tag-action from 1.71.0 to 1.73.0 Bumps [anothrNick/github-tag-action](https://github.com/anothrnick/github-tag-action) from 1.71.0 to 1.73.0. - [Release notes](https://github.com/anothrnick/github-tag-action/releases) - [Commits](https://github.com/anothrnick/github-tag-action/compare/1.71.0...1.73.0) --- updated-dependencies: - dependency-name: anothrNick/github-tag-action dependency-version: 1.73.0 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish_container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index 80432bf452..4dbd4f8bf9 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -29,7 +29,7 @@ jobs: fetch-depth: '0' - name: Bump version and push tag id: bump_tag - uses: anothrNick/github-tag-action@1.71.0 + uses: anothrNick/github-tag-action@1.73.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true From 2cbc60bd0800a1186ced0befda52ca16e9233480 Mon Sep 17 00:00:00 2001 From: Krzysztof Kochel Date: Tue, 13 May 2025 23:16:09 +0200 Subject: [PATCH 250/250] Fix after sync with upstream --- .github/workflows/build_pr_container.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index d7730e0ca8..1938e9b19e 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -228,8 +228,8 @@ jobs: context: ./sda-doa push: true tags: | - ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-doa - ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-doa + ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-doa + ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-doa labels: | org.opencontainers.image.source=${{ github.event.repository.clone_url }} org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') @@ -241,7 +241,7 @@ jobs: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true with: - image-ref: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-doa + image-ref: ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-doa format: "sarif" hide-progress: true ignore-unfixed: true