diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh
index 95abe8d675..312ca6349a 100644
--- a/.github/integration/scripts/charts/dependencies.sh
+++ b/.github/integration/scripts/charts/dependencies.sh
@@ -100,7 +100,7 @@ fi
## update values file with all credentials
if [ "$2" == "federated" ]; then
- yq -i '.global.schemaType = federated' "$values_file"
+ yq -i '.global.schemaType = "federated"' "$values_file"
fi
yq -i '
diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh
index 235590de84..2d84ebb1f7 100644
--- a/.github/integration/scripts/charts/deploy_charts.sh
+++ b/.github/integration/scripts/charts/deploy_charts.sh
@@ -50,7 +50,7 @@ if [ "$1" == "sda-mq" ]; then
--wait
if [ "$4" == "federated" ]; then
- curl -kL -u "admin:$ADMINPASS" -X PUT "$PROTOCOL://broker.127.0.0.1.nip.io/api/queues/sda/from_cega"
+ curl --retry 100 -kL -u "admin:$ADMINPASS" -d '{"durable": true}' -H "Content-Type: application/json" -X PUT "$PROTOCOL://broker.127.0.0.1.nip.io/api/queues/sda/from_cega"
fi
fi
diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml
index 747f00d246..c831981ec9 100644
--- a/.github/integration/scripts/charts/values.yaml
+++ b/.github/integration/scripts/charts/values.yaml
@@ -90,8 +90,8 @@ global:
s3ReadyPath: "/minio/health/ready"
existingClaim: inbox-pvc
reencrypt:
- host: reencrypt
- port: 50443
+ host: pipeline-sda-svc-reencrypt
+ port: 50051
sync:
api:
password: "apiuser"
diff --git a/.github/integration/sda-validator-orchestrator-integration.yml b/.github/integration/sda-validator-orchestrator-integration.yml
new file mode 100644
index 0000000000..9d02f2a9fa
--- /dev/null
+++ b/.github/integration/sda-validator-orchestrator-integration.yml
@@ -0,0 +1,366 @@
+services:
+ credentials:
+ container_name: credentials
+ command:
+ - "/bin/sh"
+ - "-c"
+ - |
+ sh /scripts/make_sda_credentials.sh;
+
+ curl -s -u guest:guest -X PUT -k "http://rabbitmq:15672/api/users/validator_orchestrator" -H "content-type:application/json" -d '{"password": "validator_orchestrator", "tags":"none"}';
+ curl -s -u guest:guest -X PUT -k "http://rabbitmq:15672/api/permissions/sda/validator_orchestrator" -H "content-type:application/json" -d '{"configure":"","write":"sda","read":".*"}';
+ depends_on:
+ postgres:
+ condition: service_healthy
+ rabbitmq:
+ condition: service_healthy
+ environment:
+ - PGPASSWORD=rootpasswd
+ image: python:3.11-slim-bookworm
+ volumes:
+ - ./scripts:/scripts
+ - shared:/shared
+
+
+ database_init:
+ container_name: database_init
+ image: postgres:15.4-alpine
+ command:
+ - "/bin/sh"
+ - "-c"
+ - |
+ pg_isready
+ psql -c "CREATE DATABASE sda_validator_orchestrator" || true;
+ psql -d sda_validator_orchestrator -c "CREATE SCHEMA sda_validator_orchestrator;" || true;
+ export PGOPTIONS="--search_path=sda_validator_orchestrator";
+ find /init/ -name *.sql -exec psql -d sda_validator_orchestrator -f {} \;
+ psql -d sda_validator_orchestrator -c "CREATE USER validator_orchestrator WITH PASSWORD 'validator_orchestrator'";
+ psql -d sda_validator_orchestrator -c "GRANT USAGE ON SCHEMA sda_validator_orchestrator TO validator_orchestrator;";
+ psql -d sda_validator_orchestrator -c "GRANT ALL ON ALL TABLES IN SCHEMA sda_validator_orchestrator TO validator_orchestrator;";
+ psql -d sda_validator_orchestrator -c "GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA sda_validator_orchestrator TO validator_orchestrator;";
+ depends_on:
+ postgres:
+ condition: service_healthy
+ restart: on-failure
+ environment:
+ - PGHOST=postgres
+ - PGPORT=5432
+ - PGUSER=postgres
+ - PGPASSWORD=rootpasswd
+ volumes:
+ - ./../../sda-validator/orchestrator/database/postgres/initdb.d/:/init
+ - shared:/shared
+
+ postgres:
+ build:
+ context: ../../postgresql
+ container_name: postgres
+ environment:
+ - POSTGRES_PASSWORD=rootpasswd
+ healthcheck:
+ test: ["CMD-SHELL", "pg_isready -U postgres"]
+ interval: 10s
+ timeout: 2s
+ retries: 6
+ image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres
+ extra_hosts:
+ - "localhost:host-gateway"
+ ports:
+ - "15432:5432"
+ restart: always
+ volumes:
+ - postgres_data:/var/lib/postgresql/data
+
+ rabbitmq:
+ build:
+ context: ../../rabbitmq
+ container_name: rabbitmq
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "bash",
+ "-c",
+ "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms",
+ ]
+ interval: 10s
+ timeout: 5s
+ retries: 6
+ image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq
+ ports:
+ - "15672:15672"
+ - "5672:5672"
+ restart: always
+ volumes:
+ - rabbitmq_data:/var/lib/rabbitmq
+
+ minio:
+ image: minio/minio
+ command: server /data --console-address ":9001"
+ container_name: s3
+ environment:
+ - MINIO_ROOT_USER=access
+ - MINIO_ROOT_PASSWORD=secretKey
+ - MINIO_SERVER_URL=http://127.0.0.1:9000
+ healthcheck:
+ test: ["CMD", "curl", "-fkq", "http://localhost:9000/minio/health/live"]
+ interval: 10s
+ timeout: 2s
+ retries: 6
+ ports:
+ - "19000:9000"
+ - "19001:9001"
+ volumes:
+ - minio_data:/data
+
+ validator-orchestrator_init:
+ container_name: validator-orchestrator_init
+ image: golang:1.25-alpine
+ depends_on:
+ credentials:
+ condition: service_completed_successfully
+ command:
+ - "/bin/sh"
+ - "-c"
+ - |
+ chmod 777 -R /validators/;
+ wget -O /validators/100-200_word_validator.sif https://github.com/neicnordic/sensitive-data-archive/raw/18059f008da61d41ee54758b93f761c3ea6d318c/sda-validator/apptainer-example/100-200_word_validator.sif
+ wget -O /validators/always_success.sif https://github.com/neicnordic/sensitive-data-archive/raw/18059f008da61d41ee54758b93f761c3ea6d318c/sda-validator/apptainer-example/always_success.sif;
+ mkdir /validators/config
+ cp /config.yaml /validators/config/config.yaml;
+ echo "sda-api-token: "$(cat /shared/token) >> /validators/config/config.yaml;
+ echo '{
+ "policy": [
+ {
+ "role": "admin",
+ "path": "/admin/*",
+ "action": "(GET)|(POST)"
+ }, {
+ "role": "submission",
+ "path": "/validators",
+ "action": "GET"
+ }, {
+ "role": "submission",
+ "path": "/validate",
+ "action": "POST"
+ }, {
+ "role": "submission",
+ "path": "/result",
+ "action": "GET"
+ }
+ ],
+ "roles": [
+ {
+ "role": "admin",
+ "rolebinding": "submission"
+ },
+ {
+ "role": "admin_user@integration.test",
+ "rolebinding": "admin"
+ }, {
+ "role": "submission_user@integration.test",
+ "rolebinding": "submission"
+ }
+ ]
+ }' > /validators/config/rbac.policy
+
+ volumes:
+ - validators:/validators
+ - ./sda/config.yaml:/config.yaml
+ - shared:/shared
+
+ validator-orchestrator:
+ build:
+ args:
+ GOLANG_VERSION: ${GOLANG_VERSION:-1.20}
+ context: ../../sda
+ target: debug
+ security_opt: # Unconfine seccomp and apparmor to allow apptainer to create usernamespaces within the container
+ - seccomp=unconfined
+ - apparmor=unconfined
+ image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-validator-orchestrator
+ container_name: validator-orchestrator
+ depends_on:
+ api:
+ condition: service_started
+ database_init:
+ condition: service_completed_successfully
+ validator-orchestrator_init:
+ condition: service_completed_successfully
+ environment:
+ - LOG_LEVEL=DEBUG
+ - API_PORT=8080
+ - VALIDATOR_PATHS=/validators/100-200_word_validator.sif,/validators/always_success.sif
+ - SDA_API_URL=http://api:8080
+ - BROKER_PASSWORD=validator_orchestrator
+ - BROKER_USER=validator_orchestrator
+ - JOB_QUEUE=validation-job-queue
+ - JOB_PREPARATION_QUEUE=validation-job-preparation-queue
+ - DATABASE_HOST=postgres
+ - DATABASE_PORT=5432
+ - DATABASE_USER=validator_orchestrator
+ - DATABASE_PASSWORD=validator_orchestrator
+ - CONFIG_FILE=/validators/config/config.yaml
+ - JWT_PUB_KEY_URL=http://localhost:8800/oidc/jwk
+ - RBAC_POLICY_FILE_PATH=/validators/config/rbac.policy
+ healthcheck:
+ test:
+ [ "CMD", "/bin/grpc_health_probe", "--addr", "localhost:9000" ]
+ interval: 10s
+ timeout: 2s
+ retries: 6
+ extra_hosts:
+ - "localhost:host-gateway"
+ restart: always
+ volumes:
+ - validators:/validators
+ ports:
+ - "8082:8080"
+
+ s3inbox:
+ build:
+ args:
+ GOLANG_VERSION: ${GOLANG_VERSION:-1.20}
+ context: ../../sda
+ target: debug
+ image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}
+ command: [ sda-s3inbox ]
+ container_name: s3inbox
+ depends_on:
+ credentials:
+ condition: service_completed_successfully
+ minio:
+ condition: service_healthy
+ mock-aai:
+ condition: service_healthy
+ postgres:
+ condition: service_healthy
+ rabbitmq:
+ condition: service_healthy
+ environment:
+ - BROKER_PASSWORD=inbox
+ - BROKER_USER=inbox
+ - BROKER_ROUTINGKEY=inbox
+ - DB_PASSWORD=inbox
+ - DB_USER=inbox
+ extra_hosts:
+ - "localhost:host-gateway"
+ restart: always
+ volumes:
+ - ./sda/config.yaml:/config.yaml
+ - shared:/shared
+ ports:
+ - "18000:8000"
+ - "18001:8001"
+
+ api:
+ command: [ sda-api ]
+ container_name: api
+ depends_on:
+ reencrypt:
+ condition: service_started
+ credentials:
+ condition: service_completed_successfully
+ postgres:
+ condition: service_healthy
+ mock-aai:
+ condition: service_healthy
+ rabbitmq:
+ condition: service_healthy
+ environment:
+ - BROKER_PASSWORD=api
+ - BROKER_USER=api
+ - DB_PASSWORD=api
+ - DB_USER=api
+ extra_hosts:
+ - "localhost:host-gateway"
+ image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}
+ ports:
+ - "8090:8080"
+ restart: always
+ volumes:
+ - ./sda/config.yaml:/config.yaml
+ - ./sda/rbac.json:/rbac.json
+ - shared:/shared
+
+
+
+ reencrypt:
+ image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}
+ command: [sda-reencrypt]
+ container_name: reencrypt
+ depends_on:
+ credentials:
+ condition: service_completed_successfully
+ ports:
+ - "50051:50051"
+ restart: always
+ volumes:
+ - ./sda/config.yaml:/config.yaml
+ - shared:/shared
+
+ mock-aai:
+ container_name: ls-aai-mock
+ depends_on:
+ aai-db:
+ condition: service_healthy
+ environment:
+ - DOCKERHOST=localhost
+ healthcheck:
+ test:
+ [ "CMD", "/bin/true" ]
+ interval: 10s
+ timeout: 2s
+ retries: 6
+ image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.3-broker2.1.13-tomcat9.0-jdk21
+ ports:
+ - "8800:8080"
+ volumes:
+ - "./sda/aai-mock:/etc/lsaai-mock"
+
+
+ aai-db:
+ container_name: ls-aai-db
+ environment:
+ MYSQL_ROOT_PASSWORD: "aaiPass"
+ MYSQL_ROOT_HOST: "%"
+ MYSQL_DATABASE: "aai"
+ MYSQL_USER: "aai"
+ MYSQL_PASSWORD: "aaiPass"
+ healthcheck:
+ test: [ "CMD", "mysqladmin", "ping", "-h", "localhost" ]
+ interval: 10s
+ timeout: 2s
+ retries: 6
+ image: mysql/mysql-server:latest
+ volumes:
+ - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql
+
+ integration_test:
+ container_name: tester
+ command:
+ - "/bin/sh"
+ - "-c"
+ - |
+ cd /tests;
+ go test -v ./...
+ depends_on:
+ validator-orchestrator:
+ condition: service_healthy
+ s3inbox:
+ condition: service_started
+ extra_hosts:
+ - "localhost:host-gateway"
+ image: golang:1.25-alpine
+ profiles:
+ - tests
+ volumes:
+ - shared:/shared
+ - ./tests/validator-orchestrator:/tests
+
+volumes:
+ minio_data:
+ postgres_data:
+ rabbitmq_data:
+ shared:
+ validators:
diff --git a/.github/integration/tests/sda/70_rotate_key_test.sh b/.github/integration/tests/sda/70_rotate_key_test.sh
index c952d4b6fe..4a8536be1a 100644
--- a/.github/integration/tests/sda/70_rotate_key_test.sh
+++ b/.github/integration/tests/sda/70_rotate_key_test.sh
@@ -130,6 +130,19 @@ curl -s -u guest:guest "http://rabbitmq:15672/api/exchanges/sda/sda/publish" \
-H 'Content-Type: application/json;charset=UTF-8' \
-d "$rotatekey_body" | jq
+# check that rotate key doesn't fail
+echo "waiting for rotatekey to complete"
+RETRY_TIMES=0
+until [ "$(curl -su guest:guest http://rabbitmq:15672/api/queues/sda/rotatekey/ | jq -r '.messages_ready')" -eq 0 ]; do
+ echo "waiting for rotatekey to complete"
+ RETRY_TIMES=$((RETRY_TIMES + 1))
+ if [ "$RETRY_TIMES" -eq 30 ]; then
+ echo "::error::Time out while waiting for rotatekey to complete"
+ exit 1
+ fi
+ sleep 2
+done
+
# check DB for updated key hash in sda.files
rotatekeyHash=$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.encryption_keys where description='this is the rotatekey key';")
if [ "$(psql -U postgres -h postgres -d sda -At -c "select key_hash from sda.files where id='$fileID';" | grep -c "$rotatekeyHash")" -ne 1 ];
@@ -138,6 +151,13 @@ then
exit 1
fi
+echo "verifying header backup in sda.file_headers_backup"
+backupCount=$(psql -U postgres -h postgres -d sda -At -c "SELECT count(*) FROM sda.file_headers_backup WHERE file_id='$fileID';")
+if [ "$backupCount" -lt 1 ]; then
+ echo "::error::No backup found in sda.file_headers_backup for file $fileID"
+ exit 1
+fi
+
# check that files were re-verified
echo "waiting for re-verify to complete"
RETRY_TIMES=0
diff --git a/.github/integration/tests/validator-orchestrator/go.mod b/.github/integration/tests/validator-orchestrator/go.mod
new file mode 100644
index 0000000000..cd4302ddbc
--- /dev/null
+++ b/.github/integration/tests/validator-orchestrator/go.mod
@@ -0,0 +1,39 @@
+module validator-orchestrator-tester
+
+go 1.25.0
+
+require (
+ github.com/aws/aws-sdk-go-v2 v1.39.6
+ github.com/aws/aws-sdk-go-v2/config v1.31.20
+ github.com/aws/aws-sdk-go-v2/credentials v1.18.24
+ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2
+ github.com/golang-jwt/jwt/v5 v5.3.0
+ github.com/google/uuid v1.6.0
+ github.com/neicnordic/crypt4gh v1.14.0
+ github.com/stretchr/testify v1.11.1
+)
+
+require (
+ filippo.io/edwards25519 v1.1.0 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 // indirect
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 // indirect
+ github.com/aws/smithy-go v1.23.2 // indirect
+ github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect
+ github.com/pmezard/go-difflib v1.0.0 // indirect
+ golang.org/x/crypto v0.45.0 // indirect
+ golang.org/x/sys v0.38.0 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+)
diff --git a/.github/integration/tests/validator-orchestrator/go.sum b/.github/integration/tests/validator-orchestrator/go.sum
new file mode 100644
index 0000000000..fdfa907be7
--- /dev/null
+++ b/.github/integration/tests/validator-orchestrator/go.sum
@@ -0,0 +1,64 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/aws/aws-sdk-go-v2 v1.39.6 h1:2JrPCVgWJm7bm83BDwY5z8ietmeJUbh3O2ACnn+Xsqk=
+github.com/aws/aws-sdk-go-v2 v1.39.6/go.mod h1:c9pm7VwuW0UPxAEYGyTmyurVcNrbF6Rt/wixFqDhcjE=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3 h1:DHctwEM8P8iTXFxC/QK0MRjwEpWQeM9yzidCRjldUz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.3/go.mod h1:xdCzcZEtnSTKVDOmUZs4l/j3pSV6rpo1WXl5ugNsL8Y=
+github.com/aws/aws-sdk-go-v2/config v1.31.20 h1:/jWF4Wu90EhKCgjTdy1DGxcbcbNrjfBHvksEL79tfQc=
+github.com/aws/aws-sdk-go-v2/config v1.31.20/go.mod h1:95Hh1Tc5VYKL9NJ7tAkDcqeKt+MCXQB1hQZaRdJIZE0=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.24 h1:iJ2FmPT35EaIB0+kMa6TnQ+PwG5A1prEdAw+PsMzfHg=
+github.com/aws/aws-sdk-go-v2/credentials v1.18.24/go.mod h1:U91+DrfjAiXPDEGYhh/x29o4p0qHX5HDqG7y5VViv64=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13 h1:T1brd5dR3/fzNFAQch/iBKeX07/ffu/cLu+q+RuzEWk=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.13/go.mod h1:Peg/GBAQ6JDt+RoBf4meB1wylmAipb7Kg2ZFakZTlwk=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7 h1:u8danF+A2Zv//pFZvj5V23v/6XG4AxuSVup5s6nxSnI=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.7/go.mod h1:uvLIvU8iJPEU5so7b6lLDNArWpOX6sRBfL5wBABmlfc=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13 h1:a+8/MLcWlIxo1lF9xaGt3J/u3yOZx+CdSveSNwjhD40=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.13/go.mod h1:oGnKwIYZ4XttyU2JWxFrwvhF6YKiK/9/wmE3v3Iu9K8=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13 h1:HBSI2kDkMdWz4ZM7FjwE7e/pWDEZ+nR95x8Ztet1ooY=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.13/go.mod h1:YE94ZoDArI7awZqJzBAZ3PDD2zSfuP7w6P2knOzIn8M=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13 h1:eg/WYAa12vqTphzIdWMzqYRVKKnCboVPRlvaybNCqPA=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.13/go.mod h1:/FDdxWhz1486obGrKKC1HONd7krpk38LBt+dutLcN9k=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3 h1:x2Ibm/Af8Fi+BH+Hsn9TXGdT+hKbDd5XOTZxTMxDk7o=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.3/go.mod h1:IW1jwyrQgMdhisceG8fQLmQIydcT/jWY21rFhzgaKwo=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4 h1:NvMjwvv8hpGUILarKw7Z4Q0w1H9anXKsesMxtw++MA4=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.4/go.mod h1:455WPHSwaGj2waRSpQp7TsnpOnBfw8iDfPfbwl7KPJE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13 h1:kDqdFvMY4AtKoACfzIGD8A0+hbT41KTKF//gq7jITfM=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.13/go.mod h1:lmKuogqSU3HzQCwZ9ZtcqOc5XGMqtDK7OIc2+DxiUEg=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13 h1:zhBJXdhWIFZ1acfDYIhu4+LCzdUS2Vbcum7D01dXlHQ=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.13/go.mod h1:JaaOeCE368qn2Hzi3sEzY6FgAZVCIYcC2nwbro2QCh8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2 h1:DhdbtDl4FdNlj31+xiRXANxEE+eC7n8JQz+/ilwQ8Uc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.90.2/go.mod h1:+wArOOrcHUevqdto9k1tKOF5++YTe9JEcPSc9Tx2ZSw=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.3 h1:NjShtS1t8r5LUfFVtFeI8xLAHQNTa7UI0VawXlrBMFQ=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.3/go.mod h1:fKvyjJcz63iL/ftA6RaM8sRCtN4r4zl4tjL3qw5ec7k=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7 h1:gTsnx0xXNQ6SBbymoDvcoRHL+q4l/dAFsQuKfDWSaGc=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.7/go.mod h1:klO+ejMvYsB4QATfEOIXk8WAEwN4N0aBfJpvC+5SZBo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.40.2 h1:HK5ON3KmQV2HcAunnx4sKLB9aPf3gKGwVAf7xnx0QT0=
+github.com/aws/aws-sdk-go-v2/service/sts v1.40.2/go.mod h1:E19xDjpzPZC7LS2knI9E6BaRFDK43Eul7vd6rSq2HWk=
+github.com/aws/smithy-go v1.23.2 h1:Crv0eatJUQhaManss33hS5r40CG3ZFH+21XSkqMrIUM=
+github.com/aws/smithy-go v1.23.2/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU=
+github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0=
+github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo=
+github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U=
+github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/.github/integration/tests/validator-orchestrator/main_test.go b/.github/integration/tests/validator-orchestrator/main_test.go
new file mode 100644
index 0000000000..683832d591
--- /dev/null
+++ b/.github/integration/tests/validator-orchestrator/main_test.go
@@ -0,0 +1,516 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/x509"
+ "encoding/json"
+ "encoding/pem"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "testing"
+ "time"
+
+ "github.com/aws/aws-sdk-go-v2/aws"
+ awsConfig "github.com/aws/aws-sdk-go-v2/config"
+ "github.com/aws/aws-sdk-go-v2/credentials"
+ "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+ "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/golang-jwt/jwt/v5"
+ "github.com/google/uuid"
+ "github.com/neicnordic/crypt4gh/keys"
+ "github.com/neicnordic/crypt4gh/streaming"
+ "github.com/stretchr/testify/suite"
+)
+
+type TestSuite struct {
+ suite.Suite
+
+ // Config
+ url string
+ inboxURL string
+ jwtKeyFilePath string
+ c4ghKeyFilePath string
+ s3AccessKey string
+ s3SecretKey string
+
+ adminToken, submissionToken string
+}
+
+func TestTestSuite(m *testing.T) {
+ suite.Run(m, new(TestSuite))
+}
+
+func (ts *TestSuite) SetupSuite() {
+ // Note these can be changed if you want to test against different endpoints, etc
+ ts.url = "http://validator-orchestrator:8080"
+ ts.inboxURL = "http://s3inbox:8000"
+ ts.jwtKeyFilePath = "/shared/keys/jwt.key"
+ ts.c4ghKeyFilePath = "/shared/c4gh.pub.pem"
+ ts.s3AccessKey = "submission_user_integration.test"
+ ts.s3SecretKey = "secretKey"
+
+ var err error
+ ts.adminToken, err = ts.generateToken("admin_user@integration.test")
+ if err != nil {
+ ts.FailNow("failed to generate admin token", err.Error())
+ }
+
+ ts.submissionToken, err = ts.generateToken("submission_user@integration.test")
+ if err != nil {
+ ts.FailNow("failed to generate submission token", err.Error())
+ }
+
+ if err := ts.uploadFiles(); err != nil {
+ ts.FailNow("failed to upload files", err.Error())
+ }
+}
+
+func (ts *TestSuite) uploadFiles() error {
+ log.Println("uploading files")
+
+ httpClient := http.DefaultClient
+
+ awsConf, err := awsConfig.LoadDefaultConfig(context.Background(),
+ awsConfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
+ ts.s3AccessKey,
+ ts.s3SecretKey,
+ ts.submissionToken,
+ )),
+ awsConfig.WithRegion("us-east-1"),
+ awsConfig.WithBaseEndpoint(ts.inboxURL),
+ )
+ if err != nil {
+ return fmt.Errorf("failed to load aws config: %v", err)
+ }
+
+ s3Client := s3.NewFromConfig(awsConf, func(o *s3.Options) {
+ o.UsePathStyle = true
+ o.EndpointOptions.DisableHTTPS = true
+ o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
+ })
+
+ // Create an uploader with the session and default options
+ uploader := manager.NewUploader(s3Client)
+
+ files := []string{"NA12878.bam", "NA12878_20k_b37.bam", "NA12878.bai", "NA12878_20k_b37.bai"}
+ for _, fileName := range files {
+ req, _ := http.NewRequest("GET", fmt.Sprintf("https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/%s", fileName), nil)
+ req.Header.Add("Authorization", "Bearer "+string(ts.submissionToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to do http request: %v", err)
+ }
+
+ c4ghKey, err := os.ReadFile(ts.c4ghKeyFilePath)
+ if err != nil {
+ return fmt.Errorf("failed to read c4gh file: %v", err)
+ }
+
+ keyReader := bytes.NewReader(c4ghKey)
+ newReaderPublicKey, err := keys.ReadPublicKey(keyReader)
+ if err != nil {
+ return fmt.Errorf("failed to public jwt key file: %v", err)
+ }
+
+ encryptedWriter := bytes.Buffer{}
+ encryptedFileWriter, err := streaming.NewCrypt4GHWriter(&encryptedWriter, [32]byte{}, [][32]byte{newReaderPublicKey}, nil)
+ if err != nil {
+ return fmt.Errorf("failed to create c4gh writer: %v", err)
+ }
+
+ _, _ = io.Copy(encryptedFileWriter, rsp.Body)
+ _ = encryptedFileWriter.Close()
+ _ = rsp.Body.Close()
+
+ reader := bytes.NewReader(encryptedWriter.Bytes())
+
+ _, err = uploader.Upload(context.Background(), &s3.PutObjectInput{
+ Body: reader,
+ Bucket: aws.String(ts.s3AccessKey),
+ Key: aws.String(fileName),
+ ContentEncoding: aws.String("UTF-8"),
+ }, func(u *manager.Uploader) {
+ u.PartSize = 50 * 1024 * 1024
+ u.LeavePartsOnError = false
+ })
+ if err != nil {
+ return fmt.Errorf("failed to upload file: %v", err)
+ }
+ }
+
+ return nil
+}
+
+func (ts *TestSuite) generateToken(sub string) (string, error) {
+ keyPem, err := os.ReadFile(ts.jwtKeyFilePath)
+ if err != nil {
+ ts.FailNow(fmt.Sprintf("failed to read key file from path: %s", ts.jwtKeyFilePath), err)
+ }
+
+ token := jwt.NewWithClaims(jwt.SigningMethodRS256, &jwt.RegisteredClaims{
+ Subject: sub,
+ Issuer: "http://integration.test",
+ ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Second * 30)),
+ IssuedAt: jwt.NewNumericDate(time.Now()),
+ })
+
+ token.Header["kid"] = "rsa1"
+ block, _ := pem.Decode(keyPem)
+ keyRaw, err := x509.ParsePKCS8PrivateKey(block.Bytes)
+ if err != nil {
+ return "", err
+ }
+
+ return token.SignedString(keyRaw)
+}
+
+func (ts *TestSuite) TestValidatorsAsAdmin() {
+ httpClient := http.DefaultClient
+
+ req, _ := http.NewRequest("GET", fmt.Sprintf("%s/validators", ts.url), nil)
+ req.Header.Add("Authorization", "Bearer "+string(ts.adminToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ return
+ }
+
+ // Read the response body
+ resBody, err := io.ReadAll(rsp.Body)
+ if err != nil {
+ ts.FailNow("failed to read response body", err.Error())
+ }
+
+ ts.Equal(http.StatusOK, rsp.StatusCode)
+ ts.Contains(string(resBody), "validator-word-count-100-200")
+ ts.Contains(string(resBody), "validator-always-success")
+}
+func (ts *TestSuite) TestValidatorsAsSubmitter() {
+ httpClient := http.DefaultClient
+
+ req, _ := http.NewRequest("GET", fmt.Sprint(ts.url, "/validators"), nil)
+ req.Header.Add("Authorization", "Bearer "+string(ts.submissionToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ // Read the response body
+ resBody, err := io.ReadAll(rsp.Body)
+ if err != nil {
+ ts.FailNow("failed to read response body", err.Error())
+ }
+
+ ts.Equal(http.StatusOK, rsp.StatusCode)
+ ts.Contains(string(resBody), "validator-word-count-100-200")
+ ts.Contains(string(resBody), "validator-always-success")
+}
+func (ts *TestSuite) TestValidateAndResult() {
+ httpClient := http.DefaultClient
+
+ files := `["NA12878.bam", "NA12878_20k_b37.bam", "NA12878.bai", "NA12878_20k_b37.bai"]`
+ req, _ := http.NewRequest("POST", fmt.Sprintf("%s/validate", ts.url), bytes.NewReader([]byte(fmt.Sprintf(`{
+"file_paths": %s,
+"validators": ["validator-word-count-100-200", "validator-always-success"]
+}`, files))))
+ req.Header.Add("Authorization", "Bearer "+string(ts.submissionToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ resBody, err := io.ReadAll(rsp.Body)
+ if err != nil {
+ ts.FailNow("failed to read response body", err.Error())
+ }
+
+ _ = rsp.Body.Close()
+
+ parsedResponse := new(validateResponse)
+
+ ts.Equal(http.StatusOK, rsp.StatusCode)
+
+ if err := json.Unmarshal(resBody, &parsedResponse); err != nil {
+ ts.FailNow("failed to unmarshal response body", err.Error())
+ }
+
+ if _, err := uuid.Parse(parsedResponse.ValidationId); err != nil {
+ ts.Fail("failed to parse validation id in response as a uuid", err.Error())
+ }
+
+ // poll until result is no longer pending
+ var resultResponse []validatorResult
+
+ abortPollingAt := time.Now().Add(time.Second * 30)
+
+ for {
+ req, _ = http.NewRequest("GET", fmt.Sprintf("%s/result?validation_id=%s", ts.url, parsedResponse.ValidationId), nil)
+ req.Header.Add("Authorization", "Bearer "+string(ts.submissionToken))
+
+ rsp, err = httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ ts.Equal(http.StatusOK, rsp.StatusCode)
+
+ resBody, err = io.ReadAll(rsp.Body)
+ if err != nil {
+ ts.FailNow("failed to read response body", err.Error())
+ }
+ _ = rsp.Body.Close()
+
+ if err := json.Unmarshal(resBody, &resultResponse); err != nil {
+ ts.FailNow("failed to parse result response body", err.Error())
+ }
+
+ allDone := true
+ for _, result := range resultResponse {
+ if result.Result == "pending" {
+ allDone = false
+ break
+ }
+ }
+ if !allDone {
+ if time.Now().After(abortPollingAt) {
+ ts.FailNow("validate result did not complete and give non pending results within 30 seconds")
+ }
+ time.Sleep(1 * time.Second)
+ continue
+ }
+ break
+ }
+
+ ts.Len(resultResponse, 2)
+
+ for _, result := range resultResponse {
+ ts.Len(result.Files, 4)
+ switch result.ValidatorId {
+ case "validator-word-count-100-200":
+
+ ts.Len(result.Messages, 0)
+ ts.Equal("failed", result.Result)
+
+ for _, fr := range result.Files {
+ ts.Equal("failed", fr.Result)
+ ts.Contains(files, fr.Path)
+ ts.Len(fr.Messages, 1)
+ }
+
+ case "validator-always-success":
+ ts.Len(result.Messages, 0)
+ ts.Equal("passed", result.Result)
+
+ for _, fr := range result.Files {
+ ts.Equal("passed", fr.Result)
+ ts.Contains(files, fr.Path)
+ ts.Len(fr.Messages, 0)
+ }
+
+ default:
+ ts.Fail("unexpected validator id in result")
+ }
+ }
+}
+func (ts *TestSuite) TestAdminValidateAndAdminResult() {
+ httpClient := http.DefaultClient
+
+ files := `["NA12878.bam", "NA12878_20k_b37.bam", "NA12878.bai", "NA12878_20k_b37.bai"]`
+ req, _ := http.NewRequest("POST", fmt.Sprintf("%s/admin/validate", ts.url), bytes.NewReader([]byte(fmt.Sprintf(`{
+"file_paths": %s,
+"validators": ["validator-word-count-100-200", "validator-always-success"],
+"user_id": "submission_user@integration.test"
+}`, files))))
+ req.Header.Add("Authorization", "Bearer "+string(ts.adminToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ resBody, err := io.ReadAll(rsp.Body)
+ if err != nil {
+ ts.FailNow("failed to read response body", err.Error())
+ }
+
+ _ = rsp.Body.Close()
+
+ parsedResponse := new(validateResponse)
+
+ ts.Equal(http.StatusOK, rsp.StatusCode)
+
+ if err := json.Unmarshal(resBody, &parsedResponse); err != nil {
+ ts.FailNow("failed to unmarshal response body", err.Error())
+ }
+
+ if _, err := uuid.Parse(parsedResponse.ValidationId); err != nil {
+ ts.Fail("failed to parse validation id in response as a uuid", err.Error())
+ }
+
+ // poll until result is no longer pending
+ var resultResponse []validatorResult
+
+ abortPollingAt := time.Now().Add(time.Second * 30)
+
+ for {
+ req, _ = http.NewRequest("GET", fmt.Sprintf("%s/admin/result?validation_id=%s&submission_user=submission_user@integration.test", ts.url, parsedResponse.ValidationId), nil)
+ req.Header.Add("Authorization", "Bearer "+string(ts.adminToken))
+
+ rsp, err = httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ ts.Equal(http.StatusOK, rsp.StatusCode)
+
+ resBody, err = io.ReadAll(rsp.Body)
+ if err != nil {
+ ts.FailNow("failed to read response body", err.Error())
+ }
+ _ = rsp.Body.Close()
+
+ if err := json.Unmarshal(resBody, &resultResponse); err != nil {
+ ts.FailNow("failed to parse result response body", err.Error())
+ }
+
+ allDone := true
+ for _, result := range resultResponse {
+ if result.Result == "pending" {
+ allDone = false
+ break
+ }
+ }
+ if !allDone {
+ if time.Now().After(abortPollingAt) {
+ ts.FailNow("validate result did not complete and give non pending results within 30 seconds")
+ }
+ time.Sleep(1 * time.Second)
+ continue
+ }
+ break
+ }
+
+ ts.Len(resultResponse, 2)
+
+ for _, result := range resultResponse {
+ ts.Len(result.Files, 4)
+ switch result.ValidatorId {
+ case "validator-word-count-100-200":
+
+ ts.Len(result.Messages, 0)
+ ts.Equal("failed", result.Result)
+
+ for _, fr := range result.Files {
+ ts.Equal("failed", fr.Result)
+ ts.Contains(files, fr.Path)
+ ts.Len(fr.Messages, 1)
+ }
+
+ case "validator-always-success":
+ ts.Len(result.Messages, 0)
+ ts.Equal("passed", result.Result)
+
+ for _, fr := range result.Files {
+ ts.Equal("passed", fr.Result)
+ ts.Contains(files, fr.Path)
+ ts.Len(fr.Messages, 0)
+ }
+
+ default:
+ ts.Fail("unexpected validator id in result")
+ }
+ }
+}
+
+func (ts *TestSuite) TestAdminValidateAsSubmitter() {
+ httpClient := http.DefaultClient
+
+ files := `["NA12878.bam", "NA12878_20k_b37.bam", "NA12878.bai", "NA12878_20k_b37.bai"]`
+ req, _ := http.NewRequest("POST", fmt.Sprintf("%s/admin/validate", ts.url), bytes.NewReader([]byte(fmt.Sprintf(`{
+"file_paths": %s,
+"validators": ["validator-word-count-100-200", "validator-always-success"],
+"user_id": "submission_user@integration.test"
+}`, files))))
+ req.Header.Add("Authorization", "Bearer "+string(ts.submissionToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ _ = rsp.Body.Close()
+ ts.Equal(http.StatusUnauthorized, rsp.StatusCode)
+}
+
+func (ts *TestSuite) TestAdminResultAsSubmitter() {
+ httpClient := http.DefaultClient
+
+ req, _ := http.NewRequest("POST", fmt.Sprintf("%s/admin/result?validation_id=dont_matter", ts.url), nil)
+ req.Header.Add("Authorization", "Bearer "+string(ts.submissionToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ _ = rsp.Body.Close()
+ ts.Equal(http.StatusUnauthorized, rsp.StatusCode)
+}
+
+func (ts *TestSuite) TestValidateUnknownFile() {
+ httpClient := http.DefaultClient
+
+ files := `["FILE_NO_EXISTS.xml", "dir/FILE_NO_EXISTS_1.xml", "NA12878.bai", "NA12878_20k_b37.bai"]`
+ req, _ := http.NewRequest("POST", fmt.Sprintf("%s/validate", ts.url), bytes.NewReader([]byte(fmt.Sprintf(`{
+"file_paths": %s,
+"validators": ["validator-word-count-100-200", "validator-always-success"]
+}`, files))))
+ req.Header.Add("Authorization", "Bearer "+string(ts.submissionToken))
+
+ rsp, err := httpClient.Do(req)
+ if err != nil {
+ ts.FailNow("failed to DO http request", err.Error())
+ }
+
+ resBody, err := io.ReadAll(rsp.Body)
+ if err != nil {
+ ts.FailNow("failed to read response body", err.Error())
+ }
+ _ = rsp.Body.Close()
+ ts.Equal(http.StatusBadRequest, rsp.StatusCode)
+ ts.Equal(`{"error":"files: [FILE_NO_EXISTS.xml dir/FILE_NO_EXISTS_1.xml] not found"}`, string(resBody))
+}
+
+type validateResponse struct {
+ ValidationId string `json:"validation_id,omitempty"`
+}
+
+type validatorResult struct {
+ ValidatorId string `json:"validator_id,omitempty"`
+ Result string `json:"result,omitempty"`
+ StartedAt time.Time `json:"started_at,omitempty"`
+ FinishedAt time.Time `json:"finished_at,omitempty"`
+ Files []fileResult `json:"files,omitempty"`
+ Messages []resultMessage `json:"messages,omitempty"`
+}
+
+type fileResult struct {
+ Path string `json:"path,omitempty"`
+ Result string `json:"result,omitempty"`
+ Messages []resultMessage `json:"messages,omitempty"`
+}
+
+type resultMessage struct {
+ Level string `json:"level,omitempty"`
+ Time string `json:"time,omitempty"`
+ Message string `json:"message,omitempty"`
+}
diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml
index c5b847ab23..140c782068 100644
--- a/.github/workflows/build_pr_container.yaml
+++ b/.github/workflows/build_pr_container.yaml
@@ -53,6 +53,19 @@ jobs:
org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
org.opencontainers.image.revision=${{ github.sha }}
+ - name: Build container for sda-validator-orchestrator
+ uses: docker/build-push-action@v6
+ with:
+ context: ./sda-validator/orchestrator
+ push: true
+ tags: |
+ ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-validator-orchestrator
+ ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-validator-orchestrator
+ labels: |
+ org.opencontainers.image.source=${{ github.event.repository.clone_url }}
+ org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+ org.opencontainers.image.revision=${{ github.sha }}
+
- name: Build container for sensitive-data-archive
uses: docker/build-push-action@v6
with:
@@ -310,6 +323,19 @@ jobs:
run: docker compose -f .github/integration/sda-sync-integration.yml run integration_test
if: steps.changes.outputs.sync == 'true' || steps.changes.outputs.sync-api == 'true'
+ sda-validator-orchestrator:
+ needs:
+ - build_go_images
+ - build_server_images
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v6
+ - name: Test sda-validator-orchestrator
+ run: |
+ echo 'kernel.apparmor_restrict_unprivileged_userns = 0' | sudo tee /etc/sysctl.d/20-apparmor-donotrestrict.conf
+ sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0
+ docker compose -f .github/integration/sda-validator-orchestrator-integration.yml run integration_test
+
sda-doa:
needs:
- build_java_images
diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml
index 93bf2ffa20..755363c783 100644
--- a/.github/workflows/code-linter.yaml
+++ b/.github/workflows/code-linter.yaml
@@ -9,6 +9,7 @@ on:
env:
go-version: '1.24'
+ go-version-validator-orchestrator: '1.25'
jobs:
check_changes:
@@ -16,6 +17,7 @@ jobs:
sda-download: ${{ steps.changes.outputs.sda-download }}
sda-admin: ${{ steps.changes.outputs.sda-admin }}
sda: ${{ steps.changes.outputs.sda }}
+ sda-validator-orchestrator: ${{ steps.changes.outputs.sda-validator-orchestrator }}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
@@ -29,6 +31,8 @@ jobs:
- 'sda-admin/**'
sda:
- 'sda/**'
+ sda-validator-orchestrator:
+ - 'sda-validator/**'
lint_sda_download:
needs: check_changes
@@ -46,7 +50,7 @@ jobs:
uses: actions/checkout@v6
- name: Run golangci-lint
- uses: golangci/golangci-lint-action@v9.1.0
+ uses: golangci/golangci-lint-action@v9.2.0
with:
args: --timeout 5m
working-directory: sda-download
@@ -67,11 +71,32 @@ jobs:
uses: actions/checkout@v6
- name: Run golangci-lint
- uses: golangci/golangci-lint-action@v9.1.0
+ uses: golangci/golangci-lint-action@v9.2.0
with:
args: --timeout 5m
working-directory: sda
+ lint_sda_validator_orchestrator:
+ needs: check_changes
+ if: needs.check_changes.outputs.sda-validator-orchestrator == 'true'
+ name: Lint sda validator orchestrator
+ runs-on: ubuntu-latest
+ steps:
+ - name: Set up Go ${{ env.go-version-validator-orchestrator }}
+ uses: actions/setup-go@v6
+ with:
+ go-version: ${{ env.go-version-validator-orchestrator }}
+ id: go
+
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v6
+
+ - name: Run golangci-lint
+ uses: golangci/golangci-lint-action@v9.2.0
+ with:
+ args: --timeout 5m
+ working-directory: sda-validator/orchestrator
+
lint_sda_admin:
needs: check_changes
if: needs.check_changes.outputs.sda-admin == 'true'
@@ -88,7 +113,7 @@ jobs:
uses: actions/checkout@v6
- name: Run golangci-lint
- uses: golangci/golangci-lint-action@v9.1.0
+ uses: golangci/golangci-lint-action@v9.2.0
with:
args: --timeout 5m
working-directory: sda-admin
diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml
index 49fc1750bb..0c49b45b61 100644
--- a/.github/workflows/publish_container.yml
+++ b/.github/workflows/publish_container.yml
@@ -57,7 +57,7 @@ jobs:
- name: Pull PR containers
run : |
- for t in -download -postgres -rabbitmq -sftp-inbox -doa; do
+ for t in -download -postgres -rabbitmq -sftp-inbox -doa -validator-orchestrator; do
docker pull ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }}$t
done
docker pull ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }}
@@ -87,6 +87,11 @@ jobs:
docker tag ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }} ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }}
docker push ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }}
+ - name: Retag PR image for sda-validator-orchestrator
+ run: |
+ docker tag ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }}-validator-orchestrator ghcr.io/${{ github.repository }}:${{ needs.tag_release.outputs.tag }}-validator-orchestrator
+ docker push ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }}-validator-orchestrator
+
- name: Retag PR image for doa
run: |
docker tag ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }}-doa ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }}-doa
@@ -120,11 +125,11 @@ jobs:
shell: bash
- name: Create Pull Request
- uses: peter-evans/create-pull-request@v7
+ uses: peter-evans/create-pull-request@v8
with:
token: ${{ secrets.GITHUB_TOKEN }}
commit-message: Bump chart version
- author: 'Biobank Lab'
+ author: user.email "$GITHUB_ACTOR@users.noreply.github.com"
base: main
branch: bump
delete-branch: true
@@ -133,4 +138,4 @@ jobs:
Bump the appVersion to: ${{ needs.tag_release.outputs.tag }}
labels: |
automated pr
- team-reviewers: 'Biobank Lab/sensitive-data-development-collaboration'
+ team-reviewers: 'biobanklab/sensitive-data-development-collaboration'
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2a676fbdec..6d55b7a797 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -6,6 +6,7 @@ on:
env:
go-version: '1.24'
+ go-version-validator-orchestrator: '1.25'
jobs:
@@ -23,7 +24,7 @@ jobs:
with:
distribution: zulu
java-version: ${{ matrix.java-version }}
- - uses: actions/cache@v4
+ - uses: actions/cache@v5
with:
path: ~/.m2/repository
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
@@ -60,7 +61,7 @@ jobs:
go test -v -coverprofile=coverage.txt -covermode=atomic ./...
- name: Codecov
- uses: codecov/codecov-action@v5.5.1
+ uses: codecov/codecov-action@v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./sda-download/coverage.txt
@@ -95,7 +96,7 @@ jobs:
go test -v -coverprofile=coverage.txt -covermode=atomic ./...
- name: Codecov
- uses: codecov/codecov-action@v5.5.1
+ uses: codecov/codecov-action@v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./sda/coverage.txt
@@ -130,9 +131,44 @@ jobs:
go test -v -coverprofile=coverage.txt -covermode=atomic ./...
- name: Codecov
- uses: codecov/codecov-action@v5.5.1
+ uses: codecov/codecov-action@v5.5.2
with:
token: ${{ secrets.CODECOV_TOKEN }}
file: ./sda-admin/coverage.txt
flags: unittests
fail_ci_if_error: false
+
+ test_sda_validator_orchestrator:
+ name: Test SDA Validator Orchestrator
+ runs-on: ubuntu-latest
+ steps:
+
+ - name: Set up Go ${{ env.go-version-validator-orchestrator }}
+ uses: actions/setup-go@v6
+ with:
+ go-version: ${{ env.go-version-validator-orchestrator }}
+ id: go
+
+ - name: Check out code into the Go module directory
+ uses: actions/checkout@v6
+
+ - name: Get dependencies
+ run: |
+ cd ./sda-validator/orchestrator
+ go get -v -t -d ./...
+ if [ -f Gopkg.toml ]; then
+ curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+ dep ensure
+ fi
+ - name: Test
+ run: |
+ cd ./sda-validator/orchestrator
+ go test -v -coverprofile=coverage.txt -covermode=atomic ./...
+
+ - name: Codecov
+ uses: codecov/codecov-action@v5.5.2
+ with:
+ token: ${{ secrets.CODECOV_TOKEN }}
+ file: ./sda-validator/orchestrator/coverage.txt
+ flags: unittests
+ fail_ci_if_error: false
diff --git a/.github/workflows/update_trivy_cache.yml b/.github/workflows/update_trivy_cache.yml
index 13f5b288c5..1330f0090b 100644
--- a/.github/workflows/update_trivy_cache.yml
+++ b/.github/workflows/update_trivy_cache.yml
@@ -31,7 +31,7 @@ jobs:
rm javadb.tar.gz
- name: Cache DBs
- uses: actions/cache/save@v4
+ uses: actions/cache/save@v5
with:
path: ${{ github.workspace }}/.cache/trivy
key: cache-trivy-${{ steps.date.outputs.date }}
\ No newline at end of file
diff --git a/Makefile b/Makefile
index c2c1f7f324..daaee27f9d 100644
--- a/Makefile
+++ b/Makefile
@@ -19,7 +19,7 @@ bootstrap: go-version-check docker-version-check
go install golang.org/x/tools/cmd/goimports@latest
# build containers
-build-all: build-postgresql build-rabbitmq build-sda build-sda-download build-sda-sftp-inbox build-sda-admin
+build-all: build-postgresql build-rabbitmq build-sda build-sda-download build-sda-sftp-inbox build-sda-admin build-sda-validator-orchestrator
build-postgresql:
@cd postgresql && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-postgres .
build-rabbitmq:
@@ -34,6 +34,8 @@ build-sda-admin:
@cd sda-admin && go build
build-sda-doa:
@cd sda-doa && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-doa .
+build-sda-validator-orchestrator:
+ @cd sda-validator/orchestrator && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-validator-orchestrator .
go-version-check: SHELL:=/bin/bash
@@ -147,7 +149,13 @@ integrationtest-sda-doa-s3-run:
integrationtest-sda-doa-s3-down:
@PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml down -v --remove-orphans
-
+integrationtest-sda-validator-orchestrator: build-all
+ @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-validator-orchestrator-integration.yml run integration_test
+ @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-validator-orchestrator-integration.yml down -v --remove-orphans
+integrationtest-sda-validator-orchestrator-run:
+ @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-validator-orchestrator-integration.yml run integration_test
+integrationtest-sda-validator-orchestrator-down:
+ @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-validator-orchestrator-integration.yml down -v --remove-orphans
# lint go code
lint-all: lint-sda lint-sda-download lint-sda-admin
@@ -191,8 +199,9 @@ k3d-version-check:
echo "kubectl is missing";\
fi
k3d-create-cluster:
- @k3d cluster create --k3s-arg "--disable=traefik@server:0" --port "80:80@loadbalancer" --port "443:443@loadbalancer"; \
- helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace
+ @k3d cluster create --agents 2 --image rancher/k3s:v1.34.2-k3s1 \
+ --port "80:80@loadbalancer" \
+ --port "443:443@loadbalancer"
k3d-delete-cluster:
@k3d cluster delete
k3d-deploy-dependencies-federated:
diff --git a/charts/sda-mq/README.md b/charts/sda-mq/README.md
index 2a52714a52..3b433e7393 100644
--- a/charts/sda-mq/README.md
+++ b/charts/sda-mq/README.md
@@ -14,7 +14,7 @@ Parameter | Description | Default
`global.adminUser` | Username of admin user |`admin`
`global.adminPassword` | Password for admin user. |`Random if unset`
`global.ingress.annotations` | extra annotations for the ingress objects | `""`
-`global.ingress.ingressClassName` | class of the ingress controller | `"nginx"`
+`global.ingress.ingressClassName` | class of the ingress controller | `""`
`global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""`
`global.ingress.hostName` | hostname for the ingress endpoint | `""`
`global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""`
diff --git a/charts/sda-mq/templates/ingress.yaml b/charts/sda-mq/templates/ingress.yaml
index e2f659f06a..6cd68351f2 100644
--- a/charts/sda-mq/templates/ingress.yaml
+++ b/charts/sda-mq/templates/ingress.yaml
@@ -9,10 +9,6 @@ metadata:
release: {{ .Release.Name }}
heritage: {{ .Release.Service }}
annotations:
- {{- if eq "nginx" .Values.global.ingress.ingressClassName }}
- nginx.ingress.kubernetes.io/rewrite-target: "/"
- nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}"
- {{- end }}
{{- if .Values.global.ingress.clusterIssuer }}
cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}
{{- else if .Values.global.ingress.issuer }}
diff --git a/charts/sda-mq/values.yaml b/charts/sda-mq/values.yaml
index 4739bd59c3..7050afd6aa 100644
--- a/charts/sda-mq/values.yaml
+++ b/charts/sda-mq/values.yaml
@@ -5,7 +5,7 @@ global:
# extra annotations for the ingress
annotations: {}
hostName: ""
- ingressClassName: "nginx"
+ ingressClassName: ""
issuer: ""
clusterIssuer: ""
# If the certificates is generated by external providers
@@ -103,7 +103,7 @@ service:
updateStrategyType: RollingUpdate
testimage:
- repository: neicnordic/sda-helm-tests-support
+ repository: biobanklab/sda-helm-tests-support
tag: latest
pullPollicy: Always
tls:
diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md
index 18506b5800..75133cc1ee 100644
--- a/charts/sda-svc/README.md
+++ b/charts/sda-svc/README.md
@@ -45,17 +45,17 @@ Parameter | Description | Default
`global.confFilePath` | This path will be a subpath to the secretsPath | `""`
`global.deploymentType` | Deployment can be split into `external` and `internal` components, available options are `all`, `external` and `internal`. | `all`
`global.schemaType` | Standalone version requires a custom JOSN schema, available options are `federated` or `isolated`, unset defaults to federated. | `""`
-`global.ingress.deploy` | | `false`
-`global.ingress.hostName.auth` | | `""`
-`global.ingress.hostName.doa` | | `""`
-`global.ingress.hostName.s3Inbox` | | `""`
+`global.ingress.deploy` | | `false`
+`global.ingress.hostName.auth` | | `""`
+`global.ingress.hostName.doa` | | `""`
+`global.ingress.hostName.s3Inbox` | | `""`
`global.ingress.secretNames.auth` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""`
`global.ingress.secretNames.doa` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""`
`global.ingress.secretNames.s3Inbox` | The name of a manually created secret holding the certificates for the ingrewss enpoint. | `""`
`global.ingress.clusterIssuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured clusterIssuer can be specified to automate certificate configuration for the ingress endpoint. | `""`
`global.ingress.issuer` | If cert-manager is set up to request certificates to the ingress endpoints, the configured issuer can be specified to automate certificate configuration for the ingress endpoint. | `""`
-`global.ingress.annotations` | extra annotations for the ingress objects | `""`
-`global.ingress.ingressClassName` | Class of ingress controller to use | `nginx`
+`global.ingress.annotations` | extra annotations shared by all ingress definitions | `""`
+`global.ingress.ingressClassName` | Class of ingress controller to use | `""`
`global.log.format` | Log format for all services, JSON or TEXT. | `json`
`global.log.level` | Log level for all services. | `info`
`global.networkPolicy.create` | Use network isolation. | `false`
@@ -66,72 +66,72 @@ Parameter | Description | Default
`global.networkPolicy.ingressNamespace` | Namespace where the ingress is deployed. | `""`
`global.networkPolicy.ingressLabelMatch` | Match to use to allow connections from ingress controller pod. | `""`
`global.revisionHistory` | Number of revisions to keep for the option to rollback a deployment | `3`
-`global.podAnnotations` | Annotations applied to pods of all services. |`{}`
-`global.pkiService` | If an external PKI infrastructure is used set this to true. |`false`
-`global.rbacEnabled` | Use role based access control. |`true`
+`global.podAnnotations` | Annotations applied to pods of all services. | `{}`
+`global.pkiService` | If an external PKI infrastructure is used set this to true. | `false`
+`global.rbacEnabled` | Use role based access control. | `true`
`global.vaultSecrets` | If Hashicorp Vault is used for secrets management | `false`
-`global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. |`s3`
-`global.archive.s3Url` | URL to S3 archive instance. |`""`
-`global.archive.s3Bucket` | S3 archive bucket. |`""`
-`global.archive.s3Region` | S3 archive region. |`us-east-1`
-`global.archive.s3ChunkSize` | S3 chunk size in MB. |`15`
-`global.archive.s3AccessKey` | Access key to S3 archive . |`null`
-`global.archive.s3SecretKey` | Secret key to S3 archive. |`null`
-`global.archive.s3CaFile` | CA certificate to use if the S3 archive is internal. |`null`
-`global.archive.s3Port` | Port that the S3 S3 archive is available on. |`443`
-`global.archive.volumePath` | Path to the mounted `posix` volume. |`/archive`
-`global.archive.nfsServer` | URL or IP address to a NFS server. |`""`
-`global.archive.nfsPath` | Path on the NFS server for the archive. |`""`
-`global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. |`null`
-`global.backupArchive.s3Url` | URL to S3 backup archive instance. |`""`
-`global.backupArchive.s3Bucket` | S3 backup archive bucket. |`""`
-`global.backupArchive.s3Region` | S3 backup archive region. |`us-east-1`
-`global.backupArchive.s3ChunkSize` | S3 chunk size in MB. |`15`
-`global.backupArchive.s3AccessKey` | Access key to S3 backup archive . |`null`
-`global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. |`null`
-`global.backupArchive.s3CaFile` | CA certificate to use if the S3 backup archive is internal. |`null`
-`global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. |`443`
-`global.backupArchive.volumePath` | Path to the mounted `posix` volume. |`/backup`
-`global.backupArchive.nfsServer` | URL or IP address to a NFS server. |`""`
-`global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. |`""`
-`global.api.jwtPubKeyName` | Public key used to verify the JWT. |``
-`global.api.jwtSecret` | The name of the secret holding the JWT public key |``
-`global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) |``
-`global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type |`"ES256"`
-`global.auth.jwtKey` | Private key used to sign the JWT. |`""`
-`global.auth.jwtPub` | Public key ues to verify the JWT. |`""`
-`global.auth.jwtTTL` | TTL of the resigned token (hours). |`168`
-`global.auth.resignJWT` | Resign the LS-AAI JWTs. |`true`
-`global.auth.useTLS` | Run a TLS secured server. |`true`
-`global.auth.corsOrigins` | Domain name allowed for cross-domain requests. |`""`
-`global.auth.corsMethods` | Allowed cross-domain request methods. |`""`
-`global.auth.corsCreds` | Include credentials in cross-domain requests. |`false`
-`global.broker.host` | Domain name or IP address to the message broker. |`""`
-`global.broker.exchange` | Exchange to publish messages to. |`""`
-`global.broker.port` | Port for the message broker. |`5671`
-`global.broker.verifyPeer` | Use Client/Server verification. |`true`
-`global.broker.vhost` | Virtual host to connect to. |`/`
-`global.broker.password` | Shared password to the message broker. |`/`
-`global.broker.username` | Shared user to the message broker. |`/`
-`global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers |`2`
-`global.cega.host` | Full URI to the EGA user authentication service. |`""`
-`global.cega.user` | Username for the EGA user authentication service. |`""`
-`global.cega.password` | Password for the EGA user authentication service. |`""`
-`global.c4gh.privateKeys` | List of Private C4GH keys. |``
-`global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. |`""`
-`global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. |`""`
-`global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. |`""`
-`global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. |`""`
-`global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. |`""`
-`global.db.admin.secretName` | Name of the secret that holds the database admin credentials. |`""`
-`global.db.admin.passKey` | Key in the secret that holds the password. |`""`
-`global.db.admin.userkey` | Key in the secret that holds the username. |`""`
-`global.db.host` | Hostname for the database. |`""`
-`global.db.name` | Database to connect to. |`lega`
-`global.db.passIngest` | Password used for `data in` services. |`""`
-`global.db.passOutgest` | Password used for `data out` services. |`""`
-`global.db.port` | Port that the database is listening on. |`5432`
-`global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. |`verify-full`
+`global.archive.storageType` | Storage type for the data archive, available options are `s3` and `posix`. | `s3`
+`global.archive.s3Url` | URL to S3 archive instance. | `""`
+`global.archive.s3Bucket` | S3 archive bucket. | `""`
+`global.archive.s3Region` | S3 archive region. | `us-east-1`
+`global.archive.s3ChunkSize` | S3 chunk size in MB. | `15`
+`global.archive.s3AccessKey` | Access key to S3 archive . | `null`
+`global.archive.s3SecretKey` | Secret key to S3 archive. | `null`
+`global.archive.s3CaFile` | CA certificate to use if the S3 archive is internal. | `null`
+`global.archive.s3Port` | Port that the S3 S3 archive is available on. | `443`
+`global.archive.volumePath` | Path to the mounted `posix` volume. | `/archive`
+`global.archive.nfsServer` | URL or IP address to a NFS server. | `""`
+`global.archive.nfsPath` | Path on the NFS server for the archive. | `""`
+`global.backupArchive.storageType` | Storage type for the backup of the data archive, available options are `s3` and `posix`. | `null`
+`global.backupArchive.s3Url` | URL to S3 backup archive instance. | `""`
+`global.backupArchive.s3Bucket` | S3 backup archive bucket. | `""`
+`global.backupArchive.s3Region` | S3 backup archive region. | `us-east-1`
+`global.backupArchive.s3ChunkSize` | S3 chunk size in MB. | `15`
+`global.backupArchive.s3AccessKey` | Access key to S3 backup archive . | `null`
+`global.backupArchive.s3SecretKey` | Secret key to S3 backup archive. | `null`
+`global.backupArchive.s3CaFile` | CA certificate to use if the S3 backup archive is internal. | `null`
+`global.backupArchive.s3Port` | Port that the S3 S3 backup archive is available on. | `443`
+`global.backupArchive.volumePath` | Path to the mounted `posix` volume. | `/backup`
+`global.backupArchive.nfsServer` | URL or IP address to a NFS server. | `""`
+`global.backupArchive.nfsPath` | Path on the NFS server for the backup archive. | `""`
+`global.api.jwtPubKeyName` | Public key used to verify the JWT. | ``
+`global.api.jwtSecret` | The name of the secret holding the JWT public key | ``
+`global.api.rbacFileSecret` | A secret holding a JSON file named `rbac.json` containg the RBAC policies, see example in the [api.md](https://github.com/neicnordic/sensitive-data-archive/blob/main/sda/cmd/api/api.md#configure-rbac) | ``
+`global.auth.jwtAlg` | Key type to sign the JWT, available options are RS265 & ES256, Must match the key type | `"ES256"`
+`global.auth.jwtKey` | Private key used to sign the JWT. | `""`
+`global.auth.jwtPub` | Public key ues to verify the JWT. | `""`
+`global.auth.jwtTTL` | TTL of the resigned token (hours). | `168`
+`global.auth.resignJWT` | Resign the LS-AAI JWTs. | `true`
+`global.auth.useTLS` | Run a TLS secured server. | `true`
+`global.auth.corsOrigins` | Domain name allowed for cross-domain requests. | `""`
+`global.auth.corsMethods` | Allowed cross-domain request methods. | `""`
+`global.auth.corsCreds` | Include credentials in cross-domain requests. | `false`
+`global.broker.host` | Domain name or IP address to the message broker. | `""`
+`global.broker.exchange` | Exchange to publish messages to. | `""`
+`global.broker.port` | Port for the message broker. | `5671`
+`global.broker.verifyPeer` | Use Client/Server verification. | `true`
+`global.broker.vhost` | Virtual host to connect to. | `/`
+`global.broker.password` | Shared password to the message broker. | `/`
+`global.broker.username` | Shared user to the message broker. | `/`
+`global.broker.prefetchCount` | Number of messages to retrieve from the broker at the time, setting this to `1` will create a round-robin behavior between consumers | `2`
+`global.cega.host` | Full URI to the EGA user authentication service. | `""`
+`global.cega.user` | Username for the EGA user authentication service. | `""`
+`global.cega.password` | Password for the EGA user authentication service. | `""`
+`global.c4gh.privateKeys` | List of Private C4GH keys. | ``
+`global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. | `""`
+`global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. | `""`
+`global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. | `""`
+`global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. | `""`
+`global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. | `""`
+`global.db.admin.secretName` | Name of the secret that holds the database admin credentials. | `""`
+`global.db.admin.passKey` | Key in the secret that holds the password. | `""`
+`global.db.admin.userkey` | Key in the secret that holds the username. | `""`
+`global.db.host` | Hostname for the database. | `""`
+`global.db.name` | Database to connect to. | `lega`
+`global.db.passIngest` | Password used for `data in` services. | `""`
+`global.db.passOutgest` | Password used for `data out` services. | `""`
+`global.db.port` | Port that the database is listening on. | `5432`
+`global.db.sslMode` | SSL mode for the database connection, options are `verify-ca` or `verify-full`. | `verify-full`
`global.doa.enabled` | Deploy the DOA service | `false`
`global.doa.envFile` | File to source when credentials are managed by Hasicorp vault | `env`
`global.doa.serviceport` | Port that the DOA service is accessible on | `443`
@@ -160,21 +160,21 @@ Parameter | Description | Default
`global.oidc.id` | User ID to the OIDC host. | ``
`global.oidc.secret` | User credentials to the OIDC host. | ``
`global.inbox.servicePort` | The port that the inbox is accessible via. | `2222`
-`global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. |`posix`
-`global.inbox.path` | Path to the mounted `posix` volume. |`/inbox`
-`global.inbox.user` | Path to the mounted `posix` volume. |`lega`
-`global.inbox.nfsServer` | URL or IP address to a NFS server. |`""`
-`global.inbox.nfsPath` | Path on the NFS server for the inbox. |`""`
+`global.inbox.storageType` | Storage type for the inbox, available options are `s3` and `posix`. | `posix`
+`global.inbox.path` | Path to the mounted `posix` volume. | `/inbox`
+`global.inbox.user` | Path to the mounted `posix` volume. | `lega`
+`global.inbox.nfsServer` | URL or IP address to a NFS server. | `""`
+`global.inbox.nfsPath` | Path on the NFS server for the inbox. | `""`
`global.inbox.existingClaim` | Existing volume to use for the `posix` inbox. | `""`
-`global.inbox.s3Url` | URL to S3 inbox instance. |`""`
-`global.inbox.s3Port` | Port that the S3 inbox is available on. |`443`
-`global.inbox.s3Bucket` | S3 inbox bucket. |`""`
-`global.inbox.s3Region` | S3 inbox region. |`""`
-`global.inbox.s3ChunkSize` | S3 chunk size in MB. |`15`
-`global.inbox.s3AccessKey` | Access key to S3 inbox . |`null`
-`global.inbox.s3SecretKey` | Secret key to S3 inbox. |`null`
-`global.inbox.s3CaFile` | CA certificate to use if the S3 inbox is internal. |`null`
-`global.inbox.s3ReadyPath` | Endpoint to verify that the inbox is respondig. |`""`
+`global.inbox.s3Url` | URL to S3 inbox instance. | `""`
+`global.inbox.s3Port` | Port that the S3 inbox is available on. | `443`
+`global.inbox.s3Bucket` | S3 inbox bucket. | `""`
+`global.inbox.s3Region` | S3 inbox region. | `""`
+`global.inbox.s3ChunkSize` | S3 chunk size in MB. | `15`
+`global.inbox.s3AccessKey` | Access key to S3 inbox . | `null`
+`global.inbox.s3SecretKey` | Secret key to S3 inbox. | `null`
+`global.inbox.s3CaFile` | CA certificate to use if the S3 inbox is internal. | `null`
+`global.inbox.s3ReadyPath` | Endpoint to verify that the inbox is respondig. | `""`
`global.sync.api.password` | Password for authenticating to the syncAPI server | `null`
`global.sync.api.user` | User for authenticating to the syncAPI server | `null`
`global.sync.brokerQueue` | Queue to read messages from | `mapping_stream`
@@ -189,11 +189,11 @@ Parameter | Description | Default
`global.sync.remote.port` | Port that the remote syncAPI host instance is available on | `null`
`global.sync.remote.password` | Password for connecting to the remote syncAPI host | `null`
`global.sync.remote.user` | Username for connecting to the remote syncAPI host | `null`
-`global.tls.enabled` | Use TLS for all connections. |`true`
-`global.tls.issuer` | Issuer for TLS certificate creation. |`""`
-`global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""`
-`global.reencrypt.host` | gRPC host for reencryption |`""`
-`global.reencrypt.port` | port number of the gRPC host for reencryption |`"50051"`
+`global.tls.enabled` | Use TLS for all connections. | `true`
+`global.tls.issuer` | Issuer for TLS certificate creation. | `""`
+`global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. | `""`
+`global.reencrypt.host` | gRPC host for reencryption | `""`
+`global.reencrypt.port` | port number of the gRPC host for reencryption | `"50051"`
### Credentials
@@ -208,9 +208,9 @@ Parameter | Description | Default
`credentials.auth.dbPassword` | Database password for auth | `""`
`credentials.api.mqPassword` | Broker password for api | `""`
`credentials.doa.dbUser` | Database user for doa | `""`
-`credentials.doa.dbPassword` | Database password for doa| `""`
+`credentials.doa.dbPassword` | Database password for doa | `""`
`credentials.download.dbUser` | Database user for download | `""`
-`credentials.download.dbPassword` | Database password for download| `""`
+`credentials.download.dbPassword` | Database password for download | `""`
`credentials.finalize.dbUser` | Database user for finalize | `""`
`credentials.finalize.dbPassword` | Database password for finalize | `""`
`credentials.finalize.mqUser` | Broker user for finalize | `""`
@@ -219,9 +219,9 @@ Parameter | Description | Default
`credentials.inbox.mqPassword` | Broker password for inbox | `""`
`credentials.ingest.dbUser` | Database user for ingest | `""`
`credentials.ingest.dbPassword` | Database password for ingest | `""`
-`credentials.ingest.mqUser` | Broker user for ingest | `""`
+`credentials.ingest.mqUser` | Broker user for ingest | `""`
`credentials.ingest.mqPassword` | Broker password for ingest | `""`
-`credentials.intercept.mqUser` | Broker user for intercept | `""`
+`credentials.intercept.mqUser` | Broker user for intercept | `""`
`credentials.intercept.mqPassword` | Broker password for intercept | `""`
`credentials.sync.dbUser` | Database user for sync | `""`
`credentials.sync.dbPassword` | Database password for sync | `""`
@@ -244,94 +244,101 @@ Parameter | Description | Default
--------- | ----------- | -------
`api.replicaCount` | Desired number of replicas | `2`
`api.annotations` | Specific annotation for the auth pod | `{}`
-`api.livenessProbe` | Liveness definition for the api pod. |`{}`
-`api.readinessProbe` | Rediness definition for the api pod. |`{}`
-`api.resources.requests.memory` | Memory request for container. |`128Mi`
-`api.resources.requests.cpu` | CPU request for container. |`100m`
-`api.resources.limits.memory` | Memory limit for container. |`256Mi`
-`api.resources.limits.cpu` | CPU limit for container. |`250m`
-`api.tls.secretName` | Secret holding the application TLS certificates |``
+`api.ingressAnnotations | Ingress annotations specific to the api service | `{}`
+`api.livenessProbe` | Liveness definition for the api pod. | `{}`
+`api.readinessProbe` | Rediness definition for the api pod. | `{}`
+`api.resources.requests.memory` | Memory request for container. | `128Mi`
+`api.resources.requests.cpu` | CPU request for container. | `100m`
+`api.resources.limits.memory` | Memory limit for container. | `256Mi`
+`api.resources.limits.cpu` | CPU limit for container. | `1000m`
+`api.tls.secretName` | Secret holding the application TLS certificates | ``
`auth.replicaCount` | desired number of replicas | `2`
`auth.annotations` | Specific annotation for the auth pod | `{}`
-`auth.livenessProbe` | Liveness definition for the auth pod. |`{}`
-`auth.readinessProbe` | Rediness definition for the auth pod. |`{}`
-`auth.resources.requests.memory` | Memory request for container. |`128Mi`
-`auth.resources.requests.cpu` | CPU request for container. |`100m`
-`auth.resources.limits.memory` | Memory limit for container. |`256Mi`
-`auth.resources.limits.cpu` | CPU limit for container. |`250m`
+`auth.ingressAnnotations | Ingress annotations specific to the auth service | `{}`
+`auth.livenessProbe` | Liveness definition for the auth pod. | `{}`
+`auth.readinessProbe` | Rediness definition for the auth pod. | `{}`
+`auth.resources.requests.memory` | Memory request for container. | `128Mi`
+`auth.resources.requests.cpu` | CPU request for container. | `100m`
+`auth.resources.limits.memory` | Memory limit for container. | `256Mi`
+`auth.resources.limits.cpu` | CPU limit for container. | 1000m`
`doa.replicaCount` | desired number of replicas | `2`
`doa.keystorePass` | keystore password | `changeit`
`doa.annotations` | Specific annotation for the doa pod | `{}`
-`doa.livenessProbe` | Liveness definition for the doa pod. |`{}`
-`doa.readinessProbe` | Rediness definition for the doa pod. |`{}`
-`doa.resources.requests.memory` | Memory request for dataedge container. |`128Mi`
-`doa.resources.requests.cpu` | CPU request for dataedge container. |`100m`
-`doa.resources.limits.memory` | Memory limit for dataedge container. |`1024Mi`
-`doa.resources.limits.cpu` | CPU limit for dataedge container. |`2000m`
+`doa.ingressAnnotations | Ingress annotations specific to the doa service | `{}`
+`doa.livenessProbe` | Liveness definition for the doa pod. | `{}`
+`doa.readinessProbe` | Rediness definition for the doa pod. | `{}`
+`doa.resources.requests.memory` | Memory request for dataedge container. | `128Mi`
+`doa.resources.requests.cpu` | CPU request for dataedge container. | `100m`
+`doa.resources.limits.memory` | Memory limit for dataedge container. | `1024Mi`
+`doa.resources.limits.cpu` | CPU limit for dataedge container. | `2000m`
`download.replicaCount` | desired number of replicas | `2`
`download.keystorePass` | keystore password | `changeit`
`download.annotations` | Specific annotation for the dataedge pod | `{}`
-`download.livenessProbe` | Liveness definition for the download pod. |`{}`
-`download.readinessProbe` | Rediness definition for the download pod. |`{}`
-`download.resources.requests.memory` | Memory request for dataedge container. |`256Mi`
-`download.resources.requests.cpu` | CPU request for dataedge container. |`100m`
-`download.resources.limits.memory` | Memory limit for dataedge container. |`512Mi`
-`download.resources.limits.cpu` | CPU limit for dataedge container. |`1000m`
+`download.ingressAnnotations | Ingress annotations specific to the download service | `{}`
+`download.livenessProbe` | Liveness definition for the download pod. | `{}`
+`download.readinessProbe` | Rediness definition for the download pod. | `{}`
+`download.resources.requests.memory` | Memory request for dataedge container. | `256Mi`
+`download.resources.requests.cpu` | CPU request for dataedge container. | `100m`
+`download.resources.limits.memory` | Memory limit for dataedge container. | `512Mi`
+`download.resources.limits.cpu` | CPU limit for dataedge container. | `1000m`
+`download.minClientVersion` | Minimal version of the `sda-cli` to accept | `v0.3.0`
`finalize.annotations` | Specific annotation for the finalize pod | `{}`
-`finalize.resources.requests.memory` | Memory request for finalize container. |`128Mi`
-`finalize.resources.requests.cpu` | CPU request for finalize container. |`100m`
-`finalize.resources.limits.memory` | Memory limit for finalize container. |`256Mi`
-`finalize.resources.limits.cpu` | CPU limit for finalize container. |`250m`
+`finalize.resources.requests.memory` | Memory request for finalize container. | `128Mi`
+`finalize.resources.requests.cpu` | CPU request for finalize container. | `100m`
+`finalize.resources.limits.memory` | Memory limit for finalize container. | `256Mi`
+`finalize.resources.limits.cpu` | CPU limit for finalize container. | `1000m`
`ingest.replicaCount` | desired number of ingest workers | `1`
`ingest.annotations` | Specific annotation for the ingest pod | `{}`
-`ingest.resources.requests.memory` | Memory request for ingest container. |`128Mi`
-`ingest.resources.requests.cpu` | CPU request for ingest container. |`100m`
-`ingest.resources.limits.memory` | Memory limit for ingest container. |`512Mi`
-`ingest.resources.limits.cpu` | CPU limit for ingest container. |`2000m`
+`ingest.resources.requests.memory` | Memory request for ingest container. | `128Mi`
+`ingest.resources.requests.cpu` | CPU request for ingest container. | `100m`
+`ingest.resources.limits.memory` | Memory limit for ingest container. | `512Mi`
+`ingest.resources.limits.cpu` | CPU limit for ingest container. | `2000m`
`intercept.replicaCount` | desired number of intercept workers | `1`
`intercept.annotations` | Specific annotation for the intercept pod | `{}`
-`intercept.resources.requests.memory` | Memory request for intercept container. |`32Mi`
-`intercept.resources.requests.cpu` | CPU request for intercept container. |`100m`
-`intercept.resources.limits.memory` | Memory limit for intercept container. |`128Mi`
-`intercept.resources.limits.cpu` | CPU limit for intercept container. |`2000m`
-`s3Inbox.replicaCount`| desired number of S3inbox containers | `2`
+`intercept.resources.requests.memory` | Memory request for intercept container. | `32Mi`
+`intercept.resources.requests.cpu` | CPU request for intercept container. | `100m`
+`intercept.resources.limits.memory` | Memory limit for intercept container. | `128Mi`
+`intercept.resources.limits.cpu` | CPU limit for intercept container. | `1000m`
+`s3Inbox.replicaCount` | desired number of S3inbox containers | `2`
`s3Inbox.annotations` | Specific annotation for the S3inbox pod | `{}`
-`s3Inbox.livenessProbe` | Liveness definition for the s3Inbox pod. |`{}`
-`s3Inbox.readinessProbe` | Rediness definition for the S3inbox pod. |`{}`
-`s3Inbox.resources.requests.memory` | Memory request for s3Inbox container. |`128Mi`
-`s3Inbox.resources.requests.cpu` | CPU request for s3Inbox container. |`100m`
-`s3Inbox.resources.limits.memory` | Memory limit for s3Inbox container. |`1024Mi`
-`s3Inbox.resources.limits.cpu` | CPU limit for s3Inbox container. |`1000m`
-`sftpInbox.replicaCount`| desired number of sftp inbox containers | `2`
+`s3Inbox.ingressAnnotations | Ingress annotations specific to the s3Inbox service | `{}`
+`s3Inbox.livenessProbe` | Liveness definition for the s3Inbox pod. | `{}`
+`s3Inbox.readinessProbe` | Rediness definition for the S3inbox pod. | `{}`
+`s3Inbox.resources.requests.memory` | Memory request for s3Inbox container. | `128Mi`
+`s3Inbox.resources.requests.cpu` | CPU request for s3Inbox container. | `100m`
+`s3Inbox.resources.limits.memory` | Memory limit for s3Inbox container. | `1024Mi`
+`s3Inbox.resources.limits.cpu` | CPU limit for s3Inbox container. | `1000m`
+`sftpInbox.replicaCount` | desired number of sftp inbox containers | `2`
`sftpInbox.keystorePass` | sftp inbox keystore password | `changeit`
`sftpInbox.nodeHostname` | Node name if the sftp inbox needs to be deployed on a specific node | `""`
`sftpInbox.annotations` | Specific annotation for the sftp inbox pod | `{}`
-`sftpInbox.livenessProbe` | Liveness definition for the sftpInbox pod. |`{}`
-`sftpInbox.readinessProbe` | Rediness definition for the sftpInbox pod. |`{}`
-`sftpInbox.resources.requests.memory` | Memory request for sftpInbox container. |`128Mi`
-`sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. |`100m`
-`sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. |`256Mi`
-`sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. |`250m`
+`sftpInbox.livenessProbe` | Liveness definition for the sftpInbox pod. | `{}`
+`sftpInbox.readinessProbe` | Rediness definition for the sftpInbox pod. | `{}`
+`sftpInbox.resources.requests.memory` | Memory request for sftpInbox container. | `128Mi`
+`sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. | `100m`
+`sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. | `256Mi`
+`sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. | `1000m`
`sync.annotations` | Specific annotation for the sync pod | `{}`
`sync.deploy` | Set to true if the sync service should be active | `false`
-`sync.replicaCount`| desired number of sync containers | `1`
-`sync.resources.requests.memory` | Memory request for sync container. |`128Mi`
-`sync.resources.requests.cpu` | CPU request for sync container. |`100m`
-`sync.resources.limits.memory` | Memory limit for sync container. |`512Mi`
-`sync.resources.limits.cpu` | CPU limit for sync container. |`500m`
-`syncAPI.replicaCount`| desired number of syncAPI containers | `1`
-`syncAPI.annotations` | Specific annotation for the syncAPI pod | `{}`
-`syncAPI.readinessProbe` | Rediness definition for the syncAPI pod. |`{}`
-`syncAPI.resources.requests.memory` | Memory request for syncAPI container. |`64Mi`
-`syncAPI.resources.requests.cpu` | CPU request for syncAPI container. |`100m`
-`syncAPI.resources.limits.memory` | Memory limit for syncAPI container. |`256Mi`
-`syncAPI.resources.limits.cpu` | CPU limit for syncAPI container. |`500m`
-`verify.replicaCount`| desired number of verify containers | `1`
+`sync.replicaCount` | desired number of sync containers | `1`
+`sync.resources.requests.memory` | Memory request for sync container. | `128Mi`
+`sync.resources.requests.cpu` | CPU request for sync container. | `100m`
+`sync.resources.limits.memory` | Memory limit for sync container. | `512Mi`
+`sync.resources.limits.cpu` | CPU limit for sync container. | `1000m`
+`syncapi.replicaCount` | desired number of syncAPI containers | `1`
+`syncapi.annotations` | Specific annotation for the syncAPI pod | `{}`
+`syncapi.ingressAnnotations | Ingress annotations specific to the syncapi service | `{}`
+`syncapi.readinessProbe` | Rediness definition for the syncAPI pod. | `{}`
+`syncapi.resources.requests.memory` | Memory request for syncAPI container. | `64Mi`
+`syncapi.resources.requests.cpu` | CPU request for syncAPI container. | `100m`
+`syncapi.resources.limits.memory` | Memory limit for syncAPI container. | `256Mi`
+`syncapi.resources.limits.cpu` | CPU limit for syncAPI container. | `1000m`
+`verify.replicaCount` | desired number of verify containers | `1`
`verify.annotations` | Specific annotation for the verify pod | `{}`
-`verify.resources.requests.memory` | Memory request for verify container. |`128Mi`
-`verify.resources.requests.cpu` | CPU request for verify container. |`100m`
-`verify.resources.limits.memory` | Memory limit for verify container. |`512Mi`
-`verify.resources.limits.cpu` | CPU limit for verify container. |`2000m`
+`verify.resources.requests.memory` | Memory request for verify container. | `128Mi`
+`verify.resources.requests.cpu` | CPU request for verify container. | `100m`
+`verify.resources.limits.memory` | Memory limit for verify container. | `512Mi`
+`verify.resources.limits.cpu` | CPU limit for verify container. | `2000m`
`releasetest.repository` | inbox container image repository | `neicnordic/sda-helm-test-support`
`releasetest.imageTag` | inbox container image version | `latest`
`releasetest.imagePullPolicy` | inbox container image pull policy | `Always`
@@ -341,3 +348,5 @@ Parameter | Description | Default
`jobs.image` | Container image used for running the DB migration jobs | `postgres:15.4-alpine`
`jobs.setKeyHash` | Populate the key_hash table after migration from V1 | `false`
`jobs.upgradeFromV1` | Upgrade database schema from a version 1 release. | `false`
+`jobs.removeUserName` | remove username from file path. | `false`
+`jobs.removeCorrID` | remove correlation ID column from file event log table. | `false`
diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml
index 76b503329d..8094522f5a 100644
--- a/charts/sda-svc/templates/api-deploy.yaml
+++ b/charts/sda-svc/templates/api-deploy.yaml
@@ -38,6 +38,9 @@ spec:
- maxSkew: 1
whenUnsatisfiable: DoNotSchedule
topologyKey: kubernetes.io/hostname
+ labelSelector:
+ matchLabels:
+ app: {{ template "sda.fullname" . }}-api
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
diff --git a/charts/sda-svc/templates/api-ingress.yaml b/charts/sda-svc/templates/api-ingress.yaml
index a5f29d7ac5..e224b877ee 100644
--- a/charts/sda-svc/templates/api-ingress.yaml
+++ b/charts/sda-svc/templates/api-ingress.yaml
@@ -1,4 +1,5 @@
{{- if (or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType)) (not .Values.global.deploymentType)) }}
+{{- if .Values.global.ingress.deploy }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
@@ -12,19 +13,17 @@ metadata:
{{ toYaml .Values.global.ingress.labels | indent 4 }}
{{- end }}
annotations:
- {{- if eq "nginx" .Values.global.ingress.ingressClassName }}
- nginx.ingress.kubernetes.io/rewrite-target: "/"
- nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}"
- nginx.ingress.kubernetes.io/affinity: "cookie"
- {{- end }}
+{{- if .Values.global.ingress.annotations }}
+{{ toYaml .Values.global.ingress.annotations | indent 4 }}
+{{- end }}
+{{- if .Values.api.ingressAnnotations }}
+{{ toYaml .Values.api.ingressAnnotations | indent 4 }}
+{{- end }}
{{- if .Values.global.ingress.clusterIssuer }}
cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}
{{- else if .Values.global.ingress.issuer }}
cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }}
{{- end }}
-{{- if .Values.global.ingress.annotations }}
-{{ toYaml .Values.global.ingress.annotations | indent 4 }}
-{{- end }}
spec:
{{- if .Values.global.ingress.ingressClassName }}
ingressClassName: {{ .Values.global.ingress.ingressClassName }}
@@ -46,4 +45,6 @@ spec:
- {{ required "An ingress hostname is required!" .Values.global.ingress.hostName.api }}
secretName: {{ if .Values.global.ingress.secretNames.api }}{{ .Values.global.ingress.secretNames.api }}{{- else }}"{{ template "sda.fullname" . }}-ingress-api"{{- end }}
{{- end }}
+
+{{- end }}
{{- end }}
diff --git a/charts/sda-svc/templates/auth-ingress.yaml b/charts/sda-svc/templates/auth-ingress.yaml
index cf9e7c92c2..9bfebff77f 100644
--- a/charts/sda-svc/templates/auth-ingress.yaml
+++ b/charts/sda-svc/templates/auth-ingress.yaml
@@ -13,19 +13,17 @@ metadata:
{{ toYaml .Values.global.ingress.labels | indent 4 }}
{{- end }}
annotations:
- {{- if eq "nginx" .Values.global.ingress.ingressClassName }}
- nginx.ingress.kubernetes.io/rewrite-target: "/"
- nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}"
- nginx.ingress.kubernetes.io/affinity: "cookie"
- {{- end }}
+{{- if .Values.global.ingress.annotations }}
+{{ toYaml .Values.global.ingress.annotations | indent 4 }}
+{{- end }}
+{{- if .Values.auth.ingressAnnotations }}
+{{ toYaml .Values.auth.ingressAnnotations | indent 4 }}
+{{- end }}
{{- if .Values.global.ingress.clusterIssuer }}
cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}
{{- else if .Values.global.ingress.issuer }}
cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }}
{{- end }}
-{{- if .Values.global.ingress.annotations }}
-{{ toYaml .Values.global.ingress.annotations | indent 4 }}
-{{- end }}
spec:
{{- if .Values.global.ingress.ingressClassName }}
ingressClassName: {{ .Values.global.ingress.ingressClassName }}
diff --git a/charts/sda-svc/templates/doa-deploy.yaml b/charts/sda-svc/templates/doa-deploy.yaml
index e26353ae7c..6bc39ecbab 100644
--- a/charts/sda-svc/templates/doa-deploy.yaml
+++ b/charts/sda-svc/templates/doa-deploy.yaml
@@ -40,6 +40,9 @@ spec:
- maxSkew: 1
whenUnsatisfiable: DoNotSchedule
topologyKey: kubernetes.io/hostname
+ labelSelector:
+ matchLabels:
+ app: {{ template "sda.fullname" . }}-doa
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
diff --git a/charts/sda-svc/templates/doa-ingress.yaml b/charts/sda-svc/templates/doa-ingress.yaml
index 8192ea0fbc..80bbe4352c 100644
--- a/charts/sda-svc/templates/doa-ingress.yaml
+++ b/charts/sda-svc/templates/doa-ingress.yaml
@@ -14,19 +14,17 @@ metadata:
{{ toYaml .Values.global.ingress.labels | indent 4 }}
{{- end }}
annotations:
- {{- if eq "nginx" .Values.global.ingress.ingressClassName }}
- nginx.ingress.kubernetes.io/rewrite-target: "/"
- nginx.ingress.kubernetes.io/backend-protocol: "HTTPS"
- nginx.ingress.kubernetes.io/affinity: "cookie"
- {{- end }}
+{{- if .Values.global.ingress.annotations }}
+{{ toYaml .Values.global.ingress.annotations | indent 4 }}
+{{- end }}
+{{- if .Values.doa.ingressAnnotations }}
+{{ toYaml .Values.doa.ingressAnnotations | indent 4 }}
+{{- end }}
{{- if .Values.global.ingress.clusterIssuer }}
cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}
{{- else if .Values.global.ingress.issuer }}
cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }}
{{- end }}
-{{- if .Values.global.ingress.annotations }}
-{{ toYaml .Values.global.ingress.annotations | indent 4 }}
-{{- end }}
spec:
{{- if .Values.global.ingress.ingressClassName }}
ingressClassName: {{ .Values.global.ingress.ingressClassName }}
diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml
index 6d3f4ffb13..62d9d25d70 100644
--- a/charts/sda-svc/templates/download-deploy.yaml
+++ b/charts/sda-svc/templates/download-deploy.yaml
@@ -39,6 +39,9 @@ spec:
- maxSkew: 1
whenUnsatisfiable: DoNotSchedule
topologyKey: kubernetes.io/hostname
+ labelSelector:
+ matchLabels:
+ app: {{ template "sda.fullname" . }}-download
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
diff --git a/charts/sda-svc/templates/download-ingress.yaml b/charts/sda-svc/templates/download-ingress.yaml
index e4f4e3338f..4c8e61c8dd 100644
--- a/charts/sda-svc/templates/download-ingress.yaml
+++ b/charts/sda-svc/templates/download-ingress.yaml
@@ -13,19 +13,17 @@ metadata:
{{ toYaml .Values.global.ingress.labels | indent 4 }}
{{- end }}
annotations:
- {{- if eq "nginx" .Values.global.ingress.ingressClassName }}
- nginx.ingress.kubernetes.io/rewrite-target: "/"
- nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}"
- nginx.ingress.kubernetes.io/affinity: "cookie"
- {{- end }}
+{{- if .Values.global.ingress.annotations }}
+{{ toYaml .Values.global.ingress.annotations | indent 4 }}
+{{- end }}
+{{- if .Values.download.ingressAnnotations }}
+{{ toYaml .Values.download.ingressAnnotations | indent 4 }}
+{{- end }}
{{- if .Values.global.ingress.clusterIssuer }}
cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}
{{- else if .Values.global.ingress.issuer }}
cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }}
{{- end }}
-{{- if .Values.global.ingress.annotations }}
-{{ toYaml .Values.global.ingress.annotations | indent 4 }}
-{{- end }}
spec:
{{- if .Values.global.ingress.ingressClassName }}
ingressClassName: {{ .Values.global.ingress.ingressClassName }}
diff --git a/charts/sda-svc/templates/download-secrets.yaml b/charts/sda-svc/templates/download-secrets.yaml
index f649ae85ee..7c97386fe5 100644
--- a/charts/sda-svc/templates/download-secrets.yaml
+++ b/charts/sda-svc/templates/download-secrets.yaml
@@ -8,19 +8,20 @@ type: Opaque
stringData:
config.yaml: |-
app:
- {{- if .Values.global.tls.enabled }}
+ {{- if .Values.global.tls.enabled }}
port: 8443
serverCert: {{ template "tlsPath" . }}/tls.crt
serverKey: {{ template "tlsPath" . }}/tls.key
- {{- end }}
+ {{- end }}
+ minimalcliversion: {{ default "v0.3.0" .Values.download.minClientVersion }}
archive:
type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }}
- {{- if eq "s3" .Values.global.archive.storageType }}
+ {{- if eq "s3" .Values.global.archive.storageType }}
accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }}
bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }}
{{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }}
caCert: {{ template "tlsPath" . }}/ca.crt
- {{ end }}
+ {{- end }}
chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }}
{{- if .Values.global.archive.s3Port }}
port: {{ .Values.global.archive.s3Port }}
@@ -28,20 +29,20 @@ stringData:
readypath: {{ .Values.global.archive.s3ReadyPath }}
secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }}
url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }}
- {{- else }}
+ {{- else }}
location: /archive/
- {{- end }}
+ {{- end }}
{{- if .Values.global.download.serveDecrypted.c4ghKeyFile }}
c4gh:
transientKeyPath: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }}
- transientKeyPassphrase: {{ .Values.global.download.serveDecrypted.passphrade }}
+ transientKeyPassphrase: {{ .Values.global.download.serveDecrypted.passphrase }}
{{- end }}
db:
- {{- if .Values.global.tls.enabled }}
+ {{- if .Values.global.tls.enabled }}
caCert: {{ template "tlsPath" . }}/ca.crt
clientCert: {{ template "tlsPath" . }}/tls.crt
clientKey: {{ template "tlsPath" . }}/tls.key
- {{- end }}
+ {{- end }}
host: {{ .Values.global.db.host }}
database: {{ .Values.global.db.name }}
password: {{ required "DB pass is required" (include "dbPassDownload" .) }}
@@ -52,11 +53,11 @@ stringData:
host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }}
port: {{ ternary 50443 50051 .Values.global.tls.enabled }}
timeout: {{ default 30 .Values.global.reencrypt.timeout }}
- {{- if .Values.global.tls.enabled }}
+ {{- if .Values.global.tls.enabled }}
caCert: {{ template "tlsPath" . }}/ca.crt
clientCert: {{ template "tlsPath" . }}/tls.crt
clientKey: {{ template "tlsPath" . }}/tls.key
- {{- end }}
+ {{- end }}
log:
format: {{ .Values.global.log.format }}
level: {{ .Values.global.log.level }}
diff --git a/charts/sda-svc/templates/job-perform-db-optimization.yaml b/charts/sda-svc/templates/job-perform-db-optimization.yaml
new file mode 100644
index 0000000000..e5d7b4ffb9
--- /dev/null
+++ b/charts/sda-svc/templates/job-perform-db-optimization.yaml
@@ -0,0 +1,127 @@
+{{- if .Values.jobs.removeCorrID }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: optimize-db
+data:
+ optimize-db.sql: |
+ DO
+ $$
+ BEGIN
+ IF (select max(version) from sda.dbschema_version) BETWEEN 16 and 18 then
+ RAISE NOTICE 'Doing migration to schema version 21';
+
+ INSERT INTO sda.dbschema_version VALUES(19, now(), 'Create new indexes on files and file_event_log tables');
+ CREATE INDEX file_event_log_file_id_started_at_idx ON sda.file_event_log(file_id, started_at);
+ CREATE INDEX files_submission_user_submission_file_path_idx ON sda.files(submission_user, submission_file_path);
+
+ IF (SELECT COUNT(*) FROM sda.file_event_log WHERE file_id != correlation_id AND correlation_id IS NOT NULL) > 0 then
+ RAISE EXCEPTION 'Cannot proceed with migration since file_event_log rows exists where file_id != correlation_id';
+ END IF;
+ INSERT INTO sda.dbschema_version VALUES(20, now(), 'Deprecate file_event_log.correlation_id column');
+
+ ALTER TABLE sda.file_event_log DROP COLUMN correlation_id;
+
+ DROP FUNCTION IF EXISTS sda.register_file;
+ CREATE FUNCTION sda.register_file(file_id TEXT, submission_file_path TEXT, submission_user TEXT) RETURNS TEXT AS $register_file$
+ DECLARE file_uuid UUID;
+ BEGIN
+ INSERT INTO sda.files( id, submission_file_path, submission_user, encryption_method )
+ VALUES( COALESCE(CAST(NULLIF(file_id, '') AS UUID), gen_random_uuid()), submission_file_path, submission_user, 'CRYPT4GH' )
+ ON CONFLICT ON CONSTRAINT unique_ingested
+ DO UPDATE SET submission_file_path = EXCLUDED.submission_file_path,
+ submission_user = EXCLUDED.submission_user,
+ encryption_method = EXCLUDED.encryption_method
+ RETURNING id INTO file_uuid;
+
+ INSERT INTO sda.file_event_log( file_id, event, user_id )
+ VALUES (file_uuid, 'registered', submission_user);
+
+ RETURN file_uuid;
+ END;
+ $register_file$ LANGUAGE plpgsql;
+
+ INSERT INTO sda.dbschema_version VALUES(21, now(), 'Drop functions set_verified, and set_archived');
+ DROP FUNCTION IF EXISTS sda.set_verified;
+ DROP FUNCTION IF EXISTS sda.set_archived;
+
+ ELSE
+ RAISE NOTICE 'Schema migration to 21 does not apply now, skipping';
+ END IF;
+ END
+ $$
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: {{ template "sda.fullname" . }}-optimize-db-job
+ labels:
+ app.kubernetes.io/managed-by: {{ .Release.Service }}
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/version: {{ .Chart.AppVersion }}
+ helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+ release: {{ .Release.Name }}
+ annotations:
+ # This is what defines this resource as a hook.
+ # Without this line, the job is considered part of the release.
+ helm.sh/hook: pre-install,pre-upgrade
+ helm.sh/hook-weight: "1"
+ helm.sh/hook-delete-policy: hook-succeeded
+spec:
+ template:
+ metadata:
+ name: {{ template "sda.fullname" . }}-optimize-db-job
+ labels:
+ app.kubernetes.io/managed-by: {{ .Release.Service | quote }}
+ app.kubernetes.io/instance: {{ .Release.Name | quote }}
+ helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
+ release: {{ .Release.Name }}
+ spec:
+ containers:
+ - name: {{ template "sda.fullname" . }}-optimize-db-job
+ command: ["/usr/local/bin/psql"]
+ args: [
+ "-qf",
+ "/migratedb/optimize-db.sql"
+ ]
+ env:
+ - name: PGDATABASE
+ value: {{ .Values.global.db.name }}
+ - name: PGHOST
+ value: {{ .Values.global.db.host }}
+ - name: PGPASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }}
+ key: {{ default "password" .Values.global.db.admin.passKey }}
+ {{- if .Values.global.db.port }}
+ - name: PGPORT
+ value: {{ .Values.global.db.port | quote }}
+ {{- end }}
+ - name: PGUSER
+ valueFrom:
+ secretKeyRef:
+ name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }}
+ key: {{ default "username" .Values.global.db.admin.userKey }}
+ image: {{ .Values.jobs.image }}
+ imagePullPolicy: {{ .Values.image.pullPolicy }}
+ securityContext:
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: ["ALL"]
+ runAsNonRoot: true
+ seccompProfile:
+ type: "RuntimeDefault"
+ volumeMounts:
+ - name: sql
+ mountPath: /migratedb
+ restartPolicy: Never
+ securityContext:
+ runAsUser: 72
+ runAsGroup: 72
+ fsGroup: 72
+ volumes:
+ - name: sql
+ configMap:
+ name: optimize-db
+{{- end }}
diff --git a/charts/sda-svc/templates/job-set-missing-keyhash.yaml b/charts/sda-svc/templates/job-set-missing-keyhash.yaml
index 24bb4de82c..25addc599a 100644
--- a/charts/sda-svc/templates/job-set-missing-keyhash.yaml
+++ b/charts/sda-svc/templates/job-set-missing-keyhash.yaml
@@ -1,4 +1,21 @@
{{- if .Values.jobs.setKeyHash }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: set-key-hash
+data:
+ set-key-hash.sql: |
+ DO
+ $$
+ BEGIN
+ IF (select max(version) from sda.dbschema_version) > 11 then
+ RAISE NOTICE 'Fixing c4gh key references';
+ UPDATE sda.files SET key_hash = (SELECT key_hash FROM sda.encryption_keys WHERE deprecated_at IS NULL ORDER BY created_at ASC LIMIT 1)
+ WHERE header IS NOT NULL AND key_hash IS NULL;
+ END IF;
+ END
+ $$
+---
apiVersion: batch/v1
kind: Job
metadata:
@@ -13,7 +30,7 @@ metadata:
# This is what defines this resource as a hook.
# Without this line, the job is considered part of the release.
helm.sh/hook: post-install,post-upgrade
- helm.sh/hook-weight: "-5"
+ helm.sh/hook-weight: "-4"
helm.sh/hook-delete-policy: hook-succeeded
spec:
template:
@@ -72,21 +89,4 @@ spec:
- name: sql
configMap:
name: set-key-hash
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: set-key-hash
-data:
- set-key-hash.sql: |
- DO
- $$
- BEGIN
- IF (select max(version) from sda.dbschema_version) > 11 then
- RAISE NOTICE 'Fixing c4gh key references';
- UPDATE sda.files SET key_hash = (SELECT key_hash FROM sda.encryption_keys WHERE deprecated_at IS NULL ORDER BY created_at ASC LIMIT 1)
- WHERE header IS NOT NULL AND key_hash IS NULL;
- END IF;
- END
- $$
{{- end }}
diff --git a/charts/sda-svc/templates/job-upgrade-from-v1.yaml b/charts/sda-svc/templates/job-upgrade-from-v1.yaml
index 0baab227cb..e6f20b7a78 100644
--- a/charts/sda-svc/templates/job-upgrade-from-v1.yaml
+++ b/charts/sda-svc/templates/job-upgrade-from-v1.yaml
@@ -1,4 +1,44 @@
{{- if .Values.jobs.upgradeFromV1 }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: upgrade-from-v1
+data:
+ upgrade-from-v1.sql: |
+ DO
+ $$
+ BEGIN
+ IF (select max(version) from sda.dbschema_version) = 11 then
+ RAISE NOTICE 'Doing migration from schema version 11 to 17';
+
+ INSERT INTO sda.dbschema_version VALUES(12, now(), 'Add key hash table');
+ CREATE TABLE IF NOT EXISTS sda.encryption_keys (
+ key_hash TEXT PRIMARY KEY,
+ created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(),
+ deprecated_at TIMESTAMP WITH TIME ZONE,
+ description TEXT
+ );
+ ALTER TABLE sda.files ADD COLUMN IF NOT EXISTS key_hash TEXT,
+ ADD CONSTRAINT fk_files_key_hash FOREIGN KEY (key_hash) REFERENCES sda.encryption_keys(key_hash);
+
+ INSERT INTO sda.dbschema_version VALUES(14, now(), 'Add userinfo table');
+ CREATE TABLE IF NOT EXISTS sda.userinfo (
+ id TEXT PRIMARY KEY,
+ name TEXT,
+ email TEXT,
+ groups TEXT[]
+ );
+
+ INSERT INTO sda.dbschema_version VALUES(17, now(), 'Add submission user to constraint');
+ ALTER TABLE sda.files DROP CONSTRAINT unique_ingested;
+ ALTER TABLE sda.files ADD CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user);
+
+ ELSE
+ RAISE NOTICE 'Schema migration from 11 does not apply now, skipping';
+ END IF;
+ END
+ $$
+---
apiVersion: batch/v1
kind: Job
metadata:
@@ -72,44 +112,4 @@ spec:
- name: sql
configMap:
name: upgrade-from-v1
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: upgrade-from-v1
-data:
- upgrade-from-v1.sql: |
- DO
- $$
- BEGIN
- IF (select max(version) from sda.dbschema_version) = 11 then
- RAISE NOTICE 'Doing migration from schema version 11 to 17';
-
- INSERT INTO sda.dbschema_version VALUES(12, now(), 'Add key hash table');
- CREATE TABLE IF NOT EXISTS sda.encryption_keys (
- key_hash TEXT PRIMARY KEY,
- created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(),
- deprecated_at TIMESTAMP WITH TIME ZONE,
- description TEXT
- );
- ALTER TABLE sda.files ADD COLUMN IF NOT EXISTS key_hash TEXT,
- ADD CONSTRAINT fk_files_key_hash FOREIGN KEY (key_hash) REFERENCES sda.encryption_keys(key_hash);
-
- INSERT INTO sda.dbschema_version VALUES(14, now(), 'Add userinfo table');
- CREATE TABLE IF NOT EXISTS sda.userinfo (
- id TEXT PRIMARY KEY,
- name TEXT,
- email TEXT,
- groups TEXT[]
- );
-
- INSERT INTO sda.dbschema_version VALUES(17, now(), 'Add submission user to constraint');
- ALTER TABLE sda.files DROP CONSTRAINT unique_ingested;
- ALTER TABLE sda.files ADD CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user);
-
- ELSE
- RAISE NOTICE 'Schema migration from 11 does not apply now, skipping';
- END IF;
- END
- $$
{{- end }}
diff --git a/charts/sda-svc/templates/re-encrypt-deploy.yaml b/charts/sda-svc/templates/re-encrypt-deploy.yaml
index 8801bd29af..f7b5b9c8e6 100644
--- a/charts/sda-svc/templates/re-encrypt-deploy.yaml
+++ b/charts/sda-svc/templates/re-encrypt-deploy.yaml
@@ -57,7 +57,7 @@ spec:
- name: grpc
containerPort: {{ ternary 50443 50051 ( .Values.global.tls.enabled ) }}
- name: grpchealth
- containerPort: {{add ( ternary 50443 50051 ( .Values.global.tls.enabled ) ) 1 }}
+ containerPort: {{add ( ternary 50444 50052 ( .Values.global.tls.enabled ) ) 1 }}
readinessProbe:
{{ toYaml .Values.reencrypt.readinessProbe | trim | indent 10 }}
resources:
diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml
index a69a66fc4a..86174eb85b 100644
--- a/charts/sda-svc/templates/s3-inbox-deploy.yaml
+++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml
@@ -40,6 +40,9 @@ spec:
- maxSkew: 1
whenUnsatisfiable: DoNotSchedule
topologyKey: kubernetes.io/hostname
+ labelSelector:
+ matchLabels:
+ app: {{ template "sda.fullname" . }}-inbox
affinity:
podAntiAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
diff --git a/charts/sda-svc/templates/s3-inbox-ingress.yaml b/charts/sda-svc/templates/s3-inbox-ingress.yaml
index b8ff2c6d74..d75109b173 100644
--- a/charts/sda-svc/templates/s3-inbox-ingress.yaml
+++ b/charts/sda-svc/templates/s3-inbox-ingress.yaml
@@ -13,22 +13,17 @@ metadata:
{{ toYaml .Values.global.ingress.labels | indent 4 }}
{{- end }}
annotations:
- {{- if eq "nginx" .Values.global.ingress.ingressClassName }}
- nginx.ingress.kubernetes.io/rewrite-target: "/"
- nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}"
- nginx.ingress.kubernetes.io/proxy-body-size: 2000m
- nginx.ingress.kubernetes.io/proxy-read-timeout: 300s
- nginx.ingress.kubernetes.io/proxy-request-buffering: "on"
- nginx.ingress.kubernetes.io/affinity: "cookie"
- {{- end }}
+{{- if .Values.global.ingress.annotations }}
+{{ toYaml .Values.global.ingress.annotations | indent 4 }}
+{{- end }}
+{{- if .Values.s3Inbox.ingressAnnotations }}
+{{ toYaml .Values.s3Inbox.ingressAnnotations | indent 4 }}
+{{- end }}
{{- if .Values.global.ingress.clusterIssuer }}
cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}
{{- else if .Values.global.ingress.issuer }}
cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }}
{{- end }}
-{{- if .Values.global.ingress.annotations }}
-{{ toYaml .Values.global.ingress.annotations | indent 4 }}
-{{- end }}
spec:
{{- if .Values.global.ingress.ingressClassName }}
ingressClassName: {{ .Values.global.ingress.ingressClassName }}
diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml
index a0f79a4fe9..d58a4c0a83 100644
--- a/charts/sda-svc/templates/sync-secrets.yaml
+++ b/charts/sda-svc/templates/sync-secrets.yaml
@@ -1,4 +1,4 @@
-{{- if eq "isolated" .Values.global.schemaType }}
+{{- if ne "federated" .Values.global.schemaType }}
{{- if ne "" .Values.global.sync.remote.host }}
{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}}
{{- if not .Values.global.vaultSecrets }}
@@ -40,13 +40,13 @@ stringData:
host: {{ required "A valid MQ host is required" .Values.global.broker.host }}
port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }}
prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }}
- password: {{ required "MQ password is required" (include "mqPassMapper" .) }}
+ password: {{ required "MQ password is required" (include "mqPassSync" .) }}
queue: {{ default "mapping_stream" .Values.global.sync.brokerQueue }}
{{- if ne "" ( default "" .Values.global.broker.serverName ) }}
serverName: {{.Values.global.broker.serverName }}
{{- end }}
ssl: {{ .Values.global.tls.enabled }}
- user: {{ required "MQ user is required" (include "mqUserMapper" .) }}
+ user: {{ required "MQ user is required" (include "mqUserSync" .) }}
{{- if .Values.global.tls.enabled }}
verifyPeer: {{ .Values.global.broker.verifyPeer }}
{{- end }}
@@ -63,13 +63,15 @@ stringData:
{{- end }}
host: {{ .Values.global.db.host }}
database: {{ .Values.global.db.name }}
- password: {{ required "MQ password is required" (include "dbPassMapper" .) }}
+ password: {{ required "MQ password is required" (include "dbPassSync" .) }}
port: {{ .Values.global.db.port }}
sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }}
- user: {{ required "DB user is required" (include "dbUserMapper" .) }}
+ user: {{ required "DB user is required" (include "dbUserSync" .) }}
log:
format: {{ .Values.global.log.format }}
level: {{ .Values.global.log.level }}
+ schema:
+ type: {{ default "isolated" .Values.global.schemaType }}
sync:
centerPrefix: {{ .Values.global.sync.centerPrefix }}
destination:
diff --git a/charts/sda-svc/templates/syncapi-ingress.yaml b/charts/sda-svc/templates/syncapi-ingress.yaml
index c0f68b806f..5304b5d480 100644
--- a/charts/sda-svc/templates/syncapi-ingress.yaml
+++ b/charts/sda-svc/templates/syncapi-ingress.yaml
@@ -1,7 +1,7 @@
{{- if eq "isolated" .Values.global.schemaType }}
{{- if and (ne "" .Values.global.sync.api.password) (ne "" .Values.global.sync.api.user) }}
{{- if (or (or (eq "all" .Values.global.deploymentType) (eq "external" .Values.global.deploymentType)) (not .Values.global.deploymentType)) }}
-{{- if and .Values.global.ingress.deploy }}
+{{- if .Values.global.ingress.deploy }}
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
@@ -15,18 +15,17 @@ metadata:
{{ toYaml .Values.global.ingress.labels | indent 4 }}
{{- end }}
annotations:
- {{- if eq "nginx" .Values.global.ingress.ingressClassName }}
- nginx.ingress.kubernetes.io/rewrite-target: "/"
- nginx.ingress.kubernetes.io/backend-protocol: "{{ ternary "HTTPS" "HTTP" .Values.global.tls.enabled }}"
- {{- end }}
+{{- if .Values.global.ingress.annotations }}
+{{ toYaml .Values.global.ingress.annotations | indent 4 }}
+{{- end }}
+{{- if .Values.syncapi.ingressAnnotations }}
+{{ toYaml .Values.syncapi.ingressAnnotations | indent 4 }}
+{{- end }}
{{- if .Values.global.ingress.clusterIssuer }}
cert-manager.io/cluster-issuer: {{ .Values.global.ingress.clusterIssuer | quote }}
{{- else if .Values.global.ingress.issuer }}
cert-manager.io/issuer: {{ .Values.global.ingress.issuer | quote }}
{{- end }}
-{{- if .Values.global.ingress.annotations }}
-{{ toYaml .Values.global.ingress.annotations | indent 4 }}
-{{- end }}
spec:
{{- if .Values.global.ingress.ingressClassName }}
ingressClassName: {{ .Values.global.ingress.ingressClassName }}
diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml
index 9b8e884237..33875c38f2 100644
--- a/charts/sda-svc/values.yaml
+++ b/charts/sda-svc/values.yaml
@@ -32,7 +32,7 @@ global:
ingress:
deploy: true
- ingressClassName: "nginx"
+ ingressClassName: ""
hostName:
api: ""
auth: ""
@@ -51,11 +51,8 @@ global:
# issuer requires a configured cert-manager
issuer: ""
clusterIssuer: ""
- # extra annotations for the ingress
+ # @param global.ingress.annotations extra annotations shared by all ingress definitions
annotations: {}
- # |
- # kubernetes.io/ingress.class: nginx
- # kubernetes.io/tls-acme: "true"
log:
level: "info"
@@ -394,11 +391,14 @@ api:
cpu: "100m"
limits:
memory: "256Mi"
- cpu: "250m"
+ cpu: "1000m"
# Extra annotations to attach to the service pods
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
annotations: {}
+## @param api.ingressAnnotations
+## Ingress annotations specific to the api service
+ ingressAnnotations: {}
tls:
secretName: ""
@@ -427,11 +427,14 @@ auth:
cpu: "100m"
limits:
memory: "256Mi"
- cpu: "250m"
+ cpu: "1000m"
# Extra annotations to attach to the service pods
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
annotations: {}
+## @param auth.ingressAnnotations
+## Ingress annotations specific to the auth service
+ ingressAnnotations: {}
tls:
secretName: ""
@@ -462,6 +465,9 @@ doa:
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
annotations: {}
+## @param doa.ingressAnnotations
+## Ingress annotations specific to the doa service
+ ingressAnnotations: {}
tls:
secretName: ""
@@ -494,8 +500,14 @@ download:
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
annotations: {}
+## @param download.ingressAnnotations
+## Ingress annotations specific to the download service
+ ingressAnnotations: {}
tls:
secretName: ""
+## @param download.minClientVersion
+## Minimal version of the `sda-cli` to accept, defaults to `v0.3.0`
+ minClientVersion: ""
finalize:
name: finalize
@@ -506,7 +518,7 @@ finalize:
cpu: "100m"
limits:
memory: "256Mi"
- cpu: "250m"
+ cpu: "1000m"
# Extra annotations to attach to the service pods
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
@@ -540,7 +552,7 @@ intercept:
cpu: "100m"
limits:
memory: "256Mi"
- cpu: "250m"
+ cpu: "1000m"
# Extra annotations to attach to the service pods
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
@@ -570,7 +582,7 @@ reencrypt:
periodSeconds: 10
timeoutSeconds: 2
grpc:
- port: 50443
+ port: 50444
service: "reencrypt.Reencrypt"
resources:
requests:
@@ -578,7 +590,7 @@ reencrypt:
cpu: "100m"
limits:
memory: "256Mi"
- cpu: "500m"
+ cpu: "1000m"
# Extra annotations to attach to the service pods
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
@@ -616,6 +628,9 @@ s3Inbox:
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
annotations: {}
+## @param s3Inbox.ingressAnnotations
+## Ingress annotations specific to the s3Inbox service
+ ingressAnnotations: {}
tls:
secretName: ""
@@ -644,7 +659,7 @@ sftpInbox:
cpu: "100m"
limits:
memory: "256Mi"
- cpu: "250m"
+ cpu: "1000m"
keystorePass: "changeit"
nodeHostname: ""
# Extra annotations to attach to the pod
@@ -663,7 +678,7 @@ sync:
cpu: "100m"
limits:
memory: "512Mi"
- cpu: "500m"
+ cpu: "1000m"
# Extra annotations to attach to the service pods
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
@@ -684,11 +699,14 @@ syncapi:
cpu: "100m"
limits:
memory: "256Mi"
- cpu: "500m"
+ cpu: "1000m"
# Extra annotations to attach to the service pods
# This should be a multi-line string mapping directly to the a map of
# the annotations to apply to the service pods
annotations: {}
+## @param syncapi.ingressAnnotations
+## Ingress annotations specific to the syncapi service
+ ingressAnnotations: {}
tls:
secretName: ""
@@ -724,4 +742,5 @@ jobs:
image: postgres:15.4-alpine
removeUserName: false
setKeyHash: false
- upgradeFromV1: false
\ No newline at end of file
+ upgradeFromV1: false
+ removeCorrID: false
\ No newline at end of file
diff --git a/postgresql/Dockerfile b/postgresql/Dockerfile
index 6e5917c897..5ff168f274 100644
--- a/postgresql/Dockerfile
+++ b/postgresql/Dockerfile
@@ -6,7 +6,7 @@ ARG SOURCE_COMMIT
LABEL org.opencontainers.image.authors="NeIC System Developers"
LABEL org.label-schema.schema-version="1.0"
LABEL org.label-schema.build-date=$BUILD_DATE
-LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive"
+LABEL org.label-schema.vcs-url="https://github.com/biobanklab/sensitive-data-archive"
LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT
COPY initdb.d /docker-entrypoint-initdb.d
diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql
index 29ec12198e..37bf2f5385 100644
--- a/postgresql/initdb.d/01_main.sql
+++ b/postgresql/initdb.d/01_main.sql
@@ -35,7 +35,8 @@ VALUES (0, now(), 'Created with version'),
(18, now(), 'Create rotatekey role and grant it privileges to sda tables'),
(19, now(), 'Create new indexes on files and file_event_log tables'),
(20, now(), 'Deprecate file_event_log.correlation_id column and migrate data where file_id != correlation_id'),
- (21, now(), 'Drop functions set_verified, and set_archived');
+ (21, now(), 'Drop functions set_verified, and set_archived'),
+ (22, now(), 'Add file_headers_backup table for key rotation safekeeping');
-- Datasets are used to group files, and permissions are set on the dataset
-- level
@@ -197,3 +198,12 @@ CREATE TABLE dataset_event_log (
message JSONB, -- The rabbitMQ message that initiated the dataset event
event_date TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp()
);
+
+-- `file_headers_backup` stores temporary copies of headers during key rotation.
+-- Designed to be manually truncated/cleaned after key rotation is complete.
+CREATE TABLE sda.file_headers_backup (
+ file_id UUID REFERENCES sda.files(id) PRIMARY KEY,
+ header TEXT NOT NULL,
+ key_hash TEXT REFERENCES sda.encryption_keys(key_hash),
+ backup_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp()
+);
diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql
index 6fed60e316..984c1bfc2b 100644
--- a/postgresql/initdb.d/04_grants.sql
+++ b/postgresql/initdb.d/04_grants.sql
@@ -136,6 +136,7 @@ GRANT SELECT ON sda.checksums TO rotatekey;
GRANT USAGE, SELECT ON SEQUENCE sda.checksums_id_seq TO rotatekey;
GRANT SELECT ON sda.file_event_log TO rotatekey;
GRANT SELECT ON sda.encryption_keys TO rotatekey;
+GRANT INSERT, SELECT, UPDATE ON sda.file_headers_backup TO rotatekey;
--------------------------------------------------------------------------------
diff --git a/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql
index 4b7e5e4aed..327a81ba34 100644
--- a/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql
+++ b/postgresql/migratedb.d/20_deprecate_files_event_log_correlation_id.sql
@@ -12,39 +12,15 @@ BEGIN
RAISE NOTICE 'Changes: %', changes;
INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes);
- -- Migrate data where files.id != file_event_log.correlation_id
-
- -- First drop foreign key constraint so we can update values without constraint restriction
- ALTER TABLE sda.file_event_log
- DROP CONSTRAINT file_event_log_file_id_fkey;
-
- -- Update all files which have a file_event_log where file_id != correlation_id
- UPDATE sda.files AS f
- SET id = fel.correlation_id
- FROM sda.file_event_log AS fel
- WHERE f.id = fel.file_id
- AND fel.file_id != fel.correlation_id
- AND fel.correlation_id IS NOT NULL;
-
- -- Update all file_event_log where file_id != correlation_id
- UPDATE sda.file_event_log AS f
- SET file_id = fel.correlation_id
- FROM sda.file_event_log AS fel
- WHERE f.file_id = fel.file_id
- AND fel.file_id != fel.correlation_id
- AND fel.correlation_id IS NOT NULL;
-
- -- Add back the foreign key constraint
- ALTER TABLE sda.file_event_log
- ADD CONSTRAINT file_event_log_file_id_fkey FOREIGN KEY (file_id)
- REFERENCES sda.files(id);
-
+ -- Check that we have no file_event_log WHERE file_id != correlation_id before proceeding with migration
+ IF (SELECT COUNT(*) FROM sda.file_event_log WHERE file_id != correlation_id AND correlation_id IS NOT NULL) > 0 then
+ RAISE EXCEPTION 'Cannot proceed with migration since file_event_log rows exists where file_id != correlation_id';
+ END IF;
-- Update RegisterFile func
-- First drop it so we can create the updated version
DROP FUNCTION IF EXISTS sda.register_file;
-
-- Create updated function
-- Function for registering files on upload
CREATE FUNCTION sda.register_file(file_id TEXT, submission_file_path TEXT, submission_user TEXT)
diff --git a/postgresql/migratedb.d/22_add_file_headers_backup_table.sql b/postgresql/migratedb.d/22_add_file_headers_backup_table.sql
new file mode 100644
index 0000000000..0ee49de3d9
--- /dev/null
+++ b/postgresql/migratedb.d/22_add_file_headers_backup_table.sql
@@ -0,0 +1,32 @@
+DO
+$$
+DECLARE
+-- The version we know how to do migration from, at the end of a successful migration
+-- we will no longer be at this version.
+ sourcever INTEGER := 21;
+ changes VARCHAR := 'Add file_headers_backup table for key rotation safekeeping';
+BEGIN
+ IF (SELECT max(version) FROM sda.dbschema_version) = sourcever THEN
+ RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1;
+ RAISE NOTICE 'Changes: %', changes;
+
+ INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes);
+
+ CREATE TABLE IF NOT EXISTS sda.file_headers_backup (
+ file_id UUID REFERENCES sda.files(id) PRIMARY KEY,
+ header TEXT NOT NULL,
+ key_hash TEXT REFERENCES sda.encryption_keys(key_hash),
+ backup_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp()
+ );
+
+
+ -- Grant permissions to the rotatekey role
+ GRANT INSERT, SELECT ON sda.file_headers_backup TO rotatekey;
+
+ RAISE NOTICE 'Migration to version % completed successfully.', sourcever+1;
+
+ ELSE
+ RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1;
+ END IF;
+END
+$$;
\ No newline at end of file
diff --git a/rabbitmq/Dockerfile b/rabbitmq/Dockerfile
index 4a2aa194c3..06cd07e9fd 100644
--- a/rabbitmq/Dockerfile
+++ b/rabbitmq/Dockerfile
@@ -6,7 +6,7 @@ ARG SOURCE_COMMIT
LABEL org.opencontainers.image.authors="NeIC System Developers"
LABEL org.label-schema.schema-version="1.0"
LABEL org.label-schema.build-date=$BUILD_DATE
-LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sda"
+LABEL org.label-schema.vcs-url="https://github.com/biobanklab/sda"
LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT
ENV RABBITMQ_CONFIG_FILE=/var/lib/rabbitmq/rabbitmq.conf
diff --git a/rabbitmq/definitions.json b/rabbitmq/definitions.json
index f4955963ef..d21819d5f7 100644
--- a/rabbitmq/definitions.json
+++ b/rabbitmq/definitions.json
@@ -162,6 +162,19 @@
"durable": true,
"auto_delete": false,
"arguments": {}
+ }, {
+ "name": "validation-job-queue",
+ "vhost": "sda",
+ "durable": true,
+ "auto_delete": false,
+ "arguments": {}
+ },
+ {
+ "name": "validation-job-preparation-queue",
+ "vhost": "sda",
+ "durable": true,
+ "auto_delete": false,
+ "arguments": {}
},
{
"name": "unknown_schema",
@@ -259,14 +272,6 @@
"destination": "error_stream",
"routing_key": "error"
},
- {
- "source": "sda",
- "vhost": "sda",
- "destination_type": "queue",
- "arguments": {},
- "destination": "files",
- "routing_key": "files"
- },
{
"source": "sda",
"vhost": "sda",
@@ -315,6 +320,22 @@
"destination": "catch_all.dead",
"routing_key": "#"
},
+ {
+ "source": "sda",
+ "vhost": "sda",
+ "destination_type": "queue",
+ "arguments": {},
+ "destination": "validation-job-queue",
+ "routing_key": "validation-job-queue"
+ },
+ {
+ "source": "sda",
+ "vhost": "sda",
+ "destination_type": "queue",
+ "arguments": {},
+ "destination": "validation-job-preparation-queue",
+ "routing_key": "validation-job-preparation-queue"
+ },
{
"source": "sda",
"vhost": "sda",
diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml
index 131aab4e92..183c0a0c4e 100644
--- a/sda-doa/pom.xml
+++ b/sda-doa/pom.xml
@@ -5,7 +5,7 @@
org.springframework.boot
spring-boot-starter-parent
- 4.0.0
+ 4.0.1
no.uio.ifi
@@ -109,7 +109,7 @@
no.elixir
crypt4gh
- 3.0.38
+ 3.0.42
org.slf4j
@@ -120,7 +120,7 @@
no.elixir
clearinghouse
- 3.0.12
+ 3.0.16
org.slf4j
diff --git a/sda-download/api/api_test.go b/sda-download/api/api_test.go
index d52fef9361..eff6d19e3f 100644
--- a/sda-download/api/api_test.go
+++ b/sda-download/api/api_test.go
@@ -1,3 +1,4 @@
+// nolint:revive
package api
import (
diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go
index efd4734baa..87418d4a84 100644
--- a/sda-download/api/s3/s3.go
+++ b/sda-download/api/s3/s3.go
@@ -72,7 +72,7 @@ func GetBucketLocation(c *gin.Context) {
return
}
c.XML(http.StatusAccepted, LocationConstraint{
- XMLns: "http://s3.amazonaws.com/doc/2006-03-01/",
+ XMLns: "https://s3.amazonaws.com/doc/2006-03-01/",
Location: "us-west-2",
})
}
diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go
index ab9f11290d..b83547ba8b 100644
--- a/sda-download/api/s3/s3_test.go
+++ b/sda-download/api/s3/s3_test.go
@@ -72,7 +72,7 @@ func (ts *S3TestSuite) TestGetBucketLocation() {
assert.Nil(ts.T(), err, "failed to parse body from location response")
defer response.Body.Close()
- expected := "\nus-west-2"
+ expected := "\nus-west-2"
assert.Equal(ts.T(), expected, string(body), "Wrong location from S3")
}
diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go
index 53a08ae73d..25e5807b81 100644
--- a/sda-download/api/sda/sda.go
+++ b/sda-download/api/sda/sda.go
@@ -286,18 +286,19 @@ func Download(c *gin.Context) {
return
}
-
- wholeFile := true //nolint:staticcheck
- if start != 0 || end != 0 {
- wholeFile = false
- }
-
start, end, err = calculateCoords(start, end, c.GetHeader("Range"), fileDetails, c.Param("type"))
if err != nil {
log.Errorf("Byte range coordinates invalid! %v", err)
+ c.String(http.StatusBadRequest, "Range header byte coordinates invalid")
return
}
+
+ wholeFile := true //nolint:staticcheck
+ if start != 0 || end != 0 {
+ wholeFile = false
+ }
+
if c.Param("type") != "encrypted" {
// set the content-length for unencrypted files
if start == 0 && end == 0 {
diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go
index 516bae40d5..1262a066d6 100644
--- a/sda-download/api/sda/sda_test.go
+++ b/sda-download/api/sda/sda_test.go
@@ -939,3 +939,395 @@ func GenerateTestC4ghKey(t *testing.T) (string, error) {
return privateKeyFile.Name(), nil
}
+
+func TestDownload_InvalidRange_NoEnd(t *testing.T) {
+ originalCheckFilePermission := database.CheckFilePermission
+ originalGetCacheFromContext := middleware.GetCacheFromContext
+ originalGetFile := database.GetFile
+
+ // Substitute mock functions
+ database.CheckFilePermission = func(_ string) (string, error) {
+ return "dataset1", nil
+ }
+ middleware.GetCacheFromContext = func(_ *gin.Context) session.Cache {
+ return session.Cache{
+ Datasets: []string{"dataset1"},
+ }
+ }
+ database.GetFile = func(_ string) (*database.FileDownload, error) {
+ fileDetails := &database.FileDownload{
+ ArchivePath: "/mocks3/somepath",
+ ArchiveSize: 0,
+ Header: []byte("mock header"),
+ }
+
+ return fileDetails, nil
+ }
+
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = &http.Request{Method: "GET", URL: &url.URL{Path: "/mocks3/somepath", RawQuery: "filename=somepath"}}
+ c.Request.Header = http.Header{"Client-Public-Key": []string{config.Config.C4GH.PublicKeyB64},
+ "Range": []string{"bytes=0-"}}
+
+ c.Params = make(gin.Params, 1)
+ c.Params[0] = gin.Param{Key: "type", Value: "encrypted"}
+
+ Download(c)
+ response := w.Result()
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ assert.FailNow(t, err.Error())
+ }
+ _ = response.Body.Close()
+
+ assert.Equal(t, http.StatusBadRequest, response.StatusCode)
+ assert.Contains(t, string(body), "Range header byte coordinates invalid")
+
+ // Return mock functions to originals
+ database.CheckFilePermission = originalCheckFilePermission
+ middleware.GetCacheFromContext = originalGetCacheFromContext
+ database.GetFile = originalGetFile
+}
+func TestDownload_InvalidRange_NoStart(t *testing.T) {
+ originalCheckFilePermission := database.CheckFilePermission
+ originalGetCacheFromContext := middleware.GetCacheFromContext
+ originalGetFile := database.GetFile
+
+ // Substitute mock functions
+ database.CheckFilePermission = func(_ string) (string, error) {
+ return "dataset1", nil
+ }
+ middleware.GetCacheFromContext = func(_ *gin.Context) session.Cache {
+ return session.Cache{
+ Datasets: []string{"dataset1"},
+ }
+ }
+ database.GetFile = func(_ string) (*database.FileDownload, error) {
+ fileDetails := &database.FileDownload{
+ ArchivePath: "/mocks3/somepath",
+ ArchiveSize: 0,
+ Header: []byte("mock header"),
+ }
+
+ return fileDetails, nil
+ }
+
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = &http.Request{Method: "GET", URL: &url.URL{Path: "/mocks3/somepath", RawQuery: "filename=somepath"}}
+ c.Request.Header = http.Header{"Client-Public-Key": []string{config.Config.C4GH.PublicKeyB64},
+ "Range": []string{"bytes=-100"}}
+
+ c.Params = make(gin.Params, 1)
+ c.Params[0] = gin.Param{Key: "type", Value: "encrypted"}
+
+ Download(c)
+ response := w.Result()
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ assert.FailNow(t, err.Error())
+ }
+ _ = response.Body.Close()
+
+ assert.Equal(t, http.StatusBadRequest, response.StatusCode)
+ assert.Contains(t, string(body), "Range header byte coordinates invalid")
+
+ // Return mock functions to originals
+ database.CheckFilePermission = originalCheckFilePermission
+ middleware.GetCacheFromContext = originalGetCacheFromContext
+ database.GetFile = originalGetFile
+}
+func TestDownload_InvalidRange_NegativeStart(t *testing.T) {
+ originalCheckFilePermission := database.CheckFilePermission
+ originalGetCacheFromContext := middleware.GetCacheFromContext
+ originalGetFile := database.GetFile
+
+ // Substitute mock functions
+ database.CheckFilePermission = func(_ string) (string, error) {
+ return "dataset1", nil
+ }
+ middleware.GetCacheFromContext = func(_ *gin.Context) session.Cache {
+ return session.Cache{
+ Datasets: []string{"dataset1"},
+ }
+ }
+ database.GetFile = func(_ string) (*database.FileDownload, error) {
+ fileDetails := &database.FileDownload{
+ ArchivePath: "/mocks3/somepath",
+ ArchiveSize: 0,
+ Header: []byte("mock header"),
+ }
+
+ return fileDetails, nil
+ }
+
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = &http.Request{Method: "GET", URL: &url.URL{Path: "/mocks3/somepath", RawQuery: "filename=somepath"}}
+ c.Request.Header = http.Header{"Client-Public-Key": []string{config.Config.C4GH.PublicKeyB64},
+ "Range": []string{"bytes=-0-1"}}
+
+ c.Params = make(gin.Params, 1)
+ c.Params[0] = gin.Param{Key: "type", Value: "encrypted"}
+
+ Download(c)
+ response := w.Result()
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ assert.FailNow(t, err.Error())
+ }
+ _ = response.Body.Close()
+ assert.Equal(t, http.StatusBadRequest, response.StatusCode)
+ assert.Contains(t, string(body), "Range header byte coordinates invalid")
+
+ // Return mock functions to originals
+ database.CheckFilePermission = originalCheckFilePermission
+ middleware.GetCacheFromContext = originalGetCacheFromContext
+ database.GetFile = originalGetFile
+}
+
+func TestDownload_Range_FromStartOfFile(t *testing.T) {
+ originalCheckFilePermission := database.CheckFilePermission
+ originalGetCacheFromContext := middleware.GetCacheFromContext
+ originalGetFile := database.GetFile
+ originalServeUnencryptedDataTrigger := config.Config.C4GH.PublicKeyB64
+ originalC4ghPrivateKeyFilepath := config.Config.C4GH.PrivateKey
+
+ // Substitute mock functions
+ database.CheckFilePermission = func(_ string) (string, error) {
+ return "dataset1", nil
+ }
+ middleware.GetCacheFromContext = func(_ *gin.Context) session.Cache {
+ return session.Cache{
+ Datasets: []string{"dataset1"},
+ }
+ }
+ database.GetFile = func(_ string) (*database.FileDownload, error) {
+ fileDetails := &database.FileDownload{
+ ArchivePath: "/mocks3/somepath",
+ ArchiveSize: 0,
+ Header: []byte("mock header;"),
+ }
+
+ return fileDetails, nil
+ }
+
+ Backend = &mockBackend{
+ file: []byte("this is a mock file"),
+ }
+
+ // Fix to run fake GRPC server
+ faker := fakeGRPC{t: t}
+ server := httptest.NewUnstartedServer(&faker)
+ server.EnableHTTP2 = true
+ server.StartTLS()
+ defer server.Close()
+
+ // Figure out IP, port
+ serverdetails := strings.Split(server.Listener.Addr().String(), ":")
+
+ // Set up TLS for fake server
+ keyfile, err := os.CreateTemp("", "key")
+ assert.NoError(t, err, "Could not create temp file for key")
+ defer os.Remove(keyfile.Name())
+ privdata, err := x509.MarshalPKCS8PrivateKey(server.TLS.Certificates[0].PrivateKey)
+ assert.NoError(t, err, "Could not marshal private key")
+
+ privPEM := "-----BEGIN PRIVATE KEY-----\n" + base64.StdEncoding.EncodeToString(privdata) + "\n-----END PRIVATE KEY-----\n"
+ _, err = keyfile.Write([]byte(privPEM))
+ assert.NoError(t, err, "Could not write private key")
+ _ = keyfile.Close()
+
+ certfile, err := os.CreateTemp("", "cert")
+ assert.NoError(t, err, "Could not create temp file for cert")
+ defer os.Remove(certfile.Name())
+ pubPEM := "-----BEGIN CERTIFICATE-----\n" + base64.StdEncoding.EncodeToString(server.Certificate().Raw) + "\n-----END CERTIFICATE-----\n"
+ _, err = certfile.Write([]byte(pubPEM))
+ assert.NoError(t, err, "Could not write public key")
+ _ = certfile.Close()
+
+ // Configure Reencrypt to use fake server
+ config.Config.Reencrypt.Host = serverdetails[0]
+ port, _ := strconv.ParseInt(serverdetails[1], 10, 32)
+ config.Config.Reencrypt.Port = int(port)
+ config.Config.Reencrypt.CACert = certfile.Name()
+ config.Config.Reencrypt.ClientCert = certfile.Name()
+ config.Config.Reencrypt.ClientKey = keyfile.Name()
+ config.Config.Reencrypt.Timeout = 10
+
+ privateKeyFilePath, err := GenerateTestC4ghKey(t)
+ if err != nil {
+ t.FailNow()
+ }
+ viper.Set("c4gh.transientKeyPath", privateKeyFilePath)
+ viper.Set("c4gh.transientPassphrase", "password")
+ config.Config.C4GH.PrivateKey, config.Config.C4GH.PublicKeyB64, err = config.GetC4GHKeys()
+ assert.NoError(t, err, "Could not load c4gh keys")
+
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = &http.Request{Method: "GET", URL: &url.URL{Path: "/mocks3/somepath", RawQuery: "filename=somepath"}}
+ c.Request.Header = http.Header{"Client-Public-Key": []string{config.Config.C4GH.PublicKeyB64},
+ "Range": []string{"bytes=0-15"}}
+
+ c.Params = make(gin.Params, 1)
+ c.Params[0] = gin.Param{Key: "type", Value: "encrypted"}
+
+ Download(c)
+ response := w.Result()
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ assert.FailNow(t, err.Error())
+ }
+ _ = response.Body.Close()
+
+ assert.Equal(t, http.StatusOK, response.StatusCode)
+ assert.Equal(t, "mock header;this", string(body))
+
+ // Return mock functions to originals
+ database.CheckFilePermission = originalCheckFilePermission
+ middleware.GetCacheFromContext = originalGetCacheFromContext
+ database.GetFile = originalGetFile
+ config.Config.C4GH.PublicKeyB64 = originalServeUnencryptedDataTrigger
+ config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath
+ viper.Set("c4gh.transientKeyPath", "")
+ viper.Set("c4gh.transientPassphrase", "")
+ Backend = nil
+}
+
+func TestDownload_Range_FromMiddleOfFile(t *testing.T) {
+ originalCheckFilePermission := database.CheckFilePermission
+ originalGetCacheFromContext := middleware.GetCacheFromContext
+ originalGetFile := database.GetFile
+ originalServeUnencryptedDataTrigger := config.Config.C4GH.PublicKeyB64
+ originalC4ghPrivateKeyFilepath := config.Config.C4GH.PrivateKey
+
+ // Substitute mock functions
+ database.CheckFilePermission = func(_ string) (string, error) {
+ return "dataset1", nil
+ }
+ middleware.GetCacheFromContext = func(_ *gin.Context) session.Cache {
+ return session.Cache{
+ Datasets: []string{"dataset1"},
+ }
+ }
+ database.GetFile = func(_ string) (*database.FileDownload, error) {
+ fileDetails := &database.FileDownload{
+ ArchivePath: "/mocks3/somepath",
+ ArchiveSize: 0,
+ Header: []byte("mock header;"),
+ }
+
+ return fileDetails, nil
+ }
+
+ Backend = &mockBackend{
+ file: []byte("this is a mock file"),
+ }
+
+ // Fix to run fake GRPC server
+ faker := fakeGRPC{t: t}
+ server := httptest.NewUnstartedServer(&faker)
+ server.EnableHTTP2 = true
+ server.StartTLS()
+ defer server.Close()
+
+ // Figure out IP, port
+ serverdetails := strings.Split(server.Listener.Addr().String(), ":")
+
+ // Set up TLS for fake server
+ keyfile, err := os.CreateTemp("", "key")
+ assert.NoError(t, err, "Could not create temp file for key")
+ defer os.Remove(keyfile.Name())
+ privdata, err := x509.MarshalPKCS8PrivateKey(server.TLS.Certificates[0].PrivateKey)
+ assert.NoError(t, err, "Could not marshal private key")
+
+ privPEM := "-----BEGIN PRIVATE KEY-----\n" + base64.StdEncoding.EncodeToString(privdata) + "\n-----END PRIVATE KEY-----\n"
+ _, err = keyfile.Write([]byte(privPEM))
+ assert.NoError(t, err, "Could not write private key")
+ _ = keyfile.Close()
+
+ certfile, err := os.CreateTemp("", "cert")
+ assert.NoError(t, err, "Could not create temp file for cert")
+ defer os.Remove(certfile.Name())
+ pubPEM := "-----BEGIN CERTIFICATE-----\n" + base64.StdEncoding.EncodeToString(server.Certificate().Raw) + "\n-----END CERTIFICATE-----\n"
+ _, err = certfile.Write([]byte(pubPEM))
+ assert.NoError(t, err, "Could not write public key")
+ _ = certfile.Close()
+
+ // Configure Reencrypt to use fake server
+ config.Config.Reencrypt.Host = serverdetails[0]
+ port, _ := strconv.ParseInt(serverdetails[1], 10, 32)
+ config.Config.Reencrypt.Port = int(port)
+ config.Config.Reencrypt.CACert = certfile.Name()
+ config.Config.Reencrypt.ClientCert = certfile.Name()
+ config.Config.Reencrypt.ClientKey = keyfile.Name()
+ config.Config.Reencrypt.Timeout = 10
+
+ privateKeyFilePath, err := GenerateTestC4ghKey(t)
+ if err != nil {
+ t.FailNow()
+ }
+ viper.Set("c4gh.transientKeyPath", privateKeyFilePath)
+ viper.Set("c4gh.transientPassphrase", "password")
+ config.Config.C4GH.PrivateKey, config.Config.C4GH.PublicKeyB64, err = config.GetC4GHKeys()
+ assert.NoError(t, err, "Could not load c4gh keys")
+
+ w := httptest.NewRecorder()
+ c, _ := gin.CreateTestContext(w)
+ c.Request = &http.Request{Method: "GET", URL: &url.URL{Path: "/mocks3/somepath", RawQuery: "filename=somepath"}}
+ c.Request.Header = http.Header{"Client-Public-Key": []string{config.Config.C4GH.PublicKeyB64},
+ "Range": []string{"bytes=12-25"}}
+
+ c.Params = make(gin.Params, 1)
+ c.Params[0] = gin.Param{Key: "type", Value: "encrypted"}
+
+ Download(c)
+ response := w.Result()
+ body, err := io.ReadAll(response.Body)
+ if err != nil {
+ assert.FailNow(t, err.Error())
+ }
+ _ = response.Body.Close()
+
+ assert.Equal(t, http.StatusOK, response.StatusCode)
+ assert.Equal(t, "this is a mock", string(body))
+
+ // Return mock functions to originals
+ database.CheckFilePermission = originalCheckFilePermission
+ middleware.GetCacheFromContext = originalGetCacheFromContext
+ database.GetFile = originalGetFile
+ config.Config.C4GH.PublicKeyB64 = originalServeUnencryptedDataTrigger
+ config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath
+ viper.Set("c4gh.transientKeyPath", "")
+ viper.Set("c4gh.transientPassphrase", "")
+ Backend = nil
+}
+
+type mockBackend struct {
+ file []byte
+}
+
+func (m *mockBackend) GetFileSize(_ string) (int64, error) {
+ return int64(len(m.file)), nil
+}
+
+func (m *mockBackend) NewFileReader(_ string) (io.ReadCloser, error) {
+ return io.NopCloser(bytes.NewReader(m.file)), nil
+}
+
+type mockReedSeekCloser struct {
+ io.ReadSeeker
+ io.Closer
+}
+
+func (m *mockBackend) NewFileReadSeeker(_ string) (io.ReadSeekCloser, error) {
+ return mockReedSeekCloser{
+ ReadSeeker: io.ReadSeeker(bytes.NewReader(m.file)),
+ }, nil
+}
+
+func (m *mockBackend) NewFileWriter(_ string) (io.WriteCloser, error) {
+ panic("not expected to be used by unit tests")
+}
diff --git a/sda-download/go.mod b/sda-download/go.mod
index 1ce8f5894d..c963b8e227 100644
--- a/sda-download/go.mod
+++ b/sda-download/go.mod
@@ -16,10 +16,10 @@ require (
github.com/sirupsen/logrus v1.9.3
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
- golang.org/x/crypto v0.45.0
+ golang.org/x/crypto v0.47.0
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6
- google.golang.org/grpc v1.77.0
- google.golang.org/protobuf v1.36.10
+ google.golang.org/grpc v1.78.0
+ google.golang.org/protobuf v1.36.11
)
require (
@@ -56,8 +56,8 @@ require (
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
- github.com/quic-go/qpack v0.5.1 // indirect
- github.com/quic-go/quic-go v0.54.1 // indirect
+ github.com/quic-go/qpack v0.6.0 // indirect
+ github.com/quic-go/quic-go v0.57.0 // indirect
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/segmentio/asm v1.2.0 // indirect
@@ -69,16 +69,13 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
- go.uber.org/mock v0.5.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.20.0 // indirect
- golang.org/x/mod v0.29.0 // indirect
- golang.org/x/net v0.47.0 // indirect
- golang.org/x/sync v0.18.0 // indirect
- golang.org/x/sys v0.38.0 // indirect
- golang.org/x/text v0.31.0 // indirect
- golang.org/x/tools v0.38.0 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
+ golang.org/x/net v0.48.0 // indirect
+ golang.org/x/sys v0.40.0 // indirect
+ golang.org/x/text v0.33.0 // indirect
+ golang.org/x/tools v0.40.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/sda-download/go.sum b/sda-download/go.sum
index ab80b13783..543469b94e 100644
--- a/sda-download/go.sum
+++ b/sda-download/go.sum
@@ -113,12 +113,12 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
-github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
-github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
-github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
-github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
+github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8=
github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8=
github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc=
@@ -173,36 +173,32 @@ go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6
go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
-go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
-go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI=
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
-golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -214,43 +210,45 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
+golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
+golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
-google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM=
-google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
+google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/sda-validator/apptainer-example/100-200_word_validator.def b/sda-validator/apptainer-example/100-200_word_validator.def
new file mode 100755
index 0000000000..1c88ebd3bf
--- /dev/null
+++ b/sda-validator/apptainer-example/100-200_word_validator.def
@@ -0,0 +1,109 @@
+BootStrap: docker
+From: alpine:3.22
+
+%post
+ apk add --no-cache jq bash
+
+%environment
+ export LC_ALL=C
+
+%runscript
+ #!/bin/bash
+ if [ "$1" != "--describe" ]; then
+
+ allPass=true
+ output='{
+ "files": ['
+
+ anyFiles=false
+
+ for k in $(jq '.files | keys | .[]' /mnt/input/input.json); do
+ anyFiles=true
+ value=$(jq -r ".files[$k]" /mnt/input/input.json);
+ path=$(jq -r '.path' <<< "$value");
+
+ if [ -f $path ]; then
+ wordCount=$(wc -w < $path)
+
+ if (($wordCount >= 100 && $wordCount <= 200)); then
+ output+="
+ {
+ \"path\": \"$path\",
+ \"result\": \"passed\"
+ },"
+ else
+
+ allPass=false
+
+ output+="
+ {
+ \"path\": \"$path\",
+ \"result\": \"failed\",
+ \"messages\": [
+ {
+ \"level\": \"error\",
+ \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+ \"message\": \"file has $wordCount words, which is not between 100 and 200\"
+ }
+ ]
+ },"
+ fi
+ else
+
+ allPass=false
+
+ output+="
+ {
+ \"path\": \"$path\",
+ \"result\": \"failed\",
+ \"messages\": [
+ {
+ \"level\": \"error\",
+ \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+ \"message\": \"file does not exists\"
+ }
+ ]
+ },"
+ fi
+ done
+
+ if [ $anyFiles = true ]; then
+ output=${output::-1}
+
+ if [ $allPass = true ]; then
+ output+="
+ ],
+ \"result\": \"passed\"
+ }"
+ else
+ output+="
+ ],
+ \"result\": \"failed\"
+ }"
+ fi
+ else
+ output+="
+ ],
+ \"result\": \"failed\",
+ \"messages\": [
+ {
+ \"level\": \"error\",
+ \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+ \"message\": \"No files submitted to be validated\"
+ }
+ ]
+ }"
+ fi
+
+ echo $output > /mnt/output/result.json
+
+ else
+ echo '{
+ "validatorId": "validator-word-count-100-200",
+ "name": "File word count validator",
+ "description": "Checks that the word count of a file is between 100 and 200 words",
+ "version": "1.0.0",
+ "mode": "file",
+ "pathSpecification": ["*"]
+ }'
+ fi
\ No newline at end of file
diff --git a/sda-validator/apptainer-example/100-200_word_validator.sif b/sda-validator/apptainer-example/100-200_word_validator.sif
new file mode 100755
index 0000000000..45f83f5420
Binary files /dev/null and b/sda-validator/apptainer-example/100-200_word_validator.sif differ
diff --git a/sda-validator/apptainer-example/README.md b/sda-validator/apptainer-example/README.md
new file mode 100644
index 0000000000..fe52a386af
--- /dev/null
+++ b/sda-validator/apptainer-example/README.md
@@ -0,0 +1,11 @@
+# Apptainer examples
+This contains 2 example [Apptainer](https://apptainer.org/) validators following the [Validator Framework Specification](https://github.com/GenomicDataInfrastructure/validation-framework/blob/main/validator-specification.md)
+
+* [100-200 Word Validator](100-200_word_validator.def) validates that a file has between 100 and 200 words
+* [Always Success Validator](always_success.def) always returns success("passed") on validations, unless no files are provided.
+
+## Build and compile
+```bash
+apptainer build 100-200_word_validator.sif 100-200_word_validator.def
+apptainer build always_success.sif always_success.def
+```
\ No newline at end of file
diff --git a/sda-validator/apptainer-example/always_success.def b/sda-validator/apptainer-example/always_success.def
new file mode 100755
index 0000000000..65abaf59ca
--- /dev/null
+++ b/sda-validator/apptainer-example/always_success.def
@@ -0,0 +1,90 @@
+BootStrap: docker
+From: alpine:3.22
+
+%post
+ apk add --no-cache jq bash
+
+%environment
+ export LC_ALL=C
+
+%runscript
+ #!/bin/bash
+ if [ "$1" != "--describe" ]; then
+
+ allPass=true
+ output='{
+ "files": ['
+
+ anyFiles=false
+
+ for k in $(jq '.files | keys | .[]' /mnt/input/input.json); do
+ anyFiles=true
+ value=$(jq -r ".files[$k]" /mnt/input/input.json);
+ path=$(jq -r '.path' <<< "$value");
+
+ if [ -f $path ]; then
+
+ output+="
+ {
+ \"path\": \"$path\",
+ \"result\": \"passed\"
+ },"
+ else
+
+ allPass=false
+
+ output+="
+ {
+ \"path\": \"$path\",
+ \"result\": \"failed\",
+ \"messages\": [
+ {
+ \"level\": \"error\",
+ \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+ \"message\": \"file does not exists\"
+ }
+ ]
+ },"
+ fi
+ done
+
+ if [ $anyFiles = true ]; then
+ output=${output::-1}
+
+ if [ $allPass = true ]; then
+ output+="
+ ],
+ \"result\": \"passed\"
+ }"
+ else
+ output+="
+ ],
+ \"result\": \"failed\"
+ }"
+ fi
+ else
+ output+="
+ ],
+ \"result\": \"failed\",
+ \"messages\": [
+ {
+ \"level\": \"error\",
+ \"time\": \"$(date +"%Y-%m-%dT%H:%M:%S%z")\",
+ \"message\": \"No files submitted to be validated\"
+ }
+ ]
+ }"
+ fi
+
+ echo $output > /mnt/output/result.json
+
+ else
+ echo '{
+ "validatorId": "validator-always-success",
+ "name": "Always success validator",
+ "description": "Validator always succeed",
+ "version": "1.0.0",
+ "mode": "file",
+ "pathSpecification": ["*"]
+ }'
+ fi
\ No newline at end of file
diff --git a/sda-validator/apptainer-example/always_success.sif b/sda-validator/apptainer-example/always_success.sif
new file mode 100755
index 0000000000..e5052da1b2
Binary files /dev/null and b/sda-validator/apptainer-example/always_success.sif differ
diff --git a/sda-validator/orchestrator/.dockerignore b/sda-validator/orchestrator/.dockerignore
new file mode 100644
index 0000000000..9897215e5c
--- /dev/null
+++ b/sda-validator/orchestrator/.dockerignore
@@ -0,0 +1,5 @@
+swagger_v1.yml
+README.md
+.kube
+Dockerfile
+.dockerignore
\ No newline at end of file
diff --git a/sda-validator/orchestrator/.kube/01_service.yaml b/sda-validator/orchestrator/.kube/01_service.yaml
new file mode 100644
index 0000000000..641c5b1b2e
--- /dev/null
+++ b/sda-validator/orchestrator/.kube/01_service.yaml
@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: sda-validator-orchestrator
+ labels:
+ app: sda-validator-orchestrator
+spec:
+ ports:
+ - name: api
+ port: 8080
+ targetPort: api
+ protocol: TCP
+ selector:
+ app: sda-validator-orchestrator
diff --git a/sda-validator/orchestrator/.kube/02_rbac_config_map.yaml b/sda-validator/orchestrator/.kube/02_rbac_config_map.yaml
new file mode 100644
index 0000000000..eee74b97d4
--- /dev/null
+++ b/sda-validator/orchestrator/.kube/02_rbac_config_map.yaml
@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: sda-validator-orchestrator-rbac
+data:
+ policy.json: |-
+ {
+ "policy": [
+ {
+ "role": "admin",
+ "path": "/admin/*",
+ "action": "(GET)|(POST)"
+ }, {
+ "role": "submission",
+ "path": "/validators",
+ "action": "GET"
+ }, {
+ "role": "submission",
+ "path": "/validate",
+ "action": "POST"
+ }, {
+ "role": "submission",
+ "path": "/result",
+ "action": "GET"
+ }
+ ],
+ "roles": [
+ {
+ "role": "admin",
+ "rolebinding": "submission"
+ },
+ {
+ "role": "testu@lifescience-ri.eu",
+ "rolebinding": "admin"
+ }, {
+ "role": "requester@demo.org",
+ "rolebinding": "admin"
+ }
+ ]
+ }
diff --git a/sda-validator/orchestrator/.kube/03_secret.yaml b/sda-validator/orchestrator/.kube/03_secret.yaml
new file mode 100644
index 0000000000..58f0d777bf
--- /dev/null
+++ b/sda-validator/orchestrator/.kube/03_secret.yaml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Secret
+metadata:
+ name: sda-validator-orchestrator
+type: Opaque
+stringData:
+ sdaApiToken: ""
+ brokerPassword: ""
+ pgPassword: ""
+
diff --git a/sda-validator/orchestrator/.kube/04_deployment.yaml b/sda-validator/orchestrator/.kube/04_deployment.yaml
new file mode 100644
index 0000000000..1da1f9ba10
--- /dev/null
+++ b/sda-validator/orchestrator/.kube/04_deployment.yaml
@@ -0,0 +1,109 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: sda-validator-orchestrator
+ labels:
+ app: sda-validator-orchestrator
+spec:
+ selector:
+ matchLabels:
+ app: sda-validator-orchestrator
+ template:
+ metadata:
+ labels:
+ app: sda-validator-orchestrator
+ spec:
+ initContainers:
+ - name: validator-downloader
+ image: golang:1.25-alpine
+ command: [ "/bin/sh", "-c" ]
+ args:
+ - wget -O /validators/100-200_word_validator.sif https://raw.githubusercontent.com/neicnordic/sensitive-data-archive/refs/heads/feature/validator-orchestrator/sda-validator/apptainer-example/100-200_word_validator.sif;
+ wget -O /validators/always_success.sif https://raw.githubusercontent.com/neicnordic/sensitive-data-archive/refs/heads/feature/validator-orchestrator/sda-validator/apptainer-example/always_success.sif;
+ volumeMounts:
+ - name: validators
+ mountPath: /validators
+ containers:
+ - name: sda-validator-orchestrator
+ image: ghcr.io/biobanklab/sensitive-data-archive/sda-validator/orchestrator:latest
+ imagePullPolicy: IfNotPresent
+ env:
+ - name: LOG_LEVEL
+ value: "DEBUG"
+ - name: API_PORT
+ value: "8080"
+ - name: VALIDATOR_PATHS
+ value: "/validators/100-200_word_validator.sif,/validators/always_success.sif"
+ - name: SDA_API_URL
+ value: "http://pipeline-sda-svc-api.default:8080"
+ - name: SDA_API_TOKEN
+ valueFrom:
+ secretKeyRef:
+ name: sda-validator-orchestrator
+ key: sdaApiToken
+ - name: BROKER_SSL
+ value: "false"
+ - name: BROKER_HOST
+ value: "broker-sda-mq"
+ - name: BROKER_PORT
+ value: "5672"
+ - name: BROKER_VHOST
+ value: "/sda"
+ - name: BROKER_EXCHANGE
+ value: "sda"
+ - name: BROKER_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: sda-validator-orchestrator
+ key: brokerPassword
+ - name: BROKER_USER
+ value: "admin"
+ - name: JOB_QUEUE
+ value: "validation-job-queue"
+ - name: JOB_PREPARATION_QUEUE
+ value: "validation-job-preparation-queue"
+ - name: JWT_PUB_KEY_PATH
+ value: "/.secrets/jwt/"
+ - name: DATABASE_HOST
+ value: "postgres-sda-db"
+ - name: DATABASE_PORT
+ value: "5432"
+ - name: DATABASE_USER
+ value: "postgres"
+ - name: DATABASE_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: sda-validator-orchestrator
+ key: pgPassword
+ - name: DATABASE_SCHEMA
+ value: sda_validator_orchestrator
+ ports:
+ - name: api
+ containerPort: 8080
+ protocol: TCP
+ securityContext:
+ runAsUser: 65534
+ runAsGroup: 65534
+ volumeMounts:
+ - name: validators
+ mountPath: /validators/
+ - name: rbac
+ mountPath: "/rbac"
+ readOnly: true
+ - mountPath: /.secrets/jwt
+ name: jwt
+ volumes:
+ - name: validators
+ emptyDir: {}
+ - name: rbac
+ configMap:
+ name: sda-validator-orchestrator-rbac
+ - name: jwt
+ projected:
+ defaultMode: 420
+ sources:
+ - secret:
+ items:
+ - key: jwt.pub
+ path: jwt.pub
+ name: jwk
diff --git a/sda-validator/orchestrator/Dockerfile b/sda-validator/orchestrator/Dockerfile
new file mode 100644
index 0000000000..090f774639
--- /dev/null
+++ b/sda-validator/orchestrator/Dockerfile
@@ -0,0 +1,51 @@
+FROM debian:trixie-slim AS apptainer_builder
+
+RUN apt update \
+ && apt install -y wget \
+ && cd /tmp \
+ && wget https://github.com/apptainer/apptainer/releases/download/v1.4.3/apptainer_1.4.3-trixie+_amd64.deb \
+ && apt install -y ./apptainer_1.4.3-trixie+_amd64.deb libsubid5 squashfs-tools
+
+COPY ./apptainer.conf /etc/apptainer/apptainer.conf
+
+FROM golang:1.25-alpine AS app_builder
+
+ENV CGO_ENABLED=0
+
+COPY . .
+
+RUN go build -buildvcs=false -o ./sda-validator-orchestrator
+
+FROM gcr.io/distroless/static-debian12
+
+ARG BUILD_DATE
+ARG SOURCE_COMMIT
+
+LABEL maintainer="NeIC System Developers"
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.build-date=$BUILD_DATE
+LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive"
+LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT
+
+# Add health probe binary such that it can be used for docker, etc
+COPY --from=ghcr.io/grpc-ecosystem/grpc-health-probe:v0.4.42 /ko-app/grpc-health-probe /bin/grpc_health_probe
+COPY --from=app_builder /go/sda-validator-orchestrator /usr/bin/
+COPY --from=app_builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+
+COPY --from=apptainer_builder /bin/apptainer /bin/apptainer
+COPY --from=apptainer_builder /etc/apptainer/ /etc/apptainer/
+COPY --from=apptainer_builder /var/lib/apptainer /var/lib/apptainer
+
+# Copy libraries needed by apptainer and any executables that apptainer needs
+COPY --from=apptainer_builder /lib64/ld-linux-x86-64.so.2 /lib64/ld-linux-x86-64.so.2
+COPY --from=apptainer_builder /lib/x86_64-linux-gnu /lib/x86_64-linux-gnu
+COPY --from=apptainer_builder /usr/libexec/apptainer/bin/starter /libexec/apptainer/bin/starter
+
+COPY --from=apptainer_builder /bin/unsquashfs /bin/unsquashfs
+COPY --from=apptainer_builder /etc/localtime /etc/localtime
+
+
+ENV PATH="/usr/local/apptainer/:$PATH"
+
+USER 65534
+CMD ["sda-validator-orchestrator"]
\ No newline at end of file
diff --git a/sda-validator/orchestrator/README.md b/sda-validator/orchestrator/README.md
new file mode 100644
index 0000000000..faca7f430c
--- /dev/null
+++ b/sda-validator/orchestrator/README.md
@@ -0,0 +1,262 @@
+# Sensitive Data Archive - validator-orchestrator
+
+The sda-validator-orchestrator is responsible for integrating with 3rd party apptainer validators, and for hosting an
+API which allows the callers to see the available validators, to invoke validation of a set of file paths
+belonging to a user, and to read the result for a specific validation request.
+
+See [swagger_v1.yml](swagger_v1.yml) for the OpenAPI definition of the ValidatorOrchestratorAPI.
+
+## High level
+
+The following sections aims to describe the sda-validator-orchestrator on a high level
+
+### Diagram
+
+
+
+### "Components"
+
+The sda-validator-orchestrator mainly consists of three "components", the HTTP server, job preparation workers, and job
+workers.
+
+#### HTTP server
+
+The HTTP server implements the [ValidatorOrchestratorAPI](swagger_v1.yml) and allows for users and admins to request the
+currently available validators,
+request a set of files to be validated with a set of validators, and to fetch the results of a validation request.
+
+The HTTP server will authenticate requests, expecting a Bearer token in the "Authorization" header, if not provided, not
+valid, or not signed by any key configured by either
+the [--jwt.pub-key-path or --jwt.pub-key-url configurations](#configuration) an 401 (Unauthorized) code will be
+returned.
+The HTTP server also enforces RBAC (role based access control) towards the available APIs on
+the [ValidatorOrchestratorAPI](swagger_v1.yml), the RBAC policy is expected to be provided by
+the [rbac.policy-file-path configuration](#configuration).
+example policy
+
+```json
+{
+ "policy": [
+ {
+ "role": "admin",
+ "path": "/admin/validate",
+ "action": "POST"
+ },
+ {
+ "role": "admin",
+ "path": "/admin/result",
+ "action": "GET"
+ },
+ {
+ "role": "submission",
+ "path": "/validators",
+ "action": "GET"
+ },
+ {
+ "role": "submission",
+ "path": "/validate",
+ "action": "POST"
+ },
+ {
+ "role": "submission",
+ "path": "/result",
+ "action": "GET"
+ }
+ ],
+ "roles": [
+ {
+ "role": "admin",
+ "rolebinding": "submission"
+ },
+ {
+ "role": "testu@lifescience-ri.eu",
+ "rolebinding": "admin"
+ },
+ {
+ "role": "EXAMPLE_TOKEN_SUBJECT",
+ "rolebinding": "admin"
+ },
+ {
+ "role": "EXAMPLE_TOKEN_SUBJECT_2",
+ "rolebinding": "submission"
+ }
+ ]
+}
+```
+
+The port the HTTP server is hosted on can be configured with the [--api-port configuration](#configuration).
+
+The HTTP server communicates with the [sda-api](../../sda/cmd/api/api.md) using the `/users/${USER}/files` API to ensure
+requested files exists, that they belong to the user, and to get additional information about the files such as their "
+sda id" and the file size.
+The sda-api is expected to be hosted on the URL provided by the [--sda-api-url configuration](#configuration)
+The sda-api requires authentication, the sda-validator-orchestrator will authenticate calls towards the sda-api with the
+token configured by the [--sda-api-token configuration](#configuration).
+
+The HTTP server implementation will publish job preparations messages to the rabbitmq queue configured by
+the [--job-preparation-queue configuration](#configuration).
+
+#### Job Preparation Worker
+
+Current main responsibility is to download the files that are to be validated into a created directory for this
+validation request in the configured validation work directory[see validation-work-dir configuration](#configuration).
+
+Files are downloaded from the [sda-api](../../sda/cmd/api/api.md) using the `/users/${USER}/file/${FILE_ID}` API which
+is expected to be hosted on the URL provided by the [--sda-api-url configuration](#configuration)
+The sda-api requires authentication, the sda-validator-orchestrator will authenticate calls towards the sda-api with the
+token configured by the [--sda-api-token configuration](#configuration).
+
+Before downloading the files the job preparation workers will ensure that sufficient space is available in the volume,
+and if it can not reserve space for all files, it will abort and reconsume the message until other validations has
+finished such that sufficient space can be reserved for the requested files.
+
+Once files has been downloaded it will send a validation job for the files for each validator requested.
+
+Amount of job preparation workers in a sda-validator-orchestrator can be configured by
+the [--job-preparation-worker-count configuration](#configuration).
+Job preparation workers will consume from the rabbitmq queue specified by
+the [--job-preparation-queue configuration](#configuration).
+Job preparation workers will publish job messages to the rabbitmq queue specified by
+the [--job-queue configuration](#configuration).
+
+#### Job worker
+
+The main responsibly of a job worker is to invoke the 3rd Party Validators(Apptainer) with the required inputs and to
+read the result and store it in the [file_validation_job table](#postgres) postgres database.
+
+After each job is completed it checks if all jobs in a validation are finished and cleans up the files from the file
+system for the validation.
+
+Amount of job workers in a sda-validator-orchestrator can be configured by
+the [--job-worker-count configuration](#configuration)
+Job workers will consume from the rabbitmq queue specified by the [--job-queue configuration](#configuration)
+
+### Postgres
+
+The sda-validator-orchestrator requires a Postgres database connection, this connection is setup with
+the [--database.* configurations](#configuration).
+And [file_validation_job table](database/postgres/initdb.d/01_create_table_file_validation_job.sql) is expected to
+exist in the database && schema provided in the configuration.
+
+### Rabbitmq Broker
+
+The sda-validator-orchestrator requires a rabbitmq connection, this connection is setup with
+the [--broker.* configurations](#configuration)..
+
+Each job preparation and job worker will start consuming messages from the configured queues based
+on [--job-preparation-queue configuration](#configuration) and [--job-queue configuration](#configuration).
+It will publish messages to the exchange configured by the [--broker.exchange configuration]](#configuration) with routing keys based on
+the [--job-preparation-queue configuration](#configuration) and [--job-queue configuration](#configuration).
+
+### File system
+
+The sda-validator-orchestrator expects a file system to be available in the [--validation-work-dir](#configuration),
+this is the directory where files will be downloaded and stored for the 3rd Party Validators to access.
+
+It is recommended that this volume is persistent such that the sda-validator-orchestrator is more resilient for
+restarts, etc, and for scaling.
+
+The sda-validator-orchestrator will ensure that sufficient space is available in
+the [--validation-work-dir](#configuration) volume by reserving space before proceeding with the downloading of the
+file, and one set of files for a validation can not exceed the [--validation-file-size-limit](#configuration)
+configuration.
+
+## Configuration
+
+| Name: | Env variable: | Type: | Usage: | Default Value: |
+|--------------------------------|------------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------|
+| --log.level | LOG_LEVEL | string | Set the log level, supported levels: PANIC, FATAL, ERROR, WARN, INFO, DEBUG, TRACE | INFO |
+| --api.port | API_PORT | int | Port to host the ValidationAPI server at | 8080 |
+| --api.server-cert | API_SERVER_CERT | string | Path to the server cert file to be used when hosting the server with TLS, required if api.server-key set | |
+| --api.server-key | API_SERVER_KEY | string | Path to the server key file to be used when hosting the server with TLS, required if api.server-cert set | |
+| --broker.ca-cert | BROKER_CA_CERT | string | The broker ca cert | |
+| --broker.client-cert | BROKER_CLIENT_CERT | string | The cert the client will use in communication with the broker | |
+| --broker.client-key | BROKER_CLIENT_KEY | string | The key for the client cert the client will use in communication with the broker | |
+| --broker.exchange | BROKER_EXCHANGE | string | The exchange the client will use when publishing messages | |
+| --broker.host | BROKER_HOST | string | The host the broker is served on | |
+| --broker.password | BROKER_PASSWORD | string | Password to used to authenticate with in communication with broker | |
+| --broker.port | BROKER_PORT | int | The port the broker is served on | 0 |
+| --broker.prefetch-count | BROKER_PREFETCH_COUNT | int | How many messages the broker will try to keep on the network for the consumers before receiving delivery acks | 2 |
+| --broker.server-name | BROKER_SERVER_NAME | string | ServerName is used to verify the hostname on the returned certificates if ssl is enabled | |
+| --broker.ssl | BROKER_SSL | bool | If the broker connection should use ssl | true |
+| --broker.user | BROKER_USER | string | Username to used to authenticate with in communication with broker | |
+| --broker.verify-peer | BROKER_VERIFY_PEER | bool | If the broker connection should use verify-peer, if true client cert, and client key needs to be provided | false |
+| --broker.vhost | BROKER_VHOST | string | The virtual host name to connect to | |
+| --config-file | CONFIG_FILE | string | Set the direct path to the config file | |
+| --config-path | CONFIG_PATH | string | Set the path viper will look for the config file at | . |
+| --database.ca-cert | DATABASE_CA_CERT | string | The database ca cert | |
+| --database.client-cert | DATABASE_CLIENT_CERT | string | The cert the client will use in communication with the database | |
+| --database.client-key | DATABASE_CLIENT_KEY | string | The key for the client cert the client will use in communication with the database | |
+| --database.host | DATABASE_HOST | string | The host the postgres database is served on | |
+| --database.name | DATABASE_NAME | string | Database to connect to | sda_validator_orchestrator |
+| --database.password | DATABASE_PASSWORD | string | Password to used to authenticate with in communication with database | |
+| --database.port | DATABASE_PORT | int | The port the database is served on | 0 |
+| --database.schema | DATABASE_SCHEMA | string | Database schema to use as search path | sda_validator_orchestrator |
+| --database.ssl-mode | DATABASE_SSL_MODE | string | The database ssl mode | disable |
+| --database.user | DATABASE_USER | string | Username to used to authenticate with in communication with database | |
+| --job-preparation-queue | JOB_PREPARATION_QUEUE | string | The queue for job preparation workers | |
+| --job-preparation-worker-count | JOB_PREPARATION_WORKER_COUNT | int | Amount of job preparation workers to run | 1 |
+| --job-queue | JOB_QUEUE | string | The queue for validation job workers | |
+| --job-worker-count | JOB_WORKER_COUNT | int | Amount of job workers to run | 2 |
+| --jwt.pub-key-path | JWT_PUB_KEY_PATH | string | Local file containing jwk for authentication for API authentication | |
+| --jwt.pub-key-url | JWT_PUB_KEY_URL | string | Url for fetching the elixir JWK for API authentication | |
+| --rbac.policy-file-path | RBAC_POLICY_FILE_PATH | string | Path to file containing rbac policy | /rbac/rbac.json |
+| --sda-api-token | SDA_API_TOKEN | string | Token to authenticate when calling the sda-api service | |
+| --sda-api-url | SDA_API_URL | string | Url to the sda-api service | |
+| --validation-file-size-limit | VALIDATION_FILE_SIZE_LIMIT | string | The human readable size limit of files in a single validation, this should equal the size of the size of the validation-work-dir. Supported abbreviations: B, kB, MB, GB, TB, PB, EB, ZB, YB | 100GB |
+| --validation-work-dir | VALIDATION_WORK_DIR | string | Directory where application will manage data to be used for validation | /validators |
+| --validator-paths | VALIDATOR_PATHS | strings | The paths to the available validators, in comma separated list | [] |
+
+## Open API generation
+
+To generate a go-gin-server template and helper structs, run the following commands, this command generates some
+additional files which are not needed and are removed as part of the following command
+
+``` bash
+rm -rf api/openapi_interface/*
+openapi-generator-cli generate -g go-gin-server -i swagger_v1.yml -o api/openapi_interface --openapi-normalizer SET_TAGS_FOR_ALL_OPERATIONS=validator_orchestrator --additional-properties=interfaceOnly=true
+rm -rf api/openapi_interface/.openapi-generator
+rm -rf api/openapi_interface/api
+rm -rf api/openapi_interface/Dockerfile
+rm -rf api/openapi_interface/go.*
+rm -rf api/openapi_interface/main.go
+rm -rf api/openapi_interface/.openapi-generator-ignore
+rm -rf api/openapi_interface/go/README.md
+mv api/openapi_interface/go/* api/openapi_interface/
+rm -rf api/openapi_interface/go/
+```
+
+## Build and deploy
+
+The [Dockerfile](Dockerfile) will produce an image containing the sda-validator-orchestrator and only its dependant
+binaries by utilising the [distroless image](gcr.io/distroless/static-debian12.
+
+The main dependant binary is [Apptainer](#apptainer) which is described in its section.
+
+To deploy the image see the [.kube folder](.kube) for an example of the kubernetes resources which populates
+the [sda-validator-orchestrator configuration](#configuration), and mounts an ephemeral volume as the validation work
+directory, in a production environment its is recommended that this volume is persistent such that the
+sda-validator-orchestrator is more resilient for restarts, etc, and for scaling.
+
+### Apptainer
+
+[Apptainer](https://apptainer.org/) is downloaded into the final image from a precompiled debian 13 binary, which then
+utilises the [apptainer.conf](apptainer.conf) to override the following default configration:
+
+- allow setuid = no (default: yes)
+ - Reason: Only allow apptainer to run in unprivileged user namespaces.
+- config resolv_conf = no (default: yes)
+ - Reason: Apptainer should not have access to internet, so no reason to config resolv.conf.
+
+And to enforce a strict security posture by limiting the container's access solely to essential validation directories, effectively isolating it from the host environment and other user resources. The following has been changed
+- mount proc = no (default: yes)
+- mount sys = no (default: yes)
+- mount dev = no (default: yes)
+- mount home = no (default: yes)
+- mount tmp = no (default: yes)
+- mount slave = no (default: yes)
+- enable fusemount = no (default: yes)
+ - Reason: Fuse mounting is not needed
+- Removal of following bind paths:
+ - bind path = /etc/localtime
+ - bind path = /etc/hosts
\ No newline at end of file
diff --git a/sda-validator/orchestrator/api/api.go b/sda-validator/orchestrator/api/api.go
new file mode 100644
index 0000000000..7147df26e8
--- /dev/null
+++ b/sda-validator/orchestrator/api/api.go
@@ -0,0 +1,359 @@
+package api
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/google/uuid"
+ "github.com/lestrrat-go/jwx/v2/jwt"
+ openapi "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/api/openapi_interface"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/broker"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/validators"
+ log "github.com/sirupsen/logrus"
+)
+
+type validatorAPIImpl struct {
+ sdaAPIURL string
+ sdaAPIToken string
+ validationFileSizeLimit int64
+ validationJobPreparationQueue string
+ broker broker.AMQPBrokerI
+}
+
+func NewValidatorAPIImpl(options ...func(*validatorAPIImpl)) (openapi.ValidatorOrchestratorAPI, error) {
+ impl := &validatorAPIImpl{}
+
+ for _, option := range options {
+ option(impl)
+ }
+
+ if impl.sdaAPIURL == "" {
+ return nil, errors.New("sdaAPIURL is required")
+ }
+ if impl.sdaAPIToken == "" {
+ return nil, errors.New("sdaAPIToken is required")
+ }
+ if impl.validationFileSizeLimit == 0 {
+ return nil, errors.New("validationFileSizeLimit is required")
+ }
+ if impl.validationJobPreparationQueue == "" {
+ return nil, errors.New("validationJobPreparationQueue is required")
+ }
+ if impl.broker == nil {
+ return nil, errors.New("broker is required")
+ }
+
+ return impl, nil
+}
+
+func (api *validatorAPIImpl) AdminResultGet(c *gin.Context) {
+ api.result(c, c.Query("validation_id"), nil)
+}
+
+func (api *validatorAPIImpl) ResultGet(c *gin.Context) {
+ token, ok := c.Get("token")
+ if !ok {
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+ userID := token.(jwt.Token).Subject()
+
+ api.result(c, c.Query("validation_id"), &userID)
+}
+
+func (api *validatorAPIImpl) result(c *gin.Context, validationID string, userID *string) {
+ if _, err := uuid.Parse(validationID); err != nil {
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Invalid validation id: %s", validationID)})
+
+ return
+ }
+
+ validationResult, err := database.ReadValidationResult(c, validationID, userID)
+
+ if err != nil {
+ log.Errorf("failed to read validation result: %v", err)
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+
+ if validationResult == nil {
+ c.JSON(http.StatusNotFound, gin.H{"error": fmt.Sprintf("No validation with id: %s found for the given user", validationID)})
+
+ return
+ }
+
+ var rsp []*openapi.ResultResponseInner
+
+ for _, validatorResult := range validationResult.ValidatorResults {
+ vr := &openapi.ResultResponseInner{
+ ValidatorId: validatorResult.ValidatorID,
+ Result: validatorResult.Result,
+ StartedAt: validatorResult.StartedAt,
+ FinishedAt: validatorResult.FinishedAt,
+ Files: make([]openapi.ResultResponseInnerFilesInner, len(validatorResult.Files)),
+ Messages: make([]openapi.ResultResponseInnerFilesInnerMessagesInner, len(validatorResult.Messages)),
+ }
+
+ for i, fileResult := range validatorResult.Files {
+ fr := openapi.ResultResponseInnerFilesInner{
+ Path: fileResult.FilePath,
+ Result: fileResult.Result,
+ Messages: make([]openapi.ResultResponseInnerFilesInnerMessagesInner, len(fileResult.Messages)),
+ }
+ for j, message := range fileResult.Messages {
+ fr.Messages[j] = openapi.ResultResponseInnerFilesInnerMessagesInner{
+ Level: message.Level,
+ Time: message.Time,
+ Message: message.Message,
+ }
+ }
+ vr.Files[i] = fr
+ }
+
+ for j, message := range validatorResult.Messages {
+ vr.Messages[j] = openapi.ResultResponseInnerFilesInnerMessagesInner{
+ Level: message.Level,
+ Time: message.Time,
+ Message: message.Message,
+ }
+ }
+
+ rsp = append(rsp, vr)
+ }
+
+ c.JSON(200, rsp)
+}
+
+func (api *validatorAPIImpl) AdminValidatePost(c *gin.Context) {
+ token, ok := c.Get("token")
+ if !ok {
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+ adminID := token.(jwt.Token).Subject()
+ request := new(openapi.AdminValidateRequest)
+ if err := c.ShouldBindJSON(request); err != nil {
+ log.Errorf("failed to bind request to json error: %v", err)
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+
+ return
+ }
+
+ if request.UserId == "" {
+ c.AbortWithStatus(http.StatusBadRequest)
+
+ return
+ }
+
+ api.validate(c, request.UserId, adminID, request.FilePaths, request.Validators)
+}
+
+// ValidatePost handles the POST /validate
+func (api *validatorAPIImpl) ValidatePost(c *gin.Context) {
+ token, ok := c.Get("token")
+ if !ok {
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+ userID := token.(jwt.Token).Subject()
+ request := new(openapi.ValidateRequest)
+ if err := c.ShouldBindJSON(request); err != nil {
+ log.Errorf("failed to bind request to json error: %v", err)
+ c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()})
+
+ return
+ }
+
+ api.validate(c, userID, userID, request.FilePaths, request.Validators)
+}
+
+func (api *validatorAPIImpl) validate(c *gin.Context, userID, triggeredBy string, requestedFilePaths, requestedValidators []string) {
+ var unsupportedValidators []string
+ var requiresFileContent bool
+ for _, requestedValidator := range requestedValidators {
+ validatorDescription, ok := validators.Validators[requestedValidator]
+ if !ok {
+ unsupportedValidators = append(unsupportedValidators, requestedValidator)
+
+ continue
+ }
+ requiresFileContent = validatorDescription.RequiresFileContent() || requiresFileContent
+ }
+ if len(unsupportedValidators) > 0 {
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("%v are not supported validators", unsupportedValidators)})
+
+ return
+ }
+
+ userFiles, err := api.getUserFiles(userID, requestedFilePaths)
+ if err != nil {
+ log.Errorf("failed to get user files due to: %v", err)
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+
+ if len(userFiles.missingFiles) > 0 {
+ c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("files: %v not found", userFiles.missingFiles)})
+
+ return
+ }
+
+ if requiresFileContent && userFiles.sumFilesSize > api.validationFileSizeLimit {
+ c.JSON(http.StatusBadRequest, gin.H{"error": "requested files exceed the file size limit"})
+
+ return
+ }
+
+ validationID := uuid.NewString()
+
+ tx, err := database.BeginTransaction(c)
+ if err != nil {
+ log.Errorf("failed to begin transaction due to: %v", err)
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+ defer func() {
+ if err := tx.Rollback(); err != nil {
+ log.Errorf("failed to rollback transactions due to: %v", err)
+ }
+ }()
+
+ now := time.Now()
+ for _, validatorID := range requestedValidators {
+ for _, file := range userFiles.fileInformation {
+ if err := tx.InsertFileValidationJob(c, &model.InsertFileValidationJobParameters{
+ ValidationID: validationID,
+ ValidatorID: validatorID,
+ FileID: file.FileID,
+ FilePath: file.FilePath,
+ SubmissionUser: userID,
+ TriggeredBy: triggeredBy,
+ FileSubmissionSize: file.SubmissionFileSize,
+ StartedAt: now,
+ }); err != nil {
+ log.Errorf("failed to insert file validation job due to: %v", err)
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+ }
+ }
+
+ if err := tx.Commit(); err != nil {
+ log.Errorf("failed to commit the transaction due to: %v", err)
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+
+ jobPreparationMessage := &model.JobPreparationMessage{ValidationID: validationID}
+ msg, err := json.Marshal(jobPreparationMessage)
+ if err != nil {
+ log.Errorf("failed to marshal job perparation message due to: %v", err)
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+
+ if err := api.broker.PublishMessage(c, api.validationJobPreparationQueue, msg); err != nil {
+ log.Errorf("failed to publish job perparation message due to: %v", err)
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+
+ c.JSON(200, &openapi.ValidatePost200Response{ValidationId: validationID})
+}
+
+type getUserFilesResponse struct {
+ fileInformation map[string]*model.FileInformation
+ sumFilesSize int64
+ missingFiles []string
+}
+
+func (api *validatorAPIImpl) getUserFiles(userID string, requestedFilePaths []string) (*getUserFilesResponse, error) {
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/users/%s/files", api.sdaAPIURL, userID), nil)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create the request, reason: %v", err)
+ }
+
+ // TODO how to handle auth in better way, TBD #989
+ req.Header.Add("Authorization", "Bearer "+api.sdaAPIToken)
+ req.Header.Add("Content-Type", "application/json")
+
+ // Send the request
+ client := &http.Client{}
+ res, err := client.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("failed to get response, reason: %v", err)
+ }
+ defer res.Body.Close()
+
+ // Check the status code
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("server returned status %d: url: %s", res.StatusCode, req.URL.String())
+ }
+
+ // Read the response body
+ resBody, err := io.ReadAll(res.Body)
+ if err != nil {
+ return nil, fmt.Errorf("failed to read response body, reason: %v", err)
+ }
+
+ var userFiles []*model.UserFilesResponse
+
+ if err := json.Unmarshal(resBody, &userFiles); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal response body, reason: %v", err)
+ }
+
+ rsp := &getUserFilesResponse{
+ fileInformation: make(map[string]*model.FileInformation),
+ }
+
+ for _, filePath := range requestedFilePaths {
+ fileFound := false
+ for _, userFile := range userFiles {
+ userFile.InboxPath = strings.TrimSuffix(userFile.InboxPath, ".c4gh")
+ if filePath == userFile.InboxPath {
+ rsp.fileInformation[filePath] = &model.FileInformation{
+ FileID: userFile.FileID,
+ FilePath: userFile.InboxPath,
+ SubmissionFileSize: userFile.SubmissionFileSize,
+ }
+ fileFound = true
+ rsp.sumFilesSize += userFile.SubmissionFileSize
+
+ break
+ }
+ }
+ if !fileFound {
+ rsp.missingFiles = append(rsp.missingFiles, filePath)
+ }
+ }
+
+ return rsp, nil
+}
+
+func (api *validatorAPIImpl) ValidatorsGet(c *gin.Context) {
+ rsp := make([]string, 0)
+
+ for validatorID := range validators.Validators {
+ rsp = append(rsp, validatorID)
+ }
+
+ c.JSON(200, rsp)
+}
diff --git a/sda-validator/orchestrator/api/api_test.go b/sda-validator/orchestrator/api/api_test.go
new file mode 100644
index 0000000000..e3746ef6be
--- /dev/null
+++ b/sda-validator/orchestrator/api/api_test.go
@@ -0,0 +1,617 @@
+package api
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/google/uuid"
+ "github.com/lestrrat-go/jwx/v2/jwt"
+ openapi "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/api/openapi_interface"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/validators"
+ amqp "github.com/rabbitmq/amqp091-go"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/suite"
+)
+
+type ValidatorAPITestSuite struct {
+ suite.Suite
+
+ tempDir string
+ ginEngine *gin.Engine
+ httpTestServer *httptest.Server
+
+ mockDatabase *mockDatabase
+ mockBroker *mockBroker
+}
+
+func (ts *ValidatorAPITestSuite) SetupSuite() {
+ ts.mockDatabase = &mockDatabase{}
+ ts.mockBroker = &mockBroker{}
+
+ ts.httpTestServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ switch req.RequestURI {
+ case "/users/test_user/files", "/users/different_user/files":
+ // Set the response status code
+ w.WriteHeader(http.StatusOK)
+ // Set the response body
+ _, _ = w.Write([]byte(`[
+{
+ "FileID": "test-file-id-1",
+ "InboxPath": "testFile1",
+ "submissionFileSize": 1024
+},{
+ "FileID": "test-file-id-2",
+ "InboxPath": "testFile2",
+ "submissionFileSize": 1024
+},{
+ "FileID": "test-file-id-3",
+ "InboxPath": "testFile3",
+ "submissionFileSize": 1024
+},{
+ "FileID": "test-file-id-4",
+ "InboxPath": "test_dir/testFile4",
+ "submissionFileSize": 1024
+},{
+ "FileID": "test-file-id-5",
+ "InboxPath": "test_dir/testFile5",
+ "submissionFileSize": 1024
+}
+]`))
+ default:
+ // Set the response status code
+ w.WriteHeader(http.StatusInternalServerError)
+ // Set the response body
+ _, _ = fmt.Fprint(w, "unexpected path called")
+ }
+ }))
+
+ database.RegisterDatabase(ts.mockDatabase)
+
+ ts.ginEngine = gin.Default()
+ ts.ginEngine.Use(mockAuthenticator)
+
+ ts.ginEngine = openapi.NewRouterWithGinEngine(ts.ginEngine,
+ openapi.ApiHandleFunctions{
+ ValidatorOrchestratorAPI: &validatorAPIImpl{
+ sdaAPIURL: ts.httpTestServer.URL,
+ sdaAPIToken: "mock-sdaAPIToken",
+ validationFileSizeLimit: 1024 * 4,
+ validationJobPreparationQueue: "job-preparation-queue",
+ broker: ts.mockBroker,
+ }})
+
+ validators.Validators = map[string]*validators.ValidatorDescription{
+ "mock-validator": {
+ ValidatorID: "mock-validator",
+ Name: "mock validator",
+ Description: "Validator for mocking",
+ Version: "v0.0.0",
+ Mode: "file",
+ PathSpecification: nil,
+ ValidatorPath: "/mock-validator.sif",
+ },
+ }
+}
+
+func (ts *ValidatorAPITestSuite) SetupTest() {
+ ts.tempDir = ts.T().TempDir()
+ // Reset any Asserts and On() on mocks from previous tests
+ *ts.mockDatabase = mockDatabase{}
+ *ts.mockBroker = mockBroker{}
+}
+
+func (ts *ValidatorAPITestSuite) TearDownSuite() {
+ ts.httpTestServer.Close()
+}
+
+func TestValidatorAPITestSuite(t *testing.T) {
+ suite.Run(t, new(ValidatorAPITestSuite))
+}
+
+type mockDatabase struct {
+ mock.Mock
+}
+
+func (m *mockDatabase) Commit() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) Rollback() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) BeginTransaction(_ context.Context) (database.Transaction, error) {
+ _ = m.Called()
+
+ return m, nil
+}
+
+func (m *mockDatabase) Close() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) ReadValidationResult(_ context.Context, validationID string, userID *string) (*model.ValidationResult, error) {
+ args := m.Called(validationID, userID)
+
+ return args.Get(0).(*model.ValidationResult), args.Error(1)
+}
+
+func (m *mockDatabase) ReadValidationInformation(_ context.Context, _ string) (*model.ValidationInformation, error) {
+ // Function not needed for unit test, but to implement interface
+ panic("database.ReadValidationInformation call not expected in unit tests")
+}
+
+func (m *mockDatabase) InsertFileValidationJob(_ context.Context, params *model.InsertFileValidationJobParameters) error {
+ args := m.Called(params.ValidationID, params.ValidatorID, params.FileID, params.FilePath, params.FileSubmissionSize, params.SubmissionUser, params.TriggeredBy, params.StartedAt.Format(time.RFC3339))
+
+ return args.Error(0)
+}
+
+func (m *mockDatabase) UpdateFileValidationJob(_ context.Context, _ *model.UpdateFileValidationJobParameters) error {
+ // Function not needed for unit test, but to implement interface
+ panic("database.UpdateFileValidationJob call not expected in unit tests")
+}
+
+func (m *mockDatabase) AllValidationJobsDone(_ context.Context, _ string) (bool, error) {
+ // Function not needed for unit test, but to implement interface
+ panic("database.AllValidationJobsDone call not expected in unit tests")
+}
+
+func (m *mockDatabase) UpdateAllValidationJobFilesOnError(_ context.Context, _ string, _ *model.Message) error {
+ panic("database.UpdateAllValidationJobFilesOnError call not expected in unit tests")
+}
+
+type mockBroker struct {
+ mock.Mock
+}
+
+func (m *mockBroker) PublishMessage(_ context.Context, destination string, body []byte) error {
+ args := m.Called(destination, body)
+
+ return args.Error(0)
+}
+
+func (m *mockBroker) Subscribe(_ context.Context, queue, consumerID string, handleFunc func(context.Context, amqp.Delivery) error) error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.subscribe call not expected in unit tests")
+}
+
+func (m *mockBroker) Close() error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.close call not expected in unit tests")
+}
+
+func (m *mockBroker) Monitor() chan *amqp.Error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.Monitor call not expected in unit tests")
+}
+
+func mockAuthenticator(c *gin.Context) {
+ token := jwt.New()
+ if err := token.Set("sub", "test_user"); err != nil {
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+ c.Set("token", token)
+}
+
+func (ts *ValidatorAPITestSuite) TestNewValidatorAPIImpl() {
+ impl, err := NewValidatorAPIImpl(
+ SdaAPIURL("mock-url"),
+ SdaAPIToken("mock-token"),
+ ValidationFileSizeLimit(1),
+ ValidationJobPreparationQueue("mock-queue"),
+ Broker(ts.mockBroker),
+ )
+ ts.NoError(err)
+ ts.NotNil(impl)
+}
+func (ts *ValidatorAPITestSuite) TestInitWorkers_NoSdaApiUrl() {
+ impl, err := NewValidatorAPIImpl(
+ SdaAPIToken("mock-token"),
+ ValidationFileSizeLimit(1),
+ ValidationJobPreparationQueue("mock-queue"),
+ Broker(ts.mockBroker),
+ )
+ ts.EqualError(err, "sdaAPIURL is required")
+ ts.Nil(impl)
+}
+func (ts *ValidatorAPITestSuite) TestInitWorkers_NoSdaApiToken() {
+ impl, err := NewValidatorAPIImpl(
+ SdaAPIURL("mock-url"),
+ ValidationFileSizeLimit(1),
+ ValidationJobPreparationQueue("mock-queue"),
+ Broker(ts.mockBroker),
+ )
+ ts.EqualError(err, "sdaAPIToken is required")
+ ts.Nil(impl)
+}
+func (ts *ValidatorAPITestSuite) TestInitWorkers_NoValidationFileSizeLimit() {
+ impl, err := NewValidatorAPIImpl(
+ SdaAPIURL("mock-url"),
+ SdaAPIToken("mock-token"),
+ ValidationJobPreparationQueue("mock-queue"),
+ Broker(ts.mockBroker),
+ )
+ ts.EqualError(err, "validationFileSizeLimit is required")
+ ts.Nil(impl)
+}
+func (ts *ValidatorAPITestSuite) TestInitWorkers_NoValidationJobPreparationQueue() {
+ impl, err := NewValidatorAPIImpl(
+ SdaAPIURL("mock-url"),
+ SdaAPIToken("mock-token"),
+ ValidationFileSizeLimit(1),
+ Broker(ts.mockBroker),
+ )
+ ts.EqualError(err, "validationJobPreparationQueue is required")
+ ts.Nil(impl)
+}
+func (ts *ValidatorAPITestSuite) TestInitWorkers_NoBroker() {
+ impl, err := NewValidatorAPIImpl(
+ SdaAPIURL("mock-url"),
+ SdaAPIToken("mock-token"),
+ ValidationFileSizeLimit(1),
+ ValidationJobPreparationQueue("mock-queue"),
+ )
+ ts.EqualError(err, "broker is required")
+ ts.Nil(impl)
+}
+
+func (ts *ValidatorAPITestSuite) TestValidatePost_MissingValidator() {
+ w := httptest.NewRecorder()
+ body, err := json.Marshal(&openapi.ValidateRequest{
+ FilePaths: []string{"123", "abc"},
+ Validators: []string{"abc-validator"},
+ })
+ if err != nil {
+ ts.FailNow(err.Error(), "failed to prepare validate request")
+ }
+ req, _ := http.NewRequest("POST", "/validate", bytes.NewReader(body))
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusBadRequest, w.Code)
+ ts.Equal(`{"error":"[abc-validator] are not supported validators"}`, w.Body.String())
+}
+
+func (ts *ValidatorAPITestSuite) TestValidatePost_FilesNotFound() {
+ w := httptest.NewRecorder()
+ body, err := json.Marshal(&openapi.ValidateRequest{
+ FilePaths: []string{"testFile1", "testFile2", "test_dir/testFile8", "testFile10"},
+ Validators: []string{"mock-validator"},
+ })
+ if err != nil {
+ ts.FailNow(err.Error(), "failed to prepare validate request")
+ }
+ req, _ := http.NewRequest("POST", "/validate", bytes.NewReader(body))
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusBadRequest, w.Code)
+ ts.Equal(`{"error":"files: [test_dir/testFile8 testFile10] not found"}`, w.Body.String())
+}
+
+func (ts *ValidatorAPITestSuite) TestValidatePost_ExceedValidationFileSizeLimit() {
+ w := httptest.NewRecorder()
+ body, err := json.Marshal(&openapi.ValidateRequest{
+ FilePaths: []string{"testFile1", "testFile2", "testFile3", "test_dir/testFile4", "test_dir/testFile5"},
+ Validators: []string{"mock-validator"},
+ })
+ if err != nil {
+ ts.FailNow(err.Error(), "failed to prepare validate request")
+ }
+ req, _ := http.NewRequest("POST", "/validate", bytes.NewReader(body))
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusBadRequest, w.Code)
+ ts.Equal(`{"error":"requested files exceed the file size limit"}`, w.Body.String())
+}
+func (ts *ValidatorAPITestSuite) TestAdminValidatePost_NoTokenInContext() {
+ ginEngine := openapi.NewRouter(openapi.ApiHandleFunctions{
+ ValidatorOrchestratorAPI: &validatorAPIImpl{
+ sdaAPIURL: ts.httpTestServer.URL,
+ sdaAPIToken: "mock-sdaAPIToken",
+ validationFileSizeLimit: 1024 * 4,
+ validationJobPreparationQueue: "job-preparation-queue",
+ broker: ts.mockBroker,
+ }})
+
+ ts.ginEngine.Use(func(c *gin.Context) {
+ c.Set("token", nil)
+ })
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("POST", "/admin/validate", nil)
+ ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusInternalServerError, w.Code)
+}
+
+func (ts *ValidatorAPITestSuite) TestValidatePost_NoTokenInContext() {
+ ginEngine := openapi.NewRouter(openapi.ApiHandleFunctions{
+ ValidatorOrchestratorAPI: &validatorAPIImpl{
+ sdaAPIURL: ts.httpTestServer.URL,
+ sdaAPIToken: "mock-sdaAPIToken",
+ validationFileSizeLimit: 1024 * 4,
+ validationJobPreparationQueue: "job-preparation-queue",
+ broker: ts.mockBroker,
+ }})
+
+ ts.ginEngine.Use(func(c *gin.Context) {
+ c.Set("token", nil)
+ })
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("POST", "/validate", nil)
+ ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusInternalServerError, w.Code)
+}
+
+func (ts *ValidatorAPITestSuite) TestValidatePost() {
+ ts.mockDatabase.On("Rollback").Return(nil)
+ ts.mockDatabase.On("BeginTransaction").Return(nil)
+ ts.mockDatabase.On("Commit").Return(nil)
+ ts.mockDatabase.On("InsertFileValidationJob", mock.Anything, "mock-validator", mock.Anything, mock.Anything, int64(1024), "test_user", "test_user", mock.Anything).Return(nil)
+ ts.mockBroker.On("PublishMessage", "job-preparation-queue", mock.Anything).Return(nil)
+
+ w := httptest.NewRecorder()
+ body, err := json.Marshal(&openapi.ValidateRequest{
+ FilePaths: []string{"testFile1", "testFile2", "test_dir/testFile5"},
+ Validators: []string{"mock-validator"},
+ })
+ if err != nil {
+ ts.FailNow(err.Error(), "failed to prepare validate request")
+ }
+ req, _ := http.NewRequest("POST", "/validate", bytes.NewReader(body))
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+
+ validateResult := new(openapi.ValidatePost200Response)
+ if err := json.Unmarshal(w.Body.Bytes(), validateResult); err != nil {
+ ts.FailNow(err.Error(), "failed to parse response body to ValidatePost200Response")
+ }
+ if _, err := uuid.Parse(validateResult.ValidationId); err != nil {
+ ts.FailNow(err.Error(), "failed to parse validation id as uuid in response")
+ }
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "InsertFileValidationJob", 3)
+ ts.mockDatabase.AssertCalled(ts.T(), "InsertFileValidationJob", mock.Anything, "mock-validator", "test-file-id-1", "testFile1", int64(1024), "test_user", "test_user", mock.Anything)
+ ts.mockDatabase.AssertCalled(ts.T(), "InsertFileValidationJob", mock.Anything, "mock-validator", "test-file-id-2", "testFile2", int64(1024), "test_user", "test_user", mock.Anything)
+ ts.mockDatabase.AssertCalled(ts.T(), "InsertFileValidationJob", mock.Anything, "mock-validator", "test-file-id-5", "test_dir/testFile5", int64(1024), "test_user", "test_user", mock.Anything)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "Commit", 1)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "Rollback", 1) // We expect rollback to have been called given its deferred to ensure tx is closed
+
+ ts.mockBroker.AssertCalled(ts.T(), "PublishMessage", "job-preparation-queue", mock.Anything)
+}
+
+func (ts *ValidatorAPITestSuite) TestAdminValidatePost() {
+ ts.mockDatabase.On("Rollback").Return(nil)
+ ts.mockDatabase.On("BeginTransaction").Return(nil)
+ ts.mockDatabase.On("Commit").Return(nil)
+ ts.mockDatabase.On("InsertFileValidationJob", mock.Anything, "mock-validator", mock.Anything, mock.Anything, int64(1024), "different_user", "test_user", mock.Anything).Return(nil)
+ ts.mockBroker.On("PublishMessage", "job-preparation-queue", mock.Anything).Return(nil)
+
+ w := httptest.NewRecorder()
+ body, err := json.Marshal(&openapi.AdminValidateRequest{
+ FilePaths: []string{"testFile2", "test_dir/testFile4", "test_dir/testFile5"},
+ Validators: []string{"mock-validator"},
+ UserId: "different_user",
+ })
+ if err != nil {
+ ts.FailNow(err.Error(), "failed to prepare validate request")
+ }
+ req, _ := http.NewRequest("POST", "/admin/validate", bytes.NewReader(body))
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+
+ validateResult := new(openapi.ValidatePost200Response)
+ if err := json.Unmarshal(w.Body.Bytes(), validateResult); err != nil {
+ ts.FailNow(err.Error(), "failed to parse response body to ValidatePost200Response")
+ }
+ if _, err := uuid.Parse(validateResult.ValidationId); err != nil {
+ ts.FailNow(err.Error(), "failed to parse validation id as uuid in response")
+ }
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "InsertFileValidationJob", 3)
+ ts.mockDatabase.AssertCalled(ts.T(), "InsertFileValidationJob", mock.Anything, "mock-validator", "test-file-id-4", "test_dir/testFile4", int64(1024), "different_user", "test_user", mock.Anything)
+ ts.mockDatabase.AssertCalled(ts.T(), "InsertFileValidationJob", mock.Anything, "mock-validator", "test-file-id-2", "testFile2", int64(1024), "different_user", "test_user", mock.Anything)
+ ts.mockDatabase.AssertCalled(ts.T(), "InsertFileValidationJob", mock.Anything, "mock-validator", "test-file-id-5", "test_dir/testFile5", int64(1024), "different_user", "test_user", mock.Anything)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "Commit", 1)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "Rollback", 1) // We expect rollback to have been called given its deferred to ensure tx is closed
+
+ ts.mockBroker.AssertCalled(ts.T(), "PublishMessage", "job-preparation-queue", mock.Anything)
+}
+
+func (ts *ValidatorAPITestSuite) TestAdminResultGet() {
+ startedAt, _ := time.Parse(time.RFC3339, time.Now().Add(-2*time.Second).Format(time.RFC3339))
+ finishedAt, _ := time.Parse(time.RFC3339, time.Now().Format(time.RFC3339))
+ testValidationResult := &model.ValidationResult{
+ ValidationID: uuid.NewString(),
+ ValidatorResults: []*model.ValidatorResult{
+ {
+ ValidatorID: "mock-validator",
+ Result: "passed",
+ StartedAt: startedAt,
+ FinishedAt: finishedAt,
+ Messages: nil,
+ Files: []*model.FileResult{
+ {
+ FilePath: "testFile1",
+ Result: "passed",
+ Messages: nil,
+ }, {
+ FilePath: "testFile2",
+ Result: "passed",
+ Messages: nil,
+ }, {
+ FilePath: "test_dir/testFile5",
+ Result: "passed",
+ Messages: nil,
+ },
+ },
+ },
+ },
+ }
+
+ ts.mockDatabase.On("ReadValidationResult", testValidationResult.ValidationID, (*string)(nil)).Return(testValidationResult, nil)
+
+ w := httptest.NewRecorder()
+
+ req, _ := http.NewRequest("GET", fmt.Sprintf("/admin/result?validation_id=%s", testValidationResult.ValidationID), nil)
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+
+ var resultResponse []openapi.ResultResponseInner
+ if err := json.Unmarshal(w.Body.Bytes(), &resultResponse); err != nil {
+ ts.FailNow(err.Error(), "failed to parse response body to []openapi.ResultResponseInner")
+ }
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "ReadValidationResult", 1)
+ ts.mockDatabase.AssertCalled(ts.T(), "ReadValidationResult", testValidationResult.ValidationID, (*string)(nil))
+
+ ts.Equal(len(testValidationResult.ValidatorResults), len(resultResponse))
+
+ ts.Equal(testValidationResult.ValidatorResults[0].Result, resultResponse[0].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].ValidatorID, resultResponse[0].ValidatorId)
+ ts.Equal(testValidationResult.ValidatorResults[0].StartedAt, resultResponse[0].StartedAt)
+ ts.Equal(testValidationResult.ValidatorResults[0].FinishedAt, resultResponse[0].FinishedAt)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Messages), len(resultResponse[0].Messages))
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files), len(resultResponse[0].Files))
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[0].Result, resultResponse[0].Files[0].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[0].FilePath, resultResponse[0].Files[0].Path)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files[0].Messages), len(resultResponse[0].Files[0].Messages))
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[1].Result, resultResponse[0].Files[1].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[1].FilePath, resultResponse[0].Files[1].Path)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files[1].Messages), len(resultResponse[0].Files[1].Messages))
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[2].Result, resultResponse[0].Files[2].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[2].FilePath, resultResponse[0].Files[2].Path)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files[2].Messages), len(resultResponse[0].Files[2].Messages))
+}
+
+func (ts *ValidatorAPITestSuite) TestResultGet_NoTokenInContext() {
+ ginEngine := openapi.NewRouter(openapi.ApiHandleFunctions{
+ ValidatorOrchestratorAPI: &validatorAPIImpl{
+ sdaAPIURL: ts.httpTestServer.URL,
+ sdaAPIToken: "mock-sdaAPIToken",
+ validationFileSizeLimit: 1024 * 4,
+ validationJobPreparationQueue: "job-preparation-queue",
+ broker: ts.mockBroker,
+ }})
+
+ ginEngine.Use(func(c *gin.Context) {
+ c.Set("token", nil)
+ })
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/result", nil)
+ ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusInternalServerError, w.Code)
+}
+
+func (ts *ValidatorAPITestSuite) TestResultGet() {
+ startedAt, _ := time.Parse(time.RFC3339, time.Now().Add(-2*time.Second).Format(time.RFC3339))
+ finishedAt, _ := time.Parse(time.RFC3339, time.Now().Format(time.RFC3339))
+ testValidationResult := &model.ValidationResult{
+ ValidationID: uuid.NewString(),
+ ValidatorResults: []*model.ValidatorResult{
+ {
+ ValidatorID: "mock-validator",
+ Result: "failed",
+ StartedAt: startedAt,
+ FinishedAt: finishedAt,
+ Messages: []*model.Message{
+ {
+ Level: "INFO",
+ Time: time.Now().Format(time.RFC3339),
+ Message: "All files passed",
+ },
+ },
+ Files: []*model.FileResult{
+ {
+ FilePath: "testFile1",
+ Result: "passed",
+ Messages: nil,
+ }, {
+ FilePath: "testFile2",
+ Result: "passed",
+ Messages: nil,
+ }, {
+ FilePath: "test_dir/testFile5",
+ Result: "failed",
+ Messages: []*model.Message{
+ {
+ Level: "error",
+ Time: time.Now().Format(time.RFC3339),
+ Message: "file failed validation",
+ },
+ },
+ },
+ },
+ },
+ },
+ }
+
+ testUser := "test_user"
+ ts.mockDatabase.On("ReadValidationResult", testValidationResult.ValidationID, &testUser).Return(testValidationResult, nil)
+
+ w := httptest.NewRecorder()
+
+ req, _ := http.NewRequest("GET", fmt.Sprintf("/result?validation_id=%s", testValidationResult.ValidationID), nil)
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+
+ var resultResponse []openapi.ResultResponseInner
+ if err := json.Unmarshal(w.Body.Bytes(), &resultResponse); err != nil {
+ ts.FailNow(err.Error(), "failed to parse response body to []openapi.ResultResponseInner")
+ }
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "ReadValidationResult", 1)
+ ts.mockDatabase.AssertCalled(ts.T(), "ReadValidationResult", testValidationResult.ValidationID, &testUser)
+
+ ts.Equal(len(testValidationResult.ValidatorResults), len(resultResponse))
+
+ ts.Equal(testValidationResult.ValidatorResults[0].Result, resultResponse[0].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].ValidatorID, resultResponse[0].ValidatorId)
+ ts.Equal(testValidationResult.ValidatorResults[0].StartedAt, resultResponse[0].StartedAt)
+ ts.Equal(testValidationResult.ValidatorResults[0].FinishedAt, resultResponse[0].FinishedAt)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Messages), len(resultResponse[0].Messages))
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files), len(resultResponse[0].Files))
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[0].Result, resultResponse[0].Files[0].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[0].FilePath, resultResponse[0].Files[0].Path)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files[0].Messages), len(resultResponse[0].Files[0].Messages))
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[1].Result, resultResponse[0].Files[1].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[1].FilePath, resultResponse[0].Files[1].Path)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files[1].Messages), len(resultResponse[0].Files[1].Messages))
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[2].Result, resultResponse[0].Files[2].Result)
+ ts.Equal(testValidationResult.ValidatorResults[0].Files[2].FilePath, resultResponse[0].Files[2].Path)
+ ts.Equal(len(testValidationResult.ValidatorResults[0].Files[2].Messages), len(resultResponse[0].Files[2].Messages))
+}
+
+func (ts *ValidatorAPITestSuite) TestValidatorsGet() {
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/validators", nil)
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+ ts.Equal(`["mock-validator"]`, w.Body.String())
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/api_validator_orchestrator.go b/sda-validator/orchestrator/api/openapi_interface/api_validator_orchestrator.go
new file mode 100644
index 0000000000..0118a871c8
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/api_validator_orchestrator.go
@@ -0,0 +1,33 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+import (
+ "github.com/gin-gonic/gin"
+)
+
+type ValidatorOrchestratorAPI interface {
+
+ // AdminResultGet Get /admin/result
+ AdminResultGet(c *gin.Context)
+
+ // AdminValidatePost Post /admin/validate
+ AdminValidatePost(c *gin.Context)
+
+ // ResultGet Get /result
+ ResultGet(c *gin.Context)
+
+ // ValidatePost Post /validate
+ ValidatePost(c *gin.Context)
+
+ // ValidatorsGet Get /validators
+ ValidatorsGet(c *gin.Context)
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/model__validate_post_200_response.go b/sda-validator/orchestrator/api/openapi_interface/model__validate_post_200_response.go
new file mode 100644
index 0000000000..500c44fda9
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/model__validate_post_200_response.go
@@ -0,0 +1,17 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+type ValidatePost200Response struct {
+
+ // The validation id
+ ValidationId string `json:"validation_id,omitempty"`
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/model_admin_validate_request.go b/sda-validator/orchestrator/api/openapi_interface/model_admin_validate_request.go
new file mode 100644
index 0000000000..b7073d86ea
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/model_admin_validate_request.go
@@ -0,0 +1,22 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+type AdminValidateRequest struct {
+
+ // The file paths to be validated
+ FilePaths []string `json:"file_paths,omitempty"`
+
+ // The validators to be executed
+ Validators []string `json:"validators,omitempty"`
+
+ UserId string `json:"user_id,omitempty"`
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner.go b/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner.go
new file mode 100644
index 0000000000..3c8e67ec58
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner.go
@@ -0,0 +1,34 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+import (
+ "time"
+)
+
+type ResultResponseInner struct {
+
+ // The id of the validator
+ ValidatorId string `json:"validator_id,omitempty"`
+
+ // Denotes the overall result of the validator validation
+ Result string `json:"result,omitempty"`
+
+ // The time the validator validation was started RFC3339 format
+ StartedAt time.Time `json:"started_at,omitempty"`
+
+ // The time the validator validation was finished RFC3339 format
+ FinishedAt time.Time `json:"finished_at,omitempty"`
+
+ Files []ResultResponseInnerFilesInner `json:"files,omitempty"`
+
+ Messages []ResultResponseInnerFilesInnerMessagesInner `json:"messages,omitempty"`
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner_files_inner.go b/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner_files_inner.go
new file mode 100644
index 0000000000..7adc00c7f7
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner_files_inner.go
@@ -0,0 +1,22 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+type ResultResponseInnerFilesInner struct {
+
+ // Path of the file
+ Path string `json:"path,omitempty"`
+
+ // Denotes the result of validation of the file
+ Result string `json:"result,omitempty"`
+
+ Messages []ResultResponseInnerFilesInnerMessagesInner `json:"messages,omitempty"`
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner_files_inner_messages_inner.go b/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner_files_inner_messages_inner.go
new file mode 100644
index 0000000000..dc1145deeb
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/model_result_response_inner_files_inner_messages_inner.go
@@ -0,0 +1,23 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+type ResultResponseInnerFilesInnerMessagesInner struct {
+
+ // Level of message
+ Level string `json:"level,omitempty"`
+
+ // Timestamp of message
+ Time string `json:"time,omitempty"`
+
+ // Message
+ Message string `json:"message,omitempty"`
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/model_validate_request.go b/sda-validator/orchestrator/api/openapi_interface/model_validate_request.go
new file mode 100644
index 0000000000..77134f699c
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/model_validate_request.go
@@ -0,0 +1,20 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+type ValidateRequest struct {
+
+ // The file paths to be validated
+ FilePaths []string `json:"file_paths,omitempty"`
+
+ // The validators to be executed
+ Validators []string `json:"validators,omitempty"`
+}
diff --git a/sda-validator/orchestrator/api/openapi_interface/routers.go b/sda-validator/orchestrator/api/openapi_interface/routers.go
new file mode 100644
index 0000000000..ca64705aa2
--- /dev/null
+++ b/sda-validator/orchestrator/api/openapi_interface/routers.go
@@ -0,0 +1,103 @@
+/*
+ * SDA validator orchestrator API
+ *
+ * This is the API for the validator orchestrator.
+ *
+ * API version: 1.0
+ * Generated by: OpenAPI Generator (https://openapi-generator.tech)
+ */
+
+// nolint
+package openapi
+
+import (
+ "net/http"
+
+ "github.com/gin-gonic/gin"
+)
+
+// Route is the information for every URI.
+type Route struct {
+ // Name is the name of this Route.
+ Name string
+ // Method is the string for the HTTP method. ex) GET, POST etc..
+ Method string
+ // Pattern is the pattern of the URI.
+ Pattern string
+ // HandlerFunc is the handler function of this route.
+ HandlerFunc gin.HandlerFunc
+}
+
+// NewRouter returns a new router.
+func NewRouter(handleFunctions ApiHandleFunctions) *gin.Engine {
+ return NewRouterWithGinEngine(gin.Default(), handleFunctions)
+}
+
+// NewRouter add routes to existing gin engine.
+func NewRouterWithGinEngine(router *gin.Engine, handleFunctions ApiHandleFunctions) *gin.Engine {
+ for _, route := range getRoutes(handleFunctions) {
+ if route.HandlerFunc == nil {
+ route.HandlerFunc = DefaultHandleFunc
+ }
+ switch route.Method {
+ case http.MethodGet:
+ router.GET(route.Pattern, route.HandlerFunc)
+ case http.MethodPost:
+ router.POST(route.Pattern, route.HandlerFunc)
+ case http.MethodPut:
+ router.PUT(route.Pattern, route.HandlerFunc)
+ case http.MethodPatch:
+ router.PATCH(route.Pattern, route.HandlerFunc)
+ case http.MethodDelete:
+ router.DELETE(route.Pattern, route.HandlerFunc)
+ }
+ }
+
+ return router
+}
+
+// Default handler for not yet implemented routes
+func DefaultHandleFunc(c *gin.Context) {
+ c.String(http.StatusNotImplemented, "501 not implemented")
+}
+
+type ApiHandleFunctions struct {
+
+ // Routes for the ValidatorOrchestratorAPI part of the API
+ ValidatorOrchestratorAPI ValidatorOrchestratorAPI
+}
+
+func getRoutes(handleFunctions ApiHandleFunctions) []Route {
+ return []Route{
+ {
+ "AdminResultGet",
+ http.MethodGet,
+ "/admin/result",
+ handleFunctions.ValidatorOrchestratorAPI.AdminResultGet,
+ },
+ {
+ "AdminValidatePost",
+ http.MethodPost,
+ "/admin/validate",
+ handleFunctions.ValidatorOrchestratorAPI.AdminValidatePost,
+ },
+ {
+ "ResultGet",
+ http.MethodGet,
+ "/result",
+ handleFunctions.ValidatorOrchestratorAPI.ResultGet,
+ },
+ {
+ "ValidatePost",
+ http.MethodPost,
+ "/validate",
+ handleFunctions.ValidatorOrchestratorAPI.ValidatePost,
+ },
+ {
+ "ValidatorsGet",
+ http.MethodGet,
+ "/validators",
+ handleFunctions.ValidatorOrchestratorAPI.ValidatorsGet,
+ },
+ }
+}
diff --git a/sda-validator/orchestrator/api/options.go b/sda-validator/orchestrator/api/options.go
new file mode 100644
index 0000000000..f085e5c085
--- /dev/null
+++ b/sda-validator/orchestrator/api/options.go
@@ -0,0 +1,33 @@
+package api
+
+import "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/broker"
+
+func SdaAPIURL(v string) func(*validatorAPIImpl) {
+ return func(impl *validatorAPIImpl) {
+ impl.sdaAPIURL = v
+ }
+}
+
+func SdaAPIToken(v string) func(*validatorAPIImpl) {
+ return func(impl *validatorAPIImpl) {
+ impl.sdaAPIToken = v
+ }
+}
+
+func ValidationJobPreparationQueue(v string) func(*validatorAPIImpl) {
+ return func(impl *validatorAPIImpl) {
+ impl.validationJobPreparationQueue = v
+ }
+}
+
+func Broker(v broker.AMQPBrokerI) func(*validatorAPIImpl) {
+ return func(impl *validatorAPIImpl) {
+ impl.broker = v
+ }
+}
+
+func ValidationFileSizeLimit(v int64) func(*validatorAPIImpl) {
+ return func(impl *validatorAPIImpl) {
+ impl.validationFileSizeLimit = v
+ }
+}
diff --git a/sda-validator/orchestrator/apptainer.conf b/sda-validator/orchestrator/apptainer.conf
new file mode 100644
index 0000000000..ad821d9080
--- /dev/null
+++ b/sda-validator/orchestrator/apptainer.conf
@@ -0,0 +1,466 @@
+# APPTAINER.CONF
+# This is the global configuration file for Apptainer. This file controls
+# what the container is allowed to do on a particular host, and as a result
+# this file must be owned by root.
+
+# ALLOW SETUID: [BOOL]
+# DEFAULT: yes
+# Should we allow users to utilize the setuid program flow within Apptainer?
+# note1: This is the default mode, and to utilize all features, this option
+# must be enabled. For example, without this option loop mounts of image
+# files will not work; only sandbox image directories, which do not need loop
+# mounts, will work (subject to note 2).
+# note2: If this option is disabled, it will rely on unprivileged user
+# namespaces which have not been integrated equally between different Linux
+# distributions.
+allow setuid = no
+
+# MAX LOOP DEVICES: [INT]
+# DEFAULT: 256
+# Set the maximum number of loop devices that Apptainer should ever attempt
+# to utilize.
+max loop devices = 256
+
+# ALLOW IPC NS: [BOOL]
+# DEFAULT: yes
+# Should we allow users to request the IPC namespace?
+allow ipc ns = yes
+
+# ALLOW PID NS: [BOOL]
+# DEFAULT: yes
+# Should we allow users to request the PID namespace? Note that for some HPC
+# resources, the PID namespace may confuse the resource manager and break how
+# some MPI implementations utilize shared memory. (note, on some older
+# systems, the PID namespace is always used)
+allow pid ns = yes
+
+# ALLOW USER NS: [BOOL]
+# DEFAULT: yes
+# Should we allow users to request the USER namespace?
+allow user ns = yes
+
+# ALLOW UTS NS: [BOOL]
+# DEFAULT: yes
+# Should we allow users to request the UTS namespace?
+allow uts ns = yes
+
+# CONFIG PASSWD: [BOOL]
+# DEFAULT: yes
+# If /etc/passwd exists within the container, this will automatically append
+# an entry for the calling user.
+config passwd = yes
+
+# CONFIG GROUP: [BOOL]
+# DEFAULT: yes
+# If /etc/group exists within the container, this will automatically append
+# group entries for the calling user.
+config group = yes
+
+# CONFIG RESOLV_CONF: [BOOL]
+# DEFAULT: yes
+# If there is a bind point within the container, use the host's
+# /etc/resolv.conf.
+config resolv_conf = no
+
+# MOUNT PROC: [BOOL]
+# DEFAULT: yes
+# Should we automatically bind mount /proc within the container?
+mount proc = no
+
+# MOUNT SYS: [BOOL]
+# DEFAULT: yes
+# Should we automatically bind mount /sys within the container?
+mount sys = no
+
+# MOUNT DEV: [yes/no/minimal]
+# DEFAULT: yes
+# Should we automatically bind mount /dev within the container? If 'minimal'
+# is chosen, then only 'null', 'zero', 'random', 'urandom', and 'shm' will
+# be included (the same effect as the --contain options)
+mount dev = no
+
+# MOUNT DEVPTS: [BOOL]
+# DEFAULT: yes
+# Should we mount a new instance of devpts if there is a 'minimal'
+# /dev, or -C is passed? Note, this requires that your kernel was
+# configured with CONFIG_DEVPTS_MULTIPLE_INSTANCES=y, or that you're
+# running kernel 4.7 or newer.
+mount devpts = no
+
+# MOUNT HOME: [BOOL]
+# DEFAULT: yes
+# Should we automatically determine the calling user's home directory and
+# attempt to mount it's base path into the container? If the --contain option
+# is used, the home directory will be created within the session directory or
+# can be overridden with the APPTAINER_HOME or APPTAINER_WORKDIR
+# environment variables (or their corresponding command line options).
+mount home = no
+
+# MOUNT TMP: [BOOL]
+# DEFAULT: yes
+# Should we automatically bind mount /tmp and /var/tmp into the container? If
+# the --contain option is used, both tmp locations will be created in the
+# session directory or can be specified via the APPTAINER_WORKDIR
+# environment variable (or the --workingdir command line option).
+mount tmp = no
+
+# MOUNT HOSTFS: [BOOL]
+# DEFAULT: no
+# Probe for all mounted file systems that are mounted on the host, and bind
+# those into the container?
+mount hostfs = no
+
+# BIND PATH: [STRING]
+# DEFAULT: Undefined
+# Define a list of files/directories that should be made available from within
+# the container. The file or directory must exist within the container on
+# which to attach to. you can specify a different source and destination
+# path (respectively) with a colon; otherwise source and dest are the same.
+# NOTE: these are ignored if apptainer is invoked with --contain except
+# for /etc/hosts and /etc/localtime. When invoked with --contain and --net,
+# /etc/hosts would contain a default generated content for localhost resolution.
+#bind path = /etc/apptainer/default-nsswitch.conf:/etc/nsswitch.conf
+#bind path = /opt
+#bind path = /scratch
+#bind path = /etc/localtime
+#bind path = /etc/hosts
+
+# USER BIND CONTROL: [BOOL]
+# DEFAULT: yes
+# Allow users to influence and/or define bind points at runtime? This will allow
+# users to specify bind points, scratch and tmp locations. (note: User bind
+# control is only allowed if the host also supports PR_SET_NO_NEW_PRIVS)
+user bind control = yes
+
+# ENABLE FUSEMOUNT: [BOOL]
+# DEFAULT: yes
+# Allow users to mount fuse filesystems inside containers with the --fusemount
+# command line option.
+enable fusemount = no
+
+# ENABLE OVERLAY: [yes/no/driver/try]
+# DEFAULT: yes
+# Enabling this option will make it possible to specify bind paths to locations
+# that do not currently exist within the container. If 'yes', kernel overlayfs
+# will be tried but if it doesn't work, the image driver (i.e. fuse-overlayfs)
+# will be used instead. 'try' is obsolete and treated the same as 'yes'.
+# If 'driver' is chosen, overlay will always be handled by the image driver.
+# If 'no' is chosen, then no overlay type will be used for missing bind paths
+# nor for any other purpose.
+# The ENABLE UNDERLAY 'preferred' option below overrides this option for
+# creating bind paths.
+enable overlay = yes
+
+# ENABLE UNDERLAY: [yes/no/preferred]
+# DEFAULT: yes
+# Enabling this option will make it possible to specify bind paths to locations
+# that do not currently exist within the container without using any overlay
+# feature, when the '--underlay' action option is given by the user or when
+# the ENABLE OVERLAY option above is set to 'no'.
+# If 'preferred' is chosen, then underlay will always be used instead of
+# overlay for creating bind paths.
+# This option is deprecated and will be removed in a future release, because
+# the implementation is complicated and the performance is similar to
+# overlayfs and fuse-overlayfs.
+enable underlay = yes
+
+# MOUNT SLAVE: [BOOL]
+# DEFAULT: yes
+# Should we automatically propagate file-system changes from the host?
+# This should be set to 'yes' when autofs mounts in the system should
+# show up in the container.
+mount slave = no
+
+# SESSIONDIR MAXSIZE: [STRING]
+# DEFAULT: 64
+# This specifies how large the default sessiondir should be (in MB). It will
+# affect users who use the "--contain" options and don't also specify a
+# location to do default read/writes to (e.g. "--workdir" or "--home") and
+# it will also affect users of "--writable-tmpfs".
+sessiondir max size = 64
+
+# *****************************************************************************
+# WARNING
+#
+# The 'limit container' and 'allow container' directives are not effective if
+# unprivileged user namespaces are enabled. They are only effectively applied
+# when Apptainer is running using the native runtime in setuid mode, and
+# unprivileged container execution is not possible on the host.
+#
+# You must disable unprivileged user namespace creation on the host if you rely
+# on the these directives to limit container execution.
+#
+# See the 'Security' and 'Configuration Files' sections of the Admin Guide for
+# more information.
+# *****************************************************************************
+
+# LIMIT CONTAINER OWNERS: [STRING]
+# DEFAULT: NULL
+# Only allow containers to be used that are owned by a given user. If this
+# configuration is undefined (commented or set to NULL), all containers are
+# allowed to be used.
+#
+# Only effective in setuid mode, with unprivileged user namespace creation
+# disabled. Ignored for the root user.
+#limit container owners = gmk, apptainer, nobody
+
+
+# LIMIT CONTAINER GROUPS: [STRING]
+# DEFAULT: NULL
+# Only allow containers to be used that are owned by a given group. If this
+# configuration is undefined (commented or set to NULL), all containers are
+# allowed to be used.
+#
+# Only effective in setuid mode, with unprivileged user namespace creation
+# disabled. Ignored for the root user.
+#limit container groups = group1, apptainer, nobody
+
+
+# LIMIT CONTAINER PATHS: [STRING]
+# DEFAULT: NULL
+# Only allow containers to be used that are located within an allowed path
+# prefix. If this configuration is undefined (commented or set to NULL),
+# containers will be allowed to run from anywhere on the file system.
+#
+# Only effective in setuid mode, with unprivileged user namespace creation
+# disabled. Ignored for the root user.
+#limit container paths = /scratch, /tmp, /global
+
+
+# ALLOW CONTAINER ${TYPE}: [BOOL]
+# DEFAULT: yes
+# This feature limits what kind of containers that Apptainer will allow
+# users to use.
+#
+# Only effective in setuid mode, with unprivileged user namespace creation
+# disabled. Ignored for the root user. Note that some of the
+# same operations can be limited in setuid mode by the ALLOW SETUID-MOUNT
+# feature below; both types need to be "yes" to be allowed.
+#
+# Allow use of unencrypted SIF containers
+allow container sif = yes
+#
+# Allow use of encrypted SIF containers
+allow container encrypted = yes
+#
+# Allow use of non-SIF image formats
+allow container squashfs = yes
+allow container extfs = yes
+allow container dir = yes
+
+# ALLOW SETUID-MOUNT ${TYPE}: [see specific types below]
+# This feature limits what types of kernel mounts that Apptainer will
+# allow unprivileged users to use in setuid mode. Note that some of
+# the same operations can also be limited by the ALLOW CONTAINER feature
+# above; both types need to be "yes" to be allowed. Ignored for the root
+# user.
+#
+# ALLOW SETUID-MOUNT ENCRYPTED: [BOOL}
+# DEFAULT: yes
+# Allow mounting of SIF encryption using the kernel device-mapper in
+# setuid mode. If set to "no", gocryptfs (FUSE-based) encryption will be
+# used instead, which uses a different format in the SIF file, the same
+# format used in unprivileged user namespace mode.
+# allow setuid-mount encrypted = yes
+#
+# ALLOW SETUID-MOUNT SQUASHFS: [yes/no/iflimited]
+# DEFAULT: iflimited
+# Allow mounting of squashfs filesystem types by the kernel in setuid mode,
+# both inside and outside of SIF files. If set to "no", a FUSE-based
+# alternative will be used, the same one used in unprivileged user namespace
+# mode. If set to "iflimited" (the default), then if either a LIMIT CONTAINER
+# option is used above or the Execution Control List (ECL) feature is activated
+# in ecl.toml, this setting will be treated as "yes", and otherwise it will be
+# treated as "no".
+# WARNING: in setuid mode a "yes" here while still allowing users write
+# access to the underlying filesystem data enables potential attacks on
+# the kernel. On the other hand, a "no" here while attempting to limit
+# users to running only approved containers enables the users to potentially
+# override those limits using ptrace() functionality since the FUSE processes
+# run under the user's own uid. So leaving this on the default setting is
+# advised.
+# allow setuid-mount squashfs = iflimited
+#
+# ALLOW SETUID-MOUNT EXTFS: [BOOL]
+# DEFAULT: no
+# Allow mounting of extfs filesystem types by the kernel in setuid mode, both
+# inside and outside of SIF files. If set to "no", a FUSE-based alternative
+# will be used, the same one used in unprivileged user namespace mode.
+# WARNING: this filesystem type frequently has relevant kernel CVEs that take
+# a long time for vendors to patch because they are not considered to be High
+# severity since normally unprivileged users do not have write access to the
+# raw filesystem data. That leaves the kernel vulnerable to attack when
+# this option is enabled in setuid mode. That is why this option defaults to
+# "no". Change it at your own risk.
+# allow setuid-mount extfs = no
+
+# ALLOW NET USERS: [STRING]
+# DEFAULT: NULL
+# A list of non-root users that are permitted to use the CNI configurations
+# specified in the 'allow net networks' directive, and can join existing
+# network namespaces listed in the 'allow netns paths' directive.
+# By default only root may use CNI configurations, or join existing network
+# namespaces, except in the case of a fakeroot execution where only the
+# 40_fakeroot.conflist CNI configuration is used. The restriction only applies
+# when Apptainer is running in SUID mode and the user is non-root.
+#allow net users = gmk, apptainer
+
+
+# ALLOW NET GROUPS: [STRING]
+# DEFAULT: NULL
+# A list of non-root groups that are permitted to use the CNI configurations
+# specified in the 'allow net networks' directive, and can join existing
+# network namespaces listed in the 'allow netns paths' directive.
+# By default only root may use CNI configurations, or join existing network
+# namespaces, except in the case of a fakeroot execution where only the
+# 40_fakeroot.conflist CNI configuration is used. The restriction only applies
+# when Apptainer is running in SUID mode and the user is non-root.
+#allow net groups = group1, apptainer
+
+
+# ALLOW NET NETWORKS: [STRING]
+# DEFAULT: NULL
+# Specify the names of CNI network configurations that may be used by users and
+# groups listed in the allow net users / allow net groups directives. This restriction
+# only applies when Apptainer is running in SUID mode and the user is non-root.
+#allow net networks = bridge
+
+
+# ALLOW NETNS PATHS: [STRING]
+# DEFAULT: NULL
+# Specify the paths to network namespaces that may be joined by users and groups
+# listed in the allow net users / allow net groups directives. This restriction
+# only applies when Apptainer is running in SUID mode and the user is non-root.
+#allow netns paths = /var/run/netns/my_network
+
+
+# ALWAYS USE NV ${TYPE}: [BOOL]
+# DEFAULT: no
+# This feature allows an administrator to determine that every action command
+# should be executed implicitly with the --nv option (useful for GPU only
+# environments).
+always use nv = no
+
+# USE NVIDIA-NVIDIA-CONTAINER-CLI ${TYPE}: [BOOL]
+# DEFAULT: no
+# EXPERIMENTAL
+# If set to yes, Apptainer will attempt to use nvidia-container-cli to setup
+# GPUs within a container when the --nv flag is enabled.
+# If no (default), the legacy binding of entries in nvbliblist.conf will be performed.
+use nvidia-container-cli = no
+
+# ALWAYS USE ROCM ${TYPE}: [BOOL]
+# DEFAULT: no
+# This feature allows an administrator to determine that every action command
+# should be executed implicitly with the --rocm option (useful for GPU only
+# environments).
+always use rocm = no
+
+# ROOT DEFAULT CAPABILITIES: [full/file/no]
+# DEFAULT: full
+# Define default root capability set kept during runtime
+# - full: keep all capabilities (same as --keep-privs)
+# - file: keep capabilities configured for root in
+# ${prefix}/etc/apptainer/capability.json
+# - no: no capabilities (same as --no-privs)
+root default capabilities = full
+
+# MEMORY FS TYPE: [tmpfs/ramfs]
+# DEFAULT: tmpfs
+# This feature allow to choose temporary filesystem type used by Apptainer.
+# Cray CLE 5 and 6 up to CLE 6.0.UP05 there is an issue (kernel panic) when Apptainer
+# use tmpfs, so on affected version it's recommended to set this value to ramfs to avoid
+# kernel panic
+memory fs type = tmpfs
+
+# CNI CONFIGURATION PATH: [STRING]
+# DEFAULT: Undefined
+# Defines path where CNI configuration files are stored
+#cni configuration path =
+
+# CNI PLUGIN PATH: [STRING]
+# DEFAULT: Undefined
+# Defines path where CNI executable plugins are stored
+#cni plugin path =
+
+
+# BINARY PATH: [STRING]
+# DEFAULT: $PATH:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+# Colon-separated list of directories to search for many binaries. May include
+# "$PATH:", which will be replaced by the user's PATH when not running a binary
+# that may be run with elevated privileges from the setuid program flow. The
+# internal bin ${prefix}/libexec/apptainer/bin is always included, either at the
+# beginning of "$PATH:" if it is present or at the very beginning if "$PATH:" is
+# not present.
+# binary path =
+
+# MKSQUASHFS PROCS: [UINT]
+# DEFAULT: 0 (All CPUs)
+# This allows the administrator to specify the number of CPUs for mksquashfs
+# to use when building an image. The fewer processors the longer it takes.
+# To enable it to use all available CPU's set this to 0.
+# mksquashfs procs = 0
+mksquashfs procs = 0
+
+# MKSQUASHFS MEM: [STRING]
+# DEFAULT: Unlimited
+# This allows the administrator to set the maximum amount of memory for mkswapfs
+# to use when building an image. e.g. 1G for 1gb or 500M for 500mb. Restricting memory
+# can have a major impact on the time it takes mksquashfs to create the image.
+# NOTE: This functionality did not exist in squashfs-tools prior to version 4.3
+# If using an earlier version you should not set this.
+# mksquashfs mem = 1G
+
+
+# SHARED LOOP DEVICES: [BOOL]
+# DEFAULT: no
+# Allow to share same images associated with loop devices to minimize loop
+# usage and optimize kernel cache (useful for MPI)
+shared loop devices = no
+
+# IMAGE DRIVER: [STRING]
+# DEFAULT: Undefined
+# This option specifies the name of an image driver provided by a plugin that
+# will be used to handle image mounts. This will override the builtin image
+# driver which provides unprivileged image mounts for squashfs, extfs,
+# overlayfs, and gocryptfs. The overlayfs image driver will only be used
+# if the kernel overlayfs is not usable, but if the 'enable overlay' option
+# above is set to 'driver', the image driver will always be used for overlay.
+# If the driver name specified has not been registered via a plugin installation
+# the run-time will abort.
+image driver =
+
+# DOWNLOAD CONCURRENCY: [UINT]
+# DEFAULT: 3
+# This option specifies how many concurrent streams when downloading (pulling)
+# an image from cloud library.
+download concurrency = 3
+
+# DOWNLOAD PART SIZE: [UINT]
+# DEFAULT: 5242880
+# This option specifies the size of each part when concurrent downloads are
+# enabled.
+download part size = 5242880
+
+# DOWNLOAD BUFFER SIZE: [UINT]
+# DEFAULT: 32768
+# This option specifies the transfer buffer size when concurrent downloads
+# are enabled.
+download buffer size = 32768
+
+# SYSTEMD CGROUPS: [BOOL]
+# DEFAULT: yes
+# Whether to use systemd to manage container cgroups. Required for rootless cgroups
+# functionality. 'no' will manage cgroups directly via cgroupfs.
+systemd cgroups = yes
+
+# APPTHEUS SOCKET PATH: [STRING]
+# DEFAULT: /run/apptheus/gateway.sock
+# Defines apptheus socket path
+apptheus socket path = /run/apptheus/gateway.sock
+
+# ALLOW MONITORING: [BOOL]
+# DEFAULT: no
+# Allow to monitor the system resource usage of apptainer. To enable this option
+# additional tool, i.e. apptheus, is required.
+allow monitoring = no
diff --git a/sda-validator/orchestrator/config/config.go b/sda-validator/orchestrator/config/config.go
new file mode 100644
index 0000000000..cbabbfdc91
--- /dev/null
+++ b/sda-validator/orchestrator/config/config.go
@@ -0,0 +1,196 @@
+package config
+
+import (
+ "log"
+ "strings"
+
+ gounits "github.com/docker/go-units"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/config"
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+)
+
+var (
+ apiPort int
+ apiServerCert string
+ apiServerKey string
+ healthPort int
+ validatorPaths []string
+ sdaAPIURL string
+ sdaAPIToken string
+ validationWorkDir string
+
+ validationFileSizeLimit int64
+ jobWorkerCount int
+ jobQueue string
+ jobPreparationWorkerCount int
+ jobPreparationQueue string
+)
+
+func init() {
+ config.RegisterFlags(
+ &config.Flag{
+ Name: "validation-file-size-limit",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "100GB", "The human readable size limit of files in a single validation, this should equal the size of the size of the validation-work-dir. Supported abbreviations: B, kB, MB, GB, TB, PB, EB, ZB, YB")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ var err error
+ validationFileSizeLimit, err = gounits.FromHumanSize(viper.GetString(flagName))
+ if err != nil {
+ log.Fatalf("failed to parse: %s due to: %v", viper.GetString(flagName), err)
+ }
+ },
+ }, &config.Flag{
+ Name: "api.port",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Int(flagName, 8080, "Port to host the ValidationAPI server at")
+ },
+ AssignFunc: func(flagName string) {
+ apiPort = viper.GetInt(flagName)
+ },
+ }, &config.Flag{
+ Name: "api.server-cert",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Path to the server cert file to be used when hosting the server with TLS, required if api.server-key set")
+ },
+ AssignFunc: func(flagName string) {
+ apiServerCert = viper.GetString(flagName)
+ },
+ }, &config.Flag{
+ Name: "api.server-key",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Path to the server key file to be used when hosting the server with TLS, required if api.server-cert set")
+ },
+ AssignFunc: func(flagName string) {
+ apiServerKey = viper.GetString(flagName)
+ },
+ }, &config.Flag{
+ Name: "health-port",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Int(flagName, 9000, "Port to host the gRPC Health server at")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ healthPort = viper.GetInt(flagName)
+ },
+ }, &config.Flag{
+ Name: "validator-paths",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The paths to the available validators, in comma separated list")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ validatorPaths = strings.Split(viper.GetString(flagName), ",")
+ },
+ }, &config.Flag{
+ Name: "sda-api-url",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Url to the sda-api service")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ sdaAPIURL = viper.GetString(flagName)
+ },
+ }, &config.Flag{
+ Name: "sda-api-token",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Token to authenticate when calling the sda-api service")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ sdaAPIToken = viper.GetString(flagName)
+ },
+ }, &config.Flag{
+ Name: "validation-work-dir",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "/validators", "Directory where application will manage data to be used for validation")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ validationWorkDir = viper.GetString(flagName)
+ },
+ }, &config.Flag{
+ Name: "job-worker-count",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Int(flagName, 2, "Amount of job workers to run")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ jobWorkerCount = viper.GetInt(flagName)
+ },
+ }, &config.Flag{
+ Name: "job-preparation-worker-count",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Int(flagName, 1, "Amount of job preparation workers to run")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ jobPreparationWorkerCount = viper.GetInt(flagName)
+ },
+ }, &config.Flag{
+ Name: "job-queue",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The queue for validation job workers")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ jobQueue = viper.GetString(flagName)
+ },
+ }, &config.Flag{
+ Name: "job-preparation-queue",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The queue for job preparation workers")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ jobPreparationQueue = viper.GetString(flagName)
+ },
+ },
+ )
+}
+
+func SdaAPIToken() string {
+ return sdaAPIToken
+}
+
+func SdaAPIURL() string {
+ return sdaAPIURL
+}
+
+func APIPort() int {
+ return apiPort
+}
+func ValidatorPaths() []string {
+ return validatorPaths
+}
+
+func ValidationWorkDir() string {
+ return validationWorkDir
+}
+
+func JobPreparationWorkerCount() int {
+ return jobPreparationWorkerCount
+}
+func JobPreparationQueue() string {
+ return jobPreparationQueue
+}
+func JobWorkerCount() int {
+ return jobWorkerCount
+}
+func JobQueue() string {
+ return jobQueue
+}
+func ValidationFileSizeLimit() int64 {
+ return validationFileSizeLimit
+}
+func APIServerCert() string {
+ return apiServerCert
+}
+func APIServerKey() string {
+ return apiServerKey
+}
+func HealthPort() int {
+ return healthPort
+}
diff --git a/sda-validator/orchestrator/database/database.go b/sda-validator/orchestrator/database/database.go
new file mode 100644
index 0000000000..9db76c12e5
--- /dev/null
+++ b/sda-validator/orchestrator/database/database.go
@@ -0,0 +1,77 @@
+package database
+
+import (
+ "context"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+)
+
+type Transaction interface {
+ // Commit the transaction
+ Commit() error
+ // Rollback the transaction
+ Rollback() error
+ functions
+}
+
+type Database interface {
+ // BeginTransaction starts a database transaction, either commit or rollback needs to be called when done to release resources and close transaction
+ BeginTransaction(ctx context.Context) (Transaction, error)
+ // Close the database connection
+ Close() error
+ functions
+}
+
+// functions denotes the available database functions
+type functions interface {
+ // ReadValidationResult reads the validation results by a validationID and optionally a userID to check that the files validated belongs to the user
+ ReadValidationResult(ctx context.Context, validationID string, userID *string) (*model.ValidationResult, error)
+ // ReadValidationInformation returns the pending validator jobs for a validation
+ ReadValidationInformation(ctx context.Context, validationID string) (*model.ValidationInformation, error)
+
+ // InsertFileValidationJob inserts a file validation job
+ InsertFileValidationJob(ctx context.Context, insertFileValidationJobParameters *model.InsertFileValidationJobParameters) error
+ // UpdateFileValidationJob updates a file validation jobs with
+ UpdateFileValidationJob(ctx context.Context, fileValidationJobUpdateParameters *model.UpdateFileValidationJobParameters) error
+ // UpdateAllValidationJobFilesOnError updates the result of all validation_file_jobs to error by the validation id, with the validator message to provide details
+ UpdateAllValidationJobFilesOnError(ctx context.Context, validationID string, validatorMessage *model.Message) error
+ // AllValidationJobsDone checks if all validator jobs for a validation have finished
+ AllValidationJobsDone(ctx context.Context, validationID string) (bool, error)
+}
+
+var db Database
+
+// RegisterDatabase registers the database implementation to be used
+func RegisterDatabase(d Database) {
+ db = d
+}
+
+// BeginTransaction starts a database transaction, either commit or rollback needs to be called when done to release resources and close transaction
+func BeginTransaction(ctx context.Context) (Transaction, error) {
+ return db.BeginTransaction(ctx)
+}
+
+// ReadValidationResult reads the validation results by a validationID and optionally a userID to check that the files validated belongs to the user
+func ReadValidationResult(ctx context.Context, validationID string, userID *string) (*model.ValidationResult, error) {
+ return db.ReadValidationResult(ctx, validationID, userID)
+}
+
+// ReadValidationInformation returns the pending validator jobs for a validation
+func ReadValidationInformation(ctx context.Context, validationID string) (*model.ValidationInformation, error) {
+ return db.ReadValidationInformation(ctx, validationID)
+}
+
+// AllValidationJobsDone checks if all validator jobs for a validation have finished
+func AllValidationJobsDone(ctx context.Context, validationID string) (bool, error) {
+ return db.AllValidationJobsDone(ctx, validationID)
+}
+
+// UpdateAllValidationJobFilesOnError updates the result of all validation_file_jobs to error by the validation id, with the validator message to provide details
+func UpdateAllValidationJobFilesOnError(ctx context.Context, validationID string, validatorMessage *model.Message) error {
+ return db.UpdateAllValidationJobFilesOnError(ctx, validationID, validatorMessage)
+}
+
+// Close the database connection
+func Close() error {
+ return db.Close()
+}
diff --git a/sda-validator/orchestrator/database/postgres/config.go b/sda-validator/orchestrator/database/postgres/config.go
new file mode 100644
index 0000000000..23067fbc10
--- /dev/null
+++ b/sda-validator/orchestrator/database/postgres/config.go
@@ -0,0 +1,222 @@
+package postgres
+
+import (
+ "fmt"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/config"
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+)
+
+// dbConfig stores configuration for rabbitmq broker
+type dbConfig struct {
+ host string
+ port int
+ user string
+ password string
+ databaseName string
+ schema string
+ cACert string
+ sslMode string
+ clientCert string
+ clientKey string
+}
+
+var globalConf *dbConfig
+
+func init() {
+ globalConf = &dbConfig{}
+
+ config.RegisterFlags(
+ &config.Flag{
+ Name: "database.host",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The host the postgres database is served on")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.host = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.port",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Int(flagName, 0, "The port the database is served on")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.port = viper.GetInt(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.user",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Username to used to authenticate with in communication with database")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.user = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.password",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Password to used to authenticate with in communication with database")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.password = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.ssl-mode",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "disable", "The database ssl mode")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.sslMode = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.ca-cert",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The database ca cert")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.cACert = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.client-cert",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The cert the client will use in communication with the database")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.clientCert = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.client-key",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The key for the client cert the client will use in communication with the database")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.clientKey = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.name",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "sda_validator_orchestrator", "Database to connect to")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.databaseName = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "database.schema",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "sda_validator_orchestrator", "Database schema to use as search path")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.schema = viper.GetString(flagName)
+ },
+ },
+ )
+}
+
+func Host(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.host = v
+ }
+}
+func Port(v int) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.port = v
+ }
+}
+func User(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.user = v
+ }
+}
+func Password(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.password = v
+ }
+}
+func SslMode(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.sslMode = v
+ }
+}
+func CACert(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.cACert = v
+ }
+}
+func ClientCert(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.clientCert = v
+ }
+}
+func ClientKey(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.clientKey = v
+ }
+}
+func DatabaseName(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.databaseName = v
+ }
+}
+func Schema(v string) func(c *dbConfig) {
+ return func(c *dbConfig) {
+ c.schema = v
+ }
+}
+
+func (c *dbConfig) clone() *dbConfig {
+ return &dbConfig{
+ host: c.host,
+ port: c.port,
+ user: c.user,
+ password: c.password,
+ databaseName: c.databaseName,
+ schema: c.schema,
+ cACert: c.cACert,
+ sslMode: c.sslMode,
+ clientCert: c.clientCert,
+ clientKey: c.clientKey,
+ }
+}
+
+// dataSourceName builds a postgresql data source string to use with sql.Open().
+func (c *dbConfig) dataSourceName() string {
+ connInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s search_path=%s",
+ c.host, c.port, c.user, c.password, c.databaseName, c.sslMode, c.schema)
+
+ if c.sslMode == "disable" {
+ return connInfo
+ }
+
+ if c.cACert != "" {
+ connInfo = fmt.Sprintf("%s sslrootcert=%s", connInfo, c.cACert)
+ }
+
+ if c.clientCert != "" {
+ connInfo = fmt.Sprintf("%s sslcert=%s", connInfo, c.clientCert)
+ }
+
+ if c.clientKey != "" {
+ connInfo = fmt.Sprintf("%s sslkey=%s", connInfo, c.clientKey)
+ }
+
+ return connInfo
+}
diff --git a/sda-validator/orchestrator/database/postgres/db_functions.go b/sda-validator/orchestrator/database/postgres/db_functions.go
new file mode 100644
index 0000000000..98debda883
--- /dev/null
+++ b/sda-validator/orchestrator/database/postgres/db_functions.go
@@ -0,0 +1,279 @@
+package postgres
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "time"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+)
+
+const (
+ readValidationResultsQuery = "readValidationResults"
+ readValidationInformationQuery = "readValidationInformation"
+ insertFileValidationJobQuery = "insertFileValidationJob"
+ updateFileValidationJobQuery = "updateFileValidationJob"
+ allValidationJobsDoneQuery = "allValidationJobsDone"
+ updateAllValidationJobFilesOnErrorQuery = "updateAllValidationJobFilesOnError"
+)
+
+var queries = map[string]string{
+ readValidationResultsQuery: `
+SELECT validator_id, validator_result, validator_messages, started_at, finished_at, file_path, file_result, file_messages
+FROM file_validation_job
+WHERE validation_id = $1
+AND ($2::text IS NULL OR $2::text = submission_user)`,
+
+ readValidationInformationQuery: `
+SELECT validation_id, file_id, file_path, submission_file_size, validator_id, submission_user
+FROM file_validation_job
+WHERE validation_id = $1
+AND validator_result = 'pending'`,
+
+ insertFileValidationJobQuery: `
+INSERT INTO file_validation_job(validation_id, validator_id, file_id, file_path, submission_file_size, submission_user, triggered_by, started_at)
+VALUES ($1, $2, $3, $4, $5, $6, $7, $8)`,
+
+ updateFileValidationJobQuery: `
+UPDATE file_validation_job SET
+finished_at = $1, file_result = $2, validator_messages = $3, file_messages = $4, validator_result = $5
+WHERE file_id = $6
+AND validator_id = $7
+AND validation_id = $8`,
+
+ allValidationJobsDoneQuery: `
+SELECT false
+FROM file_validation_job
+WHERE validation_id = $1
+AND finished_at IS NULL`,
+
+ updateAllValidationJobFilesOnErrorQuery: `
+UPDATE file_validation_job SET
+finished_at = $1, file_result = 'error', validator_messages = $2, validator_result = 'error'
+WHERE validation_id = $3`,
+}
+
+func (db *pgDb) readValidationResult(ctx context.Context, stmt *sql.Stmt, validationID string, userID *string) (*model.ValidationResult, error) {
+ rows, err := stmt.QueryContext(ctx, validationID, userID)
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ _ = rows.Close()
+ }()
+
+ validatorResults := make(map[string]*model.ValidatorResult)
+
+ for rows.Next() {
+ var startedAt string
+ fileResult := new(model.FileResult)
+ validatorResult := new(model.ValidatorResult)
+
+ var validatorMessages, fileMessages, finishedAt sql.NullString
+
+ if err := rows.Scan(
+ &validatorResult.ValidatorID,
+ &validatorResult.Result,
+ &validatorMessages,
+ &startedAt,
+ &finishedAt,
+ &fileResult.FilePath,
+ &fileResult.Result,
+ &fileMessages); err != nil {
+ return nil, err
+ }
+
+ if fileMessages.Valid {
+ if err := json.Unmarshal([]byte(fileMessages.String), &fileResult.Messages); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal file messages: %v", err)
+ }
+ }
+
+ if readValidatorResult, ok := validatorResults[validatorResult.ValidatorID]; ok {
+ readValidatorResult.Files = append(readValidatorResult.Files, fileResult)
+
+ continue
+ }
+
+ if validatorMessages.Valid {
+ if err := json.Unmarshal([]byte(validatorMessages.String), &validatorResult.Messages); err != nil {
+ return nil, fmt.Errorf("failed to unmarshal file messages: %v", err)
+ }
+ }
+
+ validatorResult.StartedAt, err = time.Parse(time.RFC3339, startedAt)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse started at: %v", err)
+ }
+ if finishedAt.Valid {
+ validatorResult.FinishedAt, err = time.Parse(time.RFC3339, finishedAt.String)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse started at: %v", err)
+ }
+ }
+ validatorResult.Files = append(validatorResult.Files, fileResult)
+ validatorResults[validatorResult.ValidatorID] = validatorResult
+ }
+
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ // Check if any rows where found, if not return nil, nil to indicate validation result not found
+ if len(validatorResults) == 0 {
+ return nil, nil
+ }
+ validationResult := &model.ValidationResult{
+ ValidationID: validationID,
+ ValidatorResults: make([]*model.ValidatorResult, 0, len(validatorResults)),
+ }
+ for _, validatorResult := range validatorResults {
+ validationResult.ValidatorResults = append(validationResult.ValidatorResults, validatorResult)
+ }
+
+ return validationResult, nil
+}
+
+func (db *pgDb) readValidationInformation(ctx context.Context, stmt *sql.Stmt, validationID string) (*model.ValidationInformation, error) {
+ rows, err := stmt.QueryContext(ctx, validationID)
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ _ = rows.Close()
+ }()
+
+ validationInformation := new(model.ValidationInformation)
+ validatorsIDs := make(map[string]struct{})
+ files := make(map[string]*model.FileInformation)
+
+ for rows.Next() {
+ fileInformation := new(model.FileInformation)
+ var validatorsID string
+
+ if err := rows.Scan(
+ &validationInformation.ValidationID,
+ &fileInformation.FileID,
+ &fileInformation.FilePath,
+ &fileInformation.SubmissionFileSize,
+ &validatorsID,
+ &validationInformation.SubmissionUserID); err != nil {
+ return nil, err
+ }
+
+ validatorsIDs[validatorsID] = struct{}{}
+ files[fileInformation.FileID] = fileInformation
+ }
+
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ // Check if any rows where found, if not return nil, nil to indicate no validation information not found
+ if len(validatorsIDs) == 0 {
+ return nil, nil
+ }
+
+ for validatorID := range validatorsIDs {
+ validationInformation.ValidatorIDs = append(validationInformation.ValidatorIDs, validatorID)
+ }
+ for _, file := range files {
+ validationInformation.Files = append(validationInformation.Files, file)
+ }
+
+ return validationInformation, nil
+}
+
+func (db *pgDb) updateFileValidationJob(ctx context.Context, stmt *sql.Stmt, params *model.UpdateFileValidationJobParameters) error {
+ fileMessages := sql.NullString{}
+ if len(params.FileMessages) > 0 {
+ fileMessagesJSON, err := json.Marshal(params.FileMessages)
+ if err != nil {
+ return fmt.Errorf("failed to marshal file messages: %v", err)
+ }
+
+ fileMessages.Valid = true
+ fileMessages.String = string(fileMessagesJSON)
+ }
+
+ validatorMessages := sql.NullString{}
+ if len(params.ValidatorMessages) > 0 {
+ validatorMessagesJSON, err := json.Marshal(params.ValidatorMessages)
+ if err != nil {
+ return fmt.Errorf("failed to marshal validator messages: %v", err)
+ }
+
+ validatorMessages.Valid = true
+ validatorMessages.String = string(validatorMessagesJSON)
+ }
+
+ if _, err := stmt.ExecContext(ctx,
+ params.FinishedAt.Format(time.RFC3339),
+ params.FileResult,
+ validatorMessages,
+ fileMessages,
+ params.ValidatorResult,
+ params.FileID,
+ params.ValidatorID,
+ params.ValidationID); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (db *pgDb) allValidationJobsDone(ctx context.Context, stmt *sql.Stmt, validationID string) (bool, error) {
+ row := stmt.QueryRowContext(ctx, validationID)
+
+ if err := row.Err(); err != nil {
+ // If we got any rows, there are still pending jobs
+ if errors.Is(row.Err(), sql.ErrNoRows) {
+ return true, nil
+ }
+
+ return false, err
+ }
+
+ return false, nil
+}
+
+func (db *pgDb) insertFileValidationJob(ctx context.Context, stmt *sql.Stmt, params *model.InsertFileValidationJobParameters) error {
+ if _, err := stmt.ExecContext(ctx,
+ params.ValidationID,
+ params.ValidatorID,
+ params.FileID,
+ params.FilePath,
+ params.FileSubmissionSize,
+ params.SubmissionUser,
+ params.TriggeredBy,
+ params.StartedAt.Format(time.RFC3339)); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (db *pgDb) updateAllValidationJobFilesOnError(ctx context.Context, stmt *sql.Stmt, validationID string, validatorMessage *model.Message) error {
+ validatorMessages := &sql.NullString{}
+ if validatorMessage != nil {
+ validatorMessagesJSON, err := json.Marshal([]*model.Message{validatorMessage})
+ if err != nil {
+ return fmt.Errorf("failed to marshal validator messages: %v", err)
+ }
+
+ validatorMessages.Valid = true
+ validatorMessages.String = string(validatorMessagesJSON)
+ }
+
+ if _, err := stmt.ExecContext(ctx,
+ time.Now().Format(time.RFC3339),
+ validatorMessages,
+ validationID); err != nil {
+ return err
+ }
+
+ return nil
+}
diff --git a/sda-validator/orchestrator/database/postgres/initdb.d/01_create_table_file_validation_job.sql b/sda-validator/orchestrator/database/postgres/initdb.d/01_create_table_file_validation_job.sql
new file mode 100644
index 0000000000..5c1967581a
--- /dev/null
+++ b/sda-validator/orchestrator/database/postgres/initdb.d/01_create_table_file_validation_job.sql
@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS file_validation_job
+(
+ id SERIAL PRIMARY KEY,
+ validation_id UUID,
+ validator_id TEXT,
+ file_path TEXT,
+ file_id UUID,
+ submission_file_size BIGINT,
+ submission_user TEXT,
+ triggered_by TEXT,
+ file_result TEXT DEFAULT 'pending',
+ started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(),
+ finished_at TIMESTAMP WITH TIME ZONE,
+ file_messages JSON,
+ validator_messages JSON,
+ validator_result TEXT DEFAULT 'pending',
+
+ CONSTRAINT unique_file_validation_job UNIQUE (validation_id, validator_id, file_id)
+);
\ No newline at end of file
diff --git a/sda-validator/orchestrator/database/postgres/postgres.go b/sda-validator/orchestrator/database/postgres/postgres.go
new file mode 100644
index 0000000000..0cad7b24d9
--- /dev/null
+++ b/sda-validator/orchestrator/database/postgres/postgres.go
@@ -0,0 +1,98 @@
+package postgres
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "fmt"
+
+ _ "github.com/lib/pq" // Import pg driver
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+ log "github.com/sirupsen/logrus"
+)
+
+type pgDb struct {
+ db *sql.DB
+ config *dbConfig
+}
+
+var preparedStatements map[string]*sql.Stmt
+
+func Init(options ...func(config *dbConfig)) error {
+ dbConf := globalConf.clone()
+
+ for _, o := range options {
+ o(dbConf)
+ }
+
+ pg := &pgDb{db: nil, config: dbConf}
+
+ var err error
+ pg.db, err = sql.Open("postgres", pg.config.dataSourceName())
+ if err != nil {
+ return fmt.Errorf("failed to connect to database: %v", err)
+ }
+
+ if err := pg.db.Ping(); err != nil {
+ return fmt.Errorf("failed to connect to database: %v", err)
+ }
+
+ // Prepare the statements from the queries
+ preparedStatements = make(map[string]*sql.Stmt)
+ for queryName, query := range queries {
+ preparedStmt, err := pg.db.Prepare(query)
+ if err != nil {
+ log.Errorf("failed to prepare query: %s, due to: %v", queryName, err)
+
+ return errors.Join(fmt.Errorf("failed to prepare query: %s", queryName), err)
+ }
+ preparedStatements[queryName] = preparedStmt
+ }
+
+ database.RegisterDatabase(pg)
+
+ return nil
+}
+
+// Close terminates the connection to the database
+func (db *pgDb) Close() error {
+ if db.db == nil {
+ return nil
+ }
+
+ return db.db.Close()
+}
+
+func (db *pgDb) BeginTransaction(ctx context.Context) (database.Transaction, error) {
+ tx, err := db.db.BeginTx(ctx, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ return &pgTx{tx: tx}, nil
+}
+
+func (db *pgDb) ReadValidationResult(ctx context.Context, validationID string, userID *string) (*model.ValidationResult, error) {
+ return db.readValidationResult(ctx, preparedStatements[readValidationResultsQuery], validationID, userID)
+}
+
+func (db *pgDb) ReadValidationInformation(ctx context.Context, validationID string) (*model.ValidationInformation, error) {
+ return db.readValidationInformation(ctx, preparedStatements[readValidationInformationQuery], validationID)
+}
+
+func (db *pgDb) InsertFileValidationJob(ctx context.Context, insertFileValidationJobParameters *model.InsertFileValidationJobParameters) error {
+ return db.insertFileValidationJob(ctx, preparedStatements[insertFileValidationJobQuery], insertFileValidationJobParameters)
+}
+
+func (db *pgDb) UpdateFileValidationJob(ctx context.Context, updateFileValidationJobParameters *model.UpdateFileValidationJobParameters) error {
+ return db.updateFileValidationJob(ctx, preparedStatements[updateFileValidationJobQuery], updateFileValidationJobParameters)
+}
+
+func (db *pgDb) AllValidationJobsDone(ctx context.Context, validationID string) (bool, error) {
+ return db.allValidationJobsDone(ctx, preparedStatements[allValidationJobsDoneQuery], validationID)
+}
+
+func (db *pgDb) UpdateAllValidationJobFilesOnError(ctx context.Context, validationID string, validatorMessage *model.Message) error {
+ return db.updateAllValidationJobFilesOnError(ctx, preparedStatements[updateAllValidationJobFilesOnErrorQuery], validationID, validatorMessage)
+}
diff --git a/sda-validator/orchestrator/database/postgres/tx.go b/sda-validator/orchestrator/database/postgres/tx.go
new file mode 100644
index 0000000000..d0e4284252
--- /dev/null
+++ b/sda-validator/orchestrator/database/postgres/tx.go
@@ -0,0 +1,51 @@
+package postgres
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+)
+
+type pgTx struct {
+ tx *sql.Tx
+ pgDb
+}
+
+func (tx *pgTx) Commit() error {
+ return tx.tx.Commit()
+}
+
+func (tx *pgTx) Rollback() error {
+ err := tx.tx.Rollback()
+ if errors.Is(err, sql.ErrTxDone) {
+ return nil
+ }
+
+ return err
+}
+
+func (tx *pgTx) ReadValidationResult(ctx context.Context, validationID string, userID *string) (*model.ValidationResult, error) {
+ return tx.readValidationResult(ctx, tx.tx.Stmt(preparedStatements[readValidationResultsQuery]), validationID, userID)
+}
+
+func (tx *pgTx) ReadValidationInformation(ctx context.Context, validationID string) (*model.ValidationInformation, error) {
+ return tx.readValidationInformation(ctx, tx.tx.Stmt(preparedStatements[readValidationInformationQuery]), validationID)
+}
+
+func (tx *pgTx) InsertFileValidationJob(ctx context.Context, insertFileValidationJobParameters *model.InsertFileValidationJobParameters) error {
+ return tx.insertFileValidationJob(ctx, tx.tx.Stmt(preparedStatements[insertFileValidationJobQuery]), insertFileValidationJobParameters)
+}
+
+func (tx *pgTx) UpdateFileValidationJob(ctx context.Context, updateFileValidationJobParameters *model.UpdateFileValidationJobParameters) error {
+ return tx.updateFileValidationJob(ctx, tx.tx.Stmt(preparedStatements[updateFileValidationJobQuery]), updateFileValidationJobParameters)
+}
+
+func (tx *pgTx) AllValidationJobsDone(ctx context.Context, validationID string) (bool, error) {
+ return tx.allValidationJobsDone(ctx, tx.tx.Stmt(preparedStatements[allValidationJobsDoneQuery]), validationID)
+}
+
+func (tx *pgTx) UpdateAllValidationJobFilesOnError(ctx context.Context, validationID string, validatorMessage *model.Message) error {
+ return tx.updateAllValidationJobFilesOnError(ctx, tx.tx.Stmt(preparedStatements[allValidationJobsDoneQuery]), validationID, validatorMessage)
+}
diff --git a/sda-validator/orchestrator/docs/high_level_diagram.jpg b/sda-validator/orchestrator/docs/high_level_diagram.jpg
new file mode 100644
index 0000000000..cc862d4bc0
Binary files /dev/null and b/sda-validator/orchestrator/docs/high_level_diagram.jpg differ
diff --git a/sda-validator/orchestrator/go.mod b/sda-validator/orchestrator/go.mod
new file mode 100644
index 0000000000..cc3a3a84ce
--- /dev/null
+++ b/sda-validator/orchestrator/go.mod
@@ -0,0 +1,76 @@
+module github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator
+
+go 1.25.0
+
+require (
+ github.com/casbin/casbin/v2 v2.131.0
+ github.com/docker/go-units v0.5.0
+ github.com/gin-gonic/gin v1.11.0
+ github.com/google/uuid v1.6.0
+ github.com/lestrrat-go/jwx/v2 v2.1.6
+ github.com/lib/pq v1.10.9
+ github.com/neicnordic/crypt4gh v1.14.0
+ github.com/rabbitmq/amqp091-go v1.10.0
+ github.com/sirupsen/logrus v1.9.3
+ github.com/spf13/cobra v1.10.1
+ github.com/spf13/pflag v1.0.10
+ github.com/spf13/viper v1.21.0
+ github.com/stretchr/testify v1.11.1
+ google.golang.org/grpc v1.76.0
+)
+
+require (
+ filippo.io/edwards25519 v1.1.0 // indirect
+ github.com/bmatcuk/doublestar/v4 v4.9.1 // indirect
+ github.com/bytedance/gopkg v0.1.3 // indirect
+ github.com/bytedance/sonic v1.14.2 // indirect
+ github.com/bytedance/sonic/loader v0.4.0 // indirect
+ github.com/casbin/govaluate v1.10.0 // indirect
+ github.com/cloudwego/base64x v0.1.6 // indirect
+ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+ github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect
+ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+ github.com/fsnotify/fsnotify v1.9.0 // indirect
+ github.com/gabriel-vasile/mimetype v1.4.11 // indirect
+ github.com/gin-contrib/sse v1.1.0 // indirect
+ github.com/go-playground/locales v0.14.1 // indirect
+ github.com/go-playground/universal-translator v0.18.1 // indirect
+ github.com/go-playground/validator/v10 v10.28.0 // indirect
+ github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+ github.com/goccy/go-json v0.10.5 // indirect
+ github.com/goccy/go-yaml v1.18.0 // indirect
+ github.com/inconshreveable/mousetrap v1.1.0 // indirect
+ github.com/json-iterator/go v1.1.12 // indirect
+ github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+ github.com/leodido/go-urn v1.4.0 // indirect
+ github.com/lestrrat-go/blackmagic v1.0.4 // indirect
+ github.com/lestrrat-go/httpcc v1.0.1 // indirect
+ github.com/lestrrat-go/httprc v1.0.6 // indirect
+ github.com/lestrrat-go/iter v1.0.2 // indirect
+ github.com/lestrrat-go/option v1.0.1 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+ github.com/modern-go/reflect2 v1.0.2 // indirect
+ github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+ github.com/quic-go/qpack v0.6.0 // indirect
+ github.com/quic-go/quic-go v0.57.0 // indirect
+ github.com/sagikazarmark/locafero v0.12.0 // indirect
+ github.com/segmentio/asm v1.2.1 // indirect
+ github.com/spf13/afero v1.15.0 // indirect
+ github.com/spf13/cast v1.10.0 // indirect
+ github.com/stretchr/objx v0.5.2 // indirect
+ github.com/subosito/gotenv v1.6.0 // indirect
+ github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+ github.com/ugorji/go/codec v1.3.1 // indirect
+ go.uber.org/mock v0.6.0 // indirect
+ go.yaml.in/yaml/v3 v3.0.4 // indirect
+ golang.org/x/arch v0.22.0 // indirect
+ golang.org/x/crypto v0.45.0 // indirect
+ golang.org/x/net v0.47.0 // indirect
+ golang.org/x/sys v0.38.0 // indirect
+ golang.org/x/text v0.31.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b // indirect
+ google.golang.org/protobuf v1.36.10 // indirect
+ gopkg.in/yaml.v3 v3.0.1 // indirect
+)
diff --git a/sda-validator/orchestrator/go.sum b/sda-validator/orchestrator/go.sum
new file mode 100644
index 0000000000..6b1c880a92
--- /dev/null
+++ b/sda-validator/orchestrator/go.sum
@@ -0,0 +1,206 @@
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
+github.com/bmatcuk/doublestar/v4 v4.9.1 h1:X8jg9rRZmJd4yRy7ZeNDRnM+T3ZfHv15JiBJ/avrEXE=
+github.com/bmatcuk/doublestar/v4 v4.9.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
+github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M=
+github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM=
+github.com/bytedance/sonic v1.14.2 h1:k1twIoe97C1DtYUo+fZQy865IuHia4PR5RPiuGPPIIE=
+github.com/bytedance/sonic v1.14.2/go.mod h1:T80iDELeHiHKSc0C9tubFygiuXoGzrkjKzX2quAx980=
+github.com/bytedance/sonic/loader v0.4.0 h1:olZ7lEqcxtZygCK9EKYKADnpQoYkRQxaeY2NYzevs+o=
+github.com/bytedance/sonic/loader v0.4.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo=
+github.com/casbin/casbin/v2 v2.131.0 h1:JWkedeOpvxkBE7iIrWqf2RSMOU/xmxsxgbQ8QzsmGiQ=
+github.com/casbin/casbin/v2 v2.131.0/go.mod h1:FmcfntdXLTcYXv/hxgNntcRPqAbwOG9xsism0yXT+18=
+github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
+github.com/casbin/govaluate v1.10.0 h1:ffGw51/hYH3w3rZcxO/KcaUIDOLP84w7nsidMVgaDG0=
+github.com/casbin/govaluate v1.10.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
+github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
+github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU=
+github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc=
+github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/gabriel-vasile/mimetype v1.4.11 h1:AQvxbp830wPhHTqc1u7nzoLT+ZFxGY7emj5DR5DYFik=
+github.com/gabriel-vasile/mimetype v1.4.11/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s=
+github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w=
+github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM=
+github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
+github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
+github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
+github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
+github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
+github.com/go-playground/validator/v10 v10.28.0 h1:Q7ibns33JjyW48gHkuFT91qX48KG0ktULL6FgHdG688=
+github.com/go-playground/validator/v10 v10.28.0/go.mod h1:GoI6I1SjPBh9p7ykNE/yj3fFYbyDOpwMn5KXd+m2hUU=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw=
+github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
+github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
+github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
+github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
+github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
+github.com/lestrrat-go/blackmagic v1.0.4 h1:IwQibdnf8l2KoO+qC3uT4OaTWsW7tuRQXy9TRN9QanA=
+github.com/lestrrat-go/blackmagic v1.0.4/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw=
+github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
+github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
+github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k=
+github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo=
+github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
+github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
+github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVfecA=
+github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU=
+github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
+github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
+github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U=
+github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
+github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
+github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
+github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
+github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
+github.com/segmentio/asm v1.2.1 h1:DTNbBqs57ioxAD4PrArqftgypG4/qNpXoJx8TVXxPR0=
+github.com/segmentio/asm v1.2.1/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
+github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
+github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
+github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
+github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
+github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
+github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
+github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
+github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
+github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
+go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
+go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
+go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
+go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
+go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
+go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
+go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
+go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
+go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
+go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
+go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
+go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/arch v0.22.0 h1:c/Zle32i5ttqRXjdLyyHZESLD/bB90DCU1g9l/0YBDI=
+golang.org/x/arch v0.22.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
+golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
+golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
+gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b h1:zPKJod4w6F1+nRGDI9ubnXYhU9NSWoFAijkHkUXeTK8=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20250804133106-a7a43d27e69b/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
+google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A=
+google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c=
+google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
+google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/sda-validator/orchestrator/internal/broker/broker.go b/sda-validator/orchestrator/internal/broker/broker.go
new file mode 100644
index 0000000000..4bc764ac86
--- /dev/null
+++ b/sda-validator/orchestrator/internal/broker/broker.go
@@ -0,0 +1,224 @@
+package broker
+
+import (
+ "context"
+ "crypto/tls"
+ "crypto/x509"
+ "fmt"
+ "os"
+ "time"
+
+ amqp "github.com/rabbitmq/amqp091-go"
+ log "github.com/sirupsen/logrus"
+)
+
+// amqpBroker is a Broker that reads messages from an AMQP broker
+type amqpBroker struct {
+ connection *amqp.Connection
+ channel *amqp.Channel
+
+ config *amqpConfig
+ publishConfirmChan <-chan amqp.Confirmation
+}
+
+// NewAMQPBroker creates a new Broker that can communicate with a backend amqp server.
+func NewAMQPBroker(options ...func(*amqpConfig)) (AMQPBrokerI, error) {
+ brokerConf := globalConf.clone()
+
+ for _, o := range options {
+ o(brokerConf)
+ }
+
+ brokerURI := globalConf.buildMQURI()
+
+ var connection *amqp.Connection
+ var err error
+
+ log.Debugf("Connecting to broker host: %s:%d vhost: %s with user: %s", globalConf.host, globalConf.port, globalConf.vhost, globalConf.user)
+ if globalConf.ssl {
+ var tlsConfig *tls.Config
+ tlsConfig, err = setupTLSConfig(brokerConf)
+ if err != nil {
+ return nil, err
+ }
+ connection, err = amqp.DialTLS(brokerURI, tlsConfig)
+ } else {
+ connection, err = amqp.Dial(brokerURI)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ channel, err := connection.Channel()
+ if err != nil {
+ return nil, err
+ }
+
+ if e := channel.Confirm(false); e != nil {
+ _, _ = fmt.Printf("channel could not be put into confirm mode: %s", e)
+
+ return nil, fmt.Errorf("channel could not be put into confirm mode: %s", e)
+ }
+
+ if globalConf.prefetchCount > 0 {
+ // limit the number of messages retrieved from the queue
+ log.Debugf("prefetch count: %v", globalConf.prefetchCount)
+ if err := channel.Qos(globalConf.prefetchCount, 0, false); err != nil {
+ log.Errorf("failed to set channel QoS to %d, reason: %v", globalConf.prefetchCount, err)
+ }
+ }
+
+ confirms := channel.NotifyPublish(make(chan amqp.Confirmation, 1))
+
+ return &amqpBroker{
+ connection: connection,
+ channel: channel,
+ config: brokerConf,
+ publishConfirmChan: confirms,
+ }, nil
+}
+
+// setupTLSConfig is a helper method to setup TLS for the message broker
+func setupTLSConfig(conf *amqpConfig) (*tls.Config, error) {
+ // Read system CAs
+ systemCAs, err := x509.SystemCertPool()
+ if err != nil {
+ log.Errorf("failed to read system CAs: %v", err)
+
+ return nil, err
+ }
+
+ tlsConfig := tls.Config{
+ MinVersion: tls.VersionTLS12,
+ RootCAs: systemCAs,
+ }
+
+ // Add CAs for broker and database
+ if conf.cACert != "" {
+ cacert, err := os.ReadFile(conf.cACert)
+ if err != nil {
+ return nil, err
+ }
+ if ok := tlsConfig.RootCAs.AppendCertsFromPEM(cacert); !ok {
+ log.Warnln("No certs appended, using system certs only")
+ }
+ }
+
+ // If the server URI difers from the hostname in the certificate
+ // we need to set the hostname to match our certificates against.
+ if conf.serverName != "" {
+ tlsConfig.ServerName = conf.serverName
+ }
+
+ if conf.verifyPeer && conf.clientCert != "" && conf.clientKey != "" {
+ cert, err := os.ReadFile(conf.clientCert)
+ if err != nil {
+ return nil, err
+ }
+ key, err := os.ReadFile(conf.clientKey)
+ if err != nil {
+ return nil, err
+ }
+ certs, err := tls.X509KeyPair(cert, key)
+ if err != nil {
+ return nil, err
+ }
+ tlsConfig.Certificates = append(tlsConfig.Certificates, certs)
+ }
+
+ return &tlsConfig, nil
+}
+
+func (broker *amqpBroker) Subscribe(ctx context.Context, queue, consumerID string, handleFunc func(context.Context, amqp.Delivery) error) error {
+ messageChan, err := broker.channel.ConsumeWithContext(ctx,
+ queue, // queue
+ consumerID, // consumerID
+ false, // auto-ack
+ false, // exclusive
+ false, // no-local
+ false, // no-wait
+ nil, // args
+ )
+ if err != nil {
+ return err
+ }
+
+ for {
+ select {
+ case message, ok := <-messageChan:
+ if !ok {
+ log.Debugf("consumerID: %s, channel closed", consumerID)
+
+ return nil
+ }
+ if err := handleFunc(context.Background(), message); err != nil {
+ if err := message.Nack(false, true); err != nil {
+ log.Errorf("consumerID: %s, failed to nack message, reason: %v", consumerID, err)
+ }
+
+ continue
+ }
+ if err := message.Ack(false); err != nil {
+ log.Errorf("consumerID: %s, failed to ack message, reason: %v", consumerID, err)
+ }
+ case <-ctx.Done():
+ // Here we cancel the consumerID and let loop continue until messageChan has closed
+ if err := broker.channel.Cancel(consumerID, true); err != nil {
+ log.Errorf("failed to cancel channel for consumerID: %s, reason: %v", consumerID, err)
+ }
+ }
+ }
+}
+
+// PublishMessage sends a message to RabbitMQ queue
+func (broker *amqpBroker) PublishMessage(ctx context.Context, destination string, body []byte) error {
+ ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+ defer cancel()
+
+ err := broker.channel.PublishWithContext(
+ ctx,
+ broker.config.exchange,
+ destination,
+ false, // mandatory
+ false, // immediate
+ amqp.Publishing{
+ Headers: amqp.Table{},
+ ContentEncoding: "UTF-8",
+ ContentType: "application/json",
+ DeliveryMode: amqp.Persistent, // 1=non-persistent, 2=persistent
+ Body: body,
+ Timestamp: time.Now(),
+ },
+ )
+ if err != nil {
+ return err
+ }
+
+ confirmed := <-broker.publishConfirmChan
+ if !confirmed.Ack {
+ return fmt.Errorf("failed delivery of delivery tag: %d", confirmed.DeliveryTag)
+ }
+ log.Debugf("published message to %s, with delivery tag: %d", destination, confirmed.DeliveryTag)
+
+ return nil
+}
+
+func (broker *amqpBroker) Close() error {
+ return broker.connection.Close()
+}
+
+// Monitor listens to close events on the broker connection and channel
+func (broker *amqpBroker) Monitor() chan *amqp.Error {
+ errChan := make(chan *amqp.Error)
+
+ go func() {
+ select {
+ case err := <-broker.connection.NotifyClose(make(chan *amqp.Error, 1)):
+ errChan <- err
+ case err := <-broker.channel.NotifyClose(make(chan *amqp.Error, 1)):
+ errChan <- err
+ }
+ }()
+
+ return errChan
+}
diff --git a/sda-validator/orchestrator/internal/broker/config.go b/sda-validator/orchestrator/internal/broker/config.go
new file mode 100644
index 0000000000..6af9377814
--- /dev/null
+++ b/sda-validator/orchestrator/internal/broker/config.go
@@ -0,0 +1,259 @@
+package broker
+
+import (
+ "fmt"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/config"
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+)
+
+// amqpConfig stores configuration for rabbitmq broker
+type amqpConfig struct {
+ host string
+ port int
+ user string
+ password string
+ exchange string
+ vhost string
+ ssl bool
+ verifyPeer bool
+ cACert string
+ clientCert string
+ clientKey string
+ serverName string
+ prefetchCount int
+}
+
+var globalConf *amqpConfig
+
+func init() {
+ globalConf = &amqpConfig{}
+
+ config.RegisterFlags(
+ &config.Flag{
+ Name: "broker.host",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The host the broker is served on")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.host = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.port",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Int(flagName, 0, "The port the broker is served on")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.port = viper.GetInt(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.exchange",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The exchange the client will use when publishing messages")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.exchange = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.user",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Username to used to authenticate with in communication with broker")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.user = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.password",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Password to used to authenticate with in communication with broker")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.password = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.vhost",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The virtual host name to connect to")
+ },
+ Required: true,
+ AssignFunc: func(flagName string) {
+ globalConf.vhost = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.ssl",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Bool(flagName, true, "If the broker connection should use ssl")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.ssl = viper.GetBool(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.verify-peer",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Bool(flagName, false, "If the broker connection should use verify-peer, if true client cert, and client key needs to be provided")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.verifyPeer = viper.GetBool(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.ca-cert",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The broker ca cert")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.cACert = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.client-cert",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The cert the client will use in communication with the broker")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.clientCert = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.client-key",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "The key for the client cert the client will use in communication with the broker")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.clientKey = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.server-name",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "ServerName is used to verify the hostname on the returned certificates if ssl is enabled")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.serverName = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "broker.prefetch-count",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.Int(flagName, 2, "How many messages the broker will try to keep on the network for the consumers before receiving delivery acks")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ globalConf.prefetchCount = viper.GetInt(flagName)
+ },
+ },
+ )
+}
+
+// buildMQURI builds the MQ connection URI
+func (c *amqpConfig) buildMQURI() string {
+ protocol := "amqp"
+ if c.ssl {
+ protocol = "amqps"
+ }
+
+ return fmt.Sprintf("%s://%s:%s@%s:%d%s", protocol, c.user, c.password, c.host, c.port, c.vhost)
+}
+
+func Host(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.host = v
+ }
+}
+func Port(v int) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.port = v
+ }
+}
+func Exchange(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.exchange = v
+ }
+}
+func User(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.user = v
+ }
+}
+func Password(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.password = v
+ }
+}
+func Vhost(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.vhost = v
+ }
+}
+func Ssl(v bool) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.ssl = v
+ }
+}
+func VerifyPeer(v bool) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.verifyPeer = v
+ }
+}
+func CACert(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.cACert = v
+ }
+}
+func ClientCert(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.clientCert = v
+ }
+}
+func ClientKey(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.clientKey = v
+ }
+}
+func ServerName(v string) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.serverName = v
+ }
+}
+func PrefetchCount(v int) func(c *amqpConfig) {
+ return func(c *amqpConfig) {
+ c.prefetchCount = v
+ }
+}
+
+func (c *amqpConfig) clone() *amqpConfig {
+ return &amqpConfig{
+ host: c.host,
+ port: c.port,
+ exchange: c.exchange,
+ user: c.user,
+ password: c.password,
+ vhost: c.vhost,
+ ssl: c.ssl,
+ verifyPeer: c.verifyPeer,
+ cACert: c.cACert,
+ clientCert: c.clientCert,
+ clientKey: c.clientKey,
+ serverName: c.serverName,
+ prefetchCount: c.prefetchCount,
+ }
+}
diff --git a/sda-validator/orchestrator/internal/broker/interface_broker.go b/sda-validator/orchestrator/internal/broker/interface_broker.go
new file mode 100644
index 0000000000..bed4f12467
--- /dev/null
+++ b/sda-validator/orchestrator/internal/broker/interface_broker.go
@@ -0,0 +1,18 @@
+package broker
+
+import (
+ "context"
+
+ amqp "github.com/rabbitmq/amqp091-go"
+)
+
+type AMQPBrokerI interface {
+ // PublishMessage to the exchange configured on the broker creation with the destination as the routing key
+ PublishMessage(ctx context.Context, destination string, body []byte) error
+ // Subscribe creates a consumer on the queue, where each message will be handled with the handleFunc
+ Subscribe(ctx context.Context, queue, consumerID string, handleFunc func(context.Context, amqp.Delivery) error) error
+ // Close the broker connection
+ Close() error
+ // Monitor returns a chan watching the broker connection and channel close events
+ Monitor() chan *amqp.Error
+}
diff --git a/sda-validator/orchestrator/internal/commandexecutor/command_executor.go b/sda-validator/orchestrator/internal/commandexecutor/command_executor.go
new file mode 100644
index 0000000000..a971436df5
--- /dev/null
+++ b/sda-validator/orchestrator/internal/commandexecutor/command_executor.go
@@ -0,0 +1,15 @@
+package commandexecutor
+
+import "os/exec"
+
+// CommandExecutor is an interface to execute commands towards the os
+type CommandExecutor interface {
+ Execute(name string, args ...string) ([]byte, error)
+}
+
+type OsCommandExecutor struct {
+}
+
+func (OsCommandExecutor) Execute(name string, args ...string) ([]byte, error) {
+ return exec.Command(name, args...).Output()
+}
diff --git a/sda-validator/orchestrator/internal/config/config.go b/sda-validator/orchestrator/internal/config/config.go
new file mode 100644
index 0000000000..ef32c03317
--- /dev/null
+++ b/sda-validator/orchestrator/internal/config/config.go
@@ -0,0 +1,121 @@
+package config
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path"
+ "strings"
+ "text/tabwriter"
+
+ log "github.com/sirupsen/logrus"
+ "github.com/spf13/cobra"
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+)
+
+type Flag struct {
+ Name string
+ RegisterFunc func(cmd *pflag.FlagSet, flagName string)
+ Required bool
+ AssignFunc func(flagName string)
+}
+
+var registeredFlags []*Flag
+
+func RegisterFlags(flags ...*Flag) {
+ registeredFlags = append(registeredFlags, flags...)
+}
+
+var command = &cobra.Command{
+ Run: func(_ *cobra.Command, _ []string) {
+ // Empty func such that cobra will evaluate flags, etc
+ },
+}
+
+func init() {
+ viper.SetConfigName("config")
+
+ viper.AutomaticEnv()
+ viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_", ".", "_"))
+ viper.SetConfigType("yaml")
+
+ command.Flags().String("config-path", ".", "Set the path viper will look for the config file at")
+ command.Flags().String("config-file", "", "Set the direct path to the config file")
+ command.Flags().String("log.level", "INFO", "Set the log level, supported levels: PANIC, FATAL, ERROR, WARN, INFO, DEBUG, TRACE")
+
+ command.SetHelpFunc(func(cmd *cobra.Command, _ []string) {
+ fmt.Println("Flags:")
+ writer := tabwriter.NewWriter(os.Stdout, 1, 1, 8, ' ', 0)
+ _, _ = fmt.Fprintln(writer, " Name:\tEnv variable:\tType:\tUsage:\tDefault Value:\t")
+ cmd.Flags().VisitAll(func(flag *pflag.Flag) {
+ if flag.Name == "help" {
+ return
+ }
+
+ flagType, usage := pflag.UnquoteUsage(flag)
+ envVar := strings.ToUpper(strings.ReplaceAll(strings.ReplaceAll(flag.Name, "-", "_"), ".", "_"))
+ _, _ = fmt.Fprintf(writer, " --%s\t%s\t%s\t%s\t%v\t\n", flag.Name, envVar, flagType, usage, flag.DefValue)
+ })
+
+ _ = writer.Flush()
+
+ os.Exit(0)
+ })
+}
+
+func Load() error {
+ for _, flag := range registeredFlags {
+ flag.RegisterFunc(command.Flags(), flag.Name)
+ }
+
+ if err := viper.BindPFlags(command.Flags()); err != nil {
+ panic(err)
+ }
+
+ if err := command.Execute(); err != nil {
+ return err
+ }
+
+ if viper.IsSet("config-path") {
+ configPath := viper.GetString("config-path")
+ splitPath := strings.Split(strings.TrimLeft(configPath, "/"), "/")
+ viper.AddConfigPath(path.Join(splitPath...))
+ }
+
+ if viper.IsSet("config-file") {
+ viper.SetConfigFile(viper.GetString("config-file"))
+ }
+
+ _ = viper.ReadInConfig()
+
+ var missingFlags error
+ for _, flag := range registeredFlags {
+ if !flag.Required {
+ continue
+ }
+
+ if !viper.IsSet(flag.Name) {
+ missingFlags = errors.Join(missingFlags, fmt.Errorf("missing required flag: %s", flag.Name))
+
+ continue
+ }
+ }
+ if missingFlags != nil {
+ return missingFlags
+ }
+ for _, flag := range registeredFlags {
+ flag.AssignFunc(flag.Name)
+ }
+
+ stringLevel := viper.GetString("log.level")
+ logLevel, err := log.ParseLevel(stringLevel)
+ if err != nil {
+ log.Debugf("Log level '%s' not supported, setting to 'trace'", stringLevel)
+ logLevel = log.TraceLevel
+ }
+ log.SetLevel(logLevel)
+ log.Infof("Setting log level to '%s'", stringLevel)
+
+ return nil
+}
diff --git a/sda-validator/orchestrator/internal/ginmiddleware/authenticator/authenticator.go b/sda-validator/orchestrator/internal/ginmiddleware/authenticator/authenticator.go
new file mode 100644
index 0000000000..3887c99b16
--- /dev/null
+++ b/sda-validator/orchestrator/internal/ginmiddleware/authenticator/authenticator.go
@@ -0,0 +1,174 @@
+package authenticator
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/gin-gonic/gin"
+ "github.com/lestrrat-go/jwx/v2/jwk"
+ "github.com/lestrrat-go/jwx/v2/jws"
+ "github.com/lestrrat-go/jwx/v2/jwt"
+ log "github.com/sirupsen/logrus"
+)
+
+type Authenticator interface {
+ Authenticate() gin.HandlerFunc
+}
+
+type validateFromToken struct {
+ Keyset jwk.Set
+ config *authenticatorConfig
+}
+
+func NewAuthenticator(options ...func(config *authenticatorConfig)) (Authenticator, error) {
+ authenticator := &validateFromToken{
+ Keyset: jwk.NewSet(),
+ config: conf.clone(),
+ }
+
+ for _, option := range options {
+ option(authenticator.config)
+ }
+
+ if authenticator.config.jwtPubKeyPath != "" {
+ log.Info("authenticator add jwt public from key path")
+ if err := authenticator.readJwtPubKeyPath(); err != nil {
+ return nil, err
+ }
+ }
+ if authenticator.config.jwtPubKeyURL != "" {
+ log.Info("authenticator add jwt public key from url")
+ if err := authenticator.fetchJwtPubKeyURL(); err != nil {
+ return nil, err
+ }
+ }
+
+ return authenticator, nil
+}
+
+func (u *validateFromToken) readJwtPubKeyPath() error {
+ err := filepath.Walk(u.config.jwtPubKeyPath,
+ func(path string, info os.FileInfo, err error) error {
+ if err != nil {
+ return err
+ }
+ if info.Mode().IsRegular() {
+ log.Debug("Reading file: ", filepath.Join(filepath.Clean(u.config.jwtPubKeyPath), info.Name()))
+ keyData, err := os.ReadFile(filepath.Join(filepath.Clean(u.config.jwtPubKeyPath), info.Name()))
+ if err != nil {
+ return fmt.Errorf("key file error: %v", err)
+ }
+
+ key, err := jwk.ParseKey(keyData, jwk.WithPEM(true))
+ if err != nil {
+ return fmt.Errorf("parseKey failed: %v", err)
+ }
+
+ if err := jwk.AssignKeyID(key); err != nil {
+ return fmt.Errorf("assignKeyID failed: %v", err)
+ }
+
+ if err := u.Keyset.AddKey(key); err != nil {
+ return fmt.Errorf("failed to add key to set: %v", err)
+ }
+ }
+
+ return nil
+ })
+ if err != nil {
+ return fmt.Errorf("failed to get public key files (%v)", err)
+ }
+
+ return nil
+}
+
+func (u *validateFromToken) fetchJwtPubKeyURL() error {
+ jwkURL, err := url.ParseRequestURI(u.config.jwtPubKeyURL)
+ if err != nil || jwkURL.Scheme == "" || jwkURL.Host == "" {
+ if err != nil {
+ return err
+ }
+
+ return fmt.Errorf("jwtPubKeyURL is not a proper URL (%s)", jwkURL)
+ }
+ log.Info("jwkURL: ", u.config.jwtPubKeyURL)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+
+ u.Keyset, err = jwk.Fetch(ctx, u.config.jwtPubKeyURL)
+ if err != nil {
+ return fmt.Errorf("jwk.Fetch failed (%v) for %s", err, u.config.jwtPubKeyURL)
+ }
+
+ for it := u.Keyset.Keys(context.Background()); it.Next(context.Background()); {
+ pair := it.Pair()
+ key := pair.Value.(jwk.Key)
+ if err := jwk.AssignKeyID(key); err != nil {
+ return fmt.Errorf("AssignKeyID failed: %v", err)
+ }
+ }
+
+ return nil
+}
+
+// Authenticate verifies that the token included in the http.Request is valid
+func (u *validateFromToken) Authenticate() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ if u == nil {
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+
+ tokenString, err := readTokenFromHeader(c.GetHeader("Authorization"))
+ if err != nil {
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+
+ if tokenString == "" {
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+
+ token, err := jwt.Parse([]byte(tokenString), jwt.WithKeySet(u.Keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true))
+ if err != nil {
+ log.Warnf("failed to parse token: %v", err)
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+
+ iss, err := url.ParseRequestURI(token.Issuer())
+ if err != nil || iss.Hostname() == "" {
+ log.Warnf("failed to get issuer from token (%v)", iss)
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+
+ c.Set("token", token)
+ c.Next()
+ }
+}
+
+func readTokenFromHeader(authStr string) (string, error) {
+ headerParts := strings.Split(authStr, " ")
+ if headerParts[0] != "Bearer" {
+ return "", errors.New("authorization scheme must be bearer")
+ }
+ if len(headerParts) != 2 || headerParts[1] == "" {
+ return "", errors.New("token string is missing from authorization header")
+ }
+
+ return headerParts[1], nil
+}
diff --git a/sda-validator/orchestrator/internal/ginmiddleware/authenticator/authenticator_test.go b/sda-validator/orchestrator/internal/ginmiddleware/authenticator/authenticator_test.go
new file mode 100644
index 0000000000..4cdae30039
--- /dev/null
+++ b/sda-validator/orchestrator/internal/ginmiddleware/authenticator/authenticator_test.go
@@ -0,0 +1,269 @@
+package authenticator
+
+import (
+ "crypto/ecdsa"
+ "crypto/elliptic"
+ "crypto/rand"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/lestrrat-go/jwx/v2/jwa"
+ "github.com/lestrrat-go/jwx/v2/jwk"
+ "github.com/lestrrat-go/jwx/v2/jwt"
+ "github.com/stretchr/testify/suite"
+)
+
+type AuthenticatorTestSuite struct {
+ suite.Suite
+
+ tempDir string
+ ginEngine *gin.Engine
+
+ token []byte
+
+ httpTestServer *httptest.Server
+}
+
+func (ts *AuthenticatorTestSuite) SetupTest() {
+ ts.tempDir = ts.T().TempDir()
+ ts.ginEngine = gin.Default()
+
+ privateKeyRaw, publicKeyRaw, err := generateEs256PemKey()
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ pubKeyPath := filepath.Join(ts.tempDir, "jwt.pub")
+ if err := os.WriteFile(pubKeyPath, publicKeyRaw, 0600); err != nil {
+ ts.FailNow("failed to write PEM key to PEM", err)
+ }
+
+ ts.token, err = generateToken(privateKeyRaw)
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ ts.httpTestServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ keySet := jwk.NewSet()
+
+ publicKey, err := jwk.ParseKey(publicKeyRaw, jwk.WithPEM(true))
+ if err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+
+ return
+ }
+
+ if err := keySet.AddKey(publicKey); err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+
+ return
+ }
+
+ raw, err := json.Marshal(keySet)
+ if err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+
+ return
+ }
+
+ w.WriteHeader(http.StatusOK)
+ // Set the response body
+
+ _, _ = fmt.Fprint(w, string(raw))
+ }))
+}
+
+func generateToken(privateKeyRaw []byte) ([]byte, error) {
+ token := jwt.New()
+ token.Set("sub", "test_user")
+ token.Set("iss", "http://unit.test")
+ token.Set("exp", time.Now().Add(time.Second*30).Unix())
+ token.Set("iat", time.Now().Unix())
+
+ prKeyParsed, err := jwk.ParseKey(privateKeyRaw, jwk.WithPEM(true))
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse private key: %v", err)
+ }
+ if err := jwk.AssignKeyID(prKeyParsed); err != nil {
+ return nil, fmt.Errorf("failed to assign key id: %v", err)
+ }
+ tokenRaw, err := jwt.Sign(token, jwt.WithKey(jwa.ES256, prKeyParsed))
+ if err != nil {
+ return nil, fmt.Errorf("failed to sign token: %v", err)
+ }
+
+ return tokenRaw, nil
+}
+
+func generateEs256PemKey() ([]byte, []byte, error) {
+ key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to generate key: %v", err)
+ }
+ publickey := &key.PublicKey
+
+ // dump private key to file
+ privateKeyBytes, err := jwk.EncodePEM(key)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to encode private key to PEM: %v", err)
+ }
+
+ // dump public key to file
+ publicKeyBytes, err := jwk.EncodePEM(publickey)
+ if err != nil {
+ return nil, nil, fmt.Errorf("failed to encode public key to PEM: %v", err)
+ }
+
+ return privateKeyBytes, publicKeyBytes, nil
+}
+
+func (ts *AuthenticatorTestSuite) TearDownTest() {
+ ts.httpTestServer.Close()
+}
+func TestValidatorAPITestSuite(t *testing.T) {
+ suite.Run(t, new(AuthenticatorTestSuite))
+}
+
+func (ts *AuthenticatorTestSuite) TestAuthenticator_FromJwtPubKeyFile() {
+ newAuthenticator, err := NewAuthenticator(
+ JwtPubKeyPath(ts.tempDir),
+ )
+
+ if err != nil {
+ ts.FailNow("failed to create new authenticator", err)
+ }
+
+ ts.ginEngine.Use(newAuthenticator.Authenticate())
+
+ ts.ginEngine.GET("/", func(c *gin.Context) {
+ if _, ok := c.Get("token"); !ok {
+ c.AbortWithStatus(http.StatusBadRequest)
+
+ return
+ }
+ c.AbortWithStatus(http.StatusOK)
+ })
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/", nil)
+
+ req.Header.Add("Authorization", "Bearer "+string(ts.token))
+
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+}
+
+func (ts *AuthenticatorTestSuite) TestAuthenticator_FromJwtPubKeyUrl() {
+ newAuthenticator, err := NewAuthenticator(
+ JwtPubKeyURL(ts.httpTestServer.URL),
+ )
+
+ if err != nil {
+ ts.FailNow("failed to create new authenticator", err)
+ }
+
+ ts.ginEngine.Use(newAuthenticator.Authenticate())
+
+ ts.ginEngine.GET("/", func(c *gin.Context) {
+ if _, ok := c.Get("token"); !ok {
+ c.AbortWithStatus(http.StatusBadRequest)
+
+ return
+ }
+ c.AbortWithStatus(http.StatusOK)
+ })
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/", nil)
+
+ req.Header.Add("Authorization", "Bearer "+string(ts.token))
+
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+}
+
+func (ts *AuthenticatorTestSuite) TestAuthenticator_FromJwtPubKeyUrl_TokenSignedWithDifferentKey() {
+ newAuthenticator, err := NewAuthenticator(
+ JwtPubKeyURL(ts.httpTestServer.URL),
+ )
+
+ if err != nil {
+ ts.FailNow("failed to create new authenticator", err)
+ }
+
+ ts.ginEngine.Use(newAuthenticator.Authenticate())
+
+ ts.ginEngine.GET("/", func(c *gin.Context) {
+ if _, ok := c.Get("token"); !ok {
+ c.AbortWithStatus(http.StatusBadRequest)
+
+ return
+ }
+ c.AbortWithStatus(http.StatusOK)
+ })
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/", nil)
+
+ privateKeyRaw, _, err := generateEs256PemKey()
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ tokenWithDifferentKey, err := generateToken(privateKeyRaw)
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ req.Header.Add("Authorization", "Bearer "+string(tokenWithDifferentKey))
+
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusUnauthorized, w.Code)
+}
+
+func (ts *AuthenticatorTestSuite) TestAuthenticator_FromJwtPubKeyPath_TokenSignedWithDifferentKey() {
+ newAuthenticator, err := NewAuthenticator(
+ JwtPubKeyPath(ts.tempDir),
+ )
+ if err != nil {
+ ts.FailNow("failed to create new authenticator", err)
+ }
+
+ ts.ginEngine.Use(newAuthenticator.Authenticate())
+
+ ts.ginEngine.GET("/", func(c *gin.Context) {
+ if _, ok := c.Get("token"); !ok {
+ c.AbortWithStatus(http.StatusBadRequest)
+
+ return
+ }
+ c.AbortWithStatus(http.StatusOK)
+ })
+
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/", nil)
+
+ privateKeyRaw, _, err := generateEs256PemKey()
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ tokenWithDifferentKey, err := generateToken(privateKeyRaw)
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ req.Header.Add("Authorization", "Bearer "+string(tokenWithDifferentKey))
+
+ ts.ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusUnauthorized, w.Code)
+}
diff --git a/sda-validator/orchestrator/internal/ginmiddleware/authenticator/config.go b/sda-validator/orchestrator/internal/ginmiddleware/authenticator/config.go
new file mode 100644
index 0000000000..3c5f7327fd
--- /dev/null
+++ b/sda-validator/orchestrator/internal/ginmiddleware/authenticator/config.go
@@ -0,0 +1,56 @@
+package authenticator
+
+import (
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/config"
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+)
+
+type authenticatorConfig struct {
+ jwtPubKeyPath, jwtPubKeyURL string
+}
+
+var conf = &authenticatorConfig{}
+
+func init() {
+ config.RegisterFlags(
+ &config.Flag{
+ Name: "jwt.pub-key-url",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Url for fetching the elixir JWK for API authentication")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ conf.jwtPubKeyURL = viper.GetString(flagName)
+ },
+ },
+ &config.Flag{
+ Name: "jwt.pub-key-path",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "", "Local file containing jwk for authentication for API authentication")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ conf.jwtPubKeyPath = viper.GetString(flagName)
+ },
+ },
+ )
+}
+
+func JwtPubKeyPath(v string) func(*authenticatorConfig) {
+ return func(c *authenticatorConfig) {
+ c.jwtPubKeyPath = v
+ }
+}
+func JwtPubKeyURL(v string) func(*authenticatorConfig) {
+ return func(c *authenticatorConfig) {
+ c.jwtPubKeyURL = v
+ }
+}
+
+func (c *authenticatorConfig) clone() *authenticatorConfig {
+ return &authenticatorConfig{
+ jwtPubKeyPath: c.jwtPubKeyPath,
+ jwtPubKeyURL: c.jwtPubKeyURL,
+ }
+}
diff --git a/sda-validator/orchestrator/internal/ginmiddleware/rbac/config.go b/sda-validator/orchestrator/internal/ginmiddleware/rbac/config.go
new file mode 100644
index 0000000000..d2ca98d20f
--- /dev/null
+++ b/sda-validator/orchestrator/internal/ginmiddleware/rbac/config.go
@@ -0,0 +1,40 @@
+package rbac
+
+import (
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/config"
+ "github.com/spf13/pflag"
+ "github.com/spf13/viper"
+)
+
+type rbacConfig struct {
+ rbacPolicyFilePath string
+}
+
+var conf = &rbacConfig{}
+
+func init() {
+ config.RegisterFlags(
+ &config.Flag{
+ Name: "rbac.policy-file-path",
+ RegisterFunc: func(flagSet *pflag.FlagSet, flagName string) {
+ flagSet.String(flagName, "/rbac/policy.json", "Path to file containing rbac policy")
+ },
+ Required: false,
+ AssignFunc: func(flagName string) {
+ conf.rbacPolicyFilePath = viper.GetString(flagName)
+ },
+ },
+ )
+}
+
+func RbacPolicyFilePath(v string) func(*rbacConfig) {
+ return func(c *rbacConfig) {
+ c.rbacPolicyFilePath = v
+ }
+}
+
+func (c *rbacConfig) clone() *rbacConfig {
+ return &rbacConfig{
+ rbacPolicyFilePath: c.rbacPolicyFilePath,
+ }
+}
diff --git a/sda-validator/orchestrator/internal/ginmiddleware/rbac/jsonadapter.go b/sda-validator/orchestrator/internal/ginmiddleware/rbac/jsonadapter.go
new file mode 100644
index 0000000000..6a896f54d7
--- /dev/null
+++ b/sda-validator/orchestrator/internal/ginmiddleware/rbac/jsonadapter.go
@@ -0,0 +1,135 @@
+// Custom JOSN parser for the RBAC library used by the api
+
+package rbac
+
+import (
+ "encoding/json"
+ "errors"
+ "fmt"
+
+ "github.com/casbin/casbin/v2/model"
+ "github.com/casbin/casbin/v2/persist"
+)
+
+type Adapter struct {
+ source []byte
+ policy []CasbinRule
+}
+
+type CasbinRule struct {
+ PType string
+ V0 string
+ V1 string
+ V2 string
+}
+
+type jsonPolicy struct {
+ Policy []policy
+ Roles []roles
+}
+
+type policy struct {
+ Action string `json:"action"`
+ Path string `json:"path"`
+ Role string `json:"role"`
+}
+
+type roles struct {
+ Role string `json:"role"`
+ RoleBinding string `json:"rolebinding"`
+}
+
+const RbacModel = `[request_definition]
+r = sub, obj, act
+[policy_definition]
+p = sub, obj, act
+[role_definition]
+g = _, _
+[policy_effect]
+e = some(where (p.eft == allow))
+[matchers]
+m = (g(r.sub, p.sub) || p.sub == "*") && (keyMatch(r.obj, p.obj)||keyMatch2(r.obj, p.obj)) && regexMatch(r.act, p.act)`
+
+func NewAdapter(source []byte) *Adapter {
+ return &Adapter{policy: []CasbinRule{}, source: source}
+}
+
+func (a *Adapter) LoadPolicy(m model.Model) error {
+ err := a.loadFromBuffer(m)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (a *Adapter) loadFromBuffer(m model.Model) error {
+ if len(a.source) == 0 {
+ return nil
+ }
+
+ var input jsonPolicy
+ err := json.Unmarshal(a.source, &input)
+ if err != nil {
+ return err
+ }
+
+ for _, p := range input.Policy {
+ if err := persist.LoadPolicyLine(fmt.Sprintf("p,%s,%s,%s", p.Role, p.Path, p.Action), m); err != nil {
+ return err
+ }
+ }
+
+ for _, r := range input.Roles {
+ if err := persist.LoadPolicyLine(fmt.Sprintf("g,%s,%s", r.Role, r.RoleBinding), m); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// AddPolicy adds a policy rule to the storage.
+func (a *Adapter) AddPolicy(_ string, _ string, _ []string) error {
+ return errors.New("not implemented")
+}
+
+// RemovePolicy removes a policy rule from the storage.
+func (a *Adapter) RemovePolicy(_ string, _ string, _ []string) error {
+ return errors.New("not implemented")
+}
+
+// RemoveFilteredPolicy removes policy rules that match the filter from the storage.
+func (a *Adapter) RemoveFilteredPolicy(_ string, _ string, _ int, _ ...string) error {
+ return errors.New("not implemented")
+}
+
+// SavePolicy saves policy
+func (a *Adapter) SavePolicy(m model.Model) error {
+ a.policy = []CasbinRule{}
+ var rules []CasbinRule
+ for ptype, ast := range m["p"] {
+ for _, rule := range ast.Policy {
+ rules = append(rules, CasbinRule{PType: ptype, V0: rule[0], V1: rule[1], V2: rule[2]})
+ }
+ }
+
+ for ptype, ast := range m["g"] {
+ for _, rule := range ast.Policy {
+ rules = append(rules, CasbinRule{PType: ptype, V0: rule[0], V1: rule[1]})
+ }
+ }
+
+ a.policy = rules
+
+ return a.saveToBuffer()
+}
+
+func (a *Adapter) saveToBuffer() error {
+ data, err := json.Marshal(a.policy)
+ if err == nil {
+ a.source = data
+ }
+
+ return err
+}
diff --git a/sda-validator/orchestrator/internal/ginmiddleware/rbac/rbac.go b/sda-validator/orchestrator/internal/ginmiddleware/rbac/rbac.go
new file mode 100644
index 0000000000..e23e7b0ea5
--- /dev/null
+++ b/sda-validator/orchestrator/internal/ginmiddleware/rbac/rbac.go
@@ -0,0 +1,78 @@
+package rbac
+
+import (
+ "net/http"
+ "os"
+
+ "github.com/casbin/casbin/v2"
+ "github.com/casbin/casbin/v2/model"
+ "github.com/gin-gonic/gin"
+ "github.com/lestrrat-go/jwx/v2/jwt"
+ log "github.com/sirupsen/logrus"
+)
+
+type Rbac interface {
+ Enforce() gin.HandlerFunc
+}
+
+type rbacCasbin struct {
+ casbinEnforcer *casbin.Enforcer
+ config *rbacConfig
+}
+
+func NewRbac(options ...func(*rbacConfig)) (Rbac, error) {
+ rbac := &rbacCasbin{
+ casbinEnforcer: nil,
+ config: conf.clone(),
+ }
+
+ for _, option := range options {
+ option(rbac.config)
+ }
+
+ rbacPolicy, err := os.ReadFile(rbac.config.rbacPolicyFilePath)
+ if err != nil {
+ return nil, err
+ }
+
+ m, _ := model.NewModelFromString(RbacModel)
+ rbac.casbinEnforcer, err = casbin.NewEnforcer(m, NewAdapter(rbacPolicy))
+ if err != nil {
+ return nil, err
+ }
+
+ return rbac, nil
+}
+
+func (rbac *rbacCasbin) Enforce() gin.HandlerFunc {
+ return func(c *gin.Context) {
+ token, ok := c.Get("token")
+ if !ok {
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+
+ if _, ok := token.(jwt.Token); !ok {
+ log.Warnf("token from gin context is not a jwt.Token")
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+
+ ok, err := rbac.casbinEnforcer.Enforce(token.(jwt.Token).Subject(), c.Request.URL.Path, c.Request.Method)
+ if err != nil {
+ log.Infof("rbac enforcement failed, reason: %s", err.Error())
+ c.AbortWithStatus(http.StatusInternalServerError)
+
+ return
+ }
+ if !ok {
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+
+ c.Next()
+ }
+}
diff --git a/sda-validator/orchestrator/internal/ginmiddleware/rbac/rbac_test.go b/sda-validator/orchestrator/internal/ginmiddleware/rbac/rbac_test.go
new file mode 100644
index 0000000000..05f3b178d7
--- /dev/null
+++ b/sda-validator/orchestrator/internal/ginmiddleware/rbac/rbac_test.go
@@ -0,0 +1,189 @@
+package rbac
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/gin-gonic/gin"
+ "github.com/lestrrat-go/jwx/v2/jwt"
+ "github.com/stretchr/testify/suite"
+)
+
+type RbacTestSuite struct {
+ suite.Suite
+
+ tempDir string
+}
+
+func TestConfigTestSuite(t *testing.T) {
+ suite.Run(t, new(RbacTestSuite))
+}
+
+type mockAuthenticator struct {
+ user string
+}
+
+func (a *mockAuthenticator) authenticate(c *gin.Context) {
+ token := jwt.New()
+ if err := token.Set("sub", a.user); err != nil {
+ c.AbortWithStatus(http.StatusUnauthorized)
+
+ return
+ }
+ c.Set("token", token)
+}
+
+func (ts *RbacTestSuite) SetupSuite() {
+ ts.tempDir = ts.T().TempDir()
+
+ if err := os.WriteFile(filepath.Join(ts.tempDir, "rbac.policy"), []byte(`
+{
+ "policy": [
+ {
+ "role": "admin",
+ "path": "/admin/test",
+ "action": "(GET)|(POST)"
+ },{
+ "role": "submitter",
+ "path": "/submitter/test",
+ "action": "(GET)|(POST)"
+ }
+ ],
+ "roles": [
+ {
+ "role": "admin",
+ "rolebinding": "submission"
+ },
+ {
+ "role": "admin_test_user",
+ "rolebinding": "admin"
+ }, {
+ "role": "submitter_test_user",
+ "rolebinding": "submitter"
+ }
+ ]
+}`), 0600); err != nil {
+ ts.FailNow("failed to write rbac policy file due to %v", err)
+ }
+}
+
+func setupGinEngine(mockAuthMiddleware, rbacMiddleware func(c *gin.Context)) *gin.Engine {
+ ginEngine := gin.New()
+ ginEngine.Use(mockAuthMiddleware, rbacMiddleware)
+
+ ginEngine.GET("/admin/test", func(c *gin.Context) {
+ c.AbortWithStatus(http.StatusOK)
+ })
+ ginEngine.GET("/submitter/test", func(c *gin.Context) {
+ c.AbortWithStatus(http.StatusOK)
+ })
+
+ return ginEngine
+}
+
+func (ts *RbacTestSuite) TestRbac_AdminUser() {
+ ma := &mockAuthenticator{"admin_test_user"}
+ rbac, err := NewRbac(
+ RbacPolicyFilePath(filepath.Join(ts.tempDir, "rbac.policy")),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ ginEngine := setupGinEngine(ma.authenticate, rbac.Enforce())
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/admin/test", nil)
+
+ ginEngine.ServeHTTP(w, req)
+
+ ts.Equal(http.StatusOK, w.Code)
+}
+
+func (ts *RbacTestSuite) TestRbac_SubmitterUser_AdminEndpoint() {
+ ma := &mockAuthenticator{"submitter_test_user"}
+ rbac, err := NewRbac(
+ RbacPolicyFilePath(filepath.Join(ts.tempDir, "rbac.policy")),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ ginEngine := setupGinEngine(ma.authenticate, rbac.Enforce())
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/admin/test", nil)
+
+ ginEngine.ServeHTTP(w, req)
+ ts.Equal(http.StatusUnauthorized, w.Code)
+}
+
+func (ts *RbacTestSuite) TestRbac_SubmitterUser() {
+ ma := &mockAuthenticator{"submitter_test_user"}
+ rbac, err := NewRbac(
+ RbacPolicyFilePath(filepath.Join(ts.tempDir, "rbac.policy")),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ ginEngine := setupGinEngine(ma.authenticate, rbac.Enforce())
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/submitter/test", nil)
+
+ ginEngine.ServeHTTP(w, req)
+ ts.Equal(http.StatusOK, w.Code)
+}
+
+func (ts *RbacTestSuite) TestRbac_UnknownUser_SubmitterEndpoint() {
+ ma := &mockAuthenticator{"unknown"}
+ rbac, err := NewRbac(
+ RbacPolicyFilePath(filepath.Join(ts.tempDir, "rbac.policy")),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ ginEngine := setupGinEngine(ma.authenticate, rbac.Enforce())
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/submitter/test", nil)
+
+ ginEngine.ServeHTTP(w, req)
+ ts.Equal(http.StatusUnauthorized, w.Code)
+}
+
+func (ts *RbacTestSuite) TestRbac_NoTokenInContext() {
+ rbac, err := NewRbac(
+ RbacPolicyFilePath(filepath.Join(ts.tempDir, "rbac.policy")),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ ginEngine := setupGinEngine(func(c *gin.Context) {
+ c.Next()
+ }, rbac.Enforce())
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/submitter/test", nil)
+
+ ginEngine.ServeHTTP(w, req)
+ ts.Equal(http.StatusUnauthorized, w.Code)
+}
+
+func (ts *RbacTestSuite) TestRbac_UnknownEndpoint() {
+ ma := &mockAuthenticator{"submitter_test_user"}
+ rbac, err := NewRbac(
+ RbacPolicyFilePath(filepath.Join(ts.tempDir, "rbac.policy")),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ ginEngine := setupGinEngine(ma.authenticate, rbac.Enforce())
+ w := httptest.NewRecorder()
+ req, _ := http.NewRequest("GET", "/unknown/test", nil)
+
+ ginEngine.ServeHTTP(w, req)
+ ts.Equal(http.StatusUnauthorized, w.Code)
+}
diff --git a/sda-validator/orchestrator/internal/health/health.go b/sda-validator/orchestrator/internal/health/health.go
new file mode 100644
index 0000000000..acb4987989
--- /dev/null
+++ b/sda-validator/orchestrator/internal/health/health.go
@@ -0,0 +1,40 @@
+package health
+
+import (
+ "fmt"
+ "net"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/health"
+ healthpb "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+var healthServer *health.Server
+
+var grpcServer *grpc.Server
+
+func init() {
+ healthServer = health.NewServer()
+ healthServer.SetServingStatus("", healthpb.HealthCheckResponse_NOT_SERVING)
+}
+
+func Start(port int) error {
+ grpcServer = grpc.NewServer()
+
+ healthpb.RegisterHealthServer(grpcServer, healthServer)
+
+ listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+ if err != nil {
+ return err
+ }
+
+ return grpcServer.Serve(listener)
+}
+
+func SetServingStatus(status healthpb.HealthCheckResponse_ServingStatus) {
+ healthServer.SetServingStatus("", status)
+}
+
+func Stop() {
+ grpcServer.GracefulStop()
+}
diff --git a/sda-validator/orchestrator/jobpreparationworker/options.go b/sda-validator/orchestrator/jobpreparationworker/options.go
new file mode 100644
index 0000000000..b01bcff5f8
--- /dev/null
+++ b/sda-validator/orchestrator/jobpreparationworker/options.go
@@ -0,0 +1,54 @@
+package jobpreparationworker
+
+import "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/broker"
+
+type config struct {
+ workerCount int
+ sourceQueue string
+ destinationQueue string
+ sdaAPIURL string
+ sdaAPIToken string // TODO TBD #989
+ broker broker.AMQPBrokerI
+ validationWorkDir string
+}
+
+func WorkerCount(v int) func(*config) {
+ return func(opts *config) {
+ opts.workerCount = v
+ }
+}
+
+func SourceQueue(v string) func(*config) {
+ return func(opts *config) {
+ opts.sourceQueue = v
+ }
+}
+func DestinationQueue(v string) func(*config) {
+ return func(opts *config) {
+ opts.destinationQueue = v
+ }
+}
+
+func Broker(v broker.AMQPBrokerI) func(*config) {
+ return func(opts *config) {
+ opts.broker = v
+ }
+}
+
+func ValidationWorkDirectory(v string) func(*config) {
+ return func(opts *config) {
+ opts.validationWorkDir = v
+ }
+}
+
+func SdaAPIToken(v string) func(*config) {
+ return func(opts *config) {
+ opts.sdaAPIToken = v
+ }
+}
+
+func SdaAPIURL(v string) func(*config) {
+ return func(opts *config) {
+ opts.sdaAPIURL = v
+ }
+}
diff --git a/sda-validator/orchestrator/jobpreparationworker/worker.go b/sda-validator/orchestrator/jobpreparationworker/worker.go
new file mode 100644
index 0000000000..f121f74296
--- /dev/null
+++ b/sda-validator/orchestrator/jobpreparationworker/worker.go
@@ -0,0 +1,332 @@
+package jobpreparationworker
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "sync"
+ "time"
+
+ "github.com/neicnordic/crypt4gh/keys"
+ "github.com/neicnordic/crypt4gh/streaming"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/validators"
+ amqp "github.com/rabbitmq/amqp091-go"
+ log "github.com/sirupsen/logrus"
+)
+
+type worker struct {
+ id string
+ ctx context.Context
+ cancel context.CancelFunc
+
+ conf *config
+
+ stopCh chan struct{}
+ running bool
+}
+
+type Workers struct {
+ workers []*worker
+ conf *config
+ workerMonitorChan chan error
+}
+
+// NewWorkers initializes the workers with the given options
+func NewWorkers(opt ...func(*config)) (*Workers, error) {
+ newWorkers := &Workers{
+ conf: &config{},
+ }
+
+ for _, o := range opt {
+ o(newWorkers.conf)
+ }
+
+ if newWorkers.conf.sourceQueue == "" {
+ return nil, errors.New("sourceQueue is required")
+ }
+
+ if newWorkers.conf.destinationQueue == "" {
+ return nil, errors.New("destinationQueue is required")
+ }
+ if newWorkers.conf.sdaAPIURL == "" {
+ return nil, errors.New("sdaAPIURL is required")
+ }
+ if newWorkers.conf.sdaAPIToken == "" {
+ return nil, errors.New("sdaAPIToken is required")
+ }
+ if newWorkers.conf.broker == nil {
+ return nil, errors.New("broker is required")
+ }
+
+ if newWorkers.conf.validationWorkDir == "" {
+ return nil, errors.New("validationWorkDir is required")
+ }
+
+ newWorkers.workerMonitorChan = make(chan error, newWorkers.conf.workerCount)
+
+ for i := 0; i < newWorkers.conf.workerCount; i++ {
+ ctx, cancel := context.WithCancel(context.Background())
+ w := &worker{
+ id: fmt.Sprintf("job-preparation-worker-%d", i),
+ ctx: ctx,
+ cancel: cancel,
+ stopCh: make(chan struct{}, 1),
+ conf: newWorkers.conf,
+ running: true,
+ }
+
+ newWorkers.workers = append(newWorkers.workers, w)
+
+ go func(w *worker) {
+ // passing ctx such that we can gracefully shut down the subscribe
+ if err := newWorkers.conf.broker.Subscribe(w.ctx, newWorkers.conf.sourceQueue, w.id, w.handleFunc); err != nil {
+ log.Errorf("job worker encountered error: %v", err)
+ newWorkers.workerMonitorChan <- err
+ }
+ w.running = false
+ w.stopCh <- struct{}{}
+ }(w)
+ }
+
+ return newWorkers, nil
+}
+
+// Monitor monitors if any worker encounters an subscribe error
+func (w *Workers) Monitor() chan error {
+ if w.conf == nil {
+ noConfErr := make(chan error, 1)
+ noConfErr <- errors.New("workers have not been initialized")
+
+ return noConfErr
+ }
+
+ return w.workerMonitorChan
+}
+
+// close a worker and wait until it has closed
+func (w *worker) close() {
+ w.cancel()
+ <-w.stopCh
+ close(w.stopCh)
+}
+
+// Shutdown shutdowns and waits for all workers to have closed
+func (w *Workers) Shutdown() {
+ wg := sync.WaitGroup{}
+ for _, w := range w.workers {
+ if !w.running {
+ continue
+ }
+ wg.Go(func() {
+ w.close()
+ })
+ }
+ wg.Wait()
+ close(w.workerMonitorChan)
+}
+
+func (w *worker) handleFunc(ctx context.Context, message amqp.Delivery) (err error) {
+ jobPreparationMessage := new(model.JobPreparationMessage)
+ if err := json.Unmarshal(message.Body, jobPreparationMessage); err != nil {
+ log.Errorf("could not unmarshal message to job preparation message due to: %v", err)
+
+ return nil // returning nil so message is not nacked and reconsumed
+ }
+
+ validationInformation, err := database.ReadValidationInformation(ctx, jobPreparationMessage.ValidationID)
+ if err != nil {
+ log.Warnf("could not read validation information due to: %v", err)
+
+ return err
+ }
+
+ if validationInformation == nil {
+ log.Warnf("received job preparation message with validation id: %s, which had no file validation jobs in database", jobPreparationMessage.ValidationID)
+
+ return nil
+ }
+
+ validationDir := filepath.Join(w.conf.validationWorkDir, jobPreparationMessage.ValidationID)
+ validationFilesDir := filepath.Join(validationDir, "files")
+ if err = os.MkdirAll(validationFilesDir, 0750); err != nil {
+ log.Errorf("failed to create validation work directory: %s, error: %v", validationFilesDir, err)
+
+ return database.UpdateAllValidationJobFilesOnError(ctx, jobPreparationMessage.ValidationID, &model.Message{
+ Level: "error",
+ Time: time.Now().Format(time.RFC3339),
+ Message: "Internal error",
+ })
+ }
+ // Remove validation directory if any error is encountered
+ defer func() {
+ if err == nil {
+ return
+ }
+ if err := os.RemoveAll(filepath.Join(w.conf.validationWorkDir, jobPreparationMessage.ValidationID)); err != nil {
+ log.Errorf("failed to remove validation directory after worker encountered an error due to: %v", err)
+ }
+ }()
+
+ requiresFileContent := false
+ for _, validatorID := range validationInformation.ValidatorIDs {
+ if validatorDescription, ok := validators.Validators[validatorID]; ok && validatorDescription.RequiresFileContent() {
+ requiresFileContent = true
+
+ break
+ }
+ }
+
+ if requiresFileContent {
+ if err := w.downloadFiles(ctx, validationFilesDir, validationInformation); err != nil {
+ log.Errorf("failed to download files, error: %v", err)
+
+ if err := os.RemoveAll(filepath.Join(w.conf.validationWorkDir, jobPreparationMessage.ValidationID)); err != nil {
+ log.Errorf("failed to remove validation directory after worker encountered an error due to: %v", err)
+ }
+
+ return database.UpdateAllValidationJobFilesOnError(ctx, jobPreparationMessage.ValidationID, &model.Message{
+ Level: "error",
+ Time: time.Now().Format(time.RFC3339),
+ Message: "Internal error",
+ })
+ }
+ }
+
+ return w.sendValidatorJobs(ctx, validationDir, validationInformation)
+}
+
+func (w *worker) downloadFiles(ctx context.Context, validationFilesDir string, validationInformation *model.ValidationInformation) error {
+ files := make(map[string]*os.File)
+ // Ensure all files are closed
+ defer func(filesToClose map[string]*os.File) {
+ for _, file := range files {
+ _ = file.Close()
+ }
+ }(files)
+
+ // Reserve disk space for all files to be downloaded
+ for _, fileInformation := range validationInformation.Files {
+ fileLocalPath := filepath.Join(validationFilesDir, fileInformation.FilePath)
+ dir, _ := filepath.Split(fileLocalPath)
+
+ // Ensure any sub directories file is in is created
+ if err := os.MkdirAll(dir, 0750); err != nil {
+ log.Errorf("failed to create sub directory for file: %s, error: %v", dir, err)
+
+ return err
+ }
+
+ file, err := os.OpenFile(fileLocalPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0444)
+ if err != nil {
+ log.Errorf("failed to open file: %s, error: %v", fileLocalPath, err)
+
+ return err
+ }
+
+ if err := file.Truncate(fileInformation.SubmissionFileSize); err != nil {
+ log.Errorf("failed to truncate file: %s, error: %v", fileLocalPath, err)
+
+ return err
+ }
+ files[fileInformation.FileID] = file
+ }
+
+ publicKeyData, privateKeyData, err := keys.GenerateKeyPair()
+ if err != nil {
+ return err
+ }
+ buf := new(bytes.Buffer)
+ if err := keys.WriteCrypt4GHX25519PublicKey(buf, publicKeyData); err != nil {
+ return err
+ }
+ pubKeyBase64 := base64.StdEncoding.EncodeToString(buf.Bytes())
+
+ // Download and mount files to local
+ for fileID, file := range files {
+ if err := w.downloadFile(ctx, validationInformation.SubmissionUserID, fileID, file, pubKeyBase64, privateKeyData); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+func (w *worker) downloadFile(_ context.Context, userID, fileID string, file *os.File, pubKeyBase64 string, privateKeyData [32]byte) error {
+ req, err := http.NewRequest("GET", fmt.Sprintf("%s/users/%s/file/%s", w.conf.sdaAPIURL, userID, fileID), nil)
+ if err != nil {
+ return fmt.Errorf("failed to create the request, reason: %v", err)
+ }
+
+ // TODO how to handle auth in better way, TBD #989
+ req.Header.Add("Authorization", "Bearer "+w.conf.sdaAPIToken)
+ req.Header.Add("Content-Type", "application/json")
+ req.Header.Set("C4GH-Public-Key", pubKeyBase64)
+
+ // Send the request
+ client := &http.Client{}
+ res, err := client.Do(req)
+ if err != nil {
+ return fmt.Errorf("failed to get response, reason: %v", err)
+ }
+ // Check the status code
+ if res.StatusCode != http.StatusOK {
+ return fmt.Errorf("server returned status %d: for: %s", res.StatusCode, req.URL.String())
+ }
+ defer func() {
+ _ = res.Body.Close()
+ }()
+
+ // Decrypt file
+ crypt4GHReader, err := streaming.NewCrypt4GHReader(res.Body, privateKeyData, nil)
+ if err != nil {
+ return fmt.Errorf("could not create cryp4gh reader: %v", err)
+ }
+ defer func() {
+ _ = crypt4GHReader.Close()
+ }()
+
+ _, err = io.Copy(file, crypt4GHReader)
+ if err != nil {
+ return fmt.Errorf("could not decrypt fileID %s, userID: %s, error: %v", fileID, userID, err)
+ }
+
+ return nil
+}
+
+func (w *worker) sendValidatorJobs(ctx context.Context, validationDir string, validationInformation *model.ValidationInformation) error {
+ files := make([]string, len(validationInformation.Files))
+
+ for i, file := range validationInformation.Files {
+ files[i] = file.FilePath
+ }
+
+ for _, validatorID := range validationInformation.ValidatorIDs {
+ jobMessage := model.JobMessage{
+ ValidationID: validationInformation.ValidationID,
+ ValidatorID: validatorID,
+ ValidationDirectory: validationDir,
+ Files: make([]*model.FileInformation, len(validationInformation.Files)),
+ }
+ copy(jobMessage.Files, validationInformation.Files)
+
+ body, err := json.Marshal(jobMessage)
+ if err != nil {
+ return fmt.Errorf("failed to marshal job message due to: %s", err)
+ }
+
+ if err := w.conf.broker.PublishMessage(ctx, w.conf.destinationQueue, body); err != nil {
+ return fmt.Errorf("failed to publish message due to: %s", err)
+ }
+ }
+
+ return nil
+}
diff --git a/sda-validator/orchestrator/jobpreparationworker/worker_test.go b/sda-validator/orchestrator/jobpreparationworker/worker_test.go
new file mode 100644
index 0000000000..42d30d3402
--- /dev/null
+++ b/sda-validator/orchestrator/jobpreparationworker/worker_test.go
@@ -0,0 +1,600 @@
+package jobpreparationworker
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/neicnordic/crypt4gh/keys"
+ "github.com/neicnordic/crypt4gh/streaming"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/validators"
+ amqp "github.com/rabbitmq/amqp091-go"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/suite"
+)
+
+type JobPreparationWorkerTestSuite struct {
+ suite.Suite
+
+ tempDir string
+ httpTestServer *httptest.Server
+
+ mockDatabase *mockDatabase
+ mockBroker *mockBroker
+}
+
+func (ts *JobPreparationWorkerTestSuite) SetupSuite() {
+ ts.httpTestServer = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ switch {
+ case strings.Contains(req.RequestURI, "/users/test_user/file/"):
+ publicKey, err := base64.StdEncoding.DecodeString(req.Header.Get("C4GH-Public-Key"))
+ if err != nil || len(publicKey) == 0 {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = fmt.Fprint(w, "bad public key")
+
+ return
+ }
+
+ reader := bytes.NewReader(publicKey)
+ newReaderPublicKey, err := keys.ReadPublicKey(reader)
+ if err != nil {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = fmt.Fprint(w, "could not read public key")
+
+ return
+ }
+
+ encryptedFile := bytes.Buffer{}
+ encryptedFileWriter, err := streaming.NewCrypt4GHWriter(&encryptedFile, [32]byte{}, [][32]byte{newReaderPublicKey}, nil)
+ if err != nil {
+ w.WriteHeader(http.StatusBadRequest)
+ _, _ = fmt.Fprint(w, "could not read create crypt4gh writer")
+
+ return
+ }
+
+ file := bytes.Buffer{}
+ _, _ = file.Write([]byte(fmt.Sprintf("this is file: %s", filepath.Base(req.URL.Path))))
+ _, _ = io.Copy(encryptedFileWriter, &file)
+
+ _ = encryptedFileWriter.Close()
+ w.WriteHeader(http.StatusOK)
+ _, _ = w.Write(encryptedFile.Bytes())
+ default:
+ // Set the response status code
+ w.WriteHeader(http.StatusInternalServerError)
+ // Set the response body
+ _, _ = fmt.Fprint(w, "unexpected path called")
+ }
+ }))
+
+ validators.Validators = map[string]*validators.ValidatorDescription{
+ "mock-validator": {
+ ValidatorID: "mock-validator",
+ Name: "mock validator",
+ Description: "Validator for mocking",
+ Version: "v0.0.0",
+ Mode: "file",
+ PathSpecification: nil,
+ ValidatorPath: "/mock-validator.sif",
+ },
+ }
+}
+func (ts *JobPreparationWorkerTestSuite) SetupTest() {
+ ts.tempDir = ts.T().TempDir()
+ // Reset any Asserts and On() on mocks from previous tests
+ ts.mockDatabase = &mockDatabase{}
+ ts.mockBroker = &mockBroker{}
+ database.RegisterDatabase(ts.mockDatabase)
+}
+
+func (ts *JobPreparationWorkerTestSuite) TearDownSuite() {
+ ts.httpTestServer.Close()
+}
+
+func TestJobPreparationWorkerTestSuite(t *testing.T) {
+ suite.Run(t, new(JobPreparationWorkerTestSuite))
+}
+
+type mockDatabase struct {
+ mock.Mock
+}
+
+func (m *mockDatabase) Commit() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) Rollback() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) BeginTransaction(_ context.Context) (database.Transaction, error) {
+ _ = m.Called()
+
+ return m, nil
+}
+
+func (m *mockDatabase) Close() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) ReadValidationResult(_ context.Context, validationID string, userID *string) (*model.ValidationResult, error) {
+ // Function not needed for unit test, but to implement interface
+ panic("database.ReadValidationResult call not expected in unit tests")
+}
+
+func (m *mockDatabase) ReadValidationInformation(_ context.Context, validationID string) (*model.ValidationInformation, error) {
+ args := m.Called(validationID)
+
+ return args.Get(0).(*model.ValidationInformation), args.Error(1)
+}
+
+func (m *mockDatabase) InsertFileValidationJob(_ context.Context, _ *model.InsertFileValidationJobParameters) error {
+ // Function not needed for unit test, but to implement interface
+ panic("database.InsertFileValidationJob call not expected in unit tests")
+}
+
+func (m *mockDatabase) UpdateFileValidationJob(_ context.Context, _ *model.UpdateFileValidationJobParameters) error {
+ // Function not needed for unit test, but to implement interface
+ panic("database.UpdateFileValidationJob call not expected in unit tests")
+}
+
+func (m *mockDatabase) AllValidationJobsDone(_ context.Context, _ string) (bool, error) {
+ // Function not needed for unit test, but to implement interface
+ panic("database.AllValidationJobsDone call not expected in unit tests")
+}
+
+func (m *mockDatabase) UpdateAllValidationJobFilesOnError(ctx context.Context, validationID string, validatorMessage *model.Message) error {
+ args := m.Called(validationID, validatorMessage)
+
+ return args.Error(0)
+}
+
+type mockBroker struct {
+ mock.Mock
+ messageChans map[string]chan amqp.Delivery
+}
+
+func (m *mockBroker) PublishMessage(_ context.Context, destination string, body []byte) error {
+ args := m.Called(destination, body)
+
+ return args.Error(0)
+}
+
+func (m *mockBroker) Subscribe(ctx context.Context, queue, consumerID string, handleFunc func(context.Context, amqp.Delivery) error) error {
+ args := m.Called(queue, consumerID)
+
+ if err := args.Error(0); err != nil {
+ return err
+ }
+
+ messageChan, ok := m.messageChans[consumerID]
+ if !ok {
+ return nil
+ }
+ for {
+ select {
+ case msg, ok := <-messageChan:
+ if !ok {
+ return nil
+ }
+ if err := handleFunc(context.TODO(), msg); err != nil {
+ return errors.Join(errors.New("unexpected consumer handleFunc error"), err)
+ }
+ case <-ctx.Done():
+ return nil
+ }
+ }
+}
+
+func (m *mockBroker) Close() error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.close call not expected in unit tests")
+}
+
+func (m *mockBroker) Monitor() chan *amqp.Error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.Monitor call not expected in unit tests")
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestInitWorkers() {
+ ts.mockBroker.On("Subscribe", "job-preparation-queue", mock.Anything).Return(nil)
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ ts.NoError(err)
+ ts.Len(workers.workers, 2)
+ workers.Shutdown()
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestInitWorkers_NoValidationWorkDirectory() {
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "validationWorkDir is required")
+ ts.Nil(workers)
+}
+func (ts *JobPreparationWorkerTestSuite) TestInitWorkers_NoBroker() {
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "broker is required")
+ ts.Nil(workers)
+}
+func (ts *JobPreparationWorkerTestSuite) TestInitWorkers_NoSdaApiToken() {
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "sdaAPIToken is required")
+ ts.Nil(workers)
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestInitWorkers_NoSdaApiUrl() {
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "sdaAPIURL is required")
+ ts.Nil(workers)
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestInitWorkers_NoDestinationQueue() {
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "destinationQueue is required")
+ ts.Nil(workers)
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestInitWorkers_NoSourceQueue() {
+ workers, err := NewWorkers(
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "sourceQueue is required")
+ ts.Nil(workers)
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestStartWorkers_NoInit() {
+ workers := &Workers{}
+ select {
+ case <-time.After(2 * time.Second):
+ ts.FailNow("timeout error, expected MonitorWorker to return error")
+ case err := <-workers.Monitor():
+ ts.EqualError(err, "workers have not been initialized")
+ }
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestStartWorkers_SubscribeError() {
+ ts.mockBroker.On("Subscribe", "job-preparation-queue", mock.Anything).Return(errors.New("subscribe error"))
+
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ select {
+ case <-time.After(2 * time.Second):
+ ts.FailNow("timeout error, expected MonitorWorker to return error")
+ case err := <-workers.Monitor():
+ ts.EqualError(err, "subscribe error")
+ }
+ workers.Shutdown()
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestStartAndShutdownWorkers() {
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-preparation-worker-0": make(chan amqp.Delivery),
+ "job-preparation-worker-1": make(chan amqp.Delivery),
+ }
+ ts.mockBroker.On("Subscribe", "job-preparation-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 2)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-preparation-worker-%d", i), worker.id)
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestWorkersConsume() {
+ worker1MessageChan := make(chan amqp.Delivery)
+ worker2MessageChan := make(chan amqp.Delivery)
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-preparation-worker-0": worker1MessageChan,
+ "job-preparation-worker-1": worker2MessageChan,
+ }
+ ts.mockBroker.On("Subscribe", "job-preparation-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(ts.httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(2),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 2)
+
+ ts.mockBroker.On("PublishMessage", "job-queue", mock.Anything).Return(nil)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-preparation-worker-%d", i), worker.id)
+ }
+
+ validationInformation1 := &model.ValidationInformation{
+ ValidationID: uuid.NewString(),
+ ValidatorIDs: []string{"mock-validator"},
+ SubmissionUserID: "test_user",
+ Files: []*model.FileInformation{
+ {
+ FileID: "testFileId1",
+ FilePath: "test_dir/file1",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "testFileId2",
+ FilePath: "another_dir/file2",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "testFileId3",
+ FilePath: "file3",
+ SubmissionFileSize: 1,
+ },
+ },
+ }
+ ts.mockDatabase.On("ReadValidationInformation", validationInformation1.ValidationID).Return(validationInformation1, nil)
+
+ validationInformation2 := &model.ValidationInformation{
+ ValidationID: uuid.NewString(),
+ ValidatorIDs: []string{"mock-validator"},
+ SubmissionUserID: "test_user",
+ Files: []*model.FileInformation{
+ {
+ FileID: "testFileId11",
+ FilePath: "test_dir/file11",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "testFileId21",
+ FilePath: "another_dir/file21",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "testFileId31",
+ FilePath: "file31",
+ SubmissionFileSize: 1,
+ },
+ },
+ }
+ ts.mockDatabase.On("ReadValidationInformation", validationInformation2.ValidationID).Return(validationInformation2, nil)
+
+ message1, err := json.Marshal(&model.JobPreparationMessage{ValidationID: validationInformation1.ValidationID})
+ if err != nil {
+ ts.FailNow("failed to marshal job preparation message", err)
+ }
+ worker1MessageChan <- amqp.Delivery{
+ Body: message1,
+ }
+
+ message2, err := json.Marshal(&model.JobPreparationMessage{ValidationID: validationInformation2.ValidationID})
+ if err != nil {
+ ts.FailNow("failed to marshal job preparation message", err)
+ }
+ worker2MessageChan <- amqp.Delivery{
+ Body: message2,
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+
+ for _, file := range validationInformation1.Files {
+ fileContent, err := os.ReadFile(filepath.Join(ts.tempDir, validationInformation1.ValidationID, "files", file.FilePath))
+ if err != nil {
+ ts.Failf("failed to read file: %s, due to: %v", file.FilePath, err)
+ }
+ ts.Equal(fmt.Sprintf("this is file: %s", file.FileID), string(fileContent))
+ }
+
+ for _, file := range validationInformation2.Files {
+ fileContent, err := os.ReadFile(filepath.Join(ts.tempDir, validationInformation2.ValidationID, "files", file.FilePath))
+ if err != nil {
+ ts.Failf(err.Error(), "failed to read file: %s, due to: %v", file.FilePath, err)
+ }
+ ts.Equal(fmt.Sprintf("this is file: %s", file.FileID), string(fileContent))
+ }
+
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "PublishMessage", 2)
+
+ expectedJobMessage1, err := json.Marshal(&model.JobMessage{
+ ValidationID: validationInformation1.ValidationID,
+ ValidatorID: "mock-validator",
+ ValidationDirectory: filepath.Join(ts.tempDir, validationInformation1.ValidationID),
+ Files: validationInformation1.Files,
+ })
+ if err != nil {
+ ts.FailNow("failed to marshal job message", err)
+ }
+ ts.mockBroker.AssertCalled(ts.T(), "PublishMessage", "job-queue", expectedJobMessage1)
+
+ expectedJobMessage2, err := json.Marshal(&model.JobMessage{
+ ValidationID: validationInformation2.ValidationID,
+ ValidatorID: "mock-validator",
+ ValidationDirectory: filepath.Join(ts.tempDir, validationInformation2.ValidationID),
+ Files: validationInformation2.Files,
+ })
+ if err != nil {
+ ts.FailNow("failed to marshal job message", err)
+ }
+ ts.mockBroker.AssertCalled(ts.T(), "PublishMessage", "job-queue", expectedJobMessage2)
+
+ ts.mockBroker.AssertCalled(ts.T(), "Subscribe", "job-preparation-queue", "job-preparation-worker-0")
+ ts.mockBroker.AssertCalled(ts.T(), "Subscribe", "job-preparation-queue", "job-preparation-worker-1")
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "Subscribe", 2)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "ReadValidationInformation", 2)
+ ts.mockDatabase.AssertCalled(ts.T(), "ReadValidationInformation", validationInformation1.ValidationID)
+ ts.mockDatabase.AssertCalled(ts.T(), "ReadValidationInformation", validationInformation2.ValidationID)
+}
+
+func (ts *JobPreparationWorkerTestSuite) TestWorkersConsumeDownloadError() {
+ worker1MessageChan := make(chan amqp.Delivery)
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-preparation-worker-0": worker1MessageChan,
+ }
+ ts.mockBroker.On("Subscribe", "job-preparation-queue", mock.Anything).Return(nil)
+
+ httpTestServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+ w.WriteHeader(http.StatusInternalServerError)
+ _, _ = fmt.Fprint(w, "expected error")
+ }))
+
+ workers, err := NewWorkers(
+ SourceQueue("job-preparation-queue"),
+ DestinationQueue("job-queue"),
+ SdaAPIURL(httpTestServer.URL),
+ SdaAPIToken("mock-token"),
+ Broker(ts.mockBroker),
+ ValidationWorkDirectory(ts.tempDir),
+ WorkerCount(1),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 1)
+
+ ts.mockBroker.On("PublishMessage", "job-queue", mock.Anything).Return(nil)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-preparation-worker-%d", i), worker.id)
+ }
+
+ validationInformation1 := &model.ValidationInformation{
+ ValidationID: uuid.NewString(),
+ ValidatorIDs: []string{"mock-validator"},
+ SubmissionUserID: "test_user",
+ Files: []*model.FileInformation{
+ {
+ FileID: "testFileId1",
+ FilePath: "test_dir/file1",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "testFileId2",
+ FilePath: "another_dir/file2",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "testFileId3",
+ FilePath: "file3",
+ SubmissionFileSize: 1,
+ },
+ },
+ }
+ ts.mockDatabase.On("ReadValidationInformation", validationInformation1.ValidationID).Return(validationInformation1, nil)
+ ts.mockDatabase.On("UpdateAllValidationJobFilesOnError", validationInformation1.ValidationID, mock.Anything).Return(nil)
+
+ message1, err := json.Marshal(&model.JobPreparationMessage{ValidationID: validationInformation1.ValidationID})
+ if err != nil {
+ ts.FailNow("failed to marshal job preparation message", err)
+ }
+ worker1MessageChan <- amqp.Delivery{
+ Body: message1,
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "PublishMessage", 0)
+ ts.mockBroker.AssertCalled(ts.T(), "Subscribe", "job-preparation-queue", "job-preparation-worker-0")
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "Subscribe", 1)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "ReadValidationInformation", 1)
+ ts.mockDatabase.AssertCalled(ts.T(), "ReadValidationInformation", validationInformation1.ValidationID)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateAllValidationJobFilesOnError", validationInformation1.ValidationID, mock.Anything)
+}
diff --git a/sda-validator/orchestrator/jobworker/options.go b/sda-validator/orchestrator/jobworker/options.go
new file mode 100644
index 0000000000..4bbe8fd546
--- /dev/null
+++ b/sda-validator/orchestrator/jobworker/options.go
@@ -0,0 +1,37 @@
+package jobworker
+
+import (
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/broker"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/commandexecutor"
+)
+
+type config struct {
+ workerCount int
+ sourceQueue string
+ broker broker.AMQPBrokerI
+ commandExecutor commandexecutor.CommandExecutor
+}
+
+func WorkerCount(v int) func(*config) {
+ return func(opts *config) {
+ opts.workerCount = v
+ }
+}
+
+func SourceQueue(v string) func(*config) {
+ return func(opts *config) {
+ opts.sourceQueue = v
+ }
+}
+
+func Broker(v broker.AMQPBrokerI) func(*config) {
+ return func(opts *config) {
+ opts.broker = v
+ }
+}
+
+func CommandExecutor(v commandexecutor.CommandExecutor) func(*config) {
+ return func(opts *config) {
+ opts.commandExecutor = v
+ }
+}
diff --git a/sda-validator/orchestrator/jobworker/worker.go b/sda-validator/orchestrator/jobworker/worker.go
new file mode 100644
index 0000000000..8f82f0ca3a
--- /dev/null
+++ b/sda-validator/orchestrator/jobworker/worker.go
@@ -0,0 +1,347 @@
+package jobworker
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/commandexecutor"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/validators"
+ amqp "github.com/rabbitmq/amqp091-go"
+ log "github.com/sirupsen/logrus"
+)
+
+type worker struct {
+ id string
+ ctx context.Context
+ cancel context.CancelFunc
+
+ commandExecutor commandexecutor.CommandExecutor
+
+ stopCh chan struct{}
+ error error
+ running bool
+}
+
+type Workers struct {
+ workers []*worker
+ conf *config
+ workerMonitorChan chan error
+}
+
+// NewWorkers initializes the workers with the given options
+func NewWorkers(opt ...func(*config)) (*Workers, error) {
+ newWorkers := &Workers{
+ conf: &config{},
+ }
+
+ for _, o := range opt {
+ o(newWorkers.conf)
+ }
+
+ if newWorkers.conf.sourceQueue == "" {
+ return nil, errors.New("sourceQueue is required")
+ }
+
+ if newWorkers.conf.broker == nil {
+ return nil, errors.New("broker is required")
+ }
+
+ if newWorkers.conf.commandExecutor == nil {
+ return nil, errors.New("commandExecutor is required")
+ }
+
+ newWorkers.workerMonitorChan = make(chan error, newWorkers.conf.workerCount)
+
+ for i := 0; i < newWorkers.conf.workerCount; i++ {
+ ctx, cancel := context.WithCancel(context.Background())
+ w := &worker{
+ id: fmt.Sprintf("job-worker-%d", i),
+ ctx: ctx,
+ cancel: cancel,
+ stopCh: make(chan struct{}, 1),
+ running: true,
+ commandExecutor: newWorkers.conf.commandExecutor,
+ }
+
+ newWorkers.workers = append(newWorkers.workers, w)
+
+ go func(w *worker) {
+ // passing ctx such that we can gracefully shut down the subscribe
+ if err := newWorkers.conf.broker.Subscribe(w.ctx, newWorkers.conf.sourceQueue, w.id, w.handleFunc); err != nil {
+ log.Errorf("job worker encountered error: %v", err)
+ newWorkers.workerMonitorChan <- err
+ }
+ w.running = false
+ w.stopCh <- struct{}{}
+ }(w)
+ }
+
+ return newWorkers, nil
+}
+
+// Monitor monitors if any worker encounters an subscribe error
+func (w *Workers) Monitor() chan error {
+ if w.conf == nil {
+ noConfErr := make(chan error, 1)
+ noConfErr <- errors.New("workers have not been initialized")
+
+ return noConfErr
+ }
+
+ return w.workerMonitorChan
+}
+
+// close a worker and wait until it has closed
+func (w *worker) close() {
+ w.cancel()
+ <-w.stopCh
+ close(w.stopCh)
+}
+
+// Shutdown shutdowns and waits for all workers to have closed
+func (w *Workers) Shutdown() {
+ wg := sync.WaitGroup{}
+
+ for _, w := range w.workers {
+ if !w.running {
+ continue
+ }
+ wg.Go(func() {
+ w.close()
+ })
+ }
+
+ wg.Wait()
+ close(w.workerMonitorChan)
+}
+
+func (w *worker) handleFunc(ctx context.Context, message amqp.Delivery) (err error) {
+ jobMessage := new(model.JobMessage)
+ if err := json.Unmarshal(message.Body, jobMessage); err != nil {
+ log.Errorf("could not unmarshal message to job message due to: %v", err)
+
+ return nil // returning nil so message is not nacked and reconsumed
+ }
+
+ jobDirectory := filepath.Join(jobMessage.ValidationDirectory, jobMessage.ValidatorID)
+ // Remove job directory if any error is encountered
+ defer func() {
+ if err == nil {
+ return
+ }
+ if err := os.RemoveAll(jobDirectory); err != nil {
+ log.Errorf("failed to remove job directory after worker encountered an error due to: %v", err)
+ }
+ }()
+
+ if err := os.MkdirAll(filepath.Join(jobDirectory, "output"), 0750); err != nil {
+ log.Errorf("failed to create job output directory at: %s, error: %v", jobDirectory, err)
+
+ return err
+ }
+
+ if err := os.MkdirAll(filepath.Join(jobDirectory, "input"), 0750); err != nil {
+ log.Errorf("failed to create job input directory at: %s, error: %v", jobDirectory, err)
+
+ return err
+ }
+
+ input := &model.ValidatorInput{
+ Files: nil,
+ Paths: nil,
+ Config: nil,
+ }
+
+ validatorDescription, ok := validators.Validators[jobMessage.ValidatorID]
+ if !ok {
+ return fmt.Errorf("validator %s no longer found as a valid validator", jobMessage.ValidatorID)
+ }
+
+ for _, fileInfo := range jobMessage.Files {
+ filePathForJob := filepath.Join("/mnt/input/data", fileInfo.FilePath)
+ switch validatorDescription.Mode {
+ case "file", "file-pair":
+ input.Files = append(input.Files, &model.FileInput{Path: filePathForJob})
+ case "file-structure":
+ input.Paths = append(input.Paths, filePathForJob)
+ default:
+ return updateFileValidationJobsOnError(ctx, jobMessage, []*model.Message{{Level: "error", Message: fmt.Sprintf("validator has unknown mode: %s", validatorDescription.Mode), Time: time.Now().Format(time.RFC3339)}})
+ }
+ }
+
+ inputData, err := json.Marshal(input)
+ if err != nil {
+ log.Errorf("failed to marshal input data due to: %v", err)
+
+ return err
+ }
+ if err := os.WriteFile(filepath.Join(jobDirectory, "input", "input.json"), inputData, 0600); err != nil {
+ log.Errorf("failed to write input.json for validator due to: %v", err)
+
+ return err
+ }
+
+ // Here we mount the validatorJobDir as /mnt with the input, and output directories such that validator can access input/input.json and write a output/result.json
+ // we also mount validatorJobDir/files/ as /mnt/input/data such that the validator can access the files without the need for us to duplicate them per validator
+ _, err = w.commandExecutor.Execute(
+ "apptainer",
+ "run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ "--bind", fmt.Sprintf("%s:/mnt", jobDirectory),
+ "--bind", fmt.Sprintf("%s:/mnt/input/data", filepath.Join(jobMessage.ValidationDirectory, "files")),
+ validatorDescription.ValidatorPath)
+
+ if err != nil {
+ log.Errorf("failed to execute run command due to: %s", err)
+
+ return updateFileValidationJobsOnError(ctx, jobMessage, []*model.Message{{Level: "error", Message: fmt.Sprintf("failed to execute run command due to: %s", err), Time: time.Now().Format(time.RFC3339)}})
+ }
+
+ result, err := os.ReadFile(filepath.Join(jobDirectory, "/output/result.json"))
+ if err != nil {
+ log.Errorf("failed to read result file: %v", err)
+
+ return updateFileValidationJobsOnError(ctx, jobMessage, []*model.Message{{Level: "error", Message: fmt.Sprintf("failed to read result file: %v", err), Time: time.Now().Format(time.RFC3339)}})
+ }
+
+ validatorOutput := new(model.ValidatorOutput)
+ if err := json.Unmarshal(result, validatorOutput); err != nil {
+ log.Errorf("failed to unmarshal result file: %v", err)
+
+ return updateFileValidationJobsOnError(ctx, jobMessage, []*model.Message{{Level: "error", Message: fmt.Sprintf("failed to unmarshal result file: %v", err), Time: time.Now().Format(time.RFC3339)}})
+ }
+
+ return updateFileValidationJobs(ctx, jobMessage, validatorOutput)
+}
+func updateFileValidationJobs(ctx context.Context, jobMessage *model.JobMessage, validatorOutput *model.ValidatorOutput) error {
+ tx, err := database.BeginTransaction(ctx)
+ if err != nil {
+ log.Errorf("failed to begin transaction: %v", err)
+
+ return err
+ }
+
+ now := time.Now()
+ for _, fileInfo := range jobMessage.Files {
+ var fileResult *model.FileResult
+ for _, fr := range validatorOutput.Files {
+ filePath, _ := strings.CutPrefix(fr.FilePath, "/mnt/input/data/")
+ if filePath == fileInfo.FilePath {
+ fileResult = fr
+
+ break
+ }
+ }
+ if fileResult == nil {
+ if err := tx.UpdateFileValidationJob(ctx, &model.UpdateFileValidationJobParameters{
+ ValidationID: jobMessage.ValidationID,
+ ValidatorID: jobMessage.ValidatorID,
+ FileID: fileInfo.FileID,
+ FileResult: "error",
+ ValidatorResult: validatorOutput.Result,
+ FileMessages: []*model.Message{{Level: "error", Message: "file result not found in validator output", Time: time.Now().Format(time.RFC3339)}},
+ FinishedAt: now,
+ ValidatorMessages: validatorOutput.Messages,
+ }); err != nil {
+ log.Errorf("failed to update file validation job on file missing from result file due to: %v", err)
+
+ return err
+ }
+
+ continue
+ }
+
+ if err := tx.UpdateFileValidationJob(ctx, &model.UpdateFileValidationJobParameters{
+ ValidationID: jobMessage.ValidationID,
+ ValidatorID: jobMessage.ValidatorID,
+ FileID: fileInfo.FileID,
+ FileResult: fileResult.Result,
+ ValidatorResult: validatorOutput.Result,
+ FileMessages: fileResult.Messages,
+ FinishedAt: now,
+ ValidatorMessages: validatorOutput.Messages,
+ }); err != nil {
+ log.Errorf("failed to update file validation job due to: %v", err)
+
+ return err
+ }
+ }
+
+ if err := tx.Commit(); err != nil {
+ log.Errorf("failed to commit transaction due to: %v", err)
+
+ return err
+ }
+
+ return checkAndCleanVolume(ctx, jobMessage.ValidationID, jobMessage.ValidationDirectory)
+}
+
+func checkAndCleanVolume(ctx context.Context, validationID, validationDirectory string) error {
+ allJobsDone, err := database.AllValidationJobsDone(ctx, validationID)
+ if err != nil {
+ log.Errorf("failed to check if all validation jobs done due to: %v", err)
+
+ return err
+ }
+
+ if !allJobsDone {
+ return nil
+ }
+
+ if err := os.RemoveAll(validationDirectory); err != nil {
+ log.Errorf("failed to remove validation directory when all jobs done, due to %v", err)
+ }
+
+ return nil
+}
+
+func updateFileValidationJobsOnError(ctx context.Context, jobMessage *model.JobMessage, validatorMessages []*model.Message) error {
+ tx, err := database.BeginTransaction(ctx)
+ if err != nil {
+ log.Errorf("failed to begin transaction: %v", err)
+
+ return err
+ }
+ // Deferring rollback as if tx has been commited rollback wont be actioned
+ defer func() {
+ if err := tx.Rollback(); err != nil {
+ log.Errorf("failed to rollback transaction due to: %v", err)
+ }
+ }()
+
+ now := time.Now()
+
+ for _, fileInfo := range jobMessage.Files {
+ if err := tx.UpdateFileValidationJob(ctx, &model.UpdateFileValidationJobParameters{
+ ValidationID: jobMessage.ValidationID,
+ ValidatorID: jobMessage.ValidatorID,
+ FileID: fileInfo.FileID,
+ FileResult: "error",
+ ValidatorResult: "error",
+ FileMessages: nil,
+ FinishedAt: now,
+ ValidatorMessages: validatorMessages,
+ }); err != nil {
+ log.Errorf("failed to update file validation job due to: %v", err)
+
+ return err
+ }
+ }
+
+ if err := tx.Commit(); err != nil {
+ return err
+ }
+
+ return checkAndCleanVolume(ctx, jobMessage.ValidationID, jobMessage.ValidationDirectory)
+}
diff --git a/sda-validator/orchestrator/jobworker/worker_test.go b/sda-validator/orchestrator/jobworker/worker_test.go
new file mode 100644
index 0000000000..4aeb2c0282
--- /dev/null
+++ b/sda-validator/orchestrator/jobworker/worker_test.go
@@ -0,0 +1,757 @@
+package jobworker
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/model"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/validators"
+ amqp "github.com/rabbitmq/amqp091-go"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/suite"
+)
+
+type JobWorkerTestSuite struct {
+ suite.Suite
+
+ tempDir string
+
+ mockDatabase *mockDatabase
+ mockBroker *mockBroker
+ mockCommandExecutor *mockCommandExecutor
+}
+
+func (ts *JobWorkerTestSuite) SetupSuite() {
+ validators.Validators = map[string]*validators.ValidatorDescription{
+ "mock-validator": {
+ ValidatorID: "mock-validator",
+ Name: "mock validator",
+ Description: "Validator for mocking",
+ Version: "v0.0.0",
+ Mode: "file",
+ PathSpecification: nil,
+ ValidatorPath: "/mock-validator.sif",
+ },
+ }
+}
+func (ts *JobWorkerTestSuite) SetupTest() {
+ ts.tempDir = ts.T().TempDir()
+ // Reset any Asserts and On() on mocks from previous tests
+ ts.mockDatabase = &mockDatabase{}
+ ts.mockBroker = &mockBroker{}
+ ts.mockCommandExecutor = &mockCommandExecutor{}
+ database.RegisterDatabase(ts.mockDatabase)
+}
+
+func (ts *JobWorkerTestSuite) TearDownTest() {
+}
+
+func (ts *JobWorkerTestSuite) TearDownSuite() {
+}
+
+func TestJobPreparationWorkerTestSuite(t *testing.T) {
+ suite.Run(t, new(JobWorkerTestSuite))
+}
+
+type mockCommandExecutor struct {
+ mock.Mock
+}
+
+func (m *mockCommandExecutor) Execute(name string, args ...string) ([]byte, error) {
+ mockArgs := m.Called(name, args)
+ mockArgs.Get(0).(func())()
+
+ return nil, mockArgs.Error(1)
+}
+
+type mockDatabase struct {
+ mock.Mock
+}
+
+func (m *mockDatabase) Commit() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) Rollback() error {
+ _ = m.Called()
+
+ return nil
+}
+
+func (m *mockDatabase) BeginTransaction(_ context.Context) (database.Transaction, error) {
+ _ = m.Called()
+
+ return m, nil
+}
+
+func (m *mockDatabase) Close() error {
+ // Function not needed for unit test, but to implement interface
+ panic("database.Close call not expected in unit tests")
+}
+
+func (m *mockDatabase) ReadValidationResult(_ context.Context, _ string, _ *string) (*model.ValidationResult, error) {
+ // Function not needed for unit test, but to implement interface
+ panic("database.ReadValidationResult call not expected in unit tests")
+}
+
+func (m *mockDatabase) ReadValidationInformation(_ context.Context, _ string) (*model.ValidationInformation, error) {
+ // Function not needed for unit test, but to implement interface
+ panic("database.ReadValidationInformation call not expected in unit tests")
+}
+
+func (m *mockDatabase) InsertFileValidationJob(_ context.Context, _ *model.InsertFileValidationJobParameters) error {
+ // Function not needed for unit test, but to implement interface
+ panic("database.InsertFileValidationJob call not expected in unit tests")
+}
+
+func (m *mockDatabase) UpdateFileValidationJob(_ context.Context, params *model.UpdateFileValidationJobParameters) error {
+ args := m.Called(params.ValidationID, params.ValidatorID, params.FileID, params.FileResult, params.FileMessages, params.FinishedAt, params.ValidatorResult, params.ValidatorMessages)
+
+ return args.Error(0)
+}
+
+func (m *mockDatabase) AllValidationJobsDone(_ context.Context, validationID string) (bool, error) {
+ args := m.Called(validationID)
+
+ return args.Bool(0), args.Error(1)
+}
+
+func (m *mockDatabase) UpdateAllValidationJobFilesOnError(_ context.Context, _ string, _ *model.Message) error {
+ panic("database.UpdateAllValidationJobFilesOnError call not expected in unit tests")
+}
+
+type mockBroker struct {
+ mock.Mock
+ messageChans map[string]chan amqp.Delivery
+}
+
+func (m *mockBroker) PublishMessage(_ context.Context, _ string, _ []byte) error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.PublishMessage call not expected in unit tests")
+}
+
+func (m *mockBroker) Subscribe(ctx context.Context, queue, consumerID string, handleFunc func(context.Context, amqp.Delivery) error) error {
+ args := m.Called(queue, consumerID)
+
+ if err := args.Error(0); err != nil {
+ return err
+ }
+
+ messageChan, ok := m.messageChans[consumerID]
+ if !ok {
+ return nil
+ }
+ for {
+ select {
+ case msg, ok := <-messageChan:
+ if !ok {
+ return nil
+ }
+ if err := handleFunc(context.TODO(), msg); err != nil {
+ return errors.Join(errors.New("unexpected consumer handleFunc error"), err)
+ }
+ case <-ctx.Done():
+ return nil
+ }
+ }
+}
+
+func (m *mockBroker) Close() error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.close call not expected in unit tests")
+}
+
+func (m *mockBroker) Monitor() chan *amqp.Error {
+ // Function not needed for unit test, but to implement interface
+ panic("broker.Monitor call not expected in unit tests")
+}
+
+func (ts *JobWorkerTestSuite) TestInitWorkers() {
+ ts.mockBroker.On("Subscribe", "job-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(2),
+ )
+ ts.NoError(err)
+ ts.Len(workers.workers, 2)
+ workers.Shutdown()
+}
+
+func (ts *JobWorkerTestSuite) TestInitWorkers_NoSourceQueue() {
+ workers, err := NewWorkers(
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "sourceQueue is required")
+ ts.Nil(workers)
+}
+
+func (ts *JobWorkerTestSuite) TestInitWorkers_NoBroker() {
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "broker is required")
+ ts.Nil(workers)
+}
+
+func (ts *JobWorkerTestSuite) TestInitWorkers_NoCommandExecutor() {
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ WorkerCount(2),
+ )
+ ts.EqualError(err, "commandExecutor is required")
+ ts.Nil(workers)
+}
+
+func (ts *JobWorkerTestSuite) TestStartWorkers_NoInit() {
+ workers := &Workers{}
+ select {
+ case <-time.After(2 * time.Second):
+ ts.FailNow("timeout error, expected MonitorWorker to return error")
+ case err := <-workers.Monitor():
+ ts.EqualError(err, "workers have not been initialized")
+ }
+}
+
+func (ts *JobWorkerTestSuite) TestStartWorkers_SubscribeError() {
+ ts.mockBroker.On("Subscribe", "job-queue", mock.Anything).Return(errors.New("subscribe error"))
+
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(2),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+
+ select {
+ case <-time.After(2 * time.Second):
+ ts.FailNow("timeout error, expected MonitorWorker to return error")
+ case err := <-workers.Monitor():
+ ts.EqualError(err, "subscribe error")
+ }
+ workers.Shutdown()
+}
+
+func (ts *JobWorkerTestSuite) TestStartAndShutdownWorkers() {
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-worker-0": make(chan amqp.Delivery),
+ "job-worker-1": make(chan amqp.Delivery),
+ }
+ ts.mockBroker.On("Subscribe", "job-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(2),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 2)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-worker-%d", i), worker.id)
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+}
+
+func (ts *JobWorkerTestSuite) TestWorkersConsume() {
+ worker1MessageChan := make(chan amqp.Delivery)
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-worker-0": worker1MessageChan,
+ }
+ ts.mockBroker.On("Subscribe", "job-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(1),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 1)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-worker-%d", i), worker.id)
+ }
+
+ validationID := uuid.NewString()
+ validationDir := filepath.Join(ts.tempDir, validationID)
+
+ if err := os.MkdirAll(filepath.Join(validationDir, "files"), 0750); err != nil {
+ ts.FailNow("failed to create validation dir", err)
+ }
+ jobMessage := &model.JobMessage{
+ ValidationID: validationID,
+ ValidatorID: "mock-validator",
+ ValidationDirectory: validationDir,
+ Files: []*model.FileInformation{
+ {
+ FileID: "fileId1",
+ FilePath: "test_dir/file1",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "fileId2",
+ FilePath: "another_dir/file2",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "fileId3",
+ FilePath: "file3",
+ SubmissionFileSize: 1,
+ },
+ },
+ }
+
+ for _, file := range jobMessage.Files {
+ filePath := filepath.Join(validationDir, "files", file.FilePath)
+ fileDir := filepath.Dir(filePath)
+ if err := os.MkdirAll(fileDir, 0750); err != nil {
+ ts.FailNow("failed to create validation file dir", err)
+ }
+ if err := os.WriteFile(filePath, []byte(fmt.Sprintf("This is file: %s", file.FileID)), 0400); err != nil {
+ ts.FailNow("failed to create validation file", err)
+ }
+ }
+
+ ts.mockDatabase.On("Rollback").Return(nil)
+ ts.mockDatabase.On("BeginTransaction").Return(nil)
+ ts.mockDatabase.On("Commit").Return(nil)
+ ts.mockDatabase.On("UpdateFileValidationJob", validationID, "mock-validator", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+ ts.mockDatabase.On("AllValidationJobsDone", validationID).Return(true, nil)
+
+ expectedResult := &model.ValidatorOutput{
+ Result: "failed",
+ Files: []*model.FileResult{
+ {
+ FilePath: "/mnt/input/data/test_dir/file1",
+ Result: "failed",
+ Messages: []*model.Message{{
+ Level: "INFO",
+ Time: time.Now().Format(time.RFC3339),
+ Message: "File failed validation",
+ }},
+ }, {
+ FilePath: "/mnt/input/data/another_dir/file2",
+ Result: "passed",
+ Messages: nil,
+ }, {
+ FilePath: "/mnt/input/data/file3",
+ Result: "passed",
+ Messages: nil,
+ },
+ },
+ Messages: nil,
+ }
+ resultJSON, err := json.Marshal(expectedResult)
+ if err != nil {
+ ts.FailNow("failed to marshal expected result", err)
+ }
+
+ ts.mockCommandExecutor.On("Execute",
+ "apptainer",
+ []string{"run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ "--bind", fmt.Sprintf("%s:/mnt", filepath.Join(validationDir, "mock-validator")),
+ "--bind", fmt.Sprintf("%s:/mnt/input/data", filepath.Join(jobMessage.ValidationDirectory, "files")),
+ "/mock-validator.sif"}).Return(func() {
+ if err := os.WriteFile(filepath.Join(validationDir, "mock-validator", "output", "result.json"), resultJSON, 0400); err != nil {
+ ts.Fail("failed to create result file")
+ }
+ }, nil)
+
+ message, err := json.Marshal(jobMessage)
+ if err != nil {
+ ts.FailNow("failed to marshal job message", err)
+ }
+ worker1MessageChan <- amqp.Delivery{
+ Body: message,
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+
+ // Check validation dir was deleted
+ _, err = os.ReadDir(validationDir)
+ ts.EqualError(err, fmt.Sprintf("open %s: no such file or directory", validationDir))
+
+ ts.mockBroker.AssertCalled(ts.T(), "Subscribe", "job-queue", "job-worker-0")
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "Subscribe", 1)
+
+ ts.mockDatabase.AssertCalled(ts.T(), "AllValidationJobsDone", validationID)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "AllValidationJobsDone", 1)
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "UpdateFileValidationJob", 3)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId1", expectedResult.Files[0].Result, expectedResult.Files[0].Messages, mock.Anything, expectedResult.Result, expectedResult.Messages)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId2", expectedResult.Files[1].Result, expectedResult.Files[1].Messages, mock.Anything, expectedResult.Result, expectedResult.Messages)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId3", expectedResult.Files[2].Result, expectedResult.Files[2].Messages, mock.Anything, expectedResult.Result, expectedResult.Messages)
+}
+
+func (ts *JobWorkerTestSuite) TestWorkersConsume_ErrorOnApptainerRun() {
+ worker1MessageChan := make(chan amqp.Delivery)
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-worker-0": worker1MessageChan,
+ }
+ ts.mockBroker.On("Subscribe", "job-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(1),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 1)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-worker-%d", i), worker.id)
+ }
+
+ validationID := uuid.NewString()
+ validationDir := filepath.Join(ts.tempDir, validationID)
+
+ if err := os.MkdirAll(filepath.Join(validationDir, "files"), 0750); err != nil {
+ ts.FailNow("failed to create validation dir", err)
+ }
+ jobMessage := &model.JobMessage{
+ ValidationID: validationID,
+ ValidatorID: "mock-validator",
+ ValidationDirectory: validationDir,
+ Files: []*model.FileInformation{
+ {
+ FileID: "fileId1",
+ FilePath: "test_dir/file1",
+ SubmissionFileSize: 1,
+ },
+ },
+ }
+
+ for _, file := range jobMessage.Files {
+ filePath := filepath.Join(validationDir, "files", file.FilePath)
+ fileDir := filepath.Dir(filePath)
+ if err := os.MkdirAll(fileDir, 0750); err != nil {
+ ts.FailNow("failed to create validation file dir", err)
+ }
+ if err := os.WriteFile(filePath, []byte(fmt.Sprintf("This is file: %s", file.FileID)), 0400); err != nil {
+ ts.FailNow("failed to create validation file", err)
+ }
+ }
+
+ ts.mockDatabase.On("Rollback").Return(nil)
+ ts.mockDatabase.On("BeginTransaction").Return(nil)
+ ts.mockDatabase.On("Commit").Return(nil)
+ ts.mockDatabase.On("UpdateFileValidationJob", validationID, "mock-validator", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+ ts.mockDatabase.On("AllValidationJobsDone", validationID).Return(true, nil)
+
+ ts.mockCommandExecutor.On("Execute",
+ "apptainer",
+ []string{"run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ "--bind", fmt.Sprintf("%s:/mnt", filepath.Join(validationDir, "mock-validator")),
+ "--bind", fmt.Sprintf("%s:/mnt/input/data", filepath.Join(jobMessage.ValidationDirectory, "files")),
+ "/mock-validator.sif"}).Return(func() {}, errors.New("expected error from apptainer"))
+
+ message, err := json.Marshal(jobMessage)
+ if err != nil {
+ ts.FailNow("failed to marshal job message", err)
+ }
+ worker1MessageChan <- amqp.Delivery{
+ Body: message,
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+
+ // Check validation dir was deleted
+ _, err = os.ReadDir(validationDir)
+ ts.EqualError(err, fmt.Sprintf("open %s: no such file or directory", validationDir))
+
+ ts.mockBroker.AssertCalled(ts.T(), "Subscribe", "job-queue", "job-worker-0")
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "Subscribe", 1)
+
+ ts.mockDatabase.AssertCalled(ts.T(), "AllValidationJobsDone", validationID)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "AllValidationJobsDone", 1)
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "UpdateFileValidationJob", 1)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId1", "error", mock.Anything, mock.Anything, "error", mock.Anything)
+}
+
+func (ts *JobWorkerTestSuite) TestWorkersConsume_NoResultFileFromApptainer() {
+ worker1MessageChan := make(chan amqp.Delivery)
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-worker-0": worker1MessageChan,
+ }
+ ts.mockBroker.On("Subscribe", "job-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(1),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 1)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-worker-%d", i), worker.id)
+ }
+
+ validationID := uuid.NewString()
+ validationDir := filepath.Join(ts.tempDir, validationID)
+
+ if err := os.MkdirAll(filepath.Join(validationDir, "files"), 0750); err != nil {
+ ts.FailNow("failed to create validation dir", err)
+ }
+ jobMessage := &model.JobMessage{
+ ValidationID: validationID,
+ ValidatorID: "mock-validator",
+ ValidationDirectory: validationDir,
+ Files: []*model.FileInformation{
+ {
+ FileID: "fileId1",
+ FilePath: "test_dir/file1",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "fileId2",
+ FilePath: "another_dir/file2",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "fileId3",
+ FilePath: "file3",
+ SubmissionFileSize: 1,
+ },
+ },
+ }
+
+ for _, file := range jobMessage.Files {
+ filePath := filepath.Join(validationDir, "files", file.FilePath)
+ fileDir := filepath.Dir(filePath)
+ if err := os.MkdirAll(fileDir, 0750); err != nil {
+ ts.FailNow("failed to create validation file dir", err)
+ }
+ if err := os.WriteFile(filePath, []byte(fmt.Sprintf("This is file: %s", file.FileID)), 0400); err != nil {
+ ts.FailNow("failed to create validation file", err)
+ }
+ }
+
+ ts.mockDatabase.On("Rollback").Return(nil)
+ ts.mockDatabase.On("BeginTransaction").Return(nil)
+ ts.mockDatabase.On("Commit").Return(nil)
+ ts.mockDatabase.On("UpdateFileValidationJob", validationID, "mock-validator", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+ ts.mockDatabase.On("AllValidationJobsDone", validationID).Return(true, nil)
+
+ ts.mockCommandExecutor.On("Execute",
+ "apptainer",
+ []string{"run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ "--bind", fmt.Sprintf("%s:/mnt", filepath.Join(validationDir, "mock-validator")),
+ "--bind", fmt.Sprintf("%s:/mnt/input/data", filepath.Join(jobMessage.ValidationDirectory, "files")),
+ "/mock-validator.sif"}).Return(func() {}, nil)
+
+ message, err := json.Marshal(jobMessage)
+ if err != nil {
+ ts.FailNow("failed to marshal job message", err)
+ }
+ worker1MessageChan <- amqp.Delivery{
+ Body: message,
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+
+ // Check validation dir was deleted
+ _, err = os.ReadDir(validationDir)
+ ts.EqualError(err, fmt.Sprintf("open %s: no such file or directory", validationDir))
+
+ ts.mockBroker.AssertCalled(ts.T(), "Subscribe", "job-queue", "job-worker-0")
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "Subscribe", 1)
+
+ ts.mockDatabase.AssertCalled(ts.T(), "AllValidationJobsDone", validationID)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "AllValidationJobsDone", 1)
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "UpdateFileValidationJob", 3)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId1", "error", mock.Anything, mock.Anything, "error", mock.Anything)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId2", "error", mock.Anything, mock.Anything, "error", mock.Anything)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId3", "error", mock.Anything, mock.Anything, "error", mock.Anything)
+}
+
+func (ts *JobWorkerTestSuite) TestWorkersConsume_MissingFileInResultFile() {
+ worker1MessageChan := make(chan amqp.Delivery)
+ ts.mockBroker.messageChans = map[string]chan amqp.Delivery{
+ "job-worker-0": worker1MessageChan,
+ }
+ ts.mockBroker.On("Subscribe", "job-queue", mock.Anything).Return(nil)
+
+ workers, err := NewWorkers(
+ SourceQueue("job-queue"),
+ Broker(ts.mockBroker),
+ CommandExecutor(ts.mockCommandExecutor),
+ WorkerCount(1),
+ )
+ if err != nil {
+ ts.FailNow(err.Error())
+ }
+ ts.Len(workers.workers, 1)
+
+ for i, worker := range workers.workers {
+ ts.Equal(true, worker.running)
+ ts.Equal(fmt.Sprintf("job-worker-%d", i), worker.id)
+ }
+
+ validationID := uuid.NewString()
+ validationDir := filepath.Join(ts.tempDir, validationID)
+
+ if err := os.MkdirAll(filepath.Join(validationDir, "files"), 0750); err != nil {
+ ts.FailNow("failed to create validation dir", err)
+ }
+ jobMessage := &model.JobMessage{
+ ValidationID: validationID,
+ ValidatorID: "mock-validator",
+ ValidationDirectory: validationDir,
+ Files: []*model.FileInformation{
+ {
+ FileID: "fileId1",
+ FilePath: "test_dir/file1",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "fileId2",
+ FilePath: "another_dir/file2",
+ SubmissionFileSize: 1,
+ }, {
+ FileID: "fileId3",
+ FilePath: "file3",
+ SubmissionFileSize: 1,
+ },
+ },
+ }
+
+ for _, file := range jobMessage.Files {
+ filePath := filepath.Join(validationDir, "files", file.FilePath)
+ fileDir := filepath.Dir(filePath)
+ if err := os.MkdirAll(fileDir, 0750); err != nil {
+ ts.FailNow("failed to create validation file dir", err)
+ }
+ if err := os.WriteFile(filePath, []byte(fmt.Sprintf("This is file: %s", file.FileID)), 0400); err != nil {
+ ts.FailNow("failed to create validation file", err)
+ }
+ }
+
+ ts.mockDatabase.On("Rollback").Return(nil)
+ ts.mockDatabase.On("BeginTransaction").Return(nil)
+ ts.mockDatabase.On("Commit").Return(nil)
+ ts.mockDatabase.On("UpdateFileValidationJob", validationID, "mock-validator", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil)
+ ts.mockDatabase.On("AllValidationJobsDone", validationID).Return(true, nil)
+
+ expectedResult := &model.ValidatorOutput{
+ Result: "passed",
+ Files: []*model.FileResult{
+ {
+ FilePath: "/mnt/input/data/another_dir/file2",
+ Result: "passed",
+ Messages: nil,
+ }, {
+ FilePath: "/mnt/input/data/file3",
+ Result: "passed",
+ Messages: nil,
+ },
+ },
+ Messages: nil,
+ }
+ resultJSON, err := json.Marshal(expectedResult)
+ if err != nil {
+ ts.FailNow("failed to marshal expected result", err)
+ }
+
+ ts.mockCommandExecutor.On("Execute",
+ "apptainer",
+ []string{"run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ "--bind", fmt.Sprintf("%s:/mnt", filepath.Join(validationDir, "mock-validator")),
+ "--bind", fmt.Sprintf("%s:/mnt/input/data", filepath.Join(jobMessage.ValidationDirectory, "files")),
+ "/mock-validator.sif"}).Return(func() {
+ if err := os.WriteFile(filepath.Join(validationDir, "mock-validator", "output", "result.json"), resultJSON, 0400); err != nil {
+ ts.Fail("failed to create result file")
+ }
+ }, nil)
+
+ message, err := json.Marshal(jobMessage)
+ if err != nil {
+ ts.FailNow("failed to marshal job message", err)
+ }
+ worker1MessageChan <- amqp.Delivery{
+ Body: message,
+ }
+
+ workers.Shutdown()
+
+ for _, worker := range workers.workers {
+ ts.Equal(false, worker.running)
+ }
+
+ // Check validation dir was deleted
+ _, err = os.ReadDir(validationDir)
+ ts.EqualError(err, fmt.Sprintf("open %s: no such file or directory", validationDir))
+
+ ts.mockBroker.AssertCalled(ts.T(), "Subscribe", "job-queue", "job-worker-0")
+ ts.mockBroker.AssertNumberOfCalls(ts.T(), "Subscribe", 1)
+
+ ts.mockDatabase.AssertCalled(ts.T(), "AllValidationJobsDone", validationID)
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "AllValidationJobsDone", 1)
+
+ ts.mockDatabase.AssertNumberOfCalls(ts.T(), "UpdateFileValidationJob", 3)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId1", "error", mock.Anything, mock.Anything, expectedResult.Result, expectedResult.Messages)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId2", expectedResult.Files[0].Result, expectedResult.Files[0].Messages, mock.Anything, expectedResult.Result, expectedResult.Messages)
+ ts.mockDatabase.AssertCalled(ts.T(), "UpdateFileValidationJob", validationID, "mock-validator", "fileId3", expectedResult.Files[1].Result, expectedResult.Files[1].Messages, mock.Anything, expectedResult.Result, expectedResult.Messages)
+}
diff --git a/sda-validator/orchestrator/main.go b/sda-validator/orchestrator/main.go
new file mode 100644
index 0000000000..fec3753f76
--- /dev/null
+++ b/sda-validator/orchestrator/main.go
@@ -0,0 +1,223 @@
+package main
+
+import (
+ "context"
+ "crypto/tls"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "net/http"
+ "os"
+ "os/signal"
+ "syscall"
+ "time"
+
+ "github.com/gin-gonic/gin"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/api"
+ validatorapi "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/api/openapi_interface"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/config"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/database/postgres"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/broker"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/commandexecutor"
+ internalconfig "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/config"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/ginmiddleware/authenticator"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/ginmiddleware/rbac"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/health"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/jobpreparationworker"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/jobworker"
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/validators"
+ log "github.com/sirupsen/logrus"
+ healthpb "google.golang.org/grpc/health/grpc_health_v1"
+)
+
+func main() {
+ log.SetFormatter(&log.JSONFormatter{})
+ gin.SetMode(gin.ReleaseMode)
+
+ ctx, cancel := context.WithCancel(context.Background())
+ sigc := make(chan os.Signal, 5)
+ signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
+
+ if err := internalconfig.Load(); err != nil {
+ log.Fatalf("failed to load config due to: %v", err)
+ }
+
+ go func() {
+ if err := health.Start(config.HealthPort()); err != nil {
+ log.Fatalf("failed to start health server due to: %v", err)
+ }
+ }()
+
+ if err := validators.Init(config.ValidatorPaths()); err != nil {
+ log.Fatalf("failed to initialize validators due to: %v", err)
+ }
+
+ amqpBroker, err := broker.NewAMQPBroker()
+ if err != nil {
+ log.Fatalf("failed to create new AMPQ broker due to: %v", err)
+ }
+
+ if err := postgres.Init(); err != nil {
+ log.Fatalf("failed to initialise postgres database due to: %v", err)
+ }
+
+ jobpreparationworkers, err := jobpreparationworker.NewWorkers(
+ jobpreparationworker.WorkerCount(config.JobPreparationWorkerCount()),
+ jobpreparationworker.Broker(amqpBroker),
+ jobpreparationworker.SourceQueue(config.JobPreparationQueue()),
+ jobpreparationworker.SdaAPIToken(config.SdaAPIToken()),
+ jobpreparationworker.SdaAPIURL(config.SdaAPIURL()),
+ jobpreparationworker.DestinationQueue(config.JobQueue()),
+ jobpreparationworker.ValidationWorkDirectory(config.ValidationWorkDir()),
+ )
+ if err != nil {
+ log.Fatalf("failed to initialize job preparation workers due to: %v", err)
+ }
+
+ jobworkers, err := jobworker.NewWorkers(
+ jobworker.WorkerCount(config.JobWorkerCount()),
+ jobworker.Broker(amqpBroker),
+ jobworker.SourceQueue(config.JobQueue()),
+ jobworker.CommandExecutor(&commandexecutor.OsCommandExecutor{}),
+ )
+ if err != nil {
+ log.Fatalf("failed to initialize job preparation workers due to: %v", err)
+ }
+
+ validatorAPIImpl, err := api.NewValidatorAPIImpl(
+ api.SdaAPIURL(config.SdaAPIURL()),
+ api.SdaAPIToken(config.SdaAPIToken()),
+ api.Broker(amqpBroker),
+ api.ValidationJobPreparationQueue(config.JobPreparationQueue()),
+ api.ValidationFileSizeLimit(config.ValidationFileSizeLimit()),
+ )
+ if err != nil {
+ log.Fatalf("failed to create new validator API impl, due to: %v", err)
+ }
+
+ ginRouter := gin.New()
+
+ authMiddleware, err := authenticator.NewAuthenticator()
+ if err != nil {
+ log.Fatalf("failed to create authenticator middleware, due to: %v", err)
+ }
+ rbacMiddleware, err := rbac.NewRbac()
+ if err != nil {
+ log.Fatalf("failed to create rbac middleware, due to: %v", err)
+ }
+
+ ginMiddlewares := []gin.HandlerFunc{gin.Recovery(), authMiddleware.Authenticate(), rbacMiddleware.Enforce()}
+
+ if log.IsLevelEnabled(log.InfoLevel) {
+ ginLogger := gin.LoggerWithConfig(
+ gin.LoggerConfig{
+ Formatter: func(params gin.LogFormatterParams) string {
+ s, _ := json.Marshal(map[string]any{
+ "level": "info",
+ "method": params.Method,
+ "path": params.Path,
+ "remote_addr": params.ClientIP,
+ "status_code": params.StatusCode,
+ "time": params.TimeStamp.Format(time.RFC3339),
+ })
+
+ return string(s) + "\n"
+ },
+
+ Output: gin.DefaultWriter,
+ SkipPaths: []string{"/ready"},
+ },
+ )
+ ginMiddlewares = append([]gin.HandlerFunc{ginLogger}, ginMiddlewares...)
+ }
+
+ ginRouter.Use(ginMiddlewares...)
+
+ validatorapi.NewRouterWithGinEngine(ginRouter, validatorapi.ApiHandleFunctions{ValidatorOrchestratorAPI: validatorAPIImpl})
+
+ cfg := &tls.Config{MinVersion: tls.VersionTLS12}
+
+ srv := &http.Server{
+ Addr: fmt.Sprintf(":%d", config.APIPort()),
+ Handler: ginRouter,
+ TLSConfig: cfg,
+ TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)),
+ ReadHeaderTimeout: 20 * time.Second,
+ ReadTimeout: 1 * time.Minute,
+ WriteTimeout: 1 * time.Minute,
+ }
+
+ go func() {
+ if err := <-jobworkers.Monitor(); err != nil {
+ log.Errorf("job workers failed: %v", err)
+ cancel()
+ }
+ }()
+
+ go func() {
+ if err := <-jobpreparationworkers.Monitor(); err != nil {
+ log.Errorf("job preparation workers failed: %v", err)
+ cancel()
+ }
+ }()
+
+ go func() {
+ var err error
+ switch {
+ case config.APIServerCert() != "" && config.APIServerKey() != "":
+ log.Infof("server listening at: https://%s", srv.Addr)
+ err = srv.ListenAndServeTLS(config.APIServerCert(), config.APIServerKey())
+ default:
+ log.Infof("server listening at: http://%s", srv.Addr)
+ err = srv.ListenAndServe()
+ }
+
+ if err != nil && !errors.Is(err, http.ErrServerClosed) {
+ log.Errorf("Error starting server, due to: %v", err)
+ cancel()
+ }
+ }()
+
+ go func() {
+ if err := <-amqpBroker.Monitor(); err != nil {
+ log.Errorf("broker connection error: %v", err)
+ cancel()
+ }
+ }()
+ health.SetServingStatus(healthpb.HealthCheckResponse_SERVING)
+ select {
+ case <-sigc:
+ case <-ctx.Done():
+ }
+
+ health.SetServingStatus(healthpb.HealthCheckResponse_NOT_SERVING)
+
+ log.Info("gracefully shutting down")
+
+ log.Infof("shutting down HTTP server")
+ serverShutdownCtx, serverShutdownCancel := context.WithTimeout(ctx, 10*time.Second)
+ if err := srv.Shutdown(serverShutdownCtx); err != nil {
+ log.Errorf("failed to gracefully shutdown the server due to: %v", err)
+ }
+ serverShutdownCancel()
+
+ log.Infof("shutting down job preparation workers")
+ jobpreparationworkers.Shutdown()
+
+ log.Infof("shutting down job workers")
+ jobworkers.Shutdown()
+
+ log.Infof("shutting down broker connection")
+ if err := amqpBroker.Close(); err != nil {
+ log.Errorf("failed to gracefully shutdown broker connection")
+ }
+
+ log.Infof("shutting down database connection")
+ if err := database.Close(); err != nil {
+ log.Errorf("failed to gracefully database connection")
+ }
+
+ health.Stop()
+ os.Exit(1)
+}
diff --git a/sda-validator/orchestrator/model/models.go b/sda-validator/orchestrator/model/models.go
new file mode 100644
index 0000000000..c3f27212c4
--- /dev/null
+++ b/sda-validator/orchestrator/model/models.go
@@ -0,0 +1,84 @@
+package model
+
+import "time"
+
+type JobPreparationMessage struct {
+ ValidationID string
+}
+type JobMessage struct {
+ ValidationID string
+ ValidatorID string
+ ValidationDirectory string
+ Files []*FileInformation
+}
+
+type ValidationInformation struct {
+ ValidationID string
+ ValidatorIDs []string
+ SubmissionUserID string
+ Files []*FileInformation
+}
+type FileInformation struct {
+ FileID string
+ FilePath string
+ SubmissionFileSize int64
+}
+
+type ValidationResult struct {
+ ValidationID string
+ ValidatorResults []*ValidatorResult
+}
+type ValidatorResult struct {
+ ValidatorID string
+ Result string
+ StartedAt time.Time
+ FinishedAt time.Time
+ Messages []*Message
+ Files []*FileResult
+}
+type FileResult struct {
+ FilePath string `json:"path"`
+ Result string `json:"result"`
+ Messages []*Message `json:"messages"`
+}
+
+type Message struct {
+ Level string `json:"level"`
+ Time string `json:"time"`
+ Message string `json:"message"`
+}
+
+type ValidatorOutput struct {
+ Result string `json:"result"`
+ Files []*FileResult `json:"files"`
+ Messages []*Message `json:"messages"`
+}
+
+type ValidatorInput struct {
+ Files []*FileInput `json:"files"`
+ Paths []string `json:"paths"`
+ Config *ValidatorConfig `json:"config"`
+}
+type FileInput struct {
+ Path string `json:"path"`
+}
+type ValidatorConfig struct {
+}
+
+type UserFilesResponse struct {
+ FileID string `json:"fileID"`
+ InboxPath string `json:"inboxPath"`
+ SubmissionFileSize int64 `json:"submissionFileSize"`
+}
+
+type InsertFileValidationJobParameters struct {
+ ValidationID, ValidatorID, FileID, FilePath, SubmissionUser, TriggeredBy string
+ FileSubmissionSize int64
+ StartedAt time.Time
+}
+type UpdateFileValidationJobParameters struct {
+ ValidationID, ValidatorID, FileID, FileResult, ValidatorResult string
+ FileMessages []*Message
+ FinishedAt time.Time
+ ValidatorMessages []*Message
+}
diff --git a/sda-validator/orchestrator/swagger_v1.yml b/sda-validator/orchestrator/swagger_v1.yml
new file mode 100644
index 0000000000..6d504db617
--- /dev/null
+++ b/sda-validator/orchestrator/swagger_v1.yml
@@ -0,0 +1,213 @@
+openapi: 3.0.3
+info:
+ title: SDA validator orchestrator API
+ version: "1.0"
+ description: This is the API for the validator orchestrator.
+paths:
+ /validate:
+ post:
+ description: Request a set of files to be validated with a set of validators
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ValidateRequest"
+ responses:
+ "200":
+ description: Successful operation
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ validation_id:
+ description: "The validation id"
+ type: string
+ "400":
+ description: Bad payload
+ "401":
+ description: Authentication failure.
+ "500":
+ description: Internal application error.
+ /admin/validate:
+ post:
+ description: Request a set of files to be validated with a set of validators for a specific user
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/AdminValidateRequest"
+ responses:
+ "200":
+ description: Successful operation
+ content:
+ application/json:
+ schema:
+ type: object
+ properties:
+ validation_id:
+ description: "The validation id"
+ type: string
+ "400":
+ description: Bad payload
+ "401":
+ description: Authentication failure.
+ "500":
+ description: Internal application error.
+ /result:
+ get:
+ description: "Get the result of a validation"
+ parameters:
+ - in: path
+ name: validation_id
+ schema:
+ type: string
+ description: "The validation id to fetch result for"
+ required: true
+ responses:
+ "200":
+ description: Successful operation
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ResultResponse"
+ "500":
+ description: Internal application error.
+ /admin/result:
+ get:
+ description: "Get the result of a validation from any user"
+ parameters:
+ - in: path
+ name: validation_id
+ schema:
+ type: string
+ description: "The validation id to fetch result for"
+ required: true
+ responses:
+ "200":
+ description: Successful operation
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/ResultResponse"
+ "500":
+ description: Internal application error.
+ /validators:
+ get:
+ description: Get the available validators
+ responses:
+ "200":
+ description: Successful operation
+ content:
+ application/json:
+ schema:
+ description: "The available validators to be executed"
+ type: array
+ items:
+ type: string
+ example: [ "xml-validator", "file-structure-validator" ]
+ "500":
+ description: Internal application error.
+
+components:
+ schemas:
+ ValidateRequest:
+ type: object
+ properties:
+ file_paths:
+ description: "The file paths to be validated"
+ type: array
+ items:
+ type: string
+ example: ["dataset_001/metadata.xml", "dataset_001/datafile.bam", "dataset_001/datafile.bam.bai"]
+ validators:
+ description: "The validators to be executed"
+ type: array
+ items:
+ type: string
+ example: ["xml-validator", "file-structure-validator"]
+ AdminValidateRequest:
+ type: object
+ properties:
+ file_paths:
+ description: "The file paths to be validated"
+ type: array
+ items:
+ type: string
+ example: ["dataset_001/metadata.xml", "dataset_001/datafile.bam", "dataset_001/datafile.bam.bai"]
+ validators:
+ description: "The validators to be executed"
+ type: array
+ items:
+ type: string
+ example: ["xml-validator", "file-structure-validator"]
+ user_id:
+ type: string
+ ResultResponse:
+ type: array
+ items:
+ type: object
+ properties:
+ validator_id:
+ description: "The id of the validator"
+ type: string
+ result:
+ description: "Denotes the overall result of the validator validation"
+ type: string
+ enum: ["Pending", "Success", "Failed", "Error"]
+ started_at:
+ type: string
+ description: "The time the validator validation was started RFC3339 format"
+ format: date-time
+ finished_at:
+ type: string
+ description: "The time the validator validation was finished RFC3339 format"
+ format: date-time
+ files:
+ type: array
+ items:
+ type: object
+ properties:
+ path:
+ description: "Path of the file"
+ type: string
+ result:
+ description: "Denotes the result of validation of the file"
+ type: string
+ enum: ["Pending", "Success", "Failed", "Error"]
+ messages:
+ type: array
+ items:
+ type: object
+ properties:
+ level:
+ type: string
+ description: "Level of message"
+ time:
+ type: string
+ description: "Timestamp of message"
+ message:
+ type: string
+ description: "Message"
+ messages:
+ type: array
+ items:
+ type: object
+ properties:
+ level:
+ type: string
+ description: "Level of message"
+ time:
+ type: string
+ description: "Timestamp of message"
+ message:
+ type: string
+ description: "Message"
+
+ securitySchemes:
+ bearerAuth:
+ type: http
+ scheme: bearer
+ bearerFormat: JWT
+security:
+ - bearerAuth: []
\ No newline at end of file
diff --git a/sda-validator/orchestrator/validators/validators.go b/sda-validator/orchestrator/validators/validators.go
new file mode 100644
index 0000000000..6860a5e778
--- /dev/null
+++ b/sda-validator/orchestrator/validators/validators.go
@@ -0,0 +1,63 @@
+package validators
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/commandexecutor"
+)
+
+var Validators map[string]*ValidatorDescription
+
+type ValidatorDescription struct {
+ ValidatorID string `json:"validatorID"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Version string `json:"version"`
+ Mode string `json:"mode"`
+ PathSpecification []string `json:"pathSpecification"`
+
+ ValidatorPath string // The path this validator is available at
+}
+
+var commandExecutor commandexecutor.CommandExecutor
+
+func init() {
+ commandExecutor = commandexecutor.OsCommandExecutor{}
+ Validators = make(map[string]*ValidatorDescription)
+}
+
+func Init(validatorsPaths []string) error {
+ for _, path := range validatorsPaths {
+ if _, err := os.Stat(path); err != nil {
+ return fmt.Errorf("failed to stat file: %s, error: %v", path, err)
+ }
+
+ out, err := commandExecutor.Execute(
+ "apptainer",
+ "run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ path,
+ "--describe")
+ if err != nil {
+ return fmt.Errorf("failed to execute describe command towards path: %s, error: %v", path, err)
+ }
+
+ vd := new(ValidatorDescription)
+ if err := json.Unmarshal(out, vd); err != nil {
+ return fmt.Errorf("failed to unmarshal response from describe command towards path: %s, error: %v", path, err)
+ }
+ vd.ValidatorPath = path
+
+ Validators[vd.ValidatorID] = vd
+ }
+
+ return nil
+}
+
+func (vd *ValidatorDescription) RequiresFileContent() bool {
+ return vd.Mode != "file-structure"
+}
diff --git a/sda-validator/orchestrator/validators/validators_test.go b/sda-validator/orchestrator/validators/validators_test.go
new file mode 100644
index 0000000000..923edc5759
--- /dev/null
+++ b/sda-validator/orchestrator/validators/validators_test.go
@@ -0,0 +1,133 @@
+package validators
+
+import (
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/neicnordic/sensitive-data-archive/sda-validator/orchestrator/internal/commandexecutor"
+ "github.com/stretchr/testify/mock"
+ "github.com/stretchr/testify/suite"
+)
+
+type ValidatorsTestSuite struct {
+ suite.Suite
+
+ tempDir string
+
+ mockCommandExecutor *mockCommandExecutor
+}
+
+func (ts *ValidatorsTestSuite) SetupTest() {
+ ts.mockCommandExecutor = &mockCommandExecutor{}
+ commandExecutor = ts.mockCommandExecutor
+ ts.tempDir = ts.T().TempDir()
+
+ if err := os.WriteFile(filepath.Join(ts.tempDir, "mock-validator-1.sif"), []byte("test validator"), 0600); err != nil {
+ ts.FailNow("failed to write file", err.Error())
+ }
+ if err := os.WriteFile(filepath.Join(ts.tempDir, "mock-validator-2.sif"), []byte("test validator"), 0600); err != nil {
+ ts.FailNow("failed to write file", err.Error())
+ }
+}
+
+func (ts *ValidatorsTestSuite) TearDownTest() {
+ commandExecutor = commandexecutor.OsCommandExecutor{}
+ Validators = make(map[string]*ValidatorDescription)
+}
+
+func TestJobPreparationWorkerTestSuite(t *testing.T) {
+ suite.Run(t, new(ValidatorsTestSuite))
+}
+
+type mockCommandExecutor struct {
+ mock.Mock
+}
+
+func (m *mockCommandExecutor) Execute(name string, args ...string) ([]byte, error) {
+ mockArgs := m.Called(name, args)
+
+ if val, ok := mockArgs.Get(0).([]byte); ok {
+ return val, mockArgs.Error(1)
+ }
+
+ return nil, mockArgs.Error(1)
+}
+
+func (ts *ValidatorsTestSuite) TestInit() {
+ validatorDescription1 := &ValidatorDescription{
+ ValidatorID: "mock-validator-1",
+ Name: "mock validator 1",
+ Description: "Mock validator 1",
+ Version: "v0.0.1",
+ Mode: "mock",
+ PathSpecification: nil,
+ ValidatorPath: "*",
+ }
+ validatorDescription2 := &ValidatorDescription{
+ ValidatorID: "mock-validator-2",
+ Name: "mock validator 2",
+ Description: "Mock validator 2",
+ Version: "v0.0.2",
+ Mode: "mock",
+ PathSpecification: nil,
+ ValidatorPath: "*",
+ }
+
+ vd1Json, err := json.Marshal(validatorDescription1)
+ if err != nil {
+ ts.FailNow("failed to marshal validator description", err)
+ }
+ vd2Json, err := json.Marshal(validatorDescription2)
+ if err != nil {
+ ts.FailNow("failed to marshal validator description", err)
+ }
+
+ ts.mockCommandExecutor.On("Execute",
+ "apptainer",
+ []string{"run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ filepath.Join(ts.tempDir, "/mock-validator-1.sif"),
+ "--describe"}).Return(vd1Json, nil)
+
+ ts.mockCommandExecutor.On("Execute",
+ "apptainer",
+ []string{"run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ filepath.Join(ts.tempDir, "/mock-validator-2.sif"),
+ "--describe"}).Return(vd2Json, nil)
+
+ ts.NoError(Init([]string{filepath.Join(ts.tempDir, "/mock-validator-1.sif"), filepath.Join(ts.tempDir, "/mock-validator-2.sif")}))
+ ts.Len(Validators, 2)
+
+ vd1, ok := Validators["mock-validator-1"]
+ if !ok {
+ ts.FailNow("mock-validator-1 does not exist")
+ }
+ ts.Equal(vd1.ValidatorPath, filepath.Join(ts.tempDir, "/mock-validator-1.sif"))
+
+ vd2, ok := Validators["mock-validator-2"]
+ if !ok {
+ ts.FailNow("mock-validator-2 does not exist")
+ }
+ ts.Equal(vd2.ValidatorPath, filepath.Join(ts.tempDir, "/mock-validator-2.sif"))
+}
+
+func (ts *ValidatorsTestSuite) TestInit_Error() {
+ ts.mockCommandExecutor.On("Execute",
+ "apptainer",
+ []string{"run",
+ "--userns",
+ "--net",
+ "--network", "none",
+ "/mock-validator-1.sif",
+ "--describe"}).Return(nil, errors.New("expected error from apptainer"))
+
+ ts.EqualError(Init([]string{"/mock-validator-1.sif"}), "failed to stat file: /mock-validator-1.sif, error: stat /mock-validator-1.sif: no such file or directory")
+}
diff --git a/sda/Dockerfile b/sda/Dockerfile
index c3415e736e..cb3f28ca87 100644
--- a/sda/Dockerfile
+++ b/sda/Dockerfile
@@ -15,7 +15,7 @@ ARG SOURCE_COMMIT
LABEL org.opencontainers.image.authors="NeIC System Developers"
LABEL org.label-schema.schema-version="1.0"
LABEL org.label-schema.build-date=now()
-LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive"
+LABEL org.label-schema.vcs-url="https://github.com/biobanklab/sensitive-data-archive"
LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT
COPY --from=Build /go/sda-* /usr/local/bin/
@@ -31,7 +31,7 @@ ARG SOURCE_COMMIT
LABEL org.opencontainers.image.authors="NeIC System Developers"
LABEL org.label-schema.schema-version="1.0"
LABEL org.label-schema.build-date=now()
-LABEL org.label-schema.vcs-url="https://github.com/neicnordic/sensitive-data-archive"
+LABEL org.label-schema.vcs-url="https://github.com/biobanklab/sensitive-data-archive"
LABEL org.label-schema.vcs-ref=$SOURCE_COMMIT
COPY --from=Build /go/sda-* /usr/local/bin/
diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go
index e7d3e9e29a..9b07706e0b 100644
--- a/sda/cmd/api/api_test.go
+++ b/sda/cmd/api/api_test.go
@@ -737,6 +737,66 @@ func (s *TestSuite) TestAPIGetFiles() {
assert.NoError(s.T(), err)
}
+func (s *TestSuite) TestAPIGetFiles_SubmissionFileSize() {
+ for i := 1; i <= 2; i++ {
+ fileID, err := Conf.API.DB.RegisterFile(nil, fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", "submission_b", i), "dummy")
+ if err != nil {
+ s.FailNow("failed to register file in database")
+ }
+
+ err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", "dummy", "{}", "{}")
+ if err != nil {
+ s.FailNow("failed to update status of file in database")
+ }
+
+ if err := Conf.API.DB.SetSubmissionFileSize(fileID, int64(2*i)); err != nil {
+ s.FailNow("failed to set submission file size")
+ }
+ }
+
+ gin.SetMode(gin.ReleaseMode)
+ assert.NoError(s.T(), setupJwtAuth())
+ m, err := model.NewModelFromString(jsonadapter.Model)
+ if err != nil {
+ s.T().Logf("failure: %v", err)
+ s.FailNow("failed to setup RBAC model")
+ }
+ e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC))
+ if err != nil {
+ s.T().Logf("failure: %v", err)
+ s.FailNow("failed to setup RBAC enforcer")
+ }
+
+ // Mock request and response holders
+ w := httptest.NewRecorder()
+ r := httptest.NewRequest(http.MethodGet, "/files?path_prefix=submission_b", http.NoBody)
+ r.Header.Add("Authorization", "Bearer "+s.Token)
+
+ _, router := gin.CreateTestContext(w)
+ router.GET("/files", rbac(e), getFiles)
+
+ router.ServeHTTP(w, r)
+ okResponse := w.Result()
+ defer okResponse.Body.Close()
+ assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode)
+
+ files := []database.SubmissionFileInfo{}
+ err = json.NewDecoder(okResponse.Body).Decode(&files)
+ assert.NoError(s.T(), err, "failed to list files from DB")
+ assert.Equal(s.T(), 2, len(files))
+
+ for _, fileInfo := range files {
+ switch {
+ case strings.Contains(fileInfo.InboxPath, "001"):
+ assert.Equal(s.T(), int64(2), fileInfo.SubmissionFileSize)
+ case strings.Contains(fileInfo.InboxPath, "002"):
+ assert.Equal(s.T(), int64(4), fileInfo.SubmissionFileSize)
+ default:
+ s.Fail("unexpected file received", fileInfo.InboxPath)
+ }
+ }
+}
+
func (s *TestSuite) TestAPIGetFiles_filteredSelection() {
testUsers := []string{"dummy", "User-B", "User-C"}
for _, user := range testUsers {
diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml
index bea8b78f52..6cf78eadd8 100644
--- a/sda/cmd/api/swagger_v1.yml
+++ b/sda/cmd/api/swagger_v1.yml
@@ -472,6 +472,9 @@ components:
CreatedAt:
type: string
example: "2025-03-02T13:14:15.123Z"
+ SubmissionFileSize:
+ type: integer
+ description: The byte size of the submitted file if known
securitySchemes:
bearerAuth:
type: http
diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go
index 59bbabdc8f..4e9b9f8ff4 100644
--- a/sda/cmd/finalize/finalize.go
+++ b/sda/cmd/finalize/finalize.go
@@ -37,6 +37,11 @@ func main() {
if err != nil {
log.Fatal(err)
}
+ if db.Version < 21 {
+ log.Error("database schema v21 is required")
+ db.Close()
+ panic(err)
+ }
if conf.Backup.Type != "" && conf.Archive.Type != "" {
log.Debugln("initiating storage backends")
diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go
index 2c4a70271f..27b30e0c02 100644
--- a/sda/cmd/ingest/ingest.go
+++ b/sda/cmd/ingest/ingest.go
@@ -70,8 +70,8 @@ func main() {
sigc <- syscall.SIGINT
panic(err)
}
- if app.DB.Version < 8 {
- log.Error("database schema v8 is required")
+ if app.DB.Version < 21 {
+ log.Error("database schema v21 is required")
sigc <- syscall.SIGINT
panic(err)
}
diff --git a/sda/cmd/rotatekey/rotatekey.go b/sda/cmd/rotatekey/rotatekey.go
index 989c92e4c9..fe6bf91d76 100644
--- a/sda/cmd/rotatekey/rotatekey.go
+++ b/sda/cmd/rotatekey/rotatekey.go
@@ -67,6 +67,11 @@ func main() {
if err != nil {
panic(err)
}
+ if app.DB.Version < 22 {
+ log.Error("database schema v22 is required")
+ app.DB.Close()
+ panic("unsupported database schema version")
+ }
go func() {
<-sigc // blocks here until it receives from sigc
@@ -236,6 +241,15 @@ func (app *RotateKey) reEncryptHeader(fileID string) (ackNack, msg string, err e
}
}
+ // Backup old header before rotating
+ log.Debugf("Backing up old header for file-id: %s", fileID)
+ if err := app.DB.BackupHeader(fileID, header, oldKeyHash); err != nil {
+ msg := fmt.Sprintf("failed to backup encryption header for file %s", fileID)
+ log.Errorf("%s, reason: %v", msg, err)
+ // We Nack and requeue because if backup fails, rotation should not proceed
+ return "nackRequeue", msg, err
+ }
+
newHeader, err := reencrypt.CallReencryptHeader(header, app.PubKeyEncoded, app.Conf.RotateKey.Grpc)
if err != nil {
msg := fmt.Sprintf("failed to rotate c4gh key for file %s", fileID)
diff --git a/sda/cmd/rotatekey/rotatekey_test.go b/sda/cmd/rotatekey/rotatekey_test.go
index 4de3af6bc4..5221beecd6 100644
--- a/sda/cmd/rotatekey/rotatekey_test.go
+++ b/sda/cmd/rotatekey/rotatekey_test.go
@@ -96,7 +96,7 @@ func TestMain(m *testing.M) {
// pulls an image, creates a container based on it and runs it
rabbitmq, err := pool.RunWithOptions(&dockertest.RunOptions{
- Repository: "ghcr.io/neicnordic/sensitive-data-archive",
+ Repository: "ghcr.io/biobanklab/sensitive-data-archive",
Tag: "v3.0.0-rabbitmq",
}, func(config *docker.HostConfig) {
// set AutoRemove to true so that stopped container goes away by itself
@@ -295,6 +295,7 @@ func (ts *TestSuite) TestReEncryptHeader() {
expectedRes string
fileID string
testName string
+ verifyBackup bool
}{
{
testName: "ingested file",
@@ -302,6 +303,7 @@ func (ts *TestSuite) TestReEncryptHeader() {
expectedMgs: "",
expectedRes: "ack",
fileID: ts.fileID,
+ verifyBackup: true,
},
{
testName: "un-ingested file",
@@ -309,6 +311,7 @@ func (ts *TestSuite) TestReEncryptHeader() {
expectedMgs: fmt.Sprintf("failed to get keyhash for file with file-id: %s", newFileID),
expectedRes: "ackSendToError",
fileID: newFileID,
+ verifyBackup: false,
},
} {
ts.T().Run(test.testName, func(t *testing.T) {
@@ -316,6 +319,15 @@ func (ts *TestSuite) TestReEncryptHeader() {
assert.Equal(t, res, test.expectedRes)
assert.Equal(t, msg, test.expectedMgs)
assert.Equal(t, err, test.expectedError)
+
+ // Verify that the backup was actually created in the DB for successful cases
+ if test.verifyBackup {
+ var count int
+ query := "SELECT count(*) FROM sda.file_headers_backup WHERE file_id = $1"
+ err := ts.app.DB.DB.QueryRow(query, test.fileID).Scan(&count)
+ assert.NoError(t, err)
+ assert.GreaterOrEqual(t, count, 1, "Backup record should exist in sda.file_headers_backup")
+ }
})
}
}
diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go
index bfe066dc00..ce0d086b58 100644
--- a/sda/cmd/s3inbox/proxy.go
+++ b/sda/cmd/s3inbox/proxy.go
@@ -661,13 +661,5 @@ func (p *Proxy) storeObjectSizeInDB(path, fileID string) error {
return err
}
- if err := p.database.DB.Ping(); err != nil {
- p.database.Close()
- _ = p.database.Connect()
- }
-
- const setObjectSize = "UPDATE sda.files set submission_file_size = $1 where id = $2;"
- _, err = p.database.DB.Exec(setObjectSize, *o.ContentLength, fileID)
-
- return err
+ return p.database.SetSubmissionFileSize(fileID, *o.ContentLength)
}
diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go
index ae7898bd96..bac69ef7a1 100644
--- a/sda/cmd/s3inbox/s3inbox.go
+++ b/sda/cmd/s3inbox/s3inbox.go
@@ -51,8 +51,8 @@ func main() {
sigc <- syscall.SIGINT
panic(err)
}
- if sdaDB.Version < 4 {
- log.Error("database schema v4 is required")
+ if sdaDB.Version < 21 {
+ log.Error("database schema v21 is required")
sigc <- syscall.SIGINT
panic(err)
}
diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go
index fc0a008bd4..04e202f308 100644
--- a/sda/cmd/verify/verify.go
+++ b/sda/cmd/verify/verify.go
@@ -36,6 +36,11 @@ func main() {
if err != nil {
log.Fatal(err)
}
+ if db.Version < 21 {
+ log.Error("database schema v21 is required")
+ db.Close()
+ panic(err)
+ }
archive, err := storage.NewBackend(conf.Archive)
if err != nil {
log.Fatal(err)
diff --git a/sda/go.mod b/sda/go.mod
index 6ef93e7294..c4561b5797 100644
--- a/sda/go.mod
+++ b/sda/go.mod
@@ -3,14 +3,14 @@ module github.com/neicnordic/sensitive-data-archive
go 1.24.1
require (
- github.com/aws/aws-sdk-go-v2 v1.32.8
- github.com/aws/aws-sdk-go-v2/config v1.28.11
- github.com/aws/aws-sdk-go-v2/credentials v1.17.52
- github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48
- github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2
- github.com/aws/smithy-go v1.23.0
- github.com/casbin/casbin/v2 v2.127.0
- github.com/coreos/go-oidc/v3 v3.15.0
+ github.com/aws/aws-sdk-go-v2 v1.41.1
+ github.com/aws/aws-sdk-go-v2/config v1.32.7
+ github.com/aws/aws-sdk-go-v2/credentials v1.19.7
+ github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19
+ github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1
+ github.com/aws/smithy-go v1.24.0
+ github.com/casbin/casbin/v2 v2.135.0
+ github.com/coreos/go-oidc/v3 v3.17.0
github.com/gin-gonic/gin v1.11.0
github.com/google/uuid v1.6.0
github.com/gorilla/mux v1.8.1
@@ -25,16 +25,16 @@ require (
github.com/ory/dockertest v3.3.5+incompatible
github.com/ory/dockertest/v3 v3.12.0
github.com/pkg/errors v0.9.1
- github.com/pkg/sftp v1.13.9
+ github.com/pkg/sftp v1.13.10
github.com/rabbitmq/amqp091-go v1.10.0
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1
github.com/sirupsen/logrus v1.9.3
github.com/spf13/viper v1.21.0
github.com/stretchr/testify v1.11.1
- golang.org/x/crypto v0.45.0
- golang.org/x/oauth2 v0.31.0
- google.golang.org/grpc v1.75.1
- google.golang.org/protobuf v1.36.9
+ golang.org/x/crypto v0.47.0
+ golang.org/x/oauth2 v0.34.0
+ google.golang.org/grpc v1.78.0
+ google.golang.org/protobuf v1.36.11
)
require (
@@ -49,19 +49,20 @@ require (
github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect
github.com/andybalholm/brotli v1.1.1 // indirect
- github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect
- github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 // indirect
- github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 // indirect
- github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 // indirect
- github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect
- github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 // indirect
- github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 // indirect
- github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 // indirect
- github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 // indirect
- github.com/aws/aws-sdk-go-v2/service/sts v1.33.7 // indirect
+ github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect
+ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect
+ github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect
+ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect
+ github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect
+ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 // indirect
+ github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect
github.com/aymerick/douceur v0.2.0 // indirect
github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect
github.com/bytedance/sonic v1.14.0 // indirect
@@ -84,8 +85,7 @@ require (
github.com/gabriel-vasile/mimetype v1.4.9 // indirect
github.com/gin-contrib/sse v1.1.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.4 // indirect
- github.com/go-jose/go-jose/v4 v4.1.1 // indirect
- github.com/go-ole/go-ole v1.2.6 // indirect
+ github.com/go-jose/go-jose/v4 v4.1.3 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-playground/validator/v10 v10.27.0 // indirect
@@ -116,7 +116,6 @@ require (
github.com/lestrrat-go/httprc v1.0.6 // indirect
github.com/lestrrat-go/iter v1.0.2 // indirect
github.com/lestrrat-go/option v1.0.1 // indirect
- github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/mailgun/raymond/v2 v2.0.48 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
@@ -132,15 +131,13 @@ require (
github.com/opencontainers/runc v1.2.8 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
- github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
- github.com/quic-go/qpack v0.5.1 // indirect
- github.com/quic-go/quic-go v0.54.1 // indirect
+ github.com/quic-go/qpack v0.6.0 // indirect
+ github.com/quic-go/quic-go v0.57.0 // indirect
github.com/rogpeppe/go-internal v1.11.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sagikazarmark/locafero v0.11.0 // indirect
github.com/schollz/closestmatch v2.1.0+incompatible // indirect
github.com/segmentio/asm v1.2.0 // indirect
- github.com/shirou/gopsutil/v3 v3.24.5 // indirect
github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
@@ -148,8 +145,6 @@ require (
github.com/subosito/gotenv v1.6.0 // indirect
github.com/tdewolff/minify/v2 v2.23.6 // indirect
github.com/tdewolff/parse/v2 v2.8.1 // indirect
- github.com/tklauser/go-sysconf v0.3.12 // indirect
- github.com/tklauser/numcpus v0.6.1 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
github.com/ugorji/go/codec v1.3.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
@@ -159,19 +154,14 @@ require (
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
github.com/yosssi/ace v0.0.5 // indirect
- github.com/yusufpapurcu/wmi v1.2.4 // indirect
- go.uber.org/mock v0.5.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/arch v0.20.0 // indirect
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect
- golang.org/x/mod v0.29.0 // indirect
- golang.org/x/net v0.47.0 // indirect
- golang.org/x/sync v0.18.0 // indirect
- golang.org/x/sys v0.38.0 // indirect
- golang.org/x/text v0.31.0 // indirect
- golang.org/x/time v0.11.0 // indirect
- golang.org/x/tools v0.38.0 // indirect
- google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect
+ golang.org/x/net v0.48.0 // indirect
+ golang.org/x/sys v0.40.0 // indirect
+ golang.org/x/text v0.33.0 // indirect
+ golang.org/x/time v0.12.0 // indirect
+ google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/sda/go.sum b/sda/go.sum
index 25da9c612f..cc5fe41cef 100644
--- a/sda/go.sum
+++ b/sda/go.sum
@@ -24,44 +24,46 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
-github.com/aws/aws-sdk-go-v2 v1.32.8 h1:cZV+NUS/eGxKXMtmyhtYPJ7Z4YLoI/V8bkTdRZfYhGo=
-github.com/aws/aws-sdk-go-v2 v1.32.8/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E=
-github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00=
-github.com/aws/aws-sdk-go-v2/config v1.28.11 h1:7Ekru0IkRHRnSRWGQLnLN6i0o1Jncd0rHo2T130+tEQ=
-github.com/aws/aws-sdk-go-v2/config v1.28.11/go.mod h1:x78TpPvBfHH16hi5tE3OCWQ0pzNfyXA349p5/Wp82Yo=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.52 h1:I4ymSk35LHogx2Re2Wu6LOHNTRaRWkLVoJgWS5Wd40M=
-github.com/aws/aws-sdk-go-v2/credentials v1.17.52/go.mod h1:vAkqKbMNUcher8fDXP2Ge2qFXKMkcD74qvk1lJRMemM=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 h1:IBAoD/1d8A8/1aA8g4MBVtTRHhXRiNAgwdbo/xRM2DI=
-github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23/go.mod h1:vfENuCM7dofkgKpYzuzf1VT1UKkA/YL3qanfBn7HCaA=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 h1:XnXVe2zRyPf0+fAW5L05esmngvBpC6DQZK7oZB/z/Co=
-github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48/go.mod h1:S3wey90OrS4f7kYxH6PT175YyEcHTORY07++HurMaRM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 h1:jSJjSBzw8VDIbWv+mmvBSP8ezsztMYJGH+eKqi9AmNs=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27/go.mod h1:/DAhLbFRgwhmvJdOfSm+WwikZrCuUJiA4WgJG0fTNSw=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 h1:l+X4K77Dui85pIj5foXDhPlnqcNRG2QUyvca300lXh8=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27/go.mod h1:KvZXSFEXm6x84yE8qffKvT3x8J5clWnVFXphpohhzJ8=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo=
-github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 h1:AmB5QxnD+fBFrg9LcqzkgF/CaYvMyU/BTlejG4t1S7Q=
-github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27/go.mod h1:Sai7P3xTiyv9ZUYO3IFxMnmiIP759/67iQbU4kdmkyU=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM=
-github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 h1:iwYS40JnrBeA9e9aI5S6KKN4EB2zR4iUVYN0nwVivz4=
-github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8/go.mod h1:Fm9Mi+ApqmFiknZtGpohVcBGvpTu542VC4XO9YudRi0=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 h1:cWno7lefSH6Pp+mSznagKCgfDGeZRin66UvYUqAkyeA=
-github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8/go.mod h1:tPD+VjU3ABTBoEJ3nctu5Nyg4P4yjqSH5bJGGkY4+XE=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 h1:/Mn7gTedG86nbpjT4QEKsN1D/fThiYe1qvq7WsBGNHg=
-github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8/go.mod h1:Ae3va9LPmvjj231ukHB6UeT8nS7wTPfC3tMZSZMwNYg=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 h1:a7aQ3RW+ug4IbhoQp29NZdc7vqrzKZZfWZSaQAXOZvQ=
-github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 h1:YqtxripbjWb2QLyzRK9pByfEDvgg95gpC2AyDq4hFE8=
-github.com/aws/aws-sdk-go-v2/service/sso v1.24.9/go.mod h1:lV8iQpg6OLOfBnqbGMBKYjilBlf633qwHnBEiMSPoHY=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 h1:6dBT1Lz8fK11m22R+AqfRsFn8320K0T5DTGxxOQBSMw=
-github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8/go.mod h1:/kiBvRQXBc6xeJTYzhSdGvJ5vm1tjaDEjH+MSeRJnlY=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.7 h1:qwGa9MA8G7mBq2YphHFaygdPe5t9OA7SvaJdwWTlEds=
-github.com/aws/aws-sdk-go-v2/service/sts v1.33.7/go.mod h1:+8h7PZb3yY5ftmVLD7ocEoE98hdc8PoKS0H3wfx1dlc=
-github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE=
-github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI=
+github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
+github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4=
+github.com/aws/aws-sdk-go-v2/config v1.32.7 h1:vxUyWGUwmkQ2g19n7JY/9YL8MfAIl7bTesIUykECXmY=
+github.com/aws/aws-sdk-go-v2/config v1.32.7/go.mod h1:2/Qm5vKUU/r7Y+zUk/Ptt2MDAEKAfUtKc1+3U1Mo3oY=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.7 h1:tHK47VqqtJxOymRrNtUXN5SP/zUTvZKeLx4tH6PGQc8=
+github.com/aws/aws-sdk-go-v2/credentials v1.19.7/go.mod h1:qOZk8sPDrxhf+4Wf4oT2urYJrYt3RejHSzgAquYeppw=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19 h1:Gxj3kAlmM+a/VVO4YNsmgHGVUZhSxs0tuVwLIxZBCtM=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.19/go.mod h1:XGq5kImVqQT4HUNbbG+0Y8O74URsPNH7CGPg1s1HW5E=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1 h1:C2dUPSnEpy4voWFIq3JNd8gN0Y5vYGDo44eUE58a/p8=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.95.1/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y=
+github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk=
+github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13 h1:gd84Omyu9JLriJVCbGApcLzVR3XtmC4ZDPcAI6Ftvds=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.13/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ=
+github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ=
+github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
+github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=
github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4=
github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
@@ -71,8 +73,8 @@ github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQ
github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA=
github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
-github.com/casbin/casbin/v2 v2.127.0 h1:UGK3uO/8cOslnNqFUJ4xzm/bh+N+o45U7cSolaFk38c=
-github.com/casbin/casbin/v2 v2.127.0/go.mod h1:n4uZK8+tCMvcD6EVQZI90zKAok8iHAvEypcMJVKhGF0=
+github.com/casbin/casbin/v2 v2.135.0 h1:6BLkMQiGotYyS5yYeWgW19vxqugUlvHFkFiLnLR/bxk=
+github.com/casbin/casbin/v2 v2.135.0/go.mod h1:FmcfntdXLTcYXv/hxgNntcRPqAbwOG9xsism0yXT+18=
github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI=
github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A=
@@ -84,8 +86,8 @@ github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI
github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4=
github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE=
-github.com/coreos/go-oidc/v3 v3.15.0 h1:R6Oz8Z4bqWR7VFQ+sPSvZPQv4x8M+sJkDO5ojgwlyAg=
-github.com/coreos/go-oidc/v3 v3.15.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU=
+github.com/coreos/go-oidc/v3 v3.17.0 h1:hWBGaQfbi0iVviX4ibC7bk8OKT5qNr4klBaCHVNvehc=
+github.com/coreos/go-oidc/v3 v3.17.0/go.mod h1:wqPbKFrVnE90vty060SB40FCJ8fTHTxSwyXJqZH+sI8=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -123,14 +125,12 @@ github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk=
github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls=
github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY=
github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
-github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI=
-github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA=
+github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
+github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
-github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
-github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
@@ -161,9 +161,7 @@ github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60qA17CptGXhwfZ+k1sFNJIUNR8DdbcuUk=
github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA=
-github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -240,8 +238,6 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
-github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw=
github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
@@ -294,17 +290,15 @@ github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw=
-github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA=
+github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU=
+github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
-github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
-github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
-github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.54.1 h1:4ZAWm0AhCb6+hE+l5Q1NAL0iRn/ZrMwqHRGQiFwj2eg=
-github.com/quic-go/quic-go v0.54.1/go.mod h1:e68ZEaCdyviluZmy44P6Iey98v/Wfz6HCjQEm+l8zTY=
+github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
+github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
+github.com/quic-go/quic-go v0.57.0 h1:AsSSrrMs4qI/hLrKlTH/TGQeTMY0ib1pAOX7vA3AdqE=
+github.com/quic-go/quic-go v0.57.0/go.mod h1:ly4QBAjHA2VhdnxhojRsCUOeJwKYg+taDlos92xb1+s=
github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw=
github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o=
github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M=
@@ -323,10 +317,6 @@ github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
-github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI=
-github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk=
-github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
-github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
@@ -355,8 +345,6 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
@@ -367,10 +355,6 @@ github.com/tdewolff/parse/v2 v2.8.1 h1:J5GSHru6o3jF1uLlEKVXkDxxcVx6yzOlIVIotK4w2
github.com/tdewolff/parse/v2 v2.8.1/go.mod h1:Hwlni2tiVNKyzR1o6nUs4FOF07URA+JLBLd6dlIXYqo=
github.com/tdewolff/test v1.0.11 h1:FdLbwQVHxqG16SlkGveC0JVyrJN62COWTRyUFzfbtBE=
github.com/tdewolff/test v1.0.11/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8=
-github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
-github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
-github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
-github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA=
@@ -402,24 +386,22 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
-github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
-go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
-go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
-go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
-go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
-go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
-go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI=
-go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg=
-go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc=
-go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps=
-go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
-go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
+go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
+go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
+go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
+go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
+go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
+go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
+go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
+go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
+go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU=
-go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM=
+go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
+go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
@@ -429,12 +411,9 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI=
golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -442,11 +421,6 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
-golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
@@ -459,33 +433,22 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
-golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo=
-golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
+golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=
+golang.org/x/net v0.48.0/go.mod h1:+ndRgGjkh8FGtu1w1FGbEC31if4VrNVMuKTgcAAnQRY=
+golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw=
+golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
-golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -496,38 +459,27 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
-golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
-golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
+golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
-golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0=
-golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
+golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
+golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
@@ -537,22 +489,18 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
-golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
-golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A=
-google.golang.org/grpc v1.75.1 h1:/ODCNEuf9VghjgO3rqLcfg8fiOP0nSluljWFlDxELLI=
-google.golang.org/grpc v1.75.1/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ=
-google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=
-google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
+google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go
index 6e94c69ff3..7e72a3486c 100644
--- a/sda/internal/database/database.go
+++ b/sda/internal/database/database.go
@@ -48,11 +48,12 @@ type SyncData struct {
}
type SubmissionFileInfo struct {
- AccessionID string `json:"accessionID,omitempty"`
- FileID string `json:"fileID"`
- InboxPath string `json:"inboxPath"`
- Status string `json:"fileStatus"`
- CreateAt string `json:"createAt"`
+ AccessionID string `json:"accessionID,omitempty"`
+ FileID string `json:"fileID"`
+ InboxPath string `json:"inboxPath"`
+ Status string `json:"fileStatus"`
+ SubmissionFileSize int64 `json:"submissionFileSize,omitempty"`
+ CreateAt string `json:"createAt"`
}
type DatasetInfo struct {
diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go
index 99f754efc5..d8d858fcdd 100644
--- a/sda/internal/database/db_functions.go
+++ b/sda/internal/database/db_functions.go
@@ -366,6 +366,46 @@ func (dbs *SDAdb) getHeader(fileID string) ([]byte, error) {
return header, nil
}
+// BackupHeader takes a backup of the current encryption header before it is rotated.
+// It stores the fileID, the hex-encoded header, and the current timestamp in a backup table.
+func (dbs *SDAdb) BackupHeader(fileID string, header []byte, keyHash string) error {
+ var (
+ err error
+ count int
+ )
+
+ for count == 0 || (err != nil && count < RetryTimes) {
+ err = dbs.backupHeader(fileID, header, keyHash)
+ count++
+ }
+
+ return err
+}
+
+func (dbs *SDAdb) backupHeader(fileID string, header []byte, keyHash string) error {
+ dbs.checkAndReconnectIfNeeded()
+ db := dbs.DB
+
+ const query = `
+ INSERT INTO sda.file_headers_backup (file_id, header, key_hash, backup_at)
+ VALUES ($1, $2, $3, NOW())
+ ON CONFLICT (file_id) DO UPDATE
+ SET header = EXCLUDED.header,
+ key_hash = EXCLUDED.key_hash,
+ backup_at = EXCLUDED.backup_at;`
+
+ result, err := db.Exec(query, fileID, hex.EncodeToString(header), keyHash)
+ if err != nil {
+ return fmt.Errorf("backupHeader error: %w", err)
+ }
+
+ if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 {
+ return errors.New("failed to backup header: zero rows were inserted")
+ }
+
+ return nil
+}
+
// MarkCompleted marks the file as "COMPLETED"
func (dbs *SDAdb) SetVerified(file FileInfo, fileID string) error {
var (
@@ -840,6 +880,31 @@ func (dbs *SDAdb) getArchivePath(stableID string) (string, error) {
return archivePath, nil
}
+// SetSubmissionFileSize sets the submission file size for a file
+func (dbs *SDAdb) SetSubmissionFileSize(fileID string, submissionFileSize int64) error {
+ var err error
+ var count int
+
+ for count == 0 || (err != nil && count < RetryTimes) {
+ err = dbs.setSubmissionFileSize(fileID, submissionFileSize)
+ count++
+ }
+
+ return err
+}
+func (dbs *SDAdb) setSubmissionFileSize(fileID string, submissionFileSize int64) error {
+ dbs.checkAndReconnectIfNeeded()
+ db := dbs.DB
+ const setSubmissionFileSize = "UPDATE sda.files set submission_file_size = $1 where id = $2;"
+
+ _, err := db.Exec(setSubmissionFileSize, submissionFileSize, fileID)
+ if err != nil {
+ return err
+ }
+
+ return nil
+}
+
// GetUserFiles retrieves all the files a user submitted
func (dbs *SDAdb) GetUserFiles(userID, pathPrefix string, allData bool) ([]*SubmissionFileInfo, error) {
var err error
@@ -867,7 +932,7 @@ func (dbs *SDAdb) getUserFiles(userID, pathPrefix string, allData bool) ([]*Subm
// select all files (that are not part of a dataset) of the user, each one annotated with its latest event
const query = `
-SELECT DISTINCT ON (f.id) f.id, f.submission_file_path, f.stable_id, fel.event, f.created_at FROM sda.files AS f
+SELECT DISTINCT ON (f.id) f.id, f.submission_file_path, f.stable_id, fel.event, f.created_at, f.submission_file_size FROM sda.files AS f
LEFT JOIN sda.file_event_log AS fel ON fel.file_id = f.id
LEFT JOIN sda.file_dataset AS fd ON fd.file_id = f.id
WHERE f.submission_user = $1 AND ($2::TEXT IS NULL OR substr(f.submission_file_path, 1, $3) = $2::TEXT)
@@ -896,11 +961,16 @@ ORDER BY f.id, fel.started_at DESC;`
var accessionID sql.NullString
// Read rows into struct
fi := &SubmissionFileInfo{}
- err := rows.Scan(&fi.FileID, &fi.InboxPath, &accessionID, &fi.Status, &fi.CreateAt)
+ var submissionFileSize sql.NullInt64
+ err := rows.Scan(&fi.FileID, &fi.InboxPath, &accessionID, &fi.Status, &fi.CreateAt, &submissionFileSize)
if err != nil {
return nil, err
}
+ if submissionFileSize.Valid {
+ fi.SubmissionFileSize = submissionFileSize.Int64
+ }
+
if allData {
fi.AccessionID = accessionID.String
}
diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go
index 877f06fc07..64c0240622 100644
--- a/sda/internal/database/db_functions_test.go
+++ b/sda/internal/database/db_functions_test.go
@@ -209,6 +209,33 @@ func (suite *DatabaseTests) TestGetHeader() {
db.Close()
}
+func (suite *DatabaseTests) TestBackupHeader() {
+ db, err := NewSDAdb(suite.dbConf)
+ assert.NoError(suite.T(), err, "got %v when creating new connection", err)
+ defer db.Close()
+
+ fileID, err := db.RegisterFile(nil, "/testuser/TestBackupHeader.c4gh", "testuser")
+ assert.NoError(suite.T(), err, "failed to register file in database")
+
+ testKeyHash := "test-key-hash-123"
+ _, err = db.DB.Exec("INSERT INTO sda.encryption_keys (key_hash) VALUES ($1) ON CONFLICT DO NOTHING", testKeyHash)
+ assert.NoError(suite.T(), err, "failed to setup test encryption key")
+
+ testHeader := []byte{1, 2, 3, 4, 5}
+ err = db.BackupHeader(fileID, testHeader, testKeyHash)
+ assert.NoError(suite.T(), err, "failed to backup header")
+
+ var storedHeaderHex string
+ var storedKeyHash string
+
+ query := "SELECT header, key_hash FROM sda.file_headers_backup WHERE file_id = $1"
+ err = db.DB.QueryRow(query, fileID).Scan(&storedHeaderHex, &storedKeyHash)
+
+ assert.NoError(suite.T(), err, "failed to find backup record in database")
+ assert.Equal(suite.T(), hex.EncodeToString(testHeader), storedHeaderHex)
+ assert.Equal(suite.T(), testKeyHash, storedKeyHash)
+}
+
func (suite *DatabaseTests) TestSetVerified() {
db, err := NewSDAdb(suite.dbConf)
assert.NoError(suite.T(), err, "got (%v) when creating new connection", err)
@@ -1539,3 +1566,29 @@ func (suite *DatabaseTests) TestGetFileDetailsFromUUID_NotFound() {
db.Close()
}
+
+func (suite *DatabaseTests) TestSetSubmissionFileSize() {
+ db, err := NewSDAdb(suite.dbConf)
+ assert.NoError(suite.T(), err, "failed to create new connection")
+
+ fileID, err := db.RegisterFile(nil, "/test.file", "user")
+ if err != nil {
+ suite.FailNow("failed to register file", err)
+ }
+
+ fileSize := int64(time.Now().Nanosecond())
+ err = db.setSubmissionFileSize(fileID, fileSize)
+ if err != nil {
+ suite.FailNow("failed to set submission file size", err)
+ }
+
+ var sizeInDb int64
+ err = db.DB.QueryRow("SELECT submission_file_size FROM sda.files WHERE id=$1", fileID).Scan(&sizeInDb)
+ if err != nil {
+ suite.FailNow("failed to get submission file size from DB", err)
+ }
+
+ assert.Equal(suite.T(), fileSize, sizeInDb)
+
+ db.Close()
+}
diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go
index 3788301770..3313abfb1d 100644
--- a/sda/internal/storage/storage.go
+++ b/sda/internal/storage/storage.go
@@ -213,6 +213,8 @@ func NewS3Client(conf S3Conf) (*s3.Client, error) {
o.EndpointOptions.DisableHTTPS = strings.HasPrefix(conf.URL, "http:")
o.Region = conf.Region
o.UsePathStyle = true
+ o.RequestChecksumCalculation = aws.RequestChecksumCalculationWhenRequired
+ o.ResponseChecksumValidation = aws.ResponseChecksumValidationWhenRequired
},
)