diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000000..0bb2e47ee3 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @neicnordic/sensitive-data-development-collaboration diff --git a/.github/integration/scripts/charts/cleanup.sh b/.github/integration/scripts/charts/cleanup.sh index bbc6c3eeaa..d063854c4e 100644 --- a/.github/integration/scripts/charts/cleanup.sh +++ b/.github/integration/scripts/charts/cleanup.sh @@ -16,4 +16,6 @@ helm uninstall broker || true helm uninstall postgres || true kubectl delete secrets api-rbac broker-sda-mq-certs broker-sda-mq-test-certs postgres-sda-db-certs postgres-sda-db-test-certs || true -rm /tmp/values.yaml /tmp/c4gh.pub.pem /tmp/c4gh.sec.pem /tmp/jwt.key /tmp/jwt.pub \ No newline at end of file +kubectl delete deployment.apps/cega-nss || true +kubectl delete cm cega-nss || true +rm /tmp/values.yaml /tmp/c4gh.pub.pem /tmp/c4gh.sec.pem /tmp/jwt.key /tmp/jwt.pub /tmp/users.json || true \ No newline at end of file diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index 1b885de96b..95abe8d675 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -2,7 +2,7 @@ set -ex YQ_VERSION="v4.20.1" -C4GH_VERSION="$(curl --retry 100 -sL https://api.github.com/repos/neicnordic/crypt4gh/releases/latest | jq -r '.name')" +C4GH_VERSION="v1.14.0" random-string() { head -c 32 /dev/urandom | base64 -w0 | tr -d '/+' | fold -w 32 | head -n 1 @@ -99,13 +99,17 @@ if [ "$1" == "local" ]; then fi ## update values file with all credentials +if [ "$2" == "federated" ]; then + yq -i '.global.schemaType = federated' "$values_file" +fi + yq -i ' .global.archive.s3AccessKey = strenv(MINIO_ACCESS) | .global.archive.s3SecretKey = strenv(MINIO_SECRET) | .global.backupArchive.s3AccessKey = strenv(MINIO_ACCESS) | .global.backupArchive.s3SecretKey = strenv(MINIO_SECRET) | .global.broker.password = strenv(MQPASSWORD) | -.global.c4gh.passphrase = strenv(C4GHPASSPHRASE) | +.global.c4gh.privateKeys[0].passphrase = strenv(C4GHPASSPHRASE) | .global.db.password = strenv(PGPASSWORD) | .global.inbox.s3AccessKey = strenv(MINIO_ACCESS) | .global.inbox.s3SecretKey = strenv(MINIO_SECRET) | @@ -115,3 +119,17 @@ yq -i ' ' "$values_file" kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json" + +cat >/tmp/users.json < /bin/sh -c " - /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage; + /usr/bin/mc alias set s3 http://outbox:9000 minio miniostorage; /usr/bin/mc mb s3/lega; - exit 0; " doa: container_name: doa build: context: ../../sda-doa - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-doa depends_on: postgres: condition: service_healthy @@ -138,7 +137,6 @@ services: - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass - OUTBOX_TYPE=S3 - - SSL_ENABLED=false - ROOT_CERT_PATH=/certs/ca.crt - CERT_PATH=/certs/client.crt - CERT_KEY=/certs/client.der @@ -159,15 +157,17 @@ services: volumes: - ../../sda-doa/src:/sda-doa/src - ../../sda-doa/pom.xml:/sda-doa/pom.xml - - ../../sda-doa/settings.xml:/root/.m2/settings.xml - ./tests:/tests - encryption_files:/test - client_certs:/certs depends_on: - - doa - - mockauth - - init-bucket + doa: + condition: service_started + mockauth: + condition: service_started + init-bucket: + condition: service_completed_successfully environment: - OUTBOX_TYPE=S3 - DOA_URL=http://doa:8080 diff --git a/.github/integration/sda-posix-integration.yml b/.github/integration/sda-posix-integration.yml index 26cb4c3a68..23fec2afae 100644 --- a/.github/integration/sda-posix-integration.yml +++ b/.github/integration/sda-posix-integration.yml @@ -37,7 +37,7 @@ services: - shared:/shared reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-reencrypt ] container_name: reencrypt depends_on: diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 642b3c39cc..ab9da116d9 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -58,7 +58,7 @@ services: - rabbitmq_data:/var/lib/rabbitmq minio: - image: minio/minio:RELEASE.2023-05-18T00-05-36Z + image: minio/minio command: server /data --console-address ":9001" container_name: s3 environment: @@ -334,7 +334,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.3-broker2.1.13-tomcat9.0-jdk21 ports: - "8800:8080" volumes: diff --git a/.github/integration/sda-sync-integration.yml b/.github/integration/sda-sync-integration.yml index 2af84742e1..d9192a36a6 100644 --- a/.github/integration/sda-sync-integration.yml +++ b/.github/integration/sda-sync-integration.yml @@ -27,7 +27,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-postgres ports: - "15432:5432" restart: always @@ -49,7 +49,7 @@ services: interval: 10s timeout: 5s retries: 6 - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq ports: - "15672:15672" restart: always @@ -57,7 +57,7 @@ services: - rabbitmq_data:/var/lib/rabbitmq minio: - image: minio/minio:RELEASE.2023-05-18T00-05-36Z + image: minio/minio command: server /data --console-address ":9001" container_name: s3 environment: @@ -81,7 +81,7 @@ services: GOLANG_VERSION: ${GOLANG_VERSION:-1.20} context: ../../sda target: debug - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-s3inbox ] container_name: s3inbox depends_on: @@ -112,7 +112,7 @@ services: - "18001:8001" ingest: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-ingest ] container_name: ingest depends_on: @@ -137,7 +137,7 @@ services: - shared:/shared verify: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-verify ] container_name: verify depends_on: @@ -162,7 +162,7 @@ services: - shared:/shared finalize: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-finalize ] container_name: finalize depends_on: @@ -187,7 +187,7 @@ services: - shared:/shared mapper: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-mapper ] container_name: mapper depends_on: @@ -211,7 +211,7 @@ services: - shared:/shared sync: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-sync ] container_name: sync depends_on: @@ -235,7 +235,7 @@ services: - shared:/shared sync-api: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-syncapi ] container_name: sync-api depends_on: @@ -268,7 +268,7 @@ services: interval: 10s timeout: 2s retries: 6 - image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.3-broker2.1.13-tomcat9.0-jdk21 ports: - "8800:8080" volumes: @@ -309,7 +309,7 @@ services: - DB_USER=api extra_hosts: - "localhost:host-gateway" - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} ports: - "8090:8080" restart: always @@ -319,7 +319,7 @@ services: - shared:/shared reencrypt: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} + image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} command: [ sda-reencrypt ] container_name: reencrypt depends_on: diff --git a/.github/integration/sda/aai-mock/clients/aai-auth.yaml b/.github/integration/sda/aai-mock/clients/aai-auth.yaml index b6ec2ed326..184c76203b 100644 --- a/.github/integration/sda/aai-mock/clients/aai-auth.yaml +++ b/.github/integration/sda/aai-mock/clients/aai-auth.yaml @@ -5,4 +5,7 @@ redirect-uris: ["http://localhost:8801/oidc/login"] token-endpoint-auth-method: "client_secret_basic" scope: ["openid", "profile", "email", "ga4gh_passport_v1", "eduperson_entitlement"] grant-types: ["authorization_code"] -post-logout-redirect-uris: ["http://localhost:8801/oidc/login"] \ No newline at end of file +post-logout-redirect-uris: ["http://localhost:8801/oidc/login"] + +# How many seconds is the access token valid #30days +access-token-validity-seconds: 2592000 \ No newline at end of file diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index c14fd719e1..e5473a023f 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -13,6 +13,10 @@ archive: bucket: "archive" region: "us-east-1" +grpc: + host: "reencrypt" + port: 50051 + auth: cega: authUrl: "http://cega-nss:8443/username/" diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index 7d37530658..515d7bb456 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -45,6 +45,11 @@ "path": "/users/:username/files", "action": "GET" }, + { + "role": "submission", + "path": "/users/:username/file/:fileid", + "action": "GET" + }, { "role": "*", "path": "/datasets", @@ -64,6 +69,9 @@ { "role": "testu@lifescience-ri.eu", "rolebinding": "admin" - } + }, { + "role": "requester@demo.org", + "rolebinding": "admin" + } ] } \ No newline at end of file diff --git a/.github/integration/tests/sda/01_install_dependencies.sh b/.github/integration/tests/sda/01_install_dependencies.sh index 9020696c1a..174d96c6dd 100644 --- a/.github/integration/tests/sda/01_install_dependencies.sh +++ b/.github/integration/tests/sda/01_install_dependencies.sh @@ -13,3 +13,5 @@ for t in curl expect jq openssh-client postgresql-client xxd; do apt-get -o DPkg::Lock::Timeout=60 install -y "$t" >/dev/null fi done + +pip -q install s3cmd diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh index 8c045f66ea..e21f474bea 100644 --- a/.github/integration/tests/sda/10_upload_test.sh +++ b/.github/integration/tests/sda/10_upload_test.sh @@ -20,8 +20,6 @@ done ## truncate database psql -U postgres -h postgres -d sda -At -c "TRUNCATE TABLE sda.files, sda.encryption_keys CASCADE;" -pip -q install s3cmd - for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do curl --retry 100 -s -L -o /shared/$file "https://github.com/ga4gh/htsget-refserver/raw/main/data/gcp/gatk-test-data/wgs_bam/$file" if [ ! -f "$file.c4gh" ]; then diff --git a/.github/integration/tests/sda/20_ingest-verify_test.sh b/.github/integration/tests/sda/20_ingest-verify_test.sh index f9443cfb31..f500d68781 100644 --- a/.github/integration/tests/sda/20_ingest-verify_test.sh +++ b/.github/integration/tests/sda/20_ingest-verify_test.sh @@ -32,11 +32,15 @@ for file in NA12878.bam NA12878_20k_b37.bam NA12878.bai NA12878_20k_b37.bai; do '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))' ) + if [ "$file" = "NA12878.bam" ]; then + file="test_dummy.org/NA12878.bam" + fi + ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/"$file.c4gh" \ + --arg filepath "$file.c4gh" \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/.github/integration/tests/sda/21_cancel_test.sh b/.github/integration/tests/sda/21_cancel_test.sh index ab6fd64b23..c7482bdbad 100644 --- a/.github/integration/tests/sda/21_cancel_test.sh +++ b/.github/integration/tests/sda/21_cancel_test.sh @@ -34,7 +34,7 @@ cancel_payload=$( jq -r -c -n \ --arg type cancel \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878_20k_b37.bam.c4gh \ + --arg filepath NA12878_20k_b37.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -65,7 +65,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878_20k_b37.bam.c4gh \ + --arg filepath NA12878_20k_b37.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/.github/integration/tests/sda/22_error_test.sh b/.github/integration/tests/sda/22_error_test.sh index 461b0f5dca..ea79144e87 100644 --- a/.github/integration/tests/sda/22_error_test.sh +++ b/.github/integration/tests/sda/22_error_test.sh @@ -29,7 +29,7 @@ if [ -n "$PGSSLCERT" ]; then fi ## get correlation id from message -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NB12878.bam.c4gh';") +CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NB12878.bam.c4gh';") properties=$( jq -c -n \ @@ -50,7 +50,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NB12878.bam.c4gh \ + --arg filepath NB12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -96,7 +96,7 @@ verify_payload=$( --arg user test@dummy.com \ --arg archive_path "$CORRID" \ --arg file_id "$CORRID" \ - --arg filepath test_dummy.org/NB12878.bam.c4gh \ + --arg filepath NB12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ --argjson re_verify false \ '$ARGS.named|@base64' diff --git a/.github/integration/tests/sda/30_backup-finalize_test.sh b/.github/integration/tests/sda/30_backup-finalize_test.sh index e4a6d4f81e..6ad7828261 100644 --- a/.github/integration/tests/sda/30_backup-finalize_test.sh +++ b/.github/integration/tests/sda/30_backup-finalize_test.sh @@ -94,7 +94,7 @@ EOD # check DB for archive file names for file in NA12878.bam.c4gh NA12878.bai.c4gh NA12878_20k_b37.bam.c4gh NA12878_20k_b37.bai.c4gh; do - archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = 'test_dummy.org/$file';") + archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = '$file';") size=$(s3cmd -c direct ls s3://backup/"$archiveName" | tr -s ' ' | cut -d ' ' -f 3) if [ "$size" -eq 0 ]; then echo "Failed to get size of $file from backup site" diff --git a/.github/integration/tests/sda/31_cancel_test2.sh b/.github/integration/tests/sda/31_cancel_test2.sh index 74cce3f025..0095a06870 100644 --- a/.github/integration/tests/sda/31_cancel_test2.sh +++ b/.github/integration/tests/sda/31_cancel_test2.sh @@ -7,7 +7,7 @@ ENC_SHA=$(sha256sum NA12878.bam.c4gh | cut -d' ' -f 1) ENC_MD5=$(md5sum NA12878.bam.c4gh | cut -d' ' -f 1) ## get correlation id from message -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/NA12878.bam.c4gh';") +CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'NA12878.bam.c4gh';") properties=$( @@ -30,7 +30,7 @@ cancel_payload=$( jq -r -c -n \ --arg type cancel \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878.bam.c4gh \ + --arg filepath NA12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -67,7 +67,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878.bam.c4gh \ + --arg filepath NA12878.bam.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -110,7 +110,7 @@ accession_payload=$( jq -r -c -n \ --arg type accession \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/NA12878.bam.c4gh \ + --arg filepath NA12878.bam.c4gh \ --arg accession_id EGAF74900000001 \ --argjson decrypted_checksums "$decrypted_checksums" \ '$ARGS.named|@base64' diff --git a/.github/integration/tests/sda/32_test_race_condition.sh b/.github/integration/tests/sda/32_test_race_condition.sh index 58260e828c..eb04ba8e6f 100644 --- a/.github/integration/tests/sda/32_test_race_condition.sh +++ b/.github/integration/tests/sda/32_test_race_condition.sh @@ -14,7 +14,7 @@ ENC_SHA=$(sha256sum race_file.c4gh | cut -d' ' -f 1) ENC_MD5=$(md5sum race_file.c4gh | cut -d' ' -f 1) ## get correlation id from message -CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'test_dummy.org/race_file.c4gh';") +CORRID=$(psql -U postgres -h postgres -d sda -At -c "select id from sda.files where submission_file_path = 'race_file.c4gh';") properties=$( jq -c -n \ @@ -36,7 +36,7 @@ accession_payload=$( jq -r -c -n \ --arg type accession \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/race_file.c4gh \ + --arg filepath race_file.c4gh \ --arg accession_id EGAF74900000099 \ --argjson decrypted_checksums "$decrypted_checksums" \ '$ARGS.named|@base64' @@ -74,7 +74,7 @@ ingest_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/race_file.c4gh \ + --arg filepath race_file.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/.github/integration/tests/sda/40_mapper_test.sh b/.github/integration/tests/sda/40_mapper_test.sh index 49cb486d71..314ffa4bf2 100644 --- a/.github/integration/tests/sda/40_mapper_test.sh +++ b/.github/integration/tests/sda/40_mapper_test.sh @@ -115,7 +115,7 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.dat echo "::error::Time out while waiting for dataset to be released" exit 1 fi - sleep 2 + sleep 4 done echo "dataset released successfully" diff --git a/.github/integration/tests/sda/45_sync_test.sh b/.github/integration/tests/sda/45_sync_test.sh index 4d44aef99e..5dc6d12215 100644 --- a/.github/integration/tests/sda/45_sync_test.sh +++ b/.github/integration/tests/sda/45_sync_test.sh @@ -17,7 +17,7 @@ for file in NA12878.bai NA12878_20k_b37.bai; do echo "::error::Time out while waiting for files to be synced" exit 1 fi - sleep 2 + sleep 4 done done diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index 3e2bc143bc..2094287a13 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -e cd shared || true @@ -41,25 +41,27 @@ fi # Reupload a file under a different name, test to delete it s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh +stream_size=$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready') -echo "waiting for upload to complete" URI=http://rabbitmq:15672 if [ -n "$PGSSLCERT" ]; then URI=https://rabbitmq:15671 fi +stream_size=$((stream_size + 1)) RETRY_TIMES=0 -until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 4 ]; do - echo "waiting for upload to complete" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 30 ]; then - echo "::error::Time out while waiting for upload to complete" - exit 1 - fi - sleep 2 + +until [ $((stream_size)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/inbox | jq '.messages_ready')" ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "Upload did not complete successfully" + exit 1 + fi + sleep 2 done # get the fileId of the new file -fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NC12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NC12878.bam.c4gh") | .fileID')" output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) if [ -z "$output" ] ; then @@ -104,27 +106,23 @@ fi # Re-upload the file and use the api to ingest it, then try to delete it s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NE12878.bam.c4gh - -URI=http://rabbitmq:15672 -if [ -n "$PGSSLCERT" ]; then - URI=https://rabbitmq:15671 -fi +stream_size=$((stream_size + 1)) RETRY_TIMES=0 -until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do - echo "waiting for upload to complete" - RETRY_TIMES=$((RETRY_TIMES + 1)) - if [ "$RETRY_TIMES" -eq 3 ]; then - echo "::error::Time out while waiting for upload to complete" - #exit 1 - break - fi - sleep 2 + +until [ $((stream_size)) -eq "$(curl -s -u guest:guest $URI/api/queues/sda/inbox | jq '.messages_ready')" ]; do + echo "waiting for upload to complete" + RETRY_TIMES=$((RETRY_TIMES + 1)) + if [ "$RETRY_TIMES" -eq 30 ]; then + echo "Upload did not complete successfully" + exit 1 + fi + sleep 2 done # Ingest it new_payload=$( jq -c -n \ - --arg filepath "test_dummy.org/NE12878.bam.c4gh" \ + --arg filepath "NE12878.bam.c4gh" \ --arg user "test@dummy.org" \ '$ARGS.named' ) @@ -135,7 +133,7 @@ if [ "$resp" != "200" ]; then exit 1 fi -fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NE12878.bam.c4gh") | .fileID')" # wait for the fail to get the correct status RETRY_TIMES=0 @@ -150,9 +148,72 @@ until [ "$(psql -U postgres -h postgres -d sda -At -c "select id from sda.file_e done # Try to delete file not in inbox -fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "test_dummy.org/NE12878.bam.c4gh") | .fileID')" +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NE12878.bam.c4gh") | .fileID')" resp="$(curl -s -k -L -o /dev/null -w "%{http_code}\n" -H "Authorization: Bearer $token" -X DELETE "http://api:8080/file/test@dummy.org/$fileid")" if [ "$resp" != "404" ]; then echo "Error when deleting the file, expected 404 got: $resp" exit 1 -fi \ No newline at end of file +fi + +# Test downloading the file from the inbox +# Reupload a file under a different name, test to download it +s3cmd -c s3cfg put "NA12878.bam.c4gh" s3://test_dummy.org/NC12878.bam.c4gh +fileid="$(curl -k -L -H "Authorization: Bearer $token" "http://api:8080/users/test@dummy.org/files" | jq -r '.[] | select(.inboxPath == "NC12878.bam.c4gh") | .fileID')" +output=$(s3cmd -c s3cfg ls s3://test_dummy.org/NC12878.bam.c4gh 2>/dev/null) +if [ -z "$output" ] ; then + echo "Uploaded file NC12878.bam.c4gh not found in inbox" + exit 1 +fi + +# download the file, re-encrypted with the client key +clientPubKey="$(base64 -w0 /shared/client.pub.pem)" +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -JO)" +if [ "$resp" != "200" ]; then + echo "Error when downloading the file, expected 200 got: $resp" + exit 1 +fi + +# decrypt the downloaded file +export C4GH_PASSPHRASE=c4ghpass +if [ ! -f "NC12878.bam.c4gh" ]; then + echo "downloaded file NC12878.bam.c4gh not found" + exit 1 +fi +if [ ! -f "/shared/client.sec.pem" ]; then + echo "client key not found" + exit 1 +fi + +# copy the file to avoid overwriting the original +if ! /shared/crypt4gh decrypt -f "NC12878.bam.c4gh" -s "/shared/client.sec.pem" ; then + echo "decrypting file NC12878.bam.c4gh failed" + exit 1 +fi + +# check the file content +if [ ! -f "NC12878.bam" ]; then + echo "decrypted file NC12878.bam not found" + exit 1 +fi +if ! cmp -s "NC12878.bam" "NA12878.bam" ; then + echo "downloaded file is different from the original one" + exit 1 +fi + +# download file as a non admin user should fail +token_nonAdmin=$(grep 'access_token' /shared/s3cfg | sed -E 's/access_token="?([^"]+)"?/\1/') +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token_nonAdmin" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$fileid" -JO)" +if [ "$resp" != "401" ]; then + echo "Error when downloading the file, expected 401 got: $resp" + exit 1 +fi + +# download a non existing file should fail +badfileid="badfileid" +resp="$(curl -s -k -L -w "%{http_code}\n" -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $clientPubKey" "http://api:8080/users/test@dummy.org/file/$badfileid" -JO)" +if [ "$resp" != "404" ]; then + echo "Trying to download a non existing file, expected 404 got: $resp" + exit 1 +fi + +echo "API admin tests completed successfully" \ No newline at end of file diff --git a/.github/integration/tests/sda/92_handle_file_errors.sh b/.github/integration/tests/sda/92_handle_file_errors.sh index bc28ef3f35..7316d1bace 100644 --- a/.github/integration/tests/sda/92_handle_file_errors.sh +++ b/.github/integration/tests/sda/92_handle_file_errors.sh @@ -45,7 +45,7 @@ bad_file_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/bad.file.c4gh \ + --arg filepath bad.file.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) @@ -76,12 +76,14 @@ missing_file_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/missing.file.c4gh \ + --arg filepath missing.file.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) -CORRID=${CORRID//a/b} +FILEID=$(psql -U postgres -h postgres -d sda -At -c "SELECT DISTINCT(file_id) FROM sda.file_event_log WHERE correlation_id = '$CORRID';") +psql -U postgres -h postgres -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, correlation_id, user_id, message) VALUES('$FILEID', 'uploaded', '$CORRID', 'test@dummy.org', '{\"uploaded\": \"message\"}');" + properties=$( jq -c -n \ --argjson delivery_mode 2 \ @@ -133,7 +135,7 @@ truncated_file_payload=$( jq -r -c -n \ --arg type ingest \ --arg user test@dummy.org \ - --arg filepath test_dummy.org/truncated.c4gh \ + --arg filepath truncated.c4gh \ --argjson encrypted_checksums "$encrypted_checksums" \ '$ARGS.named|@base64' ) diff --git a/.github/integration/tests/sda/auth/requirements.txt b/.github/integration/tests/sda/auth/requirements.txt index 1cca1d478f..8073666c6d 100644 --- a/.github/integration/tests/sda/auth/requirements.txt +++ b/.github/integration/tests/sda/auth/requirements.txt @@ -1,3 +1,3 @@ -requests==2.32.0 +requests==2.32.4 tox==4.11.3 pytest==7.4.3 diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 1938e9b19e..97e738f9a2 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -31,7 +31,7 @@ jobs: string: ${{ github.repository }} - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -81,7 +81,7 @@ jobs: string: ${{ github.repository }} - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -117,7 +117,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false @@ -135,7 +135,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false @@ -167,7 +167,7 @@ jobs: string: ${{ github.repository }} - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -190,7 +190,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false @@ -207,21 +207,6 @@ jobs: sarif_file: 'inbox-results.sarif' category: sftp-inbox - - - name: create maven settings.xml - uses: s4u/maven-settings-action@v3.1.0 - with: - servers: | - [{ - "id":"github-fega-norway", - "username": "${{github.actor}}", - "password": "${{ secrets.GITHUB_TOKEN }}" - }] - - - name: Copy settings.xml to sda-doa root - shell: bash - run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml - - name: Build container for sda-doa uses: docker/build-push-action@v6 with: @@ -236,7 +221,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on doa - uses: aquasecurity/trivy-action@0.30.0 + uses: aquasecurity/trivy-action@0.33.1 env: TRIVY_SKIP_DB_UPDATE: true TRIVY_SKIP_JAVA_DB_UPDATE: true @@ -262,7 +247,7 @@ jobs: string: ${{ github.repository }} - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Test rabbitmq federation run: docker compose -f .github/integration/rabbitmq-federation.yml run federation_test @@ -280,7 +265,7 @@ jobs: string: ${{ github.repository }} - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Test postgres run: docker compose -f .github/integration/postgres.yml run tests @@ -301,7 +286,7 @@ jobs: string: ${{ github.repository }} - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Test sensitive-data-archive run: docker compose -f .github/integration/sda-${{matrix.storage}}-integration.yml run integration_test @@ -312,7 +297,7 @@ jobs: - build_server_images runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dorny/paths-filter@v3 id: changes with: @@ -335,21 +320,7 @@ jobs: storage: [s3, posix] steps: - name: Check out code - uses: actions/checkout@v4 - - - name: create maven settings.xml - uses: s4u/maven-settings-action@v3.1.0 - with: - servers: | - [{ - "id":"github-fega-norway", - "username": "${{github.actor}}", - "password": "${{ secrets.GITHUB_TOKEN }}" - }] - - - name: Copy settings.xml to sda-doa root - shell: bash - run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml + uses: actions/checkout@v5 - name: Test sda-doa for ${{ matrix.storage }} storage run: docker compose -f .github/integration/sda-doa-${{ matrix.storage }}-outbox.yml run integration_test @@ -362,14 +333,15 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: ["1.30", "1.31"] + deployment: ["federated", "isolated"] + version: ["1.32", "1.33"] tls: ["true", "false"] storage: ["posix", "s3"] exclude: - - version: "1.30" + - version: "1.32" tls: "false" storage: "posix" - - version: "1.31" + - version: "1.33" tls: "false" storage: "posix" steps: @@ -380,7 +352,7 @@ jobs: string: ${{ github.repository }} - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Install Helm uses: azure/setup-helm@v4 @@ -395,7 +367,7 @@ jobs: shell: bash - name: Deploy external services - run: bash .github/integration/scripts/charts/dependencies.sh + run: bash .github/integration/scripts/charts/dependencies.sh ${{matrix.deployment}} shell: bash - name: Deploy DB @@ -410,7 +382,7 @@ jobs: - name: Deploy MQ id: deployMQ - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} ${{matrix.tls}} + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-mq ${{ github.event.number }} ${{matrix.tls}} ${{matrix.deployment}} shell: bash - name: debug if: failure() && steps.deployMQ.outcome == 'failure' @@ -420,7 +392,7 @@ jobs: kubectl logs broker-sda-mq-0 - name: Deploy pipeline - run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} ${{matrix.tls}} ${{matrix.storage}} + run: bash .github/integration/scripts/charts/deploy_charts.sh sda-svc ${{ github.event.number }} ${{matrix.tls}} ${{matrix.storage}} ${{matrix.deployment}} shell: bash - name: Check deployment @@ -447,4 +419,10 @@ jobs: echo "## logs $svc" && kubectl logs -l role="$svc" sleep 1 done + if [ ${{matrix.deployment}} == "federated" ]; then + echo "## describe intercept" && kubectl describe pod -l role=intercept + sleep 1 + echo "## logs intercept" && kubectl logs -l role=intercept + sleep 1 + fi shell: bash \ No newline at end of file diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 56be2be57b..03d87c40b1 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -11,20 +11,39 @@ env: go-version: '1.24' jobs: - lint_download: + check_changes: + outputs: + sda-download: ${{ steps.changes.outputs.sda-download }} + sda-admin: ${{ steps.changes.outputs.sda-admin }} + sda: ${{ steps.changes.outputs.sda }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + sda-download: + - 'sda-download/**' + sda-admin: + - 'sda-admin/**' + sda: + - 'sda/**' + + lint_sda_download: + needs: check_changes + if: needs.check_changes.outputs.sda-download == 'true' name: Lint download code runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run golangci-lint uses: golangci/golangci-lint-action@v8.0.0 @@ -33,19 +52,19 @@ jobs: working-directory: sda-download lint_sda: + needs: check_changes + if: needs.check_changes.outputs.sda == 'true' name: Lint sda code runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run golangci-lint uses: golangci/golangci-lint-action@v8.0.0 @@ -54,19 +73,19 @@ jobs: working-directory: sda lint_sda_admin: + needs: check_changes + if: needs.check_changes.outputs.sda-admin == 'true' name: Lint sda-admin code runs-on: ubuntu-latest - strategy: - fail-fast: false steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run golangci-lint uses: golangci/golangci-lint-action@v8.0.0 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 57b4c8beba..d2cd99f941 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -2,11 +2,14 @@ name: "CodeQL" on: push: - branches: [main] + branches: + - main pull_request: - branches: [main] + branches: + - main paths: - '**.go' + - '**.java' schedule: - cron: '0 15 * * 4' @@ -15,18 +18,48 @@ jobs: name: Analyze runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: [ 'java', 'go' ] + steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v5 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 with: - languages: go + languages: ${{ matrix.language }} + + - name: Set up JDK 21 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 21 + + - name: Build sda-doa + if: ${{ matrix.language == 'java' }} + working-directory: ./sda-doa + run: mvn clean install -DskipTests + + - name: Set up JDK 19 + uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: 19 + + - name: Build sda-sftp-inbox + if: ${{ matrix.language == 'java' }} + working-directory: ./sda-sftp-inbox + run: mvn clean install -DskipTests - name: Autobuild + if: ${{ matrix.language == 'go' }} uses: github/codeql-action/autobuild@v3 - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{matrix.language}}" diff --git a/.github/workflows/functionality.yml b/.github/workflows/functionality.yml index 6d6c67d613..7bd95a0294 100644 --- a/.github/workflows/functionality.yml +++ b/.github/workflows/functionality.yml @@ -11,7 +11,7 @@ jobs: sftp-inbox: ${{ steps.changes.outputs.sftp-inbox }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: dorny/paths-filter@v3 id: changes with: @@ -35,12 +35,12 @@ jobs: fail-fast: false steps: - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: "3.11" - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Run setup scripts run: | @@ -64,7 +64,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Build image run: | diff --git a/.github/workflows/publish_charts.yml b/.github/workflows/publish_charts.yml index e44bd6609e..06c0dffb33 100644 --- a/.github/workflows/publish_charts.yml +++ b/.github/workflows/publish_charts.yml @@ -16,7 +16,7 @@ jobs: continue-on-error: true steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: 0 diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index a60d76bae7..00c86bb71c 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -24,15 +24,15 @@ jobs: new_tag: ${{ steps.bump_tag.outputs.new_tag }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: '0' - name: Bump version and push tag id: bump_tag - uses: anothrNick/github-tag-action@1.73.0 + uses: anothrNick/github-tag-action@1.75.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - WITH_V: true + TAG_PREFIX: v DEFAULT_BUMP: patch TAG_CONTEXT: branch @@ -46,7 +46,7 @@ jobs: packages: write steps: - name: Check out the repo - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Log in to the Github Container registry uses: docker/login-action@v3 @@ -101,7 +101,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v5 with: fetch-depth: '0' diff --git a/.github/workflows/release_sda-admin.yaml b/.github/workflows/release_sda-admin.yaml index e785a4d8db..43c7f1d387 100644 --- a/.github/workflows/release_sda-admin.yaml +++ b/.github/workflows/release_sda-admin.yaml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get version tag run: | VERSION=$(cat sda-admin/.version) @@ -37,10 +37,10 @@ jobs: - "darwin/arm64" steps: - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Set up Go - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: '>=1.22' id: go diff --git a/.github/workflows/scan-images.yml b/.github/workflows/scan-images.yml new file mode 100644 index 0000000000..c1ea49fdcc --- /dev/null +++ b/.github/workflows/scan-images.yml @@ -0,0 +1,71 @@ +name: Scan Images +on: + pull_request: + branches: + - main + paths-ignore: + - ".github/**" + - ".gitignore" + - "**/*.md" + - "charts/**" + - "Makefile" + - "sda-admin/**" + - "tools/*" + types: [ closed ] + +jobs: + scan-all: + if: github.event.pull_request.merged == true + name: Scan ${{ matrix.image-name }} Docker Image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + security-events: write + strategy: + matrix: + image-name: [ 'download', 'postgres', 'rabbitmq', 'sftp-inbox', 'doa' ] + steps: + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.33.1 + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }}-${{ matrix.image-name }} + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: '${{ matrix.image-name }}-results.sarif' + severity: "CRITICAL,HIGH" + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: '${{ matrix.image-name }}-results.sarif' + category: ${{ matrix.image-name }} + scan-sda: + if: github.event.pull_request.merged == true + name: Scan SDA Docker image + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + security-events: write + steps: + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.33.1 + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ github.repository }}:PR${{ github.event.number }} + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: 'sda-results.sarif' + severity: "CRITICAL,HIGH" + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'sda-results.sarif' + category: sda \ No newline at end of file diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index fd1781d623..caf7e1e375 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out code - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: check all scripts uses: ludeeus/action-shellcheck@master diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c488d2a90b..1f666ba6e4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,9 +17,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK ${{ matrix.java-version }} - uses: actions/setup-java@v4 + uses: actions/setup-java@v5 with: distribution: zulu java-version: ${{ matrix.java-version }} @@ -38,13 +38,13 @@ jobs: steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get dependencies run: | @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.2 + uses: codecov/codecov-action@v5.5.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -73,13 +73,13 @@ jobs: steps: - name: Set up Go ${{ env.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ env.go-version }} id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get dependencies run: | @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.2 + uses: codecov/codecov-action@v5.5.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -108,13 +108,13 @@ jobs: steps: - name: Set up Go ${{ matrix.go-version }} - uses: actions/setup-go@v5 + uses: actions/setup-go@v6 with: go-version: ${{ matrix.go-version }} id: go - name: Check out code into the Go module directory - uses: actions/checkout@v4 + uses: actions/checkout@v5 - name: Get dependencies run: | @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.4.2 + uses: codecov/codecov-action@v5.5.1 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt diff --git a/.gitignore b/.gitignore index e3ca9a9c37..d62acf38b3 100644 --- a/.gitignore +++ b/.gitignore @@ -185,7 +185,7 @@ buildNumber.properties .mvn/timing.properties .mvn/wrapper/maven-wrapper.jar .flattened-pom.xml - +settings.xml # End of https://www.gitignore.io/api/java,maven,eclipse,intellij+all db/ diff --git a/.golangci.yml b/.golangci.yml index c05076ac27..9edab1a144 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -52,6 +52,8 @@ linters: disabled: true - name: package-comments disabled: true + - name: redundant-test-main-exit # disable for now (non-critical, needs checking) + disabled: true - name: unchecked-type-assertion # disable for now disabled: true - name: unused-parameter # disable for now diff --git a/Makefile b/Makefile index 9f8282f92a..ca0f914444 100644 --- a/Makefile +++ b/Makefile @@ -172,7 +172,7 @@ k3d-version-check: SHELL:=/bin/bash k3d-version-check: @K3D_VERSION=$$(k3d version | cut -d'v' -f 3 | head -n 1); \ IFS="." read -r -a K3D_VERSION_ARR <<< "$${K3D_VERSION}"; \ - IFS="." read -r -a K3D_VERSION_REQ <<< 5.7.0; \ + IFS="." read -r -a K3D_VERSION_REQ <<< 5.8.0; \ if [[ $${K3D_VERSION_ARR[0]} -lt $${K3D_VERSION_REQ[0]} ||\ ( $${K3D_VERSION_ARR[0]} -eq $${K3D_VERSION_REQ[0]} &&\ ( $${K3D_VERSION_ARR[1]} -lt $${K3D_VERSION_REQ[1]} ||\ @@ -190,17 +190,23 @@ k3d-create-cluster: helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace k3d-delete-cluster: @k3d cluster delete -k3d-deploy-dependencies: - @bash .github/integration/scripts/charts/dependencies.sh local +k3d-deploy-dependencies-federated: + @bash .github/integration/scripts/charts/dependencies.sh local federated +k3d-deploy-dependencies-isolated: + @bash .github/integration/scripts/charts/dependencies.sh local isolated k3d-import-images: build-all @bash .github/integration/scripts/charts/import_local_images.sh k3s-default k3d-deploy-postgres: @bash .github/integration/scripts/charts/deploy_charts.sh sda-db "$$(date +%F)" false -k3d-deploy-rabbitmq: - @bash .github/integration/scripts/charts/deploy_charts.sh sda-mq "$$(date +%F)" false -k3d-deploy-sda-s3: - @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false s3 +k3d-deploy-rabbitmq-federated: + @bash .github/integration/scripts/charts/deploy_charts.sh sda-mq "$$(date +%F)" false federated +k3d-deploy-rabbitmq-isolated: + @bash .github/integration/scripts/charts/deploy_charts.sh sda-mq "$$(date +%F)" false isolated +k3d-deploy-sda-s3-federated: + @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false s3 federated +k3d-deploy-sda-s3-isolated: + @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false s3 isolated k3d-deploy-sda-posix: @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false posix k3d-cleanup-all-deployments: - @bash .github/integration/scripts/charts/cleanup.sh + @bash .github/integration/scripts/charts/cleanup.sh \ No newline at end of file diff --git a/README.md b/README.md index 9a75c7f33b..8b665a3d63 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,9 @@ To build the `sda-admin` CLI tool: ```sh $ make build-sda-admin ``` +```sh +$ make build-sda-doa +``` ### Running the services @@ -234,11 +237,14 @@ Deployment of the charts can be done as describe below in more detail, or by usi #### Makefile commands -- make k3d-deploy-dependencies - bootstrap dependencies +- make k3d-deploy-dependencies-federated - bootstrap dependencies, F-EGA use case +- make k3d-deploy-dependencies-isolated - bootstrap dependencies, standalone use case - make k3d-import-images - build and import images into the default cluster named `k3s-default` - make k3d-deploy-postgres - deploy the sda-db chart without TLS -- make k3d-deploy-rabbitmq - deploy the sda-mq chart without TLS -- make k3d-deploy-sda-s3 - deploy the sda-svc chart with S3 storage without TLS +- make k3d-deploy-rabbitmq-federated - deploy the sda-mq chart without TLS, F-EGA use case +- make k3d-deploy-rabbitmq-isolated - deploy the sda-mq chart without TLS, standalone use case +- make k3d-deploy-sda-s3-federated - deploy the sda-svc chart with S3 storage without TLS, F-EGA use case +- make k3d-deploy-sda-s3-isolated - deploy the sda-svc chart with S3 storage without TLS, standalone use case - make k3d-deploy-sda-posix - deploy the sda-svc chart with POSIX storage without TLS - make k3d-cleanup-all-deployments - Remove all deployed components and dependencies @@ -246,15 +252,15 @@ Deployment of the charts can be done as describe below in more detail, or by usi This script requires [yq](https://github.com/mikefarah/yq/releases/latest), the GO version of [crypt4gh](https://github.com/neicnordic/crypt4gh/releases/latest) as well as [xxd](https://manpages.org/xxd) and [jq](https://manpages.org/jq) to be installed. -Bootstrap the dependencies with the command: `make k3d-deploy-dependencies`. +Bootstrap the dependencies with the command: `make k3d-deploy-dependencies-[federated|isolated]`. #### Deploy the Sensitive Data Archive components Start by building and importing the required containers using the `make k3d-import-images`. -The Postgres and RabbitMQ Needs to be deployed first using the following commands: `make k3d-deploy-postgres` and `make k3d-deploy-rabbitmq`. +The Postgres and RabbitMQ Needs to be deployed first using the following commands: `make k3d-deploy-postgres` and `make k3d-deploy-rabbitmq-[federated|isolated]`. -Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`), `make k3d-deploy-sda-posix` or `make k3d-deploy-sda-s3`. +Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`), `make k3d-deploy-sda-posix` or `make k3d-deploy-sda-s3-[federated|isolated]`. #### Testing with ingress diff --git a/charts/sda-svc/README.md b/charts/sda-svc/README.md index 7ca4df4bba..18506b5800 100644 --- a/charts/sda-svc/README.md +++ b/charts/sda-svc/README.md @@ -10,6 +10,24 @@ Source repositories: Edit the values.yaml file and specify the relevant parts of the `global` section. If no shared credentials for the broker and database are used, the credentials for each service shuld be set in the `credentials` section. +While it is possible to deploy this chart with the crypt4gh keys included in the values file as base64 encoded strings, it is advisable to create the secret containing the crypt4gh keys manually. + +## Upgrading an existing Release to a new version + +A major chart version change (like v1.2.3 -> v2.0.0) indicates that there is an incompatible breaking change needing manual actions. + +### To 3.0.0 + +This version adds jobs that perform the following actions: + +- Migrates the database schema from a V1 release. +- Sets the first registered crypt4gh key to all ingested files that do not have a c4gh key referenced. +- Removes the submitters username from the submission file paths in the database. + +When upgrading from a V1 release *All these jobs* should be run. + +Unless the same queries are being executed manually by a database administrator a `Basic authentication Secret` containing the credentials to perform the upgrade needs to be created. + ### Configuration The following table lists the configurable parameters of the `sda-svc` chart and their default values. @@ -17,7 +35,7 @@ The following table lists the configurable parameters of the `sda-svc` chart and Parameter | Description | Default --------- | ----------- | ------- `image.repository` | Repository URI | `ghcr.io/neicnordic/sensitive-data-archive` -`image.tag` | Tag version to deploy | `` +`image.tag` | Tag version to deploy | `chart appVersion` `image.pullPolicy` | Image pull policy, `Always` or `IfNotPresent` | `Always` `global.secretsPath` | Path where the sensitive files can be found | `/.secrets` `global.c4ghPath` | This path will be a subpath to the secretsPath | `c4gh` @@ -99,9 +117,15 @@ Parameter | Description | Default `global.cega.host` | Full URI to the EGA user authentication service. |`""` `global.cega.user` | Username for the EGA user authentication service. |`""` `global.cega.password` | Password for the EGA user authentication service. |`""` -`global.c4gh.keyFile` | Private C4GH key. |`c4gh.key` -`global.c4gh.passphrase` | Passphrase for the private C4GH key. |`""` -`global.c4gh.publicFile` | Public key corresponding to the private key, provided in /info endpoint. |`""` +`global.c4gh.privateKeys` | List of Private C4GH keys. |`` +`global.c4gh.privateKeys.0.keyData` | The private crypt4gh key provided as a base64 encoded string. |`""` +`global.c4gh.privateKeys.0.keyName` | Filename of the private C4GH key. |`""` +`global.c4gh.privateKeys.0.passphrase` | Passphrase for the private C4GH key. |`""` +`global.c4gh.publicKey` | Public key corresponding to the private key, provided in /info endpoint. |`""` +`global.c4gh.publicKeyData` | Public key corresponding to the private key, provided as a base64 encoded string. |`""` +`global.db.admin.secretName` | Name of the secret that holds the database admin credentials. |`""` +`global.db.admin.passKey` | Key in the secret that holds the password. |`""` +`global.db.admin.userkey` | Key in the secret that holds the username. |`""` `global.db.host` | Hostname for the database. |`""` `global.db.name` | Database to connect to. |`lega` `global.db.passIngest` | Password used for `data in` services. |`""` @@ -153,7 +177,7 @@ Parameter | Description | Default `global.inbox.s3ReadyPath` | Endpoint to verify that the inbox is respondig. |`""` `global.sync.api.password` | Password for authenticating to the syncAPI server | `null` `global.sync.api.user` | User for authenticating to the syncAPI server | `null` -`global.sync.brokerQueue` | Queue to read messages from | `sync` +`global.sync.brokerQueue` | Queue to read messages from | `mapping_stream` `global.sync.centerPrefix` | Prefix for locally generated datasets | `null` `global.sync.destination.storageType` | Storage type for the sync destination, currently only supports S3 | `s3` `global.sync.destination.accesskey` | Access key to S3 sync destination | `null` @@ -168,6 +192,8 @@ Parameter | Description | Default `global.tls.enabled` | Use TLS for all connections. |`true` `global.tls.issuer` | Issuer for TLS certificate creation. |`""` `global.tls.clusterIssuer` | ClusterIssuer for TLS certificate creation. |`""` +`global.reencrypt.host` | gRPC host for reencryption |`""` +`global.reencrypt.port` | port number of the gRPC host for reencryption |`"50051"` ### Credentials @@ -218,6 +244,8 @@ Parameter | Description | Default --------- | ----------- | ------- `api.replicaCount` | Desired number of replicas | `2` `api.annotations` | Specific annotation for the auth pod | `{}` +`api.livenessProbe` | Liveness definition for the api pod. |`{}` +`api.readinessProbe` | Rediness definition for the api pod. |`{}` `api.resources.requests.memory` | Memory request for container. |`128Mi` `api.resources.requests.cpu` | CPU request for container. |`100m` `api.resources.limits.memory` | Memory limit for container. |`256Mi` @@ -225,21 +253,17 @@ Parameter | Description | Default `api.tls.secretName` | Secret holding the application TLS certificates |`` `auth.replicaCount` | desired number of replicas | `2` `auth.annotations` | Specific annotation for the auth pod | `{}` +`auth.livenessProbe` | Liveness definition for the auth pod. |`{}` +`auth.readinessProbe` | Rediness definition for the auth pod. |`{}` `auth.resources.requests.memory` | Memory request for container. |`128Mi` `auth.resources.requests.cpu` | CPU request for container. |`100m` `auth.resources.limits.memory` | Memory limit for container. |`256Mi` `auth.resources.limits.cpu` | CPU limit for container. |`250m` -`sync.annotations` | Specific annotation for the sync pod | `{}` -`sync.resources.requests.memory` | Memory request for sync container. |`128Mi` -`sync.resources.requests.cpu` | CPU request for sync container. |`100m` -`sync.resources.limits.memory` | Memory limit for sync container. |`256Mi` -`sync.resources.limits.cpu` | CPU limit for sync container. |`250m` -`sync.deploy` | Set to true if the sync service should be active | `false` `doa.replicaCount` | desired number of replicas | `2` -`doa.repository` | dataedge container image repository | `neicnordic/sda-doa` -`doa.imageTag` | dataedge container image version | `"latest"` `doa.keystorePass` | keystore password | `changeit` `doa.annotations` | Specific annotation for the doa pod | `{}` +`doa.livenessProbe` | Liveness definition for the doa pod. |`{}` +`doa.readinessProbe` | Rediness definition for the doa pod. |`{}` `doa.resources.requests.memory` | Memory request for dataedge container. |`128Mi` `doa.resources.requests.cpu` | CPU request for dataedge container. |`100m` `doa.resources.limits.memory` | Memory limit for dataedge container. |`1024Mi` @@ -247,6 +271,8 @@ Parameter | Description | Default `download.replicaCount` | desired number of replicas | `2` `download.keystorePass` | keystore password | `changeit` `download.annotations` | Specific annotation for the dataedge pod | `{}` +`download.livenessProbe` | Liveness definition for the download pod. |`{}` +`download.readinessProbe` | Rediness definition for the download pod. |`{}` `download.resources.requests.memory` | Memory request for dataedge container. |`256Mi` `download.resources.requests.cpu` | CPU request for dataedge container. |`100m` `download.resources.limits.memory` | Memory limit for dataedge container. |`512Mi` @@ -264,13 +290,14 @@ Parameter | Description | Default `ingest.resources.limits.cpu` | CPU limit for ingest container. |`2000m` `intercept.replicaCount` | desired number of intercept workers | `1` `intercept.annotations` | Specific annotation for the intercept pod | `{}` -`intercept.deploy` | Set to false in a non federated deployment | `true` `intercept.resources.requests.memory` | Memory request for intercept container. |`32Mi` `intercept.resources.requests.cpu` | CPU request for intercept container. |`100m` `intercept.resources.limits.memory` | Memory limit for intercept container. |`128Mi` `intercept.resources.limits.cpu` | CPU limit for intercept container. |`2000m` `s3Inbox.replicaCount`| desired number of S3inbox containers | `2` `s3Inbox.annotations` | Specific annotation for the S3inbox pod | `{}` +`s3Inbox.livenessProbe` | Liveness definition for the s3Inbox pod. |`{}` +`s3Inbox.readinessProbe` | Rediness definition for the S3inbox pod. |`{}` `s3Inbox.resources.requests.memory` | Memory request for s3Inbox container. |`128Mi` `s3Inbox.resources.requests.cpu` | CPU request for s3Inbox container. |`100m` `s3Inbox.resources.limits.memory` | Memory limit for s3Inbox container. |`1024Mi` @@ -279,18 +306,22 @@ Parameter | Description | Default `sftpInbox.keystorePass` | sftp inbox keystore password | `changeit` `sftpInbox.nodeHostname` | Node name if the sftp inbox needs to be deployed on a specific node | `""` `sftpInbox.annotations` | Specific annotation for the sftp inbox pod | `{}` +`sftpInbox.livenessProbe` | Liveness definition for the sftpInbox pod. |`{}` +`sftpInbox.readinessProbe` | Rediness definition for the sftpInbox pod. |`{}` `sftpInbox.resources.requests.memory` | Memory request for sftpInbox container. |`128Mi` `sftpInbox.resources.requests.cpu` | CPU request for sftpInbox container. |`100m` `sftpInbox.resources.limits.memory` | Memory limit for sftpInbox container. |`256Mi` `sftpInbox.resources.limits.cpu` | CPU limit for sftpInbox container. |`250m` -`sync.replicaCount`| desired number of sync containers | `1` `sync.annotations` | Specific annotation for the sync pod | `{}` +`sync.deploy` | Set to true if the sync service should be active | `false` +`sync.replicaCount`| desired number of sync containers | `1` `sync.resources.requests.memory` | Memory request for sync container. |`128Mi` `sync.resources.requests.cpu` | CPU request for sync container. |`100m` `sync.resources.limits.memory` | Memory limit for sync container. |`512Mi` `sync.resources.limits.cpu` | CPU limit for sync container. |`500m` `syncAPI.replicaCount`| desired number of syncAPI containers | `1` `syncAPI.annotations` | Specific annotation for the syncAPI pod | `{}` +`syncAPI.readinessProbe` | Rediness definition for the syncAPI pod. |`{}` `syncAPI.resources.requests.memory` | Memory request for syncAPI container. |`64Mi` `syncAPI.resources.requests.cpu` | CPU request for syncAPI container. |`100m` `syncAPI.resources.limits.memory` | Memory limit for syncAPI container. |`256Mi` @@ -304,3 +335,9 @@ Parameter | Description | Default `releasetest.repository` | inbox container image repository | `neicnordic/sda-helm-test-support` `releasetest.imageTag` | inbox container image version | `latest` `releasetest.imagePullPolicy` | inbox container image pull policy | `Always` + +### Jobs + +`jobs.image` | Container image used for running the DB migration jobs | `postgres:15.4-alpine` +`jobs.setKeyHash` | Populate the key_hash table after migration from V1 | `false` +`jobs.upgradeFromV1` | Upgrade database schema from a version 1 release. | `false` diff --git a/charts/sda-svc/templates/_helpers.yaml b/charts/sda-svc/templates/_helpers.yaml index 7079c71768..29e483226d 100644 --- a/charts/sda-svc/templates/_helpers.yaml +++ b/charts/sda-svc/templates/_helpers.yaml @@ -101,25 +101,29 @@ Create chart name and version as used by the chart label. {{- .Values.global.secretsPath -}} {{- end -}} +{{- define "configPath" -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.confFilePath | trimPrefix "/")) (empty .Values.global.confFilePath) -}} +{{- end -}} + {{- define "c4ghPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.c4ghPath) (empty .Values.global.c4ghPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.c4ghPath | trimPrefix "/")) (empty .Values.global.c4ghPath) -}} {{- end -}} {{- define "trustedIssPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.download.trusted.configPath) (empty .Values.global.download.trusted.configPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.download.trusted.configPath | trimPrefix "/")) (empty .Values.global.download.trusted.configPath) -}} {{- end -}} {{- define "tlsPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.tlsPath) (empty .Values.global.tlsPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.tlsPath | trimPrefix "/")) (empty .Values.global.tlsPath) -}} {{- end -}} {{- define "jwtPath" -}} -{{- ternary .Values.global.secretsPath (printf "%s/%s" .Values.global.secretsPath .Values.global.jwtPath) (empty .Values.global.jwtPath) -}} +{{- ternary .Values.global.secretsPath (printf "%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.jwtPath | trimPrefix "/")) (empty .Values.global.jwtPath) -}} {{- end -}} {{- define "confFile" -}} {{- if .Values.global.confFilePath -}} - {{- printf "%s/%s/%s" .Values.global.secretsPath .Values.global.confFilePath .Values.global.confFile }} + {{- printf "%s/%s/%s" (.Values.global.secretsPath | trimSuffix "/") (.Values.global.confFilePath | trimPrefix "/") .Values.global.confFile }} {{- else }} {{- printf "%s/%s" .Values.global.secretsPath .Values.global.confFile -}} {{ end }} diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index ae607d6011..e1867e8dfc 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -5,7 +5,7 @@ metadata: name: {{ template "sda.fullname" . }}-api labels: role: api - app: {{ template "sda.fullname" . }} + app: {{ template "sda.fullname" . }}-api chart: {{ .Chart.Name }}-{{ .Chart.Version }} component: {{ template "sda.fullname" . }}-api release: {{ .Release.Name }} @@ -71,171 +71,26 @@ spec: {{- end }} command: ["sda-api"] env: -{{- if not .Values.global.vaultSecrets }} - - name: API_RBACFILE - value: {{ template "secretsPath" . }}/rbac.json - {{- if .Values.global.tls.enabled }} - - name: API_SERVERCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: API_SERVERKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: mqUser - {{- if .Values.global.tls.enabled }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: DB_SSLMODE - value: {{ .Values.global.db.sslMode | quote }} - {{- else }} - - name: DB_SSLMODE - value: "disable" - {{- end }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbUser - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: INBOX_TYPE - {{- if eq "s3" .Values.global.inbox.storageType }} - value: "s3" - - name: INBOX_BUCKET - value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - - name: INBOX_REGION - value: {{ default "us-east-1" .Values.global.inbox.s3Region }} - - name: INBOX_URL - value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- else }} - value: "posix" - - name: INBOX_LOCATION - value: "{{ .Values.global.inbox.path }}/" - {{- end }} - {{- if eq "s3" .Values.global.inbox.storageType }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "federated" .Values.global.schemaType }} - {{- if .Values.global.api.jwtPubKeyName }} - - name: SERVER_JWTPUBKEYPATH - value: {{ include "jwtPath" . }} - {{- else }} - - name: SERVER_JWTPUBKEYURL - value: {{ required "A oidc provider is required" .Values.global.oidc.provider }}{{ .Values.global.oidc.jwkPath }} - {{- end }} -{{- else }} - - name: SERVER_CONFFILE + - name: CONFIGFILE value: {{ include "confFile" . }} -{{- end }} ports: - name: api containerPort: 8080 protocol: TCP livenessProbe: - httpGet: - port: api - path: /ready - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.api }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.api.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - port: api - path: /ready - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.api }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.api.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.api.resources | trim | indent 10 }} volumeMounts: - {{- if .Values.global.api.jwtPubKeyName }} - - name: jwt - mountPath: {{ include "jwtPath" . }} - {{- end }} {{- if not .Values.global.vaultSecrets }} - - name: rbac + - name: config mountPath: {{ template "secretsPath" . }} + {{- if .Values.global.api.jwtPubKeyName }} + - name: jwt + mountPath: {{ include "jwtPath" . }} + {{- end }} {{- end }} {{- if .Values.global.tls.enabled }} - name: tls @@ -243,13 +98,18 @@ spec: {{- end }} {{- if eq "posix" .Values.global.inbox.storageType }} - name: inbox - mountPath: {{ .Values.global.inbox.path | quote }} + mountPath: "/inbox" {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} - - name: rbac + - name: config projected: sources: + - secret: + name: {{ template "sda.fullname" . }}-api + items: + - key: config.yaml + path: config.yaml - secret: name: {{ required "a secret containing the RBAC policy is required" .Values.global.api.rbacFileSecret }} items: diff --git a/charts/sda-svc/templates/api-secrets.yaml b/charts/sda-svc/templates/api-secrets.yaml index a2a1d387a9..f29d39b30c 100644 --- a/charts/sda-svc/templates/api-secrets.yaml +++ b/charts/sda-svc/templates/api-secrets.yaml @@ -6,11 +6,84 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-api type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassAPI" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserAPI" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassAPI" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserAPI" .) | b64enc }} +stringData: + config.yaml: |- + api: + rbacFile: {{ template "secretsPath" . }}/rbac.json + {{- if .Values.global.tls.enabled }} + serverCert: {{ template "tlsPath" . }}/tls.crt + serverKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassAPI" .) }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserAPI" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if eq "verify-full" .Values.global.db.sslMode}} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassAPI" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserAPI" .) }} + grpc: + host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }} + {{- if .Values.global.reencrypt.port}} + port: {{ .Values.global.reencrypt.port }} + {{- end }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} + server: + {{- if .Values.global.api.jwtPubKeyName }} + jwtpubkeypath: {{ include "jwtPath" . }} + {{- else }} + jwtpubkeyurl: "{{ required "A oidc provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/{{ .Values.global.oidc.jwkPath | trimPrefix "/" }}" + {{- end }} --- {{- if not .Values.global.api.adminsFileSecret }} apiVersion: v1 diff --git a/charts/sda-svc/templates/auth-deploy.yaml b/charts/sda-svc/templates/auth-deploy.yaml index b7d2542f0a..e72c6c0eb6 100644 --- a/charts/sda-svc/templates/auth-deploy.yaml +++ b/charts/sda-svc/templates/auth-deploy.yaml @@ -72,148 +72,16 @@ spec: {{- end }} command: ["sda-auth"] env: - {{- if not .Values.global.vaultSecrets }} - {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} - - name: OIDC_ID - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: oidcID - - name: OIDC_SECRET - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: oidcSecret - {{- end }} - {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - - name: AUTH_CEGA_ID - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: cegaID - - name: AUTH_CEGA_SECRET - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-auth - key: cegaSecret - {{- end }} - {{- else }} - - name: SERVER_CONFFILE + - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} - {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - - name: AUTH_CEGA_AUTHURL - value: {{ .Values.global.cega.host | quote }} - {{- end }} - {{- if .Values.global.auth.corsOrigins }} - - name: AUTH_CORS_ORIGINS - value: {{ .Values.global.auth.corsOrigins | quote }} - - name: AUTH_CORS_METHODS - value: {{ .Values.global.auth.corsMethods | quote }} - - name: AUTH_CORS_CREDENTIALS - value: {{ .Values.global.auth.corsCreds | quote }} - {{- end }} - - name: AUTH_INFOTEXT - value: {{ .Values.global.auth.infoText }} - - name: AUTH_INFOURL - value: {{ .Values.global.auth.inforURL }} - {{- if .Values.global.auth.resignJwt }} - - name: AUTH_JWT_ISSUER - {{- if .Values.global.tls.enabled }} - value: "https://{{ .Values.global.ingress.hostName.auth }}" - {{- else }} - value: "http://{{ .Values.global.ingress.hostName.auth }}" - {{- end }} - - name: AUTH_JWT_PRIVATEKEY - value: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}" - - name: AUTH_JWT_SIGNATUREALG - value: {{ .Values.global.auth.jwtAlg }} - - name: AUTH_JWT_TOKENTTL - value: {{ .Values.global.auth.jwtTTL | quote }} - {{- end }} - - name: AUTH_PUBLICFILE - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.publicFile }}" - - name: AUTH_RESIGNJWT - value: {{ .Values.global.auth.resignJwt | quote }} - - name: AUTH_S3INBOX - value: {{ .Values.global.ingress.hostName.s3Inbox }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} - - name: OIDC_REDIRECTURL - value: {{ template "authRedirect" .}} - - name: OIDC_PROVIDER - value: "{{ .Values.global.oidc.provider }}" - - name: OIDC_JWKPATH - value: {{ .Values.global.oidc.jwkPath | quote }} - {{- end }} - {{- if .Values.global.tls.enabled}} - - name: SERVER_CERT - value: {{ template "tlsPath" . }}/tls.crt - - name: SERVER_KEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: DB_SSLMODE - value: {{ .Values.global.db.sslMode | quote }} - {{- else }} - - name: DB_SSLMODE - value: "disable" - {{- end }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-api - key: dbUser - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} ports: - name: auth containerPort: 8080 protocol: TCP livenessProbe: - httpGet: - port: auth - path: / - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.auth }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.auth.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - port: auth - path: / - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.auth }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.auth.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.auth.resources | trim | indent 10 }} volumeMounts: @@ -226,11 +94,14 @@ spec: mountPath: {{ template "jwtPath" . }} {{- end }} {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} volumes: - {{- if and (.Values.global.auth.resignJwt) (not .Values.global.vaultSecrets) }} + {{- if not .Values.global.vaultSecrets }} + {{- if and (.Values.global.auth.resignJwt) (not .Values.global.vaultSecrets) }} - name: jwt projected: defaultMode: 0440 @@ -240,15 +111,18 @@ spec: items: - key: {{ required "The name of the JWT signing key is needed" .Values.global.auth.jwtKey }} path: {{ .Values.global.auth.jwtKey }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} + {{- end }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-auth - name: c4gh secret: defaultMode: 0440 secretName: {{ required "A secret for the C4GH public key is needed" .Values.global.c4gh.secretName }} items: - - key: {{ required "The C4GH public key is needed" .Values.global.c4gh.publicFile }} - path: {{ .Values.global.c4gh.publicFile }} + - key: {{ required "The C4GH public key is needed" .Values.global.c4gh.publicKey }} + path: {{ .Values.global.c4gh.publicKey }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls diff --git a/charts/sda-svc/templates/auth-secrets.yaml b/charts/sda-svc/templates/auth-secrets.yaml index 81de9f9e67..abde1f0381 100644 --- a/charts/sda-svc/templates/auth-secrets.yaml +++ b/charts/sda-svc/templates/auth-secrets.yaml @@ -6,17 +6,61 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-auth type: Opaque -data: - {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} - oidcID: {{ .Values.global.oidc.id | quote | trimall "\"" | b64enc }} - oidcSecret: {{ .Values.global.oidc.secret | quote | trimall "\"" | b64enc }} - {{- end }} - {{- if or ( eq "federated" .Values.global.schemaType) ( eq "" .Values.global.schemaType) }} - cegaID: {{ .Values.global.cega.user | quote | trimall "\"" | b64enc }} - cegaSecret: {{ .Values.global.cega.password | quote | trimall "\"" | b64enc }} - {{- end }} - dbPassword: {{ required "DB password is required" (include "dbPassAuth" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserAuth" .) | b64enc }} +stringData: + config.yaml: |- + auth: + {{- if eq "federated" .Values.global.schemaType }} + cega: + authURL: {{ required "C-EGA NSS host is required" .Values.global.cega.host }} + id: {{ required "C-EGA NSS user is required" .Values.global.cega.user }} + secret: {{ required "C-EGA NSS password is required" .Values.global.cega.password }} + {{- end }} + {{- if .Values.global.auth.corsOrigins }} + cors: + credentials: {{ .Values.global.auth.corsCreds }} + methods: {{ .Values.global.auth.corsMethods }} + orogins: {{ .Values.global.auth.corsOrigins }} + {{- end }} + infotext: {{ .Values.global.auth.infoText }} + infoURL: {{ .Values.global.auth.infoURL }} + {{- if .Values.global.auth.resignJwt }} + jwt: + issuer: {{ ternary "https://" "http://" .Values.global.tls.enabled}}{{ .Values.global.ingress.hostName.auth }} + privatekey: "{{ template "jwtPath" . }}/{{ .Values.global.auth.jwtKey }}" + signaturealg: {{ .Values.global.auth.jwtAlg }} + tokenttl: {{ .Values.global.auth.jwtTTL }} + {{- end }} + publicfile: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.publicKey }}" + resignjwt: {{ .Values.global.auth.resignJwt }} + s3Inbox: {{ .Values.global.ingress.hostName.s3Inbox }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassAuth" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserAuth" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + {{- if and (ne "" .Values.global.oidc.id) (ne "" .Values.global.oidc.secret) }} + oidc: + id: {{ .Values.global.oidc.id }} + jwkPath: {{ .Values.global.oidc.jwkPath }} + provider: {{ .Values.global.oidc.provider }} + redirectURL: {{ template "authRedirect" .}} + secret: {{ .Values.global.oidc.secret }} + {{- end }} + {{- if .Values.global.tls.enabled}} + server: + cert: {{ template "tlsPath" . }}/tls.crt + key: {{ template "tlsPath" . }}/tls.key + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/doa-deploy.yaml b/charts/sda-svc/templates/doa-deploy.yaml index 49fbd9e515..705495196e 100644 --- a/charts/sda-svc/templates/doa-deploy.yaml +++ b/charts/sda-svc/templates/doa-deploy.yaml @@ -206,7 +206,7 @@ spec: - name: CERT_KEY value: {{ ternary (printf "/etc/ssl/certs/java/tls.key.der" ) (printf "%s/%s" .Values.global.secretsPath .Values.doa.tls.keyFile) (empty .Values.global.pkiService) }} - name: CRYPT4GH_PRIVATE_KEY_PATH - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}" + value: "{{ template "c4ghPath" . }}/{{ (first .Values.global.c4gh.privateKeys).keyName }}" - name: CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH value: "{{ template "c4ghPath" . }}/passphrase" - name: OPENID_CONFIGURATION_URL @@ -257,15 +257,9 @@ spec: containerPort: 8080 protocol: TCP livenessProbe: - tcpSocket: - port: doa - initialDelaySeconds: 120 - periodSeconds: 30 +{{ toYaml .Values.doa.livenessProbe | trim | indent 10 }} readinessProbe: - tcpSocket: - port: doa - initialDelaySeconds: 30 - periodSeconds: 15 +{{ toYaml .Values.doa.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.doa.resources | trim | indent 10 }} volumeMounts: @@ -315,9 +309,19 @@ spec: path: {{ .Values.global.oidc.pubKey }} {{- end }} - name: c4gh-key - secret: - secretName: {{ .Values.global.c4gh.secretName }} + projected: defaultMode: 0440 + sources: + - secret: + name: {{ .Values.global.c4gh.secretName }} + items: + - key: {{ (first .Values.global.c4gh.privateKeys).keyName }} + path: {{ (first .Values.global.c4gh.privateKeys).keyName }} + - secret: + name: {{ template "sda.fullname" . }}-doa + items: + - key: c4ghPassphrase + path: passphrase {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive diff --git a/charts/sda-svc/templates/doa-secrets.yaml b/charts/sda-svc/templates/doa-secrets.yaml index c7fb633a55..9e33d951a3 100644 --- a/charts/sda-svc/templates/doa-secrets.yaml +++ b/charts/sda-svc/templates/doa-secrets.yaml @@ -8,6 +8,7 @@ metadata: name: {{ template "sda.fullname" . }}-doa type: Opaque data: + c4ghPassphrase: {{ (first .Values.global.c4gh.privateKeys).passphrase }} dbPassword: {{ include "dbPassDoa" . | b64enc }} {{- if and .Values.global.doa.outbox.enabled }} mqPassword: {{ include "mqPassDoa" . | b64enc }} diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index 9b1a2134eb..90650a6f05 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -69,181 +69,37 @@ spec: type: "RuntimeDefault" command: ["sda-download"] env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if .Values.global.archive.s3CaFile }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/{{ .Values.global.archive.s3CaFile }} - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: OIDC_CONFIGURATION_URL - value: "{{ .Values.global.oidc.provider | trimSuffix "/" }}/.well-known/openid-configuration" - {{- if .Values.global.download.trusted.iss }} - - name: OIDC_TRUSTED_ISS - value: {{ include "trustedIssPath" . }}/{{ default "iss.json" .Values.global.download.trusted.configFile }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: DB_SSLMODE - value: {{ .Values.global.db.sslMode | quote }} - {{- else }} - - name: DB_SSLMODE - value: "disable" - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - {{- if .Values.global.tls.enabled }} - - name: GRPC_CACERT - value: {{ include "tlsPath" . }}/ca.crt - - name: GRPC_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: GRPC_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - - name: GRPC_HOST - value: {{ required "A valid grpc host is required" .Values.global.reencrypt.host | quote }} - - name: GRPC_PORT - value: {{ .Values.global.reencrypt.port | quote }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} - - name: C4GH_TRANSIENTKEYPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }} - - name: C4GH_TRANSIENTPASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the transient c4gh key is required" .Values.global.download.serveDecrypted.secretName }} - key: passphrase - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: APP_PORT - value: "8443" - - name: APP_SERVERCERT - value: "{{ template "tlsPath" . }}/tls.crt" - - name: APP_SERVERKEY - value: "{{ template "tlsPath" . }}/tls.key" - {{- else }} - - name: SESSION_SECURE - value: "false" - {{- end }} - - name: SESSION_DOMAIN - value: {{ .Values.global.ingress.hostName.download | quote }} - - name: SESSION_EXPIRATION - value: {{ .Values.global.download.sessionExpiration | quote }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-download - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-download - key: dbUser - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} ports: - name: download containerPort: {{ ternary 8443 8080 .Values.global.tls.enabled }} protocol: TCP livenessProbe: - httpGet: - port: download - path: /health - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.download }} - initialDelaySeconds: 20 - periodSeconds: 10 +{{ toYaml .Values.download.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - port: download - path: /health - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.download }} - initialDelaySeconds: 20 - periodSeconds: 10 - startupProbe: - httpGet: - path: /health - port: download - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled) }} - httpHeaders: - - name: Host - value: {{ .Values.global.ingress.hostName.download }} - failureThreshold: 30 - periodSeconds: 10 +{{ toYaml .Values.download.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.download.resources | trim | indent 10 }} volumeMounts: - {{- if not .Values.global.vaultSecrets }} + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} - name: c4gh-transient mountPath: {{ template "c4ghPath" . }} {{- end }} - name: iss mountPath: {{ template "trustedIssPath" . }} - {{- end }} - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + {{- end }} + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} - {{- end }} - {{- if eq "posix" .Values.global.archive.storageType }} + {{- end }} + {{- if eq "posix" .Values.global.archive.storageType }} - name: archive mountPath: {{ .Values.global.archive.volumePath | quote }} - {{- end }} + {{- end }} volumes: {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls @@ -258,7 +114,11 @@ spec: {{- end }} {{- end }} {{- if not .Values.global.vaultSecrets }} - {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-download + {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} - name: c4gh-transient secret: defaultMode: 0440 @@ -266,7 +126,7 @@ spec: items: - key: {{ .Values.global.download.serveDecrypted.c4ghKeyFile }} path: {{ .Values.global.download.serveDecrypted.c4ghKeyFile }} - {{- end }} + {{- end }} - name: iss secret: defaultMode: 0440 diff --git a/charts/sda-svc/templates/download-secrets.yaml b/charts/sda-svc/templates/download-secrets.yaml index 463811cfe4..f649ae85ee 100644 --- a/charts/sda-svc/templates/download-secrets.yaml +++ b/charts/sda-svc/templates/download-secrets.yaml @@ -5,10 +5,70 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-download type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassDownload" .) | b64enc }} - c4ghPassphrase: {{ .Values.global.c4gh.passphrase | b64enc }} - dbUser: {{ required "MQ user is required" (include "dbUserDownload" .) | b64enc }} +stringData: + config.yaml: |- + app: + {{- if .Values.global.tls.enabled }} + port: 8443 + serverCert: {{ template "tlsPath" . }}/tls.crt + serverKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + {{- if .Values.global.download.serveDecrypted.c4ghKeyFile }} + c4gh: + transientKeyPath: {{ template "c4ghPath" . }}/{{ .Values.global.download.serveDecrypted.c4ghKeyFile }} + transientKeyPassphrase: {{ .Values.global.download.serveDecrypted.passphrade }} + {{- end }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB pass is required" (include "dbPassDownload" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserDownload" .) }} + grpc: + host: {{ required "A valid grpc host is required" .Values.global.reencrypt.host }} + port: {{ ternary 50443 50051 .Values.global.tls.enabled }} + timeout: {{ default 30 .Values.global.reencrypt.timeout }} + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + oidc: + configuration: + url: {{ required "A oidc provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/.well-known/openid-configuration + trusted: + iss: {{ include "trustedIssPath" . }}/{{ default "iss.json" .Values.global.download.trusted.configFile }} + session: + domain: {{ .Values.global.ingress.hostName.download }} + expiration: {{ .Values.global.download.sessionExpiration }} + secure: {{ ternary true false .Values.global.tls.enabled }} --- apiVersion: v1 kind: Secret diff --git a/charts/sda-svc/templates/finalize-deploy.yaml b/charts/sda-svc/templates/finalize-deploy.yaml index 77530022ef..068f8752ad 100644 --- a/charts/sda-svc/templates/finalize-deploy.yaml +++ b/charts/sda-svc/templates/finalize-deploy.yaml @@ -71,166 +71,15 @@ spec: seccompProfile: type: "RuntimeDefault" env: -{{- if .Values.global.backupArchive.storageType }} - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: BACKUP_TYPE - {{- if eq "s3" .Values.global.backupArchive.storageType }} - value: "s3" - - name: BACKUP_URL - value: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }} - {{- if .Values.global.backupArchive.s3Port }} - - name: BACKUP_PORT - value: {{ .Values.global.backupArchive.s3Port | quote }} - {{- end }} - - name: BACKUP_BUCKET - value: {{ required "S3 backup archive bucket missing" .Values.global.backupArchive.s3Bucket }} - - name: BACKUP_REGION - value: {{ default "us-east-1" .Values.global.backupArchive.s3Region }} - - name: BACKUP_CHUNKSIZE - value: {{ .Values.global.backupArchive.s3ChunkSize | quote }} - {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} - - name: BACKUP_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: BACKUP_LOCATION - value: "{{ .Values.global.backupArchive.volumePath }}" - {{- end }} -{{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: "accession" - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_ROUTINGKEY - value: "completed" - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "sda" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "federated" .Values.global.schemaType }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - {{- if eq "s3" .Values.global.backupArchive.storageType }} - - name: BACKUP_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3backup-keys - key: s3BackupAccessKey - - name: BACKUP_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3backup-keys - key: s3BackupSecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-finalize - key: dbUser - {{ else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.finalize.resources | trim | indent 10 }} volumeMounts: + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} @@ -244,6 +93,12 @@ spec: mountPath: {{ .Values.global.backupArchive.volumePath | quote }} {{- end }} volumes: + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-finalize + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} diff --git a/charts/sda-svc/templates/finalize-secrets.yaml b/charts/sda-svc/templates/finalize-secrets.yaml index 6fb4d350bb..c8d92e3a2a 100644 --- a/charts/sda-svc/templates/finalize-secrets.yaml +++ b/charts/sda-svc/templates/finalize-secrets.yaml @@ -6,10 +6,86 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-finalize type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassFinalize" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserFinalize" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassFinalize" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserFinalize" .) | b64enc }} +stringData: + config.yaml: |- + {{- if .Values.global.backupArchive.storageType }} + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + backup: + type: {{ .Values.global.backupArchive.storageType }} + {{- if eq "s3" .Values.global.backupArchive.storageType }} + accessKey: {{ required "Accesskey required for backup archive" .Values.global.backupArchive.s3AccessKey }} + bucket: {{ required "S3 backup archive bucket not set" .Values.global.backupArchive.s3Bucket }} + {{- if and .Values.global.backupArchive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.backupArchive.s3ChunkSize }} + {{- if .Values.global.backupArchive.s3Port }} + port: {{ .Values.global.backupArchive.s3Port }} + {{- end }} + readypath: {{ .Values.global.backupArchive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for backup archive" .Values.global.backupArchive.s3SecretKey }} + url: {{ required "S3 backup archive URL missing" .Values.global.backupArchive.s3Url }} + {{- else }} + location: /backup/ + {{- end }} + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassFinalize" .) }} + queue: {{ default "accession" .Values.global.broker.finalizeQueue }} + routingKey: {{ default "completed" .Values.global.broker.finalizeRoutingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserFinalize" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassFinalize" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserFinalize" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/ingest-deploy.yaml b/charts/sda-svc/templates/ingest-deploy.yaml index 9f2fb49c2c..90de9d4674 100644 --- a/charts/sda-svc/templates/ingest-deploy.yaml +++ b/charts/sda-svc/templates/ingest-deploy.yaml @@ -72,168 +72,14 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}/" - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: "ingest" - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_ROUTINGKEY - value: "archived" - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: C4GH_FILEPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - - name: INBOX_TYPE - {{- if eq "s3" .Values.global.inbox.storageType }} - value: "s3" - - name: INBOX_BUCKET - value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - - name: INBOX_REGION - value: {{ default "us-east-1" .Values.global.inbox.s3Region }} - - name: INBOX_URL - value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- else }} - value: "posix" - - name: INBOX_LOCATION - value: "{{ .Values.global.inbox.path }}/" - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-ingest - key: dbUser - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - {{- if eq "s3" .Values.global.inbox.storageType }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - {{- end }} - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.ingest.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} @@ -251,13 +97,14 @@ spec: {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-ingest - name: c4gh secret: defaultMode: 0440 secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls diff --git a/charts/sda-svc/templates/ingest-secrets.yaml b/charts/sda-svc/templates/ingest-secrets.yaml index 74bb2d085f..d08426f88b 100644 --- a/charts/sda-svc/templates/ingest-secrets.yaml +++ b/charts/sda-svc/templates/ingest-secrets.yaml @@ -6,11 +6,86 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-ingest type: Opaque -data: - c4ghPassphrase: {{ .Values.global.c4gh.passphrase | b64enc }} - dbPassword: {{ required "DB password is required" (include "dbPassIngest" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserIngest" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassIngest" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserIngest" .) | b64enc }} +stringData: + config.yaml: |- + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassIngest" .) }} + queue: {{ default "ingest" .Values.global.broker.ingestQueue }} + routingKey: {{ default "archived" .Values.global.broker.routingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserIngest" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + c4gh: + privateKeys: + {{- range $k := .Values.global.c4gh.privateKeys }} + - filePath: {{ template "c4ghPath" $ }}/{{ $k.keyName }} + passphrase: {{ $k.passphrase }} + {{- end }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassIngest" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserIngest" .) }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/intercept-deploy.yaml b/charts/sda-svc/templates/intercept-deploy.yaml index 5c18e9d188..062ae95fb5 100644 --- a/charts/sda-svc/templates/intercept-deploy.yaml +++ b/charts/sda-svc/templates/intercept-deploy.yaml @@ -58,74 +58,36 @@ spec: {{- toYaml .Values.global.extraSecurityContext | nindent 10 -}} {{- end }} env: - {{- if not .Values.global.vaultSecrets }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-intercept - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-intercept - key: mqUser - {{- else }} - name: CONFIGFILE value: {{ template "confFile" . }} - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange | quote}} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_QUEUE - value: "from_cega" - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} resources: {{ toYaml .Values.intercept.resources | trim | indent 10 }} - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + {{- if not .Values.global.vaultSecrets }} volumeMounts: + - name: config + mountPath: {{ template "configPath" . }} + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} + {{- end }} volumes: - {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} + - name: config + projected: + sources: + - secret: + name: {{ template "sda.fullname" . }}-intercept + {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls - {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} + {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} secret: defaultMode: 0440 secretName: {{ template "sda.fullname" . }}-intercept-certs - {{- else }} + {{- else }} secret: defaultMode: 0440 secretName: {{ required "An certificate issuer or a TLS secret name is required for intercept" .Values.intercept.tls.secretName }} + {{- end }} {{- end }} - {{- end }} {{- end }} restartPolicy: Always {{- end }} diff --git a/charts/sda-svc/templates/intercept-secrets.yaml b/charts/sda-svc/templates/intercept-secrets.yaml index bd9295972e..71d65ad89f 100644 --- a/charts/sda-svc/templates/intercept-secrets.yaml +++ b/charts/sda-svc/templates/intercept-secrets.yaml @@ -7,9 +7,34 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-intercept type: Opaque -data: - mqPassword: {{ required "MQ password is required" (include "mqPassInterceptor" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserInterceptor" .) | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassInterceptor" .) }} + queue: "from_cega" + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserInterceptor" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/job-remove-user-name.yaml b/charts/sda-svc/templates/job-remove-user-name.yaml new file mode 100644 index 0000000000..650b959e45 --- /dev/null +++ b/charts/sda-svc/templates/job-remove-user-name.yaml @@ -0,0 +1,68 @@ +{{- if .Values.jobs.removeUserName }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sda.fullname" . }}-remove-user-name-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "0" + helm.sh/hook-delete-policy: hook-succeeded +spec: + template: + metadata: + name: {{ template "sda.fullname" . }}-remove-user-name-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + spec: + containers: + - name: {{ template "sda.fullname" . }}-remove-user-name-job + command: ["/usr/local/bin/psql"] + args: [ + "-qc", + "UPDATE sda.files SET submission_file_path = REPLACE(submission_file_path, CONCAT(REPLACE(submission_user, '@', '_'),'/'), '');" + ] + env: + - name: PGDATABASE + value: {{ .Values.global.db.name }} + - name: PGHOST + value: {{ .Values.global.db.host }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- if .Values.global.db.port }} + - name: PGPORT + value: {{ .Values.global.db.port | quote }} + {{- end }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + image: {{ .Values.jobs.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + restartPolicy: Never + securityContext: + runAsUser: 72 + runAsGroup: 72 + fsGroup: 72 +{{- end }} diff --git a/charts/sda-svc/templates/job-set-missing-keyhash.yaml b/charts/sda-svc/templates/job-set-missing-keyhash.yaml new file mode 100644 index 0000000000..24bb4de82c --- /dev/null +++ b/charts/sda-svc/templates/job-set-missing-keyhash.yaml @@ -0,0 +1,92 @@ +{{- if .Values.jobs.setKeyHash }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sda.fullname" . }}-set-key-hash-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: post-install,post-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: hook-succeeded +spec: + template: + metadata: + name: {{ template "sda.fullname" . }}-set-key-hash-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + spec: + containers: + - name: {{ template "sda.fullname" . }}-set-key-hash-job + command: ["/usr/local/bin/psql"] + args: [ + "-qf", + "/migratedb/set-key-hash.sql" + ] + env: + - name: PGDATABASE + value: {{ .Values.global.db.name }} + - name: PGHOST + value: {{ .Values.global.db.host }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- if .Values.global.db.port }} + - name: PGPORT + value: {{ .Values.global.db.port | quote }} + {{- end }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + image: {{ .Values.jobs.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumeMounts: + - name: sql + mountPath: /migratedb + restartPolicy: Never + securityContext: + runAsUser: 72 + runAsGroup: 72 + fsGroup: 72 + volumes: + - name: sql + configMap: + name: set-key-hash +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: set-key-hash +data: + set-key-hash.sql: | + DO + $$ + BEGIN + IF (select max(version) from sda.dbschema_version) > 11 then + RAISE NOTICE 'Fixing c4gh key references'; + UPDATE sda.files SET key_hash = (SELECT key_hash FROM sda.encryption_keys WHERE deprecated_at IS NULL ORDER BY created_at ASC LIMIT 1) + WHERE header IS NOT NULL AND key_hash IS NULL; + END IF; + END + $$ +{{- end }} diff --git a/charts/sda-svc/templates/job-upgrade-from-v1.yaml b/charts/sda-svc/templates/job-upgrade-from-v1.yaml new file mode 100644 index 0000000000..0baab227cb --- /dev/null +++ b/charts/sda-svc/templates/job-upgrade-from-v1.yaml @@ -0,0 +1,115 @@ +{{- if .Values.jobs.upgradeFromV1 }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "sda.fullname" . }}-upgrade-from-v1-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} + release: {{ .Release.Name }} + annotations: + # This is what defines this resource as a hook. + # Without this line, the job is considered part of the release. + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-weight: "-5" + helm.sh/hook-delete-policy: hook-succeeded +spec: + template: + metadata: + name: {{ template "sda.fullname" . }}-upgrade-from-v1-job + labels: + app.kubernetes.io/managed-by: {{ .Release.Service | quote }} + app.kubernetes.io/instance: {{ .Release.Name | quote }} + helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + release: {{ .Release.Name }} + spec: + containers: + - name: {{ template "sda.fullname" . }}-upgrade-from-v1-job + command: ["/usr/local/bin/psql"] + args: [ + "-qf", + "/migratedb/upgrade-from-v1.sql" + ] + env: + - name: PGDATABASE + value: {{ .Values.global.db.name }} + - name: PGHOST + value: {{ .Values.global.db.host }} + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "password" .Values.global.db.admin.passKey }} + {{- if .Values.global.db.port }} + - name: PGPORT + value: {{ .Values.global.db.port | quote }} + {{- end }} + - name: PGUSER + valueFrom: + secretKeyRef: + name: {{ required "a secret holding the DB admin credentials are needed" .Values.global.db.admin.secretName }} + key: {{ default "username" .Values.global.db.admin.userKey }} + image: {{ .Values.jobs.image }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + runAsNonRoot: true + seccompProfile: + type: "RuntimeDefault" + volumeMounts: + - name: sql + mountPath: /migratedb + restartPolicy: Never + securityContext: + runAsUser: 72 + runAsGroup: 72 + fsGroup: 72 + volumes: + - name: sql + configMap: + name: upgrade-from-v1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: upgrade-from-v1 +data: + upgrade-from-v1.sql: | + DO + $$ + BEGIN + IF (select max(version) from sda.dbschema_version) = 11 then + RAISE NOTICE 'Doing migration from schema version 11 to 17'; + + INSERT INTO sda.dbschema_version VALUES(12, now(), 'Add key hash table'); + CREATE TABLE IF NOT EXISTS sda.encryption_keys ( + key_hash TEXT PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(), + deprecated_at TIMESTAMP WITH TIME ZONE, + description TEXT + ); + ALTER TABLE sda.files ADD COLUMN IF NOT EXISTS key_hash TEXT, + ADD CONSTRAINT fk_files_key_hash FOREIGN KEY (key_hash) REFERENCES sda.encryption_keys(key_hash); + + INSERT INTO sda.dbschema_version VALUES(14, now(), 'Add userinfo table'); + CREATE TABLE IF NOT EXISTS sda.userinfo ( + id TEXT PRIMARY KEY, + name TEXT, + email TEXT, + groups TEXT[] + ); + + INSERT INTO sda.dbschema_version VALUES(17, now(), 'Add submission user to constraint'); + ALTER TABLE sda.files DROP CONSTRAINT unique_ingested; + ALTER TABLE sda.files ADD CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user); + + ELSE + RAISE NOTICE 'Schema migration from 11 does not apply now, skipping'; + END IF; + END + $$ +{{- end }} diff --git a/charts/sda-svc/templates/mapper-deploy.yaml b/charts/sda-svc/templates/mapper-deploy.yaml index 91e41c2de1..7c89f09343 100644 --- a/charts/sda-svc/templates/mapper-deploy.yaml +++ b/charts/sda-svc/templates/mapper-deploy.yaml @@ -54,133 +54,30 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: "mappings" - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - - name: INBOX_TYPE - {{- if eq "s3" .Values.global.inbox.storageType }} - value: "s3" - - name: INBOX_BUCKET - value: {{ required "S3 inbox bucket missing" .Values.global.inbox.s3Bucket }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - - name: INBOX_REGION - value: {{ default "us-east-1" .Values.global.inbox.s3Region }} - - name: INBOX_URL - value: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- else }} - value: "posix" - - name: INBOX_LOCATION - value: "{{ .Values.global.inbox.path }}/" - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "federated" .Values.global.schemaType }} - {{- if not .Values.global.vaultSecrets }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: mqUser - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-mapper - key: dbUser - {{- if eq "s3" .Values.global.inbox.storageType }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - {{- end }} - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.mapper.resources | trim | indent 10 }} volumeMounts: + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} {{- end }} - {{- if eq "posix" .Values.global.inbox.storageType }} + {{- end }} + {{- if eq "posix" .Values.global.inbox.storageType }} - name: inbox mountPath: {{ .Values.global.inbox.path | quote }} - {{- end }} + {{- end }} volumes: + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-mapper + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} diff --git a/charts/sda-svc/templates/mapper-secrets.yaml b/charts/sda-svc/templates/mapper-secrets.yaml index 3252c43f9c..c66ed0dc15 100644 --- a/charts/sda-svc/templates/mapper-secrets.yaml +++ b/charts/sda-svc/templates/mapper-secrets.yaml @@ -6,10 +6,65 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-mapper type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassMapper" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserMapper" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassMapper" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserMapper" .) | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassMapper" .) }} + queue: {{ default "mappings" .Values.global.broker.finalizeQueue }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserMapper" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassMapper" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserMapper" .) }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/re-encrypt-deploy.yaml b/charts/sda-svc/templates/re-encrypt-deploy.yaml index a5d30a185d..ee7eb02134 100644 --- a/charts/sda-svc/templates/re-encrypt-deploy.yaml +++ b/charts/sda-svc/templates/re-encrypt-deploy.yaml @@ -51,41 +51,21 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: C4GH_FILEPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} - {{- if not .Values.global.vaultSecrets }} - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} - {{- if .Values.global.tls.enabled}} - - name: GRPC_CACERT - value: {{ template "tlsPath" . }}/ca.crt - - name: GRPC_SERVERCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: GRPC_SERVERKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} ports: - name: grpc containerPort: {{ ternary 50443 50051 ( .Values.global.tls.enabled ) }} - name: grpchealth containerPort: {{add ( ternary 50443 50051 ( .Values.global.tls.enabled ) ) 1 }} readinessProbe: - initialDelaySeconds: 5 - timeoutSeconds: 2 - grpc: - port: {{add ( ternary 50443 50051 ( .Values.global.tls.enabled ) ) 1 }} - service: "reencrypt.Reencrypt" +{{ toYaml .Values.reencrypt.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.reencrypt.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} @@ -95,13 +75,14 @@ spec: {{- end }} volumes: {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-reencrypt - name: c4gh secret: defaultMode: 0440 - secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} + secretName: {{ required "A secret for the c4gh keys is required" .Values.global.c4gh.secretName }} {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls diff --git a/charts/sda-svc/templates/reencrypt-secrets.yaml b/charts/sda-svc/templates/reencrypt-secrets.yaml new file mode 100644 index 0000000000..fbf718c186 --- /dev/null +++ b/charts/sda-svc/templates/reencrypt-secrets.yaml @@ -0,0 +1,27 @@ +{{- if or (or (eq "all" .Values.global.deploymentType) (eq "internal" .Values.global.deploymentType) ) (not .Values.global.deploymentType)}} +{{- if not .Values.global.vaultSecrets }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "sda.fullname" . }}-reencrypt +type: Opaque +stringData: + config.yaml: |- + c4gh: + privateKeys: + {{- range $k := .Values.global.c4gh.privateKeys }} + - filePath: {{ template "c4ghPath" $ }}/{{ $k.keyName }} + passphrase: {{ $k.passphrase }} + {{- end }} + {{- if .Values.global.tls.enabled }} + grpc: + caCert: {{ template "tlsPath" . }}/ca.crt + serverCert: {{ template "tlsPath" . }}/tls.crt + serverKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} +{{- end }} +{{- end }} diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index bf97b7faa4..5a34704320 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -90,142 +90,16 @@ spec: {{- toYaml .Values.global.extraSecurityContext | nindent 10 -}} {{- end }} env: - {{- if not .Values.global.vaultSecrets }} - - name: INBOX_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxAccessKey - - name: INBOX_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3inbox-keys - key: s3InboxSecretKey - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-inbox - key: mqPassword - - name: BROKER_USER - value: {{ include "mqUserInbox" . | quote }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-inbox - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-inbox - key: dbUser - {{- else }} - - name: SERVER_CONFFILE + - name: CONFIGFILE value: {{ include "confFile" .}} - {{- end }} - - name: INBOX_URL - value: {{ .Values.global.inbox.s3Url | quote }} - {{- if .Values.global.inbox.s3Port }} - - name: INBOX_PORT - value: {{ .Values.global.inbox.s3Port | quote }} - {{- end }} - {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} - - name: INBOX_CACERT - value: "{{ include "tlsPath" . }}/ca.crt" - {{- end }} - {{- if .Values.global.inbox.s3Region }} - - name: INBOX_REGION - value: {{ .Values.global.inbox.s3Region | quote }} - {{- end }} - - name: INBOX_BUCKET - value: {{ .Values.global.inbox.s3Bucket | quote }} - {{- if .Values.global.inbox.s3ReadyPath }} - - name: INBOX_READYPATH - value: {{ .Values.global.inbox.s3ReadyPath }} - {{- end }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_VHOST - value: {{ include "brokerVhost" . | quote }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_ROUTINGKEY - value: "inbox" - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.tls.enabled }} - - name: SERVER_CERT - value: {{ include "tlsPath" . }}/tls.crt - - name: SERVER_KEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- if .Values.global.auth.jwtPub }} - - name: SERVER_JWTPUBKEYPATH - value: {{ include "jwtPath" . }} - {{- end }} - {{- if .Values.global.auth.resignJwt }} - {{- $_ := required "The signing pub key is required for s3inbox when auth resigns tokens" .Values.global.auth.jwtPub }} - {{- else }} - - name: SERVER_JWTPUBKEYURL - value: {{ .Values.global.oidc.provider }}{{ .Values.global.oidc.jwkPath }} - {{- end }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} ports: - name: inbox containerPort: 8000 protocol: TCP livenessProbe: - httpGet: - path: /health - port: inbox - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - failureThreshold: 1 - periodSeconds: 10 +{{ toYaml .Values.s3Inbox.livenessProbe | trim | indent 10 }} readinessProbe: - httpGet: - path: /health - port: inbox - scheme: {{ ternary "HTTPS" "HTTP" ( .Values.global.tls.enabled ) }} - failureThreshold: 1 - periodSeconds: 5 +{{ toYaml .Values.s3Inbox.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.s3Inbox.resources | trim | indent 10 }} volumeMounts: @@ -233,9 +107,13 @@ spec: - name: tls mountPath: {{ include "tlsPath" . }} {{- end }} - {{- if .Values.global.auth.jwtPub }} + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} + {{- if or .Values.global.auth.jwtPub .Values.global.oidc.jwtPub }} - name: jwt mountPath: {{ include "jwtPath" . }} + {{- end }} {{- end }} volumes: {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} @@ -250,15 +128,18 @@ spec: secretName: {{ required "An certificate issuer or a TLS secret name is required for s3inbox" .Values.s3Ibox.tls.secretName }} {{- end }} {{- end }} - {{- if .Values.global.auth.jwtPub }} + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-inbox + {{- if or .Values.global.auth.jwtPub .Values.global.oidc.jwtPub }} - name: jwt - projected: - sources: - - secret: - name: {{ .Values.global.auth.jwtSecret }} - items: - - key: {{ required "The name of the JWT signing key is needed" .Values.global.auth.jwtPub }} - path: "{{ .Values.global.ingress.hostName.auth }}.pub" + secret: + defaultMode: 0440 + secretName: {{ ternary .Values.global.auth.jwtSecret .Values.global.oidc.jwtSecret .Values.global.auth.resignJwt }} + + {{- end }} {{- end }} restartPolicy: Always {{- end }} diff --git a/charts/sda-svc/templates/s3-inbox-secrets.yaml b/charts/sda-svc/templates/s3-inbox-secrets.yaml index ab17a02507..340a6a7f86 100644 --- a/charts/sda-svc/templates/s3-inbox-secrets.yaml +++ b/charts/sda-svc/templates/s3-inbox-secrets.yaml @@ -6,11 +6,79 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-inbox type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassInbox" .) | quote | trimall "\"" | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserInbox" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassInbox" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserInbox" .) | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassInbox" .) }} + routingKey: {{ default "inbox" .Values.global.broker.inboxRoutingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserInbox" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassInbox" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserInbox" .) }} + inbox: + type: {{ required "Inbox storage type is not set" .Values.global.inbox.storageType }} + {{- if eq "s3" .Values.global.inbox.storageType }} + accessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey }} + bucket: {{ required "S3 inbox bucket not set" .Values.global.inbox.s3Bucket }} + {{- if and .Values.global.inbox.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.inbox.s3ChunkSize }} + {{- if .Values.global.inbox.s3Port }} + port: {{ .Values.global.inbox.s3Port }} + {{- end }} + readypath: {{ .Values.global.inbox.s3ReadyPath }} + secretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey }} + url: {{ required "S3 inbox URL missing" .Values.global.inbox.s3Url }} + {{- else }} + location: /inbox/ + {{- end }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + schema: + type: {{ default "federated" .Values.global.schemaType }} + server: + {{- if .Values.global.tls.enabled }} + cert: {{ include "tlsPath" . }}/tls.crt + key: {{ include "tlsPath" . }}/tls.key + {{- end }} + {{- if or .Values.global.oidc.jwtPub .Values.global.auth.resignJwt }} + jwtpubkeypath: {{ include "jwtPath" . }} + {{- if .Values.global.auth.resignJwt }} + {{- $_ := required "The signing pub key is required for s3inbox when auth resigns tokens" .Values.global.auth.jwtPub }} + {{- end }} + {{- else }} + jwtpubkeyurl: {{ required "An OIDC provider is required" .Values.global.oidc.provider | trimSuffix "/" }}/{{ required "An OIDC JWK path is required" .Values.global.oidc.jwkPath | trimPrefix "/" }} + {{- end }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/sftp-inbox-deploy.yaml b/charts/sda-svc/templates/sftp-inbox-deploy.yaml index 5d05514697..8dcd2528e0 100644 --- a/charts/sda-svc/templates/sftp-inbox-deploy.yaml +++ b/charts/sda-svc/templates/sftp-inbox-deploy.yaml @@ -154,21 +154,9 @@ spec: containerPort: 2222 protocol: TCP livenessProbe: - exec: - command: - - /bin/sh - - -c - - netstat -an | grep -q ':2222 ' - initialDelaySeconds: 120 - periodSeconds: 5 +{{ toYaml .Values.sftpInbox.livenessProbe | trim | indent 10 }} readinessProbe: - exec: - command: - - /bin/sh - - -c - - netstat -an | grep -q ':2222 ' - initialDelaySeconds: 30 - periodSeconds: 5 +{{ toYaml .Values.sftpInbox.readinessProbe | trim | indent 10 }} volumeMounts: - name: inbox mountPath: "/ega/inbox" diff --git a/charts/sda-svc/templates/shared-secrets.yaml b/charts/sda-svc/templates/shared-secrets.yaml index 49dfb9adb5..14fd283bdc 100644 --- a/charts/sda-svc/templates/shared-secrets.yaml +++ b/charts/sda-svc/templates/shared-secrets.yaml @@ -32,4 +32,22 @@ data: s3InboxAccessKey: {{ required "Accesskey required for inbox" .Values.global.inbox.s3AccessKey | quote | trimall "\"" | b64enc }} s3InboxSecretKey: {{ required "Secretkey required for inbox" .Values.global.inbox.s3SecretKey | quote | trimall "\"" | b64enc }} {{- end }} +{{- if and .Values.global.c4gh.publicKeyData (first .Values.global.c4gh.privateKeys).keyData }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.global.c4gh.secretName }} +type: Opaque +data: + {{ .Values.global.c4gh.publicKey }}: {{ .Values.global.c4gh.publicKeyData }} +{{- range $pk := .Values.global.c4gh.privateKeys }} + {{- if and $pk.keyName $pk.keyData $pk.passphrase }} + {{ $pk.keyName }}: {{ $pk.keyData }} + {{- end }} +{{- end }} + {{- if and .Values.global.c4gh.syncPubKey .Values.global.c4gh.syncPubKeyData }} + {{ .Values.global.c4gh.syncPubKey }}: {{ .Values.global.c4gh.syncPubKeyData }} + {{- end }} +{{- end }} {{- end }} diff --git a/charts/sda-svc/templates/sync-deploy.yaml b/charts/sda-svc/templates/sync-deploy.yaml index 6e4bec1ddb..536f41e6ef 100644 --- a/charts/sda-svc/templates/sync-deploy.yaml +++ b/charts/sda-svc/templates/sync-deploy.yaml @@ -73,184 +73,16 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: SYNC_CENTERPREFIX - value: {{ .Values.global.sync.centerPrefix}} - - name: SYNC_DESTINATION_TYPE - {{- if eq "s3" .Values.global.sync.destination.storageType }} - value: "s3" - - name: SYNC_DESTINATION_URL - value: {{ required "S3 sync archive URL missing" .Values.global.sync.destination.url }} - {{- if .Values.global.sync.destination.port }} - - name: SYNC_DESTINATION_PORT - value: {{ .Values.global.sync.destination.port | quote }} - {{- end }} - - name: SYNC_DESTINATION_BUCKET - value: {{ required "S3 sync archive bucket missing" .Values.global.sync.destination.bucket }} - - name: SYNC_DESTINATION_REGION - value: {{ default "us-east-1" .Values.global.sync.destination.region }} - - name: SYNC_DESTINATION_CHUNKSIZE - value: {{ .Values.global.sync.destination.s3ChunkSize | quote }} - {{- if and .Values.global.sync.destination.s3CaFile .Values.global.tls.enabled }} - - name: SYNC_DESTINATION_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- end }} - - name: C4GH_FILEPATH - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }}" - - name: C4GH_SYNCPUBKEYPATH - value: "{{ template "c4ghPath" . }}/{{ .Values.global.c4gh.syncPubKey }}" - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_QUEUE - value: {{ default "sync" .Values.global.sync.brokerQueue }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "sda" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - - name: SCHEMA_TYPE - value: {{ default "isolated" .Values.global.schemaType }} - - name: SYNC_REMOTE_HOST - value: {{ .Values.global.sync.remote.host }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - {{- if eq "s3" .Values.global.sync.destination.storageType }} - - name: SYNC_DESTINATION_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: s3AccessKey - - name: SYNC_DESTINATION_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: s3SecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: mqUser - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: dbUser - - name: SYNC_REMOTE_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: restUser - - name: SYNC_REMOTE_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-sync - key: restPassword - {{ else }} - - name: CONFIGFILE - value: {{ include "confFile" . }} - {{- end }} + - name: CONFIGFILE + value: {{ include "confFile" . }} resources: {{ toYaml .Values.sync.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} - name: c4gh mountPath: {{ template "c4ghPath" . }} + - name: config + mountPath: {{ template "configPath" . }} {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive @@ -279,10 +111,14 @@ spec: defaultMode: 0440 secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} + - key: {{ (first .Values.global.c4gh.privateKeys).keyName }} + path: {{ (first .Values.global.c4gh.privateKeys).keyName }} - key: {{ .Values.global.c4gh.syncPubKey }} path: {{ .Values.global.c4gh.syncPubKey }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-sync {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive diff --git a/charts/sda-svc/templates/sync-secrets.yaml b/charts/sda-svc/templates/sync-secrets.yaml index c8f9154aa3..a0f79a4fe9 100644 --- a/charts/sda-svc/templates/sync-secrets.yaml +++ b/charts/sda-svc/templates/sync-secrets.yaml @@ -8,18 +8,90 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-sync type: Opaque -data: - c4ghPassphrase: {{ required "Crypt4gh passphrase is required" .Values.global.c4gh.passphrase | b64enc }} - dbPassword: {{ required "DB password is required" (include "dbPassSync" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserSync" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassSync" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserSync" .) | b64enc }} - restPassword: {{ required "REST password is required" .Values.global.sync.remote.password | b64enc }} - restUser: {{ required "REST user is required" .Values.global.sync.remote.user | b64enc }} - {{- if eq "s3" .Values.global.sync.destination.storageType }} - s3AccessKey: {{ required "Accesskey required for sync destination" .Values.global.sync.destination.accessKey | quote | trimall "\"" | b64enc }} - s3SecretKey: {{ required "Secretkey required for sync destination" .Values.global.sync.destination.secretKey | quote | trimall "\"" | b64enc }} - {{- end }} +stringData: + config.yaml: |- + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassMapper" .) }} + queue: {{ default "mapping_stream" .Values.global.sync.brokerQueue }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserMapper" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + c4gh: + filePath: {{ template "c4ghPath" . }}/{{ (first .Values.global.c4gh.privateKeys).keyName }} + passphrase: {{ (first .Values.global.c4gh.privateKeys).passphrase }} + syncPubKeyPath: {{ template "c4ghPath" . }}/{{ required "remote sync public c4gh key is missing" .Values.global.c4gh.syncPubKey }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "MQ password is required" (include "dbPassMapper" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserMapper" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + sync: + centerPrefix: {{ .Values.global.sync.centerPrefix }} + destination: + type: s3 + url: {{ required "S3 sync archive URL missing" .Values.global.sync.destination.url }} + {{- if not (empty .Values.global.sync.destination.port) }} + port: {{ .Values.global.sync.destination.port }} + {{- end }} + {{- if not (empty .Values.global.sync.destination.readypath) }} + readypath: {{ .Values.global.sync.destination.readypath }} + {{- end }} + accessKey: {{ required "Accesskey required for sync destination" .Values.global.sync.destination.accessKey }} + secretKey: {{ required "SecretKey required for sync destination" .Values.global.sync.destination.secretKey }} + bucket: {{ required "S3 sync destination bucket missing" .Values.global.sync.destination.bucket }} + region: {{ default "us-east-1" .Values.global.sync.destination.region }} + remote: + host: {{ required "remote sync API host is required" .Values.global.sync.remote.host }} + password: {{ required "remote sync API password is required" .Values.global.sync.remote.password }} + {{- if not (empty .Values.global.sync.remote.port) }} + port: {{ .Values.global.sync.remote.port }} + {{- end }} + user: {{ required "remote sync API user is required" .Values.global.sync.remote.user }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/syncapi-deploy.yaml b/charts/sda-svc/templates/syncapi-deploy.yaml index 2a9cd16a43..6e0f474a96 100644 --- a/charts/sda-svc/templates/syncapi-deploy.yaml +++ b/charts/sda-svc/templates/syncapi-deploy.yaml @@ -73,100 +73,32 @@ spec: {{- toYaml .Values.global.extraSecurityContext | nindent 10 -}} {{- end }} env: - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: mqUser - - name: SYNC_API_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: password - - name: SYNC_API_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-syncapi - key: user - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.host | quote }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ include "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ include "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ include "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - {{- if .Values.global.tls.enabled }} - - name: SERVER_CERT - value: {{ template "tlsPath" . }}/tls.crt - - name: SERVER_KEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- if .Values.global.sync.api.accessionRouting }} - - name: SYNC_API_ACCESSIONROUTING - value: {{ .Values.global.sync.api.accessionRouting | quote }} - {{- end }} - {{- if .Values.global.sync.api.ingestRouting }} - - name: SYNC_API_INGESTROUTING - value: {{ .Values.global.sync.api.ingestRouting | quote }} - {{- end }} - {{- if .Values.global.sync.api.mappingRouting }} - - name: SYNC_API_MAPPINGROUTING - value: {{ .Values.global.sync.api.mappingRouting | quote }} - {{- end }} ports: - name: syncapi containerPort: 8080 protocol: TCP readinessProbe: - tcpSocket: - port: syncapi - initialDelaySeconds: 10 - periodSeconds: 5 +{{ toYaml .Values.syncapi.readinessProbe | trim | indent 10 }} resources: {{ toYaml .Values.syncapi.resources | trim | indent 10 }} volumeMounts: + {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} + {{- end }} {{- if .Values.global.tls.enabled }} - name: tls mountPath: {{ template "tlsPath" . }} {{- end }} volumes: + {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-syncapi + {{- end }} {{- if and (not .Values.global.pkiService) .Values.global.tls.enabled }} - name: tls {{- if or .Values.global.tls.clusterIssuer .Values.global.tls.issuer }} diff --git a/charts/sda-svc/templates/syncapi-secrets.yaml b/charts/sda-svc/templates/syncapi-secrets.yaml index 78a5bfcf63..3cdc05852c 100644 --- a/charts/sda-svc/templates/syncapi-secrets.yaml +++ b/charts/sda-svc/templates/syncapi-secrets.yaml @@ -8,11 +8,43 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-syncapi type: Opaque -data: - mqPassword: {{ required "MQ password is required" (include "mqPassSync" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserSync" .) | b64enc }} - password: {{ required "Sync API password is required" .Values.global.sync.api.password | b64enc }} - user: {{ required "Sync API username is required" .Values.global.sync.api.user | b64enc }} +stringData: + config.yaml: |- + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassMapper" .) }} + queue: {{ default "mappings" .Values.global.broker.finalizeQueue }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserMapper" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} + {{- if .Values.global.tls.enabled }} + server: + cert: {{ template "tlsPath" . }}/tls.crt + key: {{ template "tlsPath" . }}/tls.key + {{- end }} + sync: + api: + password: {{ required "sync API password is required" .Values.global.sync.api.password }} + user: {{ required "sync API user is required" .Values.global.sync.api.user }} {{- end }} {{- end }} {{- end }} diff --git a/charts/sda-svc/templates/verify-deploy.yaml b/charts/sda-svc/templates/verify-deploy.yaml index fed66d48a0..23f30efe1a 100644 --- a/charts/sda-svc/templates/verify-deploy.yaml +++ b/charts/sda-svc/templates/verify-deploy.yaml @@ -55,136 +55,14 @@ spec: seccompProfile: type: "RuntimeDefault" env: - - name: ARCHIVE_TYPE - {{- if eq "s3" .Values.global.archive.storageType }} - value: "s3" - - name: ARCHIVE_URL - value: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} - {{- if .Values.global.archive.s3Port }} - - name: ARCHIVE_PORT - value: {{ .Values.global.archive.s3Port | quote }} - {{- end }} - - name: ARCHIVE_BUCKET - value: {{ required "S3 archive bucket missing" .Values.global.archive.s3Bucket }} - - name: ARCHIVE_REGION - value: {{ default "us-east-1" .Values.global.archive.s3Region }} - - name: ARCHIVE_CHUNKSIZE - value: {{ .Values.global.archive.s3ChunkSize | quote }} - {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} - - name: ARCHIVE_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- end }} - {{- else }} - value: "posix" - - name: ARCHIVE_LOCATION - value: "{{ .Values.global.archive.volumePath }}" - {{- end }} - - name: BROKER_SSL - value: {{ .Values.global.tls.enabled | quote }} - {{- if .Values.global.tls.enabled }} - - name: BROKER_VERIFYPEER - value: {{ .Values.global.broker.verifyPeer | quote }} - - name: BROKER_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if .Values.global.broker.verifyPeer }} - - name: BROKER_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: BROKER_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: BROKER_EXCHANGE - value: {{ default "sda" .Values.global.broker.exchange }} - - name: BROKER_HOST - value: {{ required "A valid MQ host is required" .Values.global.broker.host | quote }} - - name: BROKER_PORT - value: {{ .Values.global.broker.port | quote }} - - name: BROKER_PREFETCHCOUNT - value: {{ .Values.global.broker.prefetchCount | quote }} - - name: BROKER_QUEUE - value: "archived" - - name: BROKER_ROUTINGKEY - value: "verified" - {{- if .Values.global.broker.serverName }} - - name: BROKER_SERVERNAME - value: {{ .Values.global.broker.serverName | quote }} - {{- end }} - - name: BROKER_VHOST - value: {{ .Values.global.broker.vhost | quote }} - - name: C4GH_FILEPATH - value: {{ template "c4ghPath" . }}/{{ .Values.global.c4gh.keyFile }} - {{- if .Values.global.tls.enabled }} - - name: DB_CACERT - value: {{ template "tlsPath" . }}/ca.crt - {{- if ne "verify-none" .Values.global.db.sslMode }} - - name: DB_CLIENTCERT - value: {{ template "tlsPath" . }}/tls.crt - - name: DB_CLIENTKEY - value: {{ template "tlsPath" . }}/tls.key - {{- end }} - {{- end }} - - name: DB_DATABASE - value: {{ default "lega" .Values.global.db.name | quote }} - - name: DB_HOST - value: {{ required "A valid DB host is required" .Values.global.db.host | quote }} - - name: DB_PORT - value: {{ .Values.global.db.port | quote }} - - name: DB_SSLMODE - value: {{ template "dbSSLmode" . }} - {{- if .Values.global.log.format }} - - name: LOG_FORMAT - value: {{ .Values.global.log.format | quote }} - {{- end }} - {{- if .Values.global.log.level }} - - name: LOG_LEVEL - value: {{ .Values.global.log.level | quote }} - {{- end }} - {{- if not .Values.global.vaultSecrets }} - {{- if eq "s3" .Values.global.archive.storageType }} - - name: ARCHIVE_ACCESSKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveAccessKey - - name: ARCHIVE_SECRETKEY - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-s3archive-keys - key: s3ArchiveSecretKey - {{- end }} - - name: BROKER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: mqPassword - - name: BROKER_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: mqUser - - name: C4GH_PASSPHRASE - valueFrom: - secretKeyRef: - name: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - key: passphrase - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: dbPassword - - name: DB_USER - valueFrom: - secretKeyRef: - name: {{ template "sda.fullname" . }}-verify - key: dbUser - {{- else }} - name: CONFIGFILE value: {{ include "confFile" . }} - {{- end }} resources: {{ toYaml .Values.verify.resources | trim | indent 10 }} volumeMounts: {{- if not .Values.global.vaultSecrets }} + - name: config + mountPath: {{ template "configPath" . }} - name: c4gh mountPath: {{ template "c4ghPath" . }} {{- end }} @@ -210,13 +88,14 @@ spec: {{- end }} {{- end }} {{- if not .Values.global.vaultSecrets }} + - name: config + secret: + defaultMode: 0440 + secretName: {{ template "sda.fullname" . }}-verify - name: c4gh secret: defaultMode: 0440 secretName: {{ required "A secret for the c4gh key is required" .Values.global.c4gh.secretName }} - items: - - key: {{ .Values.global.c4gh.keyFile }} - path: {{ .Values.global.c4gh.keyFile }} {{- end }} {{- if eq "posix" .Values.global.archive.storageType }} - name: archive diff --git a/charts/sda-svc/templates/verify-secrets.yaml b/charts/sda-svc/templates/verify-secrets.yaml index cc3b1fe0bc..7fde154e2d 100644 --- a/charts/sda-svc/templates/verify-secrets.yaml +++ b/charts/sda-svc/templates/verify-secrets.yaml @@ -7,10 +7,70 @@ kind: Secret metadata: name: {{ template "sda.fullname" . }}-verify type: Opaque -data: - dbPassword: {{ required "DB password is required" (include "dbPassVerify" .) | b64enc }} - dbUser: {{ required "DB user is required" (include "dbUserVerify" .) | b64enc }} - mqPassword: {{ required "MQ password is required" (include "mqPassVerify" .) | b64enc }} - mqUser: {{ required "MQ user is required" (include "mqUserVerify" .) | b64enc }} +stringData: + config.yaml: |- + archive: + type: {{ required "Archive storage type is not set" .Values.global.archive.storageType }} + {{- if eq "s3" .Values.global.archive.storageType }} + accessKey: {{ required "Accesskey required for archive" .Values.global.archive.s3AccessKey }} + bucket: {{ required "S3 archive bucket not set" .Values.global.archive.s3Bucket }} + {{- if and .Values.global.archive.s3CaFile .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{ end }} + chunksize: {{ default 15 .Values.global.archive.s3ChunkSize }} + {{- if .Values.global.archive.s3Port }} + port: {{ .Values.global.archive.s3Port }} + {{- end }} + readypath: {{ .Values.global.archive.s3ReadyPath }} + secretKey: {{ required "Secretkey required for archive" .Values.global.archive.s3SecretKey }} + url: {{ required "S3 archive URL missing" .Values.global.archive.s3Url }} + {{- else }} + location: /archive/ + {{- end }} + broker: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + {{- if .Values.global.broker.verifyPeer }} + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + {{- end }} + exchange: {{ default "sda" .Values.global.broker.exchange }} + host: {{ required "A valid MQ host is required" .Values.global.broker.host }} + port: {{ default (ternary 5671 5672 .Values.global.tls.enabled) .Values.global.broker.port }} + prefetchCount: {{ default 1 .Values.global.broker.prefetchCount }} + password: {{ required "MQ password is required" (include "mqPassVerify" .) }} + queue: {{ default "archived" .Values.global.broker.verifyQueue }} + routingKey: {{ default "verified" .Values.global.broker.routingKey }} + {{- if ne "" ( default "" .Values.global.broker.serverName ) }} + serverName: {{.Values.global.broker.serverName }} + {{- end }} + ssl: {{ .Values.global.tls.enabled }} + user: {{ required "MQ user is required" (include "mqUserVerify" .) }} + {{- if .Values.global.tls.enabled }} + verifyPeer: {{ .Values.global.broker.verifyPeer }} + {{- end }} + vhost: {{ include "brokerVhost" . }} + c4gh: + privateKeys: + {{- range $k := .Values.global.c4gh.privateKeys }} + - filePath: {{ template "c4ghPath" $ }}/{{ $k.keyName }} + passphrase: {{ $k.passphrase }} + {{- end }} + db: + {{- if .Values.global.tls.enabled }} + caCert: {{ template "tlsPath" . }}/ca.crt + clientCert: {{ template "tlsPath" . }}/tls.crt + clientKey: {{ template "tlsPath" . }}/tls.key + {{- end }} + host: {{ .Values.global.db.host }} + database: {{ .Values.global.db.name }} + password: {{ required "DB user is required" (include "dbPassVerify" .) }} + port: {{ .Values.global.db.port }} + sslmode: {{ ternary .Values.global.db.sslMode "disable" .Values.global.tls.enabled }} + user: {{ required "DB user is required" (include "dbUserVerify" .) }} + log: + format: {{ .Values.global.log.format }} + level: {{ .Values.global.log.level }} {{- end }} {{- end }} diff --git a/charts/sda-svc/values.yaml b/charts/sda-svc/values.yaml index 096ab8464f..9b8e884237 100644 --- a/charts/sda-svc/values.yaml +++ b/charts/sda-svc/values.yaml @@ -9,7 +9,7 @@ global: # Path where the sensitive files can be found, default is "/.secrets". # TLS certificates or C4GH key locations can be set using global.tlsPath or global.c4ghPath respectively, # this path will be a subpath to the secretsPath. - secretsPath: /.secrets + secretsPath: "/.secrets" c4ghPath: "c4gh" tlsPath: "tls" jwtPath: "jwt" @@ -188,15 +188,22 @@ global: password: "" c4gh: - secretName: "" - # name of the c4gh key file. - keyFile: "" - # name of the public file, required for testing - publicFile: "" - passphrase: "" - backupPubKey: "" - + secretName: + publicKey: + publicKeyData: + privateKeys: + - keyName: + passphrase: + keyData: + syncPubKey: + syncPubKeyData: db: + admin: + # These credentials need at least dbOwner permissions + # in order to perform schema migrations. + secretName: "" + passKey: "" + userKey: "" host: "" name: "sda" user: "" @@ -366,6 +373,21 @@ credentials: api: name: api replicaCount: 2 + livenessProbe: + # httpGet: + # port: api + # path: /ready + # scheme: "HTTPS" + # initialDelaySeconds: 20 + # periodSeconds: 60 + # timeoutSeconds: 30 + readinessProbe: + httpGet: + port: api + path: /ready + scheme: "HTTPS" + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "128Mi" @@ -383,6 +405,22 @@ api: auth: name: auth replicaCount: 2 + livenessProbe: + # httpGet: + # port: auth + # path: / + # scheme: "HTTPS" + # httpHeaders: + # initialDelaySeconds: 20 + # periodSeconds: 60 + # timeoutSeconds: 30 + readinessProbe: + httpGet: + port: auth + path: / + scheme: "HTTPS" + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "128Mi" @@ -399,10 +437,17 @@ auth: doa: name: doa - repository: ghcr.io/neicnordic/sda-doa - imageTag: v1.6.62 imagePullPolicy: IfNotPresent replicaCount: 2 + livenessProbe: + # tcpSocket: + # port: doa + # initialDelaySeconds: 120 + # periodSeconds: 30 + readinessProbe: + tcpSocket: + port: doa + periodSeconds: 15 resources: requests: memory: "256Mi" @@ -423,6 +468,21 @@ doa: download: name: download replicaCount: 2 + livenessProbe: + # httpGet: + # path: /health + # port: download + # scheme: "HTTPS" + # initialDelaySeconds: 20 + # periodSeconds: 60 + # timeoutSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: download + scheme: "HTTPS" + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "256Mi" @@ -472,7 +532,6 @@ ingest: secretName: "" intercept: - deploy: true name: ingest replicaCount: 1 resources: @@ -507,6 +566,12 @@ mapper: reencrypt: replicaCount: 1 + readinessProbe: + periodSeconds: 10 + timeoutSeconds: 2 + grpc: + port: 50443 + service: "reencrypt.Reencrypt" resources: requests: memory: "128Mi" @@ -524,6 +589,22 @@ reencrypt: s3Inbox: name: s3Inbox replicaCount: 2 + livenessProbe: + # httpGet: + # path: /health + # port: inbox + # scheme: "HTTPS" + # failureThreshold: 1 + # periodSeconds: 60 + # timeoutSeconds: 55 + readinessProbe: + httpGet: + path: /health + port: inbox + scheme: "HTTPS" + failureThreshold: 1 + periodSeconds: 10 + timeoutSeconds: 9 resources: requests: memory: "128Mi" @@ -541,6 +622,22 @@ s3Inbox: sftpInbox: name: sftpInbox replicaCount: 2 + livenessProbe: + # exec: + # command: + # - /bin/sh + # - -c + # - netstat -an | grep -q ':2222 ' + # initialDelaySeconds: 120 + # periodSeconds: 30 + # timeoutSeconds: 25 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - netstat -an | grep -q ':2222 ' + periodSeconds: 5 resources: requests: memory: "128Mi" @@ -577,6 +674,10 @@ sync: syncapi: name: sync-api replicaCount: 1 + readinessProbe: + tcpSocket: + port: syncapi + periodSeconds: 5 resources: requests: memory: "64Mi" @@ -604,7 +705,6 @@ releasetest: verify: replicaCount: 1 repository: ghcr.io/biobanklab/sda-pipeline - imageTag: v0.4.27 imagePullPolicy: IfNotPresent resources: requests: @@ -619,3 +719,9 @@ verify: annotations: {} tls: secretName: "" + +jobs: + image: postgres:15.4-alpine + removeUserName: false + setKeyHash: false + upgradeFromV1: false \ No newline at end of file diff --git a/postgresql/initdb.d/04_grants.sql b/postgresql/initdb.d/04_grants.sql index c9778aa4ed..a04fe93f3e 100644 --- a/postgresql/initdb.d/04_grants.sql +++ b/postgresql/initdb.d/04_grants.sql @@ -44,6 +44,7 @@ GRANT INSERT ON sda.file_event_log TO ingest; GRANT SELECT ON sda.file_event_log TO ingest; GRANT USAGE, SELECT ON SEQUENCE sda.file_event_log_id_seq TO ingest; GRANT SELECT ON sda.encryption_keys TO ingest; +GRANT INSERT ON sda.encryption_keys TO ingest; -- legacy schema GRANT USAGE ON SCHEMA local_ega TO ingest; diff --git a/sda-doa/Dockerfile b/sda-doa/Dockerfile index 10ae01974d..e8908b716b 100644 --- a/sda-doa/Dockerfile +++ b/sda-doa/Dockerfile @@ -5,8 +5,6 @@ COPY pom.xml . RUN mkdir -p /root/.m2 && \ mkdir /root/.m2/repository -COPY settings.xml /root/.m2 - RUN mvn dependency:go-offline --no-transfer-progress COPY src/ /src/ diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 5b42709df8..a7d98e50cc 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.5 + 3.5.5 no.uio.ifi @@ -67,18 +67,18 @@ io.jsonwebtoken jjwt-api - 0.12.6 + 0.13.0 io.jsonwebtoken jjwt-impl - 0.12.6 + 0.13.0 runtime io.jsonwebtoken jjwt-jackson - 0.12.6 + 0.13.0 runtime @@ -110,7 +110,7 @@ no.elixir crypt4gh - 3.0.24 + 3.0.33 org.slf4j @@ -121,7 +121,7 @@ no.elixir clearinghouse - 2.0.18 + 3.0.8 org.slf4j @@ -132,17 +132,10 @@ org.yaml snakeyaml - 2.4 + 2.5 - - - github-fega-norway - https://maven.pkg.github.com/ELIXIR-NO/FEGA-Norway - - - diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java index 0ca74c911d..944a2ee073 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/MetadataController.java @@ -13,6 +13,7 @@ import org.springframework.web.bind.annotation.RestController; import jakarta.servlet.http.HttpServletRequest; + import java.util.Set; /** @@ -38,9 +39,14 @@ public class MetadataController { @SuppressWarnings("unchecked") @GetMapping("/datasets") public ResponseEntity datasets() { - log.info("User has permissions to list datasets"); - Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); - return ResponseEntity.ok(metadataService.datasets(datasetIds)); + try { + log.info("User has permissions to list datasets"); + Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); + return ResponseEntity.ok(metadataService.datasets(datasetIds)); + } catch (Exception e) { + log.error("Error in listing datasets", e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error while listing datasets: " + e.getMessage()); + } } /** @@ -52,13 +58,18 @@ public ResponseEntity datasets() { @SuppressWarnings("unchecked") @GetMapping("/datasets/{datasetId}/files") public ResponseEntity files(@PathVariable(value = "datasetId") String datasetId) { - Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); - if (!datasetIds.contains(datasetId)) { - log.info("User doesn't have permissions to list files in the requested dataset: {}", datasetId); - return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); + try { + Set datasetIds = (Set) request.getAttribute(AAIAspect.DATASETS); + if (!datasetIds.contains(datasetId)) { + log.info("User doesn't have permissions to list files in the requested dataset: {}", datasetId); + return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); + } + log.info("User has permissions to list files in the requested dataset: {}", datasetId); + return ResponseEntity.ok(metadataService.files(datasetId)); + } catch (Exception e) { + log.error("Error in listing files in the dataset: {}", datasetId, e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error while listing files in the dataset: " + e.getMessage()); } - log.info("User has permissions to list files in the requested dataset: {}", datasetId); - return ResponseEntity.ok(metadataService.files(datasetId)); } } diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java index 6123f13acb..e4aceccf06 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/rest/StreamingController.java @@ -59,6 +59,9 @@ public ResponseEntity files(@RequestHeader(value = "Public-Key", required = f } catch (AuthException e) { log.info("User doesn't have permissions to download requested file: {}", fileId); return ResponseEntity.status(HttpStatus.FORBIDDEN).build(); + } catch (Exception e) { + log.error("Error in streaming file: {}", fileId, e); + return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR).body("Error in streaming file: " + e.getMessage()); } } diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java index e22b223c7e..916aa88bf6 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java @@ -3,9 +3,9 @@ import com.google.gson.Gson; import com.google.gson.JsonObject; import lombok.extern.slf4j.Slf4j; -import no.uio.ifi.clearinghouse.Clearinghouse; -import no.uio.ifi.clearinghouse.model.Visa; -import no.uio.ifi.clearinghouse.model.VisaType; +import no.elixir.clearinghouse.Clearinghouse; +import no.elixir.clearinghouse.model.Visa; +import no.elixir.clearinghouse.model.VisaType; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index 3aec599b3f..ed680004dd 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -260,7 +260,7 @@ func TestTokenMiddleware_Success_FromCache(t *testing.T) { // nolint:bodyclose for _, c := range w.Result().Cookies() { if c.Name == config.Config.Session.Name { - t.Errorf("TestTokenMiddleware_Success_FromCache failed, got a session cookie, when should not have") + t.Error("TestTokenMiddleware_Success_FromCache failed, got a session cookie, when should not have") } } diff --git a/sda-download/dev_utils/compose-no-tls.yml b/sda-download/dev_utils/compose-no-tls.yml index cb74174a93..de5c402270 100644 --- a/sda-download/dev_utils/compose-no-tls.yml +++ b/sda-download/dev_utils/compose-no-tls.yml @@ -30,7 +30,7 @@ services: interval: 5s timeout: 20s retries: 3 - image: minio/minio:RELEASE.2021-11-24T23-19-33Z + image: minio/minio ports: - "9000:9000" @@ -42,7 +42,7 @@ services: condition: service_healthy entrypoint: > /bin/sh -c " - /usr/bin/mc config host add s3 http://s3:9000 access secretkey; + /usr/bin/mc alias set add s3 http://s3:9000 access secretkey; /usr/bin/mc mb s3/archive; exit 0; " diff --git a/sda-download/dev_utils/compose-sda.yml b/sda-download/dev_utils/compose-sda.yml index 7a619a1123..15a11fe495 100644 --- a/sda-download/dev_utils/compose-sda.yml +++ b/sda-download/dev_utils/compose-sda.yml @@ -51,7 +51,7 @@ services: interval: 5s timeout: 20s retries: 3 - image: minio/minio:RELEASE.2020-06-03T22-13-49Z + image: minio/minio ports: - "9000:9000" volumes: @@ -66,7 +66,7 @@ services: entrypoint: > /bin/sh -c " sleep 10; - /usr/bin/mc config host add s3 https://s3:9000 access secretkey; + /usr/bin/mc alias set s3 https://s3:9000 access secretkey; /usr/bin/mc mb s3/archive; exit 0; " diff --git a/sda-download/dev_utils/compose.yml b/sda-download/dev_utils/compose.yml index 3a491beddc..0f84f7fc2e 100644 --- a/sda-download/dev_utils/compose.yml +++ b/sda-download/dev_utils/compose.yml @@ -54,7 +54,7 @@ services: interval: 5s timeout: 20s retries: 3 - image: minio/minio:RELEASE.2020-06-03T22-13-49Z + image: minio/minio ports: - "9000:9000" volumes: @@ -70,7 +70,7 @@ services: condition: service_healthy entrypoint: > /bin/sh -c " - /usr/bin/mc config host add s3 https://s3:9000 access secretkey; + /usr/bin/mc alias set s3 https://s3:9000 access secretkey; /usr/bin/mc mb s3/archive; exit 0; " diff --git a/sda-download/dev_utils/config-notls.yaml b/sda-download/dev_utils/config-notls.yaml index 13b90a638a..f67d8d7f5a 100644 --- a/sda-download/dev_utils/config-notls.yaml +++ b/sda-download/dev_utils/config-notls.yaml @@ -19,8 +19,11 @@ grpc: port: 50051 c4gh: - passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" + privateKeys: + - filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" db: host: "localhost" diff --git a/sda-download/dev_utils/config.yaml b/sda-download/dev_utils/config.yaml index 062299df62..a75f9bfec3 100644 --- a/sda-download/dev_utils/config.yaml +++ b/sda-download/dev_utils/config.yaml @@ -41,8 +41,11 @@ session: name: "sda_session_key" c4gh: - passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" + privateKeys: + - filepath: "./dev_utils/c4gh.sec.pem" + passphrase: "oaagCP1YgAZeEyl2eJAkHv9lkcWXWFgm" transientPassphrase: "" transientKeyPath: "" diff --git a/sda-download/go.mod b/sda-download/go.mod index 73eae28d65..be1582d707 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -1,50 +1,47 @@ module github.com/neicnordic/sda-download -go 1.24.0 - -toolchain go1.24.1 +go 1.24.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go v1.55.7 + github.com/aws/aws-sdk-go v1.55.8 github.com/dgraph-io/ristretto v1.0.0 - github.com/gin-gonic/gin v1.10.0 + github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.10.9 github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.38.0 - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - google.golang.org/grpc v1.72.0 - google.golang.org/protobuf v1.36.6 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + golang.org/x/crypto v0.42.0 + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.8 ) require ( filippo.io/edwards25519 v1.1.0 // indirect - github.com/bytedance/sonic v1.12.2 // indirect - github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/bytedance/sonic v1.13.2 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.5 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.22.1 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.3 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect @@ -54,27 +51,27 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.12.0 // indirect - github.com/spf13/cast v1.7.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.10.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect - golang.org/x/tools v0.25.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/arch v0.17.0 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/tools v0.36.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index e4383bafcb..ad2264e99f 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -3,20 +3,19 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= -github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= -github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= -github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/aws/aws-sdk-go v1.55.8 h1:JRmEUbU52aJQZ2AjX4q4Wu7t4uZjOu71uyNmaWlUkJQ= +github.com/aws/aws-sdk-go v1.55.8/go.mod h1:ZkViS9AqA6otK+JBBNH2++sx1sgxrPKcSzPPvQkUtXk= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cevatbarisyilmaz/ara v0.0.4 h1:SGH10hXpBJhhTlObuZzTuFn1rrdmjQImITXnZVPSodc= github.com/cevatbarisyilmaz/ara v0.0.4/go.mod h1:BfFOxnUd6Mj6xmcvRxHN3Sr21Z1T3U2MYkYOmoQe4Ts= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -34,16 +33,16 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= -github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -52,16 +51,16 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -75,8 +74,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/sqlstruct v0.0.0-20201105191214-5f3e10d3ab46/go.mod h1:yyMNCyc/Ib3bDTKd379tNMpB/7/H5TjM2Y9QJ5THLbE= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -107,8 +106,8 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U= github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -118,8 +117,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= -github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= @@ -127,17 +126,17 @@ github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df h1:S77Pf5fIG github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df/go.mod h1:dcuzJZ83w/SqN9k4eQqwKYMgmKWzg/KzJAURBhRL1tc= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= -github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -148,8 +147,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= @@ -160,26 +159,26 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= -golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= +golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= @@ -189,8 +188,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -205,38 +204,40 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= -golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= -golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= +golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= +golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= -google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= -google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index 6f18c2ca18..ca8d608130 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -114,7 +114,7 @@ func TestNewDB(t *testing.T) { err := CatchNewDBPanic() if err == nil { - t.Errorf("NewDB did not report error when it should.") + t.Error("NewDB did not report error when it should.") } db, mock, _ := sqlmock.New(sqlmock.MonitorPingsOption(true)) diff --git a/sda-download/internal/storage/seekable.go b/sda-download/internal/storage/seekable.go index 845d150a70..5a6a36156c 100644 --- a/sda-download/internal/storage/seekable.go +++ b/sda-download/internal/storage/seekable.go @@ -186,7 +186,6 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { go r.prefetchAt(r.currentOffset) return offset, nil - case io.SeekCurrent: if r.currentOffset+offset < 0 { return r.currentOffset, fmt.Errorf("invalid offset %v from %v would be be before start", offset, r.currentOffset) @@ -199,7 +198,6 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { go r.prefetchAt(r.currentOffset) return r.currentOffset, nil - case io.SeekEnd: if r.objectSize+offset < 0 { return r.currentOffset, fmt.Errorf("invalid offset %v from end in %v bytes object, would be before file start", offset, r.objectSize) @@ -212,9 +210,9 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { go r.prefetchAt(r.currentOffset) return r.currentOffset, nil + default: + return r.currentOffset, errors.New("bad whence") } - - return r.currentOffset, errors.New("bad whence") } // removeFromOutstanding removes a prefetch from the list of outstanding prefetches once it's no longer active diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index e52ee1ebbd..37324fd37d 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.5 + 3.5.5 @@ -29,7 +29,7 @@ org.junit junit-bom - 5.12.2 + 5.13.4 pom import @@ -98,17 +98,17 @@ org.apache.sshd sshd-sftp - 2.15.0 + 2.16.0 commons-io commons-io - 2.19.0 + 2.20.0 com.amazonaws aws-java-sdk-s3 - 1.12.783 + 1.12.791 com.google.guava @@ -123,7 +123,7 @@ org.bouncycastle bcprov-jdk18on - 1.80 + 1.81 @@ -144,7 +144,7 @@ com.hierynomus sshj - 0.39.0 + 0.40.0 test @@ -161,7 +161,7 @@ org.yaml snakeyaml - 2.4 + 2.5 junit diff --git a/sda-sftp-inbox/src/test/resources/rsa.sec b/sda-sftp-inbox/src/test/resources/rsa.sec index 55bb962c4c..55a3c8c154 100644 --- a/sda-sftp-inbox/src/test/resources/rsa.sec +++ b/sda-sftp-inbox/src/test/resources/rsa.sec @@ -1,54 +1,50 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: AES-256-CBC,D0F80CA3EA0ACF3698A232F9F2D82B54 - -8BqfqBi5JjcuC8d01M+zGmyH8402V6yN+tui6kUIdKPyJvIq3nyaczXpngfp9oay -G+8RW2QlgLpouoAXQ354cCj/V5mpTVDRRKjZGbo3oaIjARuuRRfXJz4M95zR5iMh -RlOUoTAtpumoHStd0TNwhsBQtZwdjBxw8vHeAmA/hWpD82tTOhTtFmWmfMmZrilu -T4roC8jEsyfqoBGkhKPm0RM34xkP/tgJ63QqRLhUd2wbyvF4I9jtQumNAxLUbRi+ -sBeLJteLXg7z3JiaKTdYCYdecoNPFEzo9vHF0HKEVBGvbhnUCJTyqdqMzc0SohT/ -NG89RvdD75ryBVVKVsNj/fbTvC5Fg0XYHMW4DQ8qmnVO2MpHojpj+RWM7chIPwEX -ndBGLUJk3aCdz/0qMxN06v7NdT/bQglApVMTOXNMz5TjmtSMTGlXH7DJeY8vwwQa -t+RPnZSLCA4IoXGXH/ODlEilPQuGIQ458HLYBJjjBaMWGF3j80vZXFQyq752JrNH -3fbEfwEw08TNIcFIeuXBa1gx7cWhSqArSh/wdx0ke9FfbBTitue6GBFOOeABdCjd -gt188i+XAGxJQfUz4GCug3t7AFiYNgG4hizhTdGgpuGEYgkMxm5FK8RkeMyBof6A -ESODLfgbuG1GGU6ohOL/lQQ/1PC/a2Fm3H1VWsdMGthPyfJ+B/CHD0W8RKUa3fpQ -I9qqOLpRD2eEWA+FVZXC3xkPRkhXEId5O5jyKAsU2FNhZqL64UkEpWlkd+nQhdJE -/Gkd+wxrTu2dO//AJVFZLGLTKBqipebEolR4uOWlc+AoQaxZtdedl87jFNa5AdQa -tNQ6P5uy/X13EkkOYlFYWxwDmj9hlCcCaAUbExVy77lMBRmBjAqSm6IYRnMjZWfj -x2zh0ilCpoEuetcQc0+EaLBqzrBBOr4vVopr4rKfZ4SSYq13HWIFZre5XHxx3ddl -FJXIoKUniUCB/0O7krbTtPbKrv6bgl+77VJPN8G2t0Pz7DPUzEOxFH12D/LxvMqh -2EslcN/pRLWQLC26loybmSq0UXL1BW6UbtoTeGidLvmalSFLpkFkZhu/lgvM5ith -bwABkBfIaFoAismpCkYIyEK5mscYi1E4ziiwu+U83yaOILOUK66M6XaNMElWVVLq -fe68BwBUl0GTHgQmEYcPntQUiHNsAhc0HG6qrSYcD/c2uIC8aTYiPo8VbLz/qKhn -qDeDvI9Y/IVapB/tZ6c7mEiKiW2o4GvDA/647yOv1qMXjsU8ME12/VciV+1KL4AZ -eeCas/4G2FtLTDvzJZhJtbncFiSl9Q/RitvQBkVWc16vApPpMUHVokBl3ZsbiT8c -RBKo3V1h6ekSr6YM3wsNiD65LnQr4HP+PkDbbAOYSo1a4EAnuAg6sD08alN96ajm -bXucuXmxoiUdNSWH/mWCXo48C7bVFcpimXJnExf7E/ebccmGeQe78Zu4d4W3uofh -HfNGBcZLFRgPPEDwf0GZqjhuMA7AyWXckZtEGU7Hd03GwLz8Q/spknpontTfd1cH -xeu6hMvtMz934vPyyXSMrXWUK2wnQSWh2wQPXC2tPr0HVsZdwlwhIftNFWlTlHqL -aednAapHGIA1vJxn2dT1cvJympy+f+nJFvQYi6rO54Fj0AfwCASDiIUge+G/CVKA -OdDd46pcmzJr4peRiAJ4smTTWFLK+sJ1hAy4wmocsdhn6+6QRxSv/ayXOZ9ZxMRi -yOVHTIIq/c9b8GNvMZt75sQr8eh12VSWyWskwat1GqKNpx/AAhgGxnxxVGVYRVI+ -dgPLaioKKQGtlEhefxFGUg4NLc1AH70smzPn/44YqSbcMC5zDgObNaGI62X9ELgE -7v8BBgsH0i9HFEqaDd2jk1na0tXBg6nRpu4Jf/Va0DhQ2/aT0td4jtMckCw4lfHB -/xfcRwN6j7smorGMd4DT8rgrhTFK2qcQYYtaFF8i92+OTBE0byIXi4JNhxQoz6Vr -VeApj94au3KLG3L4+nkZp+Pmk5ayPQr2t+Ni1B75cdfKHnO0+yfkEBR7j+n9v4aA -hEjAOr/e6FMcuk0/oT1j0svJcWAvPlneFgfmBv8eEby0E7jmBinafqn5MktjMpDd -OccHpC4/P2c6z5w787t52X99UNWaUahoPZGkQzexm+A1XQc3s8JS5TFpoH9fO0OW -buiUXAurpSE7UUJ4ygWAZiHCHsfQJbd585m+3Xd5B188m4okXvvXF6sWksdrUMS2 -01tj29xMm2g0TTXro9ZsbEMpOIQOP/NMZY9+iDqbQ7+lpkHUcDaQVndJsbsJGbZA -7+b+OgSv1mMQvrbcbSSInsDEP6vkdrOgWvVpYDSbhIMnEMpzQLMfGymiN/IAOvAj -xfUz24wnhhdUmk7we5yrqActuWDxE64W9nabo6Le1lNpVg1NY2ppAylcomc27rzh -97NzIPDZSWpHVr5l6LADlb6yw/II55vtmRKAq3fM6NZUyyM4oZ3FQOx+3N/z1JML -lMwKwzqGOqplS4KN1hurB/BGqAhqiKGXjTWfDWMBtESLkRZZp3c2U7wZbbOaN++y -3pUO0HenycRivYcThh6Qlc6jwBubDCV0oqXb1JCyfnfYoQ8G9hfxPOPbXdhvdGnI -wLx0KLDUrxArk1goachtA/ldTbktz5oAlViUDc1Rv195jR1sqIScCgcad9e0fFdN -azF6ZJVKy4/MZOEXDOgZPYg3rmuxAssthOR+kZRoSAvxXfK39iYc6LCLFRJVfFz1 -EhwIRDds7GmW/JY9VNUwO4iDCrwsVX9ZClWx600itJMqM5lPIhB1hLZZkzHex/uo -NeeJkelUXakkCul5W8aCTnNhvggdxwCx74uZWGKqMGbrQm3bUdUIJKWrwPd6UYEY -i7hlapy4w4j87z2hiAxNyP2hK/neL4E6y2qERKsxl4/UpcPOzP/R1bh1ZRnNlMEq -pwzQ1o6BrQX4nxuNudaglGEpExeaGZxb555CqHFXswM3ft+5J8F3NG2SdOzt+k/y -rmessOyro/b9XWdt9+XJBo0FKS/DuKwMoXetoAMhiFvifEMdhz8ZOdV6nCRpjFw6 -/X4X0iQ/vxaKAWyrPApU9YUY/Y+IZTaBHVyCgYqWds4Ri12sqK1lRjhm0ycgwGlQ ------END RSA PRIVATE KEY----- +-----BEGIN OPENSSH PRIVATE KEY----- +b3BlbnNzaC1rZXktdjEAAAAACmFlczI1Ni1jdHIAAAAGYmNyeXB0AAAAGAAAABDlggJkmc +9n/V4c070nKIL8AAAAGAAAAAEAAAIXAAAAB3NzaC1yc2EAAAADAQABAAACAQC/MfPjl2bE +Y860QjfNC9phNeZ8FqSeJoP8WnWfO05AsOV/xC2RV6HlYCC/7QQ8Sa9ulrLse/7EUo3qtO +F3J7/xNxEugVCInA2GvSWrI2fHIPOfjl6TdqjwzT3RwfNFl7RJPHeO40MlC83O1o3MdxXN +YrFQUCbQsoLOdeBsqKsQc0b8DaGxUhAcle45M4cppbu1omhQgsJyLmiYUu7vZvjLF267wE +AuysUgAi3m4aE1egN+N896XasWdZjzWWUVshBLJU5NuaGO9PkSv/i1Lkj5syZOXCgrmTfW +UbVz3GMzokUG+rZoWqnZNpbBzxqqYaGIZssSGwjzpFWPAgfVsfDW0Qq0bLqZsszallx28e +8r7fd8O068U0E7u/gMa6ct+nqW7TMuetPUap0SKdAsevgW6KcUJUnyb91Nrq3P552HqYYM +8RcDapxvKvPCZSnHNFTF48shA0AcNGFuYVFichbFdttP+P/Cxx8aqyt4as77JwwW4/JdO9 +393kKUwKjk5gCl1yMLS8iq2eQcXGKHztKfBh4yxl4ZaEXl+1Ceqj4SBm1/pJRgP0PQfi/I +kvUMA+W90oJ/XUdrbV7+k2jzWMR3yD5SX+3F2CFXjIj228UX07Gfgh/UkPHErIaity8lQ2 +Qk+czrVgqdER9hOWELhcmMpx3ZBgxywhGeUjgbs6v8SQAAB1BWtRJsI0i/MF5cJaX3y7Gc +9kcG5XavsDuZQjp7DYQ91So7BRX7Jp4aCJD3DXsT/4uQKE+9feLfAKB1nxBrKuDN6mo+F7 +vsfHUAZM1nXBZGe02Y8sX02eer9w/jz2aEV1ovbFEpMbrZhjRy4HYDUAdLvhO3GFhAjtEe +p9pj0W5URgGOvEG/QxR1xskecPb9apYsDKjCGe+ROtbmiPMu7apdgU+IsBw1Dpe8MOYu/W +FW6xLhgx88V4QmbztjtwfoZDRqX9fJNBu8PJqaVAFVxVgBGn726K1a9VPljflqHsMLef2i +m/d/t8c8xY01oKWQ9cypg+BnYcZQ/N+T7vSbhH6/SvcXaR2AwfbRUOkpjiKQ3Oh9sX724c +TtoOHHB24a7sU3XeooogoieGFjJW0caMIjOJ6SjXtaEzmjy7W4pQaI8wFl/Pi7mm5xkI7l +1wsLhS4l4GuPooxRcTk02fj/97PM7dbhA0zDG1PGJx1teCZOx0ca29bAsU7pyoez1/TU5i +r/nXQNnMpNM4bzBNqy+tqEIRDbzg2MSErJeG4q+ov4GJLSAp6YoU0OsWgxEpM6So4vXq2m +DVyP25TqJMjjemqI4E0vTqFYVbI9HsPmNLyPbDXm5dVj/IMqEFcB+N18rr1TvNggH62qIX +m/I0JqWtfuIDUHan9+sA0HL8wCdGUxiyZ153v/nFRIqUWY35AxtyGmi0UJFF9yd5PNz3iM +/MklE0s3dYBRTo3/XejWokzTnNeVkh3IDx+oBvqEGPNTOV0r2qZN1TUGrsV9ahgiR4Cn1U +J88OUkfkshH7kqZs+WnvOvIhj9wFNmWEGv0oPBhz7ew7lK+4xp+p1mwCYgJhS+eae3sbcw +VM9Dg1BxC7xNpHY0c2C4LsUIdrfW/LVJRRePjWBAzmFFRRrMZKd/ojWiYUTQad6F6+obh/ +NeJ3PPfBGmg9merJbZSf2w+R7PTd/49Npid2XGTidWieUMylQYCMHMz1g1q2SXHR/HUCLM +3ezM81f0CNm9QRVuRZ1Z+L7H9/X/ULwoog5+IHnNa71lboi/DKPSENW08MV5Nhv8I1yF6V +2zvkevGUSSs9AtPlVWDRBXPr+rgGKcR4MY8p4oMEJwSbMTiuhLU1Scq4/uggZu6Gb0qyAG +59ReewfAG7y6ZzwiNGvcOWsUSgKkoVLWksCbUX2IbwojhP63ZX/8fide1Ls5dmEVbVKp6Y +n/eu/FumDM50bWm6Zwy/krFU64eVfYD5YRgJeDbC0cjG8M2o/UASYarINFlmHWVkVa1/zl +VEMd1ZnEDHXlrcPndIt4m+XXuuUiNbV/jmsbEWl7T2SM0ijunBZtZzjvZD0XxEu9xno0MB ++0tWLHK+WtXR3uk53vwFZgaJfYjb8scWf81o5pjAfNzDzR+2z0f4UlTi+xVnCWfa8is3yQ +jg3/7QR+KD6PIg2Zlf/B4A0oX8RwyeGEsYt3bmITJiQganPR3gc6zs1AL4dbBseC38V4Fr +UuROgiG6TDcgBsfJdoTPeESZK808fDO3ywylMRjQZ9GL6UJJfe7iP6FQrwR6SjFcPEYdO5 +a63ylDVdS+kY0IL56b25U0whakJj+QIEEdGB6e0iLUGlZ9ggaAufLOc4lbPtHwSpZCu5u5 +3BVjc/b3xTRFym4N+qLR+jPatRfkMeIdFKNItgXT15wK9UdTJNT4gTASLGZvIRzXMjUH0k +LVDru6h52yD9bU8OjOBAMS6eM83SXItXKy1OhQ3ESqADXjoD7hHLfo00kvOd9VHEVIc4zZ +FSCVMSFOUFo99ulLJoUJlF8gh/q7VSHGCPeq/rUlNvfRIrt+ieW68dOuyQKxxFZQ8Hzv8U +7EZT7Bx5OEMSXhnR+GC+m2OKamBjjJgZTG+t43y+n696h1jJu8hW7WdecrHeVdQe2rDKeP +I2vzTK0dlOPXXZcxN2L2V4GfO6LTm7wl3gJX9lAZ/AVEZHkbgGEbOoM+1hzXJWIo7NMiDk +MjVqSejabEtz5HU43wYvjpHhifbNClD5qaKi5L3q+f/5C1FOZjuWcuwgXMcZnYPtmDIWdl +C5Ir5MUWdYZdHvhoVq+320jSyHNinCr9qXZVRLH7CX3bc4n78YZ4qZQNlrp0850Fyrod7x +qNuut2hzx/s9CkMT6HvOIEPb4BjWRU9WQJRo+h4KIXOEXgsEnnLwQ75dnPi8cQlywnHAd7 +DnicsM5P6kcisWAe5qjmgu3s7j6PdV9fz4OWJdB+B9xTIdYyijQh8dlfpPIsTeYkV2ZLS7 +hKZRld4Fv1ESdf8TzwkGp1l2wn2zD8ZfL3opVb98S3E7CxLzHSkcIbT+NNe/3X8L2uLjxR +As427SYRR5Z3aqRVYtktWKRKZ8lsMJZqfVKIEU/+70FccDA/+mdETXDK2rQ8wlVtEjA3l2 +Tlsw5aQXriNFA19I4nlQC18oD/nijAQYNyro5goRkB41ZCO6G54MN2k8ETYRVCUf5L5vDH +0nP6FOt+47Rri8u2UUS2m278M= +-----END OPENSSH PRIVATE KEY----- diff --git a/sda-sftp-inbox/src/test/resources/rsa.ssh b/sda-sftp-inbox/src/test/resources/rsa.ssh index fe47a6cd92..a29f900c1d 100644 --- a/sda-sftp-inbox/src/test/resources/rsa.ssh +++ b/sda-sftp-inbox/src/test/resources/rsa.ssh @@ -1 +1 @@ -ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCqSXigbq5S/aPGE412BmHOYaLEQilVzXvjKRWVLOcZOGrElTjtiu7/Ezqdg4B3bVvKGC4TLUKDflO7qM+QnmpSbEFLz7suGz+0XzdEi6En1ihdoPOKDEvRlqZhNHlf3qjoatX9ScQa6vMuKhDwfneZNfhOzUzxJ8UihXYGHzoVvt4CGkSVSSFwj9dUmg2bG3hGL6i6w/F9rjCO4t9u3Rh/TFMBzcGIq+ft0BOM8XqSWmUDn/hYdt1Xkxnz1D0QHS5SjVmqnzb4r60pkgKSzQ3M9ECIz/ktVFCVZxH61dAeqko2196WHfe+zn0PpYfnMjhqtJlV4gstdz5C2g3IUKL2jxP0APDXib6lZuNSSiXT/1wiWE4jJDLtGVnauKDMerpJlcwl0zawMcCxqIxjY4OrfP+/k35JfDM1o3VJ8EZMtd4bCdG6w9iFzu/Yyjl91ryFHvPY/J4wuzErbzBUDfri18dKXbgH7EES7J6jbKJcmgVRWaEHpwogGcPHvJHxMDTOuwLxaNHkMxm2WTaFG4Q5uC2ZJhPPX40k/sn/paE3zgX0u7VIwczOr+cBqNNW4lqbGqooTn5RhHnXfpKRFZNz8fAdL+EPg0Y3hoPvZtRl04bvc4d5Ajj86Do5B/Wq2qAzDZoTtspPhcf7RnO3L+EzikshZcjES6I8IxCCppkMrQ== \ No newline at end of file +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/MfPjl2bEY860QjfNC9phNeZ8FqSeJoP8WnWfO05AsOV/xC2RV6HlYCC/7QQ8Sa9ulrLse/7EUo3qtOF3J7/xNxEugVCInA2GvSWrI2fHIPOfjl6TdqjwzT3RwfNFl7RJPHeO40MlC83O1o3MdxXNYrFQUCbQsoLOdeBsqKsQc0b8DaGxUhAcle45M4cppbu1omhQgsJyLmiYUu7vZvjLF267wEAuysUgAi3m4aE1egN+N896XasWdZjzWWUVshBLJU5NuaGO9PkSv/i1Lkj5syZOXCgrmTfWUbVz3GMzokUG+rZoWqnZNpbBzxqqYaGIZssSGwjzpFWPAgfVsfDW0Qq0bLqZsszallx28e8r7fd8O068U0E7u/gMa6ct+nqW7TMuetPUap0SKdAsevgW6KcUJUnyb91Nrq3P552HqYYM8RcDapxvKvPCZSnHNFTF48shA0AcNGFuYVFichbFdttP+P/Cxx8aqyt4as77JwwW4/JdO9393kKUwKjk5gCl1yMLS8iq2eQcXGKHztKfBh4yxl4ZaEXl+1Ceqj4SBm1/pJRgP0PQfi/IkvUMA+W90oJ/XUdrbV7+k2jzWMR3yD5SX+3F2CFXjIj228UX07Gfgh/UkPHErIaity8lQ2Qk+czrVgqdER9hOWELhcmMpx3ZBgxywhGeUjgbs6v8SQ== \ No newline at end of file diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index b57981b1a7..b0bfe22554 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -9,10 +9,12 @@ import ( "encoding/json" "errors" "fmt" + "io" "math" "net/http" "os" "os/signal" + "path" "strings" "syscall" "time" @@ -22,14 +24,19 @@ import ( "github.com/gin-gonic/gin" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/neicnordic/crypt4gh/keys" + "github.com/neicnordic/crypt4gh/model/headers" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" + "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" ) type dataset struct { @@ -39,9 +46,10 @@ type dataset struct { } var ( - Conf *config.Config - err error - auth *userauth.ValidateFromToken + Conf *config.Config + err error + auth *userauth.ValidateFromToken + auditLogger *log.Logger ) func main() { @@ -94,7 +102,39 @@ func setup(conf *config.Config) *http.Server { log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) // nolint # FIXME Fatal should only be called from main } - r := gin.Default() + r := gin.New() + r.Use(gin.Recovery()) + + // Enable default Gin logger in debug mode for detailed request logging during development + if log.GetLevel() == log.DebugLevel { + r.Use(gin.LoggerWithConfig(gin.LoggerConfig{ + SkipPaths: []string{"/ready"}, + })) + } + + // Enable structured JSON logging in info mode for clean, parseable logs in production + if log.GetLevel() == log.InfoLevel { + r.Use(gin.LoggerWithConfig( + gin.LoggerConfig{ + Formatter: func(params gin.LogFormatterParams) string { + s, _ := json.Marshal(map[string]any{ + "level": "info", + "method": params.Method, + "path": params.Path, + "remote_addr": params.ClientIP, + "status_code": params.StatusCode, + "time": params.TimeStamp.Format(time.RFC3339), + }) + + return string(s) + "\n" + }, + + Output: gin.DefaultWriter, + SkipPaths: []string{"/ready"}, + }, + )) + } + r.GET("/ready", readinessResponse) r.GET("/files", rbac(e), getFiles) r.GET("/datasets", rbac(e), listDatasets) @@ -104,16 +144,18 @@ func setup(conf *config.Config) *http.Server { r.POST("/c4gh-keys/deprecate/*keyHash", rbac(e), deprecateC4ghHash) // Deprecate a given key hash r.DELETE("/file/:username/:fileid", rbac(e), deleteFile) // Delete a file from inbox // submission endpoints below here - r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file - r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file - r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file - r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset - r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible - r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset - r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status - r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specififc user - r.GET("/users", rbac(e), listActiveUsers) // Lists all users - r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user + r.POST("/file/ingest", rbac(e), ingestFile) // start ingestion of a file + r.POST("/file/accession", rbac(e), setAccession) // assign accession ID to a file + r.PUT("/file/verify/:accession", rbac(e), reVerifyFile) // trigger reverification of a file + r.POST("/dataset/create", rbac(e), createDataset) // maps a set of files to a dataset + r.POST("/dataset/release/*dataset", rbac(e), releaseDataset) // Releases a dataset to be accessible + r.PUT("/dataset/verify/*dataset", rbac(e), reVerifyDataset) // Re-verify all files in the dataset + r.GET("/datasets/list", rbac(e), listAllDatasets) // Lists all datasets with their status + r.GET("/datasets/list/:username", rbac(e), listUserDatasets) // Lists datasets with their status for a specific user + r.GET("/users", rbac(e), listActiveUsers) // Lists all users + r.GET("/users/:username/files", rbac(e), listUserFiles) // Lists all unmapped files for a user + r.GET("/users/:username/file/:fileid", rbac(e), downloadFile) // Download a file from a users inbox + cfg := &tls.Config{MinVersion: tls.VersionTLS12} srv := &http.Server{ @@ -194,28 +236,59 @@ func checkDB(db *database.SDAdb, timeout time.Duration) error { return db.DB.PingContext(ctx) } +func auditLog(auditFields log.Fields) { + auditFields["audit"] = true + Conf.API.AuditLogger.WithFields(auditFields).Info("Incoming audit event") +} + func rbac(e *casbin.Enforcer) gin.HandlerFunc { return func(c *gin.Context) { token, err := auth.Authenticate(c.Request) if err != nil { - log.Debugln("bad token") + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "authentication error": err.Error(), + "path": c.Request.URL.Path, + }) + } c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": err.Error()}) return } - ok, err := e.Enforce(token.Subject(), c.Request.URL.String(), c.Request.Method) + ok, err := e.Enforce(token.Subject(), c.Request.URL.Path, c.Request.Method) if err != nil { + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "authorization": "error (err.Error())", + "user": token.Subject(), + "path": c.Request.URL.Path, + }) + } log.Debugf("rbac enforcement failed, reason: %s\n", err.Error()) c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) return } if !ok { + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "authorization": "failed", + "user": token.Subject(), + "path": c.Request.URL.Path, + }) + } c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"error": "not authorized"}) return } + + if Conf.API.AuditLogger != nil { + auditLog(log.Fields{ + "user": token.Subject(), + "path": c.Request.URL.Path, + }) + } log.Debugln("authorized") } } @@ -232,7 +305,7 @@ func getFiles(c *gin.Context) { return } - files, err := Conf.API.DB.GetUserFiles(token.Subject(), false) + files, err := Conf.API.DB.GetUserFiles(token.Subject(), c.Query("path_prefix"), false) if err != nil { // something went wrong with querying or parsing rows c.JSON(502, err.Error()) @@ -319,8 +392,8 @@ func deleteFile(c *gin.Context) { return } - var retryTimes = 5 - for count := 1; count <= retryTimes; count++ { + filePath = helper.UnanonymizeFilepath(filePath, submissionUser) + for count := 1; count <= 5; count++ { err = inbox.RemoveFile(filePath) if err == nil { break @@ -344,6 +417,123 @@ func deleteFile(c *gin.Context) { c.Status(http.StatusOK) } +// reencryptHeader re-encrypts the header of a file using the public key +// provided in the request header and returns the new header. The function uses +// gRPC to communicate with the re-encrypt service and handles TLS configuration +// if needed. The function also handles the case where the CA certificate is +// provided for secure communication. +func reencryptHeader(oldHeader []byte, c4ghPubKey string) ([]byte, error) { + var opts []grpc.DialOption + switch { + case Conf.API.Grpc.ClientCreds != nil: + opts = append(opts, grpc.WithTransportCredentials(Conf.API.Grpc.ClientCreds)) + default: + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + } + + conn, err := grpc.NewClient(fmt.Sprintf("%s:%d", Conf.API.Grpc.Host, Conf.API.Grpc.Port), opts...) + if err != nil { + log.Errorf("failed to connect to the reencrypt service, reason: %s", err) + + return nil, err + } + defer conn.Close() + + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(Conf.API.Grpc.Timeout)*time.Second) + defer cancel() + + c := reencrypt.NewReencryptClient(conn) + res, err := c.ReencryptHeader(ctx, &reencrypt.ReencryptRequest{Oldheader: oldHeader, Publickey: c4ghPubKey}) + if err != nil { + return nil, err + } + + return res.Header, nil +} + +// The downloadFile function download a file re-encrypted with the public key +// provided in the request header from the inbox. It retrieves the file path +// from the database using the file ID and user ID. +func downloadFile(c *gin.Context) { + // Get the public key from the request header. + c4ghPubKey := c.GetHeader("C4GH-Public-Key") + + pubKey, err := base64.StdEncoding.DecodeString(c4ghPubKey) + if err != nil || len(pubKey) == 0 { + log.Errorf("bad public key, error: %v", err) + c.AbortWithStatusJSON(http.StatusBadRequest, "bad public key") + + return + } + + inbox, err := storage.NewBackend(Conf.Inbox) + if err != nil { + log.Errorf("failed to initialize inbox backend, reason: %v", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, "storage backend error") + + return + } + + // Retrieve the actual file path for the user's file. + fileID := strings.TrimPrefix(c.Param("fileid"), "/") + filePath, err := Conf.API.DB.GetInboxFilePathFromID( + strings.TrimPrefix(c.Param("username"), "/"), + fileID, + ) + if err != nil { + log.Errorf("getting file path from fileID (%s) failed, reason: %v", fileID, err) + c.AbortWithStatusJSON(http.StatusNotFound, "failed to retrieve inbox file path") + + return + } + + // Get inbox file handle #noqa + file, err := inbox.NewFileReader( + helper.UnanonymizeFilepath( + filePath, + strings.TrimPrefix(c.Param("username"), "/"), + ), + ) + if err != nil { + log.Errorf("inbox file %s not found or failed to read, %s", filePath, err.Error()) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to read inbox file") + + return + } + + // get the header of the crypt4gh file + header, err := headers.ReadHeader(file) + if err != nil { + log.Errorf("failed to read header for fileID %s, reason: %v", fileID, err) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to read the start of the file") + + return + } + + newHeader, err := reencryptHeader(header, c4ghPubKey) + if err != nil { + log.Errorf("failed to reencrypt header, reason: %v", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to reencrypt header") + + return + } + + // Set the headers for the response. + c.Header("Content-Type", "application/octet-stream") + c.Header("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(filePath))) + + reader := io.MultiReader(bytes.NewReader(newHeader), file) + _, err = io.Copy(c.Writer, reader) + if err != nil { + log.Errorf("error occurred while sending stream, reason: %v", err) + c.AbortWithStatusJSON(http.StatusInternalServerError, "failed to stream data to client") + + return + } + + c.Status(http.StatusOK) +} + func setAccession(c *gin.Context) { var accession schema.IngestionAccession if err := c.BindJSON(&accession); err != nil { @@ -413,7 +603,7 @@ func createDataset(c *gin.Context) { } if len(dataset.AccessionIDs) == 0 { - c.AbortWithStatusJSON(http.StatusBadRequest, "at least one accessionID is reqired") + c.AbortWithStatusJSON(http.StatusBadRequest, "at least one accessionID is required") return } @@ -542,7 +732,8 @@ func listUserFiles(c *gin.Context) { username = strings.TrimPrefix(username, "/") username = strings.TrimSuffix(username, "/files") log.Debugln(username) - files, err := Conf.API.DB.GetUserFiles(username, true) + + files, err := Conf.API.DB.GetUserFiles(username, c.Query("path_prefix"), true) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index 0501f3c583..1a124c8134 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -19,6 +19,12 @@ Endpoints: [{"inboxPath":"requester_demo.org/data/file1.c4gh","fileStatus":"uploaded","createAt":"2023-11-13T10:12:43.144242Z"}] ``` + It is possible to limit the returned results by supplying a base path prefix in the query. In this case only files that have a path that starts with `submission-1` will be returned. + + ```bash + curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/files?path_prefix=submission-1 + ``` + If the `token` is invalid, 401 is returned. - `/datasets` @@ -215,11 +221,35 @@ Admin endpoints are only available to a set of whitelisted users specified in th curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/users/submitter@example.org/files ``` + It is possible to limit the returned results by supplying a base path prefix in the query. In this case only files that have a path that starts with `submission-1` will be returned. + + ```bash + curl -H "Authorization: Bearer $token" -X GET https://HOSTNAME/users/submitter@example.org/files?path_prefix=submission-1 + ``` + - Error codes - `200` Query execute ok. - `401` Token user is not in the list of admins. - `500` Internal error due to DB failure. +- `/users/:username/file/:fileid` + - accepts `GET` requests. + - downloads a file from the inbox, re-encrypted with the client’s public key provided in the request header. + - the public key provided in the header should be `base64` encoded. + + - Error codes + - `200` Query execute ok. + - `400` Client public key not provided or not valid. + - `401` Token user is not in the list of admins. + - `404` File not found + - `500` Internal error due to Inbox, Reencrypt, DB, MQ or streaming failures. + + Example: + + ```bash + curl -H "Authorization: Bearer $token" -H "C4GH-Public-Key: $base64_encoded_public_key" -X GET https://HOSTNAME/users/submitter@example.org/file/c2acecc6-f208-441c-877a-2670e4cbb040 + ``` + - `/c4gh-keys/add` - accepts `POST` requests with the hex hash of the key and its description - registers the key hash in the database. diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index c8a38b4387..85f57fb799 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -2,15 +2,20 @@ package main import ( "bytes" + "context" "crypto/sha256" "database/sql" + "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "io" + "net" "net/http" "net/http/httptest" "os" "path" + "path/filepath" "runtime" "strconv" "strings" @@ -22,22 +27,53 @@ import ( "github.com/gin-gonic/gin" "github.com/google/uuid" _ "github.com/lib/pq" + "github.com/neicnordic/crypt4gh/keys" + "github.com/neicnordic/crypt4gh/streaming" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/jsonadapter" + "github.com/neicnordic/sensitive-data-archive/internal/reencrypt" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" + "google.golang.org/grpc" ) var dbPort, mqPort, OIDCport int var BrokerAPI string +type server struct { + reencrypt.UnimplementedReencryptServer + c4ghPrivateKeyList []*[32]byte +} + +func (s *server) ReencryptHeader(ctx context.Context, req *reencrypt.ReencryptRequest) (*reencrypt.ReencryptResponse, error) { + // Mock response based on your needs + mockedResponse := &reencrypt.ReencryptResponse{ + Header: []byte("predefined header response"), + } + + return mockedResponse, nil +} + +func splitHostPort(addr string) (string, int, error) { + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + return "", 0, fmt.Errorf("invalid address format: %w", err) + } + port, err := strconv.Atoi(portStr) + if err != nil { + return "", 0, fmt.Errorf("invalid port in address '%s': %w", addr, err) + } + + return host, port, nil +} + func TestMain(m *testing.M) { if _, err := os.Stat("/.dockerenv"); err == nil { m.Run() @@ -229,15 +265,36 @@ func TestMain(m *testing.M) { os.Exit(code) } +type UserKey struct { + PublicKey [32]byte + PrivateKey [32]byte + PrivateKeyPath string + PubKeyBase64 string +} + +type GrpcListener struct { + Listener net.Listener + gs *grpc.Server + serverOpts []grpc.ServerOption +} + type TestSuite struct { suite.Suite - Path string - PublicPath string - PrivatePath string - KeyName string - RBAC []byte - Token string - User string + Path string + PublicPath string + PrivatePath string + KeyName string + RBAC []byte + Token string + User string + PubKeyList [][32]byte + PrivateKey *[32]byte + UserKey UserKey + FileData []byte + FileHeader []byte + GoodC4ghFile string + BadC4ghFile string + GrpcListener GrpcListener } func (s *TestSuite) TestShutdown() { @@ -358,6 +415,9 @@ func (s *TestSuite) SetupSuite() { Conf = c + Conf.API.Host = "0.0.0.0" + Conf.API.Port = 8080 + log.Print("Setup DB for my test") Conf.Database = database.DBConf{ Host: "localhost", @@ -388,12 +448,105 @@ func (s *TestSuite) SetupSuite() { {"role":"submission","path":"/file/accession","action":"POST"}, {"role":"submission","path":"/users","action":"GET"}, {"role":"submission","path":"/users/:username/files","action":"GET"}, + {"role":"submission","path":"/users/:username/file/:fileid","action":"GET"}, {"role":"*","path":"/files","action":"GET"}], "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"admin"}]}`) + + Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") + if err != nil { + s.FailNow("failed to create temp folder") + } + + s.UserKey.PublicKey, s.UserKey.PrivateKey, err = keys.GenerateKeyPair() + if err != nil { + s.T().FailNow() + } + + s.UserKey.PrivateKeyPath = s.Path + "/user.key" + key, err := os.Create(s.UserKey.PrivateKeyPath) + if err != nil { + s.T().FailNow() + } + if err := keys.WriteCrypt4GHX25519PrivateKey(key, s.UserKey.PrivateKey, []byte("password")); err != nil { + s.T().FailNow() + } + + buf := new(bytes.Buffer) + if err := keys.WriteCrypt4GHX25519PublicKey(buf, s.UserKey.PublicKey); err != nil { + s.T().FailNow() + } + s.UserKey.PubKeyBase64 = base64.StdEncoding.EncodeToString(buf.Bytes()) + + Conf.API.Grpc = config.Grpc{ + Host: "localhost", + Port: 50051, + Timeout: 30, + } + + s.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") + s.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") + + err = os.MkdirAll(path.Join(Conf.Inbox.Posix.Location, s.User), 0750) + assert.NoError(s.T(), err, "failed to create inbox directory") + + // Create test files in the inbox + fileContent := []byte("This is the content of the test file.") + contentReader := bytes.NewReader(fileContent) + s.GoodC4ghFile = path.Join(Conf.Inbox.Posix.Location, s.User, "test_download.c4gh") + s.BadC4ghFile = path.Join(Conf.Inbox.Posix.Location, s.User, "badc4ghfile.c4gh") + + outFile, err := os.Create(s.GoodC4ghFile) + if err != nil { + s.FailNow("failed to create encrypted test file") + } + defer outFile.Close() + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + s.FailNow("failed to create private c4gh key") + } + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, s.PubKeyList, nil) + if err != nil { + s.FailNow("failed to create c4gh writer") + } + + _, err = io.Copy(crypt4GHWriter, contentReader) + if err != nil { + s.FailNow("failed to write predefined data to encrypted test file") + } + crypt4GHWriter.Close() + + err = os.WriteFile(s.BadC4ghFile, []byte("bad c4gh content"), 0600) + if err != nil { + s.FailNow("failed to write content to file") + } + + // create a gPRC listener for tests using the reencrypt service + s.GrpcListener.Listener, err = net.Listen("tcp", "localhost:0") + if err != nil { + s.T().FailNow() + } + + go func() { + var opts []grpc.ServerOption + s.GrpcListener.gs = grpc.NewServer(opts...) + reencrypt.RegisterReencryptServer(s.GrpcListener.gs, &server{c4ghPrivateKeyList: Conf.ReEncrypt.C4ghPrivateKeyList}) + if err := s.GrpcListener.gs.Serve(s.GrpcListener.Listener); err != nil { + s.T().Fail() + } + }() } func (s *TestSuite) TearDownSuite() { assert.NoError(s.T(), os.RemoveAll(s.Path)) + assert.NoError(s.T(), os.RemoveAll(Conf.Inbox.Posix.Location)) + if s.GrpcListener.gs != nil { + s.GrpcListener.gs.GracefulStop() + } + if s.GrpcListener.Listener != nil { + s.GrpcListener.Listener.Close() + } } func (s *TestSuite) SetupTest() { Conf.Database = database.DBConf{ @@ -547,11 +700,67 @@ func (s *TestSuite) TestAPIGetFiles() { assert.Equal(s.T(), fileInfo.Status, latestStatus) case file2: assert.Equal(s.T(), fileInfo.Status, "registered") + default: + s.Fail("unknown inbox path") } } assert.NoError(s.T(), err) } +func (s *TestSuite) TestAPIGetFiles_filteredSelection() { + testUsers := []string{"dummy", "User-B", "User-C"} + for _, user := range testUsers { + sub := "submission_a" + + for i := range 5 { + if i == 2 { + sub = "submission_b" + } + + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + if err != nil { + s.FailNow("failed to register file in database") + } + + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + if err != nil { + s.FailNow("failed to update satus of file in database") + } + } + } + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") + } + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") + } + + // Mock request and response holders + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/files?path_prefix=submission_b", http.NoBody) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.GET("/files", rbac(e), getFiles) + + router.ServeHTTP(w, r) + okResponse := w.Result() + defer okResponse.Body.Close() + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + + files := []database.SubmissionFileInfo{} + err = json.NewDecoder(okResponse.Body).Decode(&files) + assert.NoError(s.T(), err, "failed to list files from DB") + assert.Equal(s.T(), 3, len(files)) +} + func TestApiTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } @@ -560,6 +769,160 @@ func testEndpoint(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) } +func (s *TestSuite) TestGinLogLevel_Debug() { + // Capture log output from gin in a buffer + var buf bytes.Buffer + gin.DefaultWriter = &buf + gin.DefaultErrorWriter = &buf + + log.SetLevel(log.DebugLevel) + + // A specific port is enforced here so we don't have a conflict when running the LogLevel_Info test + Conf.API.Port = 8081 + srv := setup(Conf) + go func() { + if err := srv.ListenAndServe(); err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failure from API server") + } + }() + + // Allow some time for startup + time.Sleep(500 * time.Millisecond) + + assert.NoError(s.T(), setupJwtAuth()) + + // Send a request towards previously started *http.Server + client := http.Client{Timeout: 5 * time.Second} + r, err := http.NewRequest("GET", fmt.Sprintf("http://%s:%d/files", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance") + } + + r.Header.Add("Authorization", "Bearer "+s.Token) + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput := buf.String() + + // Remove leading and/or trailing newlines and spaces + lines := strings.Split(strings.TrimSpace(logOutput), "\n") + assert.Greater(s.T(), len(lines), 0) + + // Pick the last produced output from GIN + logOutput = lines[len(lines)-1] + assert.Contains(s.T(), logOutput, "[GIN]") + + buf.Reset() + r, err = http.NewRequest("GET", fmt.Sprintf("http://%s:%d/ready", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance for /ready") + } + + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request to /ready") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput = buf.String() + + lines = strings.Split(strings.TrimSpace(logOutput), "\n") + fmt.Println("lines : ", lines) + fmt.Println("len(lines) : ", len(lines)) + if len(lines) > 1 { + assert.NotContains(s.T(), logOutput[len(logOutput)-1], "[GIN]") + } else { + s.T().Log("No log output from /ready, considered PASS on log level debug") + } +} + +func (s *TestSuite) TestGinLogLevel_Info() { + // Capture log output from gin in a buffer + var buf bytes.Buffer + gin.DefaultWriter = &buf + gin.DefaultErrorWriter = &buf + + log.SetLevel(log.InfoLevel) + + // A specific port is enforced here so we don't have a conflict when running the LogLevel_Debug test + Conf.API.Port = 8082 + srv := setup(Conf) + go func() { + if err := srv.ListenAndServe(); err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failure from API server") + } + }() + + // Allow some time for startup + time.Sleep(500 * time.Millisecond) + + assert.NoError(s.T(), setupJwtAuth()) + + // Send a request towards previously started *http.Server + client := http.Client{Timeout: 5 * time.Second} + r, err := http.NewRequest("GET", fmt.Sprintf("http://%s:%d/files", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance") + } + + r.Header.Add("Authorization", "Bearer "+s.Token) + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput := buf.String() + + // Remove leading and/or trailing newlines and spaces + lines := strings.Split(strings.TrimSpace(logOutput), "\n") + assert.Greater(s.T(), len(lines), 0) + + // Pick the last produced output from GIN + logOutput = lines[len(lines)-1] + assert.NotContains(s.T(), logOutput, "[GIN]") + + buf.Reset() + r, err = http.NewRequest("GET", fmt.Sprintf("http://%s:%d/ready", Conf.API.Host, Conf.API.Port), nil) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to create new request instance for /ready") + } + + _, err = client.Do(r) // nolint: bodyclose + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to execute HTTP request to /ready") + } + + // Allow logs to flush + time.Sleep(500 * time.Millisecond) + logOutput = buf.String() + + lines = strings.Split(strings.TrimSpace(logOutput), "\n") + fmt.Println("lines : ", lines) + fmt.Println("len(lines) : ", len(lines)) + if len(lines) > 1 { + assert.NotContains(s.T(), logOutput[len(logOutput)-1], "[GIN]") + } else { + s.T().Log("No log output from /ready, considered PASS on log level info") + } +} + func (s *TestSuite) TestRBAC() { gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) @@ -1162,7 +1525,7 @@ func (s *TestSuite) TestCreateDataset_MissingAccessionIDs() { response.Body.Close() assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(s.T(), string(body), "at least one accessionID is reqired") + assert.Contains(s.T(), string(body), "at least one accessionID is required") } func (s *TestSuite) TestCreateDataset_WrongIDs() { @@ -1558,6 +1921,61 @@ func (s *TestSuite) TestListUserFiles() { assert.Contains(s.T(), files[0].AccessionID, "accession_user_example.org_0") } +func (s *TestSuite) TestListUserFiles_filteredSelection() { + testUsers := []string{"user_example.org", "User-B", "User-C"} + for _, user := range testUsers { + sub := "submission_a" + + for i := range 5 { + if i == 2 { + sub = "submission_b" + } + + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("%s/TestGetUserFiles-00%d.c4gh", sub, i), strings.ReplaceAll(user, "_", "@")) + if err != nil { + s.FailNow("failed to register file in database") + } + + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") + if err != nil { + s.FailNow("failed to update satus of file in database") + } + } + } + + gin.SetMode(gin.ReleaseMode) + assert.NoError(s.T(), setupJwtAuth()) + m, err := model.NewModelFromString(jsonadapter.Model) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") + } + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) + if err != nil { + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") + } + + // Mock request and response holders + w := httptest.NewRecorder() + r := httptest.NewRequest(http.MethodGet, "/users/user@example.org/files?path_prefix=submission_b", http.NoBody) + r.Header.Add("Authorization", "Bearer "+s.Token) + + _, router := gin.CreateTestContext(w) + router.GET("/users/:username/files", rbac(e), listUserFiles) + + router.ServeHTTP(w, r) + okResponse := w.Result() + defer okResponse.Body.Close() + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + + files := []database.SubmissionFileInfo{} + err = json.NewDecoder(okResponse.Body).Decode(&files) + assert.NoError(s.T(), err, "failed to list user files from DB") + assert.Equal(s.T(), 3, len(files)) + assert.Contains(s.T(), files[0].Status, "uploaded") +} + func (s *TestSuite) TestAddC4ghHash() { gin.SetMode(gin.ReleaseMode) assert.NoError(s.T(), setupJwtAuth()) @@ -2152,3 +2570,193 @@ func (s *TestSuite) TestReVerifyDataset_wrongDatasetName() { defer okResponse.Body.Close() assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } + +func (s *TestSuite) TestDownloadFile() { + mockServerAddress := s.GrpcListener.Listener.Addr().String() + Conf.API.Grpc.Host, Conf.API.Grpc.Port, err = splitHostPort(mockServerAddress) + if err != nil { + s.T().Fatal("failed to split host and port:", err) + } + + gin.SetMode(gin.ReleaseMode) + r := gin.Default() + + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + // Register the file in the database + fileID, err := Conf.API.DB.RegisterFile(filepath.Base(s.GoodC4ghFile), s.User) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + // Mock request to download the file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) + assert.Equal(s.T(), "application/octet-stream", resp.Header.Get("Content-Type")) + + // Check the content of the response + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err, "failed to read response body") + assert.Contains(s.T(), string(body), "predefined header respons") +} + +func (s *TestSuite) TestDownloadFile_badPublicKey() { + r := gin.Default() + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + // Mock request to download a file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, "d35a6b53-a5d8-4b1b-921f-0a2829b8d2f2") + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", "invalid_key") + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) + + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) + assert.Contains(s.T(), string(body), "bad public key") +} + +func (s *TestSuite) TestDownloadFile_fileIDNotExist() { + r := gin.Default() + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + // Mock a request to download a file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, "d35a6b53-a5d8-4b1b-921f-0a2829b8d2f2") + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) + + assert.Equal(s.T(), http.StatusNotFound, resp.StatusCode) + assert.Contains(s.T(), string(body), "failed to retrieve inbox file path") +} + +func (s *TestSuite) TestDownloadFile_fileNotExist() { + r := gin.Default() + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + // Register a file in the database (but don't create the actual file) + filePath := fmt.Sprintf("/%v/nonexistent.c4gh", s.User) + fileID, err := Conf.API.DB.RegisterFile(filePath, s.User) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + // Mock request to download the file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) + + assert.Equal(s.T(), http.StatusInternalServerError, resp.StatusCode) + assert.Contains(s.T(), string(body), "failed to read inbox file") +} + +func (s *TestSuite) TestDownloadFile_badC4ghFile() { + r := gin.Default() + r.GET("/users/:username/file/:fileid", func(c *gin.Context) { + downloadFile(c) + }) + ts := httptest.NewServer(r) + defer ts.Close() + + // Register a file in the database (but don't create the actual file) + fileID, err := Conf.API.DB.RegisterFile(filepath.Base(s.BadC4ghFile), s.User) + assert.NoError(s.T(), err, "failed to register file in database") + err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "failed to update satus of file in database") + + // Mock request to download the file + downloadURL := fmt.Sprintf("%s/users/%s/file/%s", ts.URL, s.User, fileID) + req, err := http.NewRequest("GET", downloadURL, nil) + assert.NoError(s.T(), err) + req.Header.Set("C4GH-Public-Key", s.UserKey.PubKeyBase64) + + client := &http.Client{} + resp, err := client.Do(req) + assert.NoError(s.T(), err) + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) + + assert.Equal(s.T(), http.StatusInternalServerError, resp.StatusCode) + assert.Contains(s.T(), string(body), "failed to read the start of the file") +} + +func (s *TestSuite) TestReencryptHeader() { + mockServerAddress := s.GrpcListener.Listener.Addr().String() + log.Debugf("Mock server address: %s", mockServerAddress) + Conf.API.Grpc.Host, Conf.API.Grpc.Port, err = splitHostPort(mockServerAddress) + if err != nil { + s.T().Fatal("failed to split host and port:", err) + } + + // Call the function under test + newHeader, err := reencryptHeader(s.FileHeader, s.UserKey.PubKeyBase64) + if err != nil { + s.T().Fatal("reencryptHeader failed:", err) + } + + // Validate the returned header + expectedHeader := []byte("predefined header response") + assert.Equal(s.T(), expectedHeader, newHeader, "returned header does not match the expected header") +} + +func (s *TestSuite) TestReencryptHeader_failedToConnect() { + // Mock the server address to an invalid one + Conf.API.Grpc.Host, Conf.API.Grpc.Port, _ = splitHostPort("localhost:9999") + + newHeader, err := reencryptHeader(s.FileHeader, s.UserKey.PubKeyBase64) + if err == nil { + s.T().Fatal("Expected an error due to failed connection, but got nil") + } + assert.Equal(s.T(), newHeader, []uint8([]byte(nil)), "expected header to be nil") + assert.ErrorContains(s.T(), err, "connection refused") +} diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml index f0efc433de..4904de3494 100644 --- a/sda/cmd/api/swagger_v1.yml +++ b/sda/cmd/api/swagger_v1.yml @@ -300,6 +300,12 @@ paths: schema: type: string required: true + - in: query + name: path_prefix + schema: + type: string + required: false + description: The inbox path prefix to filter entries on. responses: "200": content: @@ -315,6 +321,36 @@ paths: description: Authentication failure "500": description: Internal application error + /users/{userName}/file/{fileID}: + get: + description: Download a file from the inbox re-encrypted with the user public key. + parameters: + - in: path + name: userName + schema: + type: string + required: true + - in: path + name: fileID + schema: + type: string + required: true + responses: + "200": + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successful operation + "400": + description: User public key not provided or not valid. + "401": + description: Authentication failure. + "404": + description: File not found. + "500": + description: Internal application error. components: schemas: C4ghKeyAdd: diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index df2096cac9..8f7ca8acfb 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -4,7 +4,6 @@ package main import ( "encoding/json" - "errors" "fmt" "io" @@ -77,7 +76,7 @@ func main() { log.Debugf("Received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-accession) failed, reason: %v ", err) + log.Errorf("validation of incoming message (ingestion-accession) failed, correlation-id: %s, reason: %v ", delivered.CorrelationId, err) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -90,9 +89,9 @@ func main() { // If the file has been canceled by the uploader, don't spend time working on it. status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, reason: %v", err) + log.Errorf("failed to get file status, correlation-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -100,7 +99,7 @@ func main() { switch status { case "disabled": - log.Infof("file with correlation ID: %s is disabled, stopping work", delivered.CorrelationId) + log.Infof("file with correlation-id: %s is disabled, aborting work", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -110,14 +109,14 @@ func main() { case "verified": case "enabled": case "ready": - log.Infof("File with correlation ID %s is already marked as ready.", delivered.CorrelationId) + log.Infof("File with correlation-id: %s is already marked as ready.", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking message, reason: %v", err) } continue default: - log.Warnf("file with correlation ID: %s is not verified yet, stopping work", delivered.CorrelationId) + log.Warnf("file with correlation-id: %s is not verified yet, aborting work", delivered.CorrelationId) if err := delivered.Nack(false, true); err != nil { log.Errorf("Failed acking canceled work, reason: %v", err) } @@ -127,9 +126,9 @@ func main() { fileID, err := db.GetFileID(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get ID for file, reason: %v", err) + log.Errorf("failed to get file-id for file with correlation-id: %s, reason: %v", delivered.CorrelationId, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -144,16 +143,16 @@ func main() { completeMsg, _ := json.Marshal(&c) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-completion.json", conf.Broker.SchemasPath), completeMsg) if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%v)", err) + log.Errorf("Validation of outgoing message ingestion-completion failed, reason: (%v). Message body: %s\n", err, string(completeMsg)) continue } accessionIDExists, err := db.CheckAccessionIDExists(message.AccessionID, fileID) if err != nil { - log.Errorf("CheckAccessionIdExists failed, reason: %v ", err) + log.Errorf("CheckAccessionIdExists failed, file-id: %s, reason: %v ", fileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -161,7 +160,7 @@ func main() { switch accessionIDExists { case "duplicate": - log.Debugf("Seems accession ID already exists (corr-id: %s, accessionid: %s", delivered.CorrelationId, message.AccessionID) + log.Errorf("accession ID already exists in the system, file-id: %s, accession-id: %s\n", fileID, message.AccessionID) // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ Error: "There is a conflict regarding the file accessionID", @@ -172,22 +171,22 @@ func main() { // Send the message to an error queue so it can be analyzed. if e := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); e != nil { - log.Errorf("failed to publish message, reason: (%v)", err) + log.Errorf("failed to publish message, reason: %v", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%v)", err) + log.Errorf("failed to Ack message, reason: %v", err) } continue case "same": - log.Infoln("file already has a stable ID, marking it as ready") + log.Infof("file already has a stable ID, marking it as ready, file-id: %s", fileID) default: if conf.Backup.Type != "" && conf.Archive.Type != "" { if err = backupFile(delivered); err != nil { - log.Errorf("Failed to backup file with corrID: %v, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to backup file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -195,9 +194,9 @@ func main() { } if err := db.SetAccessionID(message.AccessionID, fileID); err != nil { - log.Errorf("Failed to set accessionID for file with corrID: %v, reason: %v", delivered.CorrelationId, err) + log.Errorf("failed to set accessionID for file, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, reason: %v", err) } continue @@ -206,25 +205,25 @@ func main() { // Mark file as "ready" if err := db.UpdateFileEventLog(fileID, "ready", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, reason: (%v)", err) + log.Errorf("set status ready failed, file-id: %s, reason: %v", fileID, err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, reason: %v", err) } continue } if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, completeMsg); err != nil { - log.Errorf("failed to publish message, reason: (%v)", err) + log.Errorf("failed to publish message, reason: %v", err) if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%v)", err) + log.Errorf("failed to Nack message, reason: %v", err) } continue } if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%v)", err) + log.Errorf("failed to Ack message, reason: %v", err) } } }() @@ -234,12 +233,12 @@ func main() { func backupFile(delivered amqp.Delivery) error { log.Debug("Backup initiated") - fileUUID, err := db.GetFileID(delivered.CorrelationId) + fileID, err := db.GetFileID(delivered.CorrelationId) if err != nil { return fmt.Errorf("failed to get ID for file, reason: %s", err.Error()) } - filePath, fileSize, err := db.GetArchived(fileUUID) + filePath, fileSize, err := db.GetArchived(fileID) if err != nil { return fmt.Errorf("failed to get file archive information, reason: %v", err) } @@ -251,7 +250,7 @@ func backupFile(delivered amqp.Delivery) error { } if diskFileSize != int64(fileSize) { - return errors.New("file size in archive does not match database for archive file") + return fmt.Errorf("archive file size does not match registered file size, (disk size: %d, db size: %d)", diskFileSize, fileSize) } file, err := archive.NewFileReader(filePath) @@ -273,7 +272,7 @@ func backupFile(delivered amqp.Delivery) error { } // Mark file as "backed up" - if err := db.UpdateFileEventLog(fileUUID, "backed up", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { + if err := db.UpdateFileEventLog(fileID, "backed up", delivered.CorrelationId, "finalize", "{}", string(delivered.Body)); err != nil { return fmt.Errorf("UpdateFileEventLog failed, reason: (%v)", err) } diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index 6c66889fb6..5c64923567 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -8,13 +8,13 @@ import ( "crypto/sha256" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "os" "os/signal" "strings" "syscall" - "time" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/crypt4gh/model/headers" @@ -22,6 +22,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" @@ -75,11 +76,15 @@ func main() { panic(err) } app.ArchiveKeyList, err = config.GetC4GHprivateKeys() - if err != nil { - log.Error(err) + if err != nil || len(app.ArchiveKeyList) == 0 { sigc <- syscall.SIGINT + panic(errors.New("no C4GH private keys configured")) + } + + if err := app.registerC4GHKey(); err != nil { panic(err) } + app.Archive, err = storage.NewBackend(app.Conf.Archive) if err != nil { log.Error(err) @@ -112,24 +117,6 @@ func main() { log.Info("starting ingest service") go func() { - start := time.Now() - for i := 1; i > 0; i++ { - h, err := app.DB.ListKeyHashes() - if err != nil { - log.Errorln(err.Error()) - } - if len(h) != 0 { - break - } - - time.Sleep(time.Duration(30 * time.Second)) - if time.Since(start).Seconds() >= float64(300) { - log.Errorln("no crypt4gh key hash registered, restarting") - forever <- false - } - log.Errorln("no crypt4gh key hash registered") - } - messages, err := app.MQ.GetMessages(app.Conf.Broker.Queue) if err != nil { log.Panic(err) @@ -140,7 +127,7 @@ func main() { message := schema.IngestionTrigger{} err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.Conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-trigger) failed, reason: (%s)", err.Error()) + log.Errorf("validation of incoming message (ingestion-trigger) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Message validation failed", @@ -150,10 +137,10 @@ func main() { body, _ := json.Marshal(infoErrorMessage) if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: %v", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) + log.Errorf("Failed acking canceled work, reason: %v", err) } continue @@ -172,23 +159,23 @@ func main() { default: log.Errorln("unexpected ingest message type") if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message for reason: (%s)", err.Error()) + log.Errorf("failed to reject message, reason: %v", err) } } switch ackNack { case "ack": if err := delivered.Ack(false); err != nil { - log.Errorf("failed to ack message for reason: (%s)", err.Error()) + log.Errorf("failed to ack message, reason: %v", err) } case "nack": if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) + log.Errorf("failed to Nack message, reason: %v", err) } default: // will catch `reject`s, failures that should not be requeued. if err := delivered.Reject(false); err != nil { - log.Errorf("failed to reject message for reason: (%s)", err.Error()) + log.Errorf("failed to reject message, reason: %v", err) } } } @@ -197,10 +184,27 @@ func main() { <-forever } +func (app *Ingest) registerC4GHKey() error { + h, err := app.DB.ListKeyHashes() + if err != nil { + return err + } + if len(h) == 0 { + for num, key := range app.ArchiveKeyList { + publicKey := keys.DerivePublicKey(*key) + if err := app.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), fmt.Sprintf("bootstrapped key: %d", num)); err != nil { + return err + } + } + } + + return nil +} + func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { - fileUUID, err := app.DB.GetFileID(correlationID) + fileID, err := app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file from message (correlationID: %s), reason: %s", correlationID, err.Error()) + log.Errorf("failed to get file-id for file from message (correlation-id: %s), reason: %s", correlationID, err.Error()) if strings.Contains(err.Error(), "sql: no rows in result set") { return "reject" } @@ -209,8 +213,8 @@ func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrig } m, _ := json.Marshal(message) - if err := app.DB.UpdateFileEventLog(fileUUID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set event log status for file: %s", correlationID) + if err := app.DB.UpdateFileEventLog(fileID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to update event log for file with id : %s", fileID) return "nack" } @@ -222,7 +226,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig var fileID string status, err := app.DB.GetFileStatus(correlationID) if err != nil && err.Error() != "sql: no rows in result set" { - log.Errorf("failed to get status for file, reason: (%s)", err.Error()) + log.Errorf("failed to get status for file, correlation-id: %s, reason: (%s)", correlationID, err.Error()) return "nack" } @@ -231,14 +235,14 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig case "disabled": fileID, err = app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) + log.Errorf("failed to get file-id for file, correlation-id: %s, reason: %s", correlationID, err.Error()) return "nack" } fileInfo, err := app.DB.GetFileInfo(fileID) if err != nil { - log.Errorf("failed to get info for file: %s, reason: %s", fileID, err.Error()) + log.Errorf("failed to get info for file, file-id: %s, reason: %s", fileID, err.Error()) return "nack" } @@ -246,15 +250,15 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // What if the file in the inbox is different this time? // Check uploaded checksum in the DB against the checksum of the file. // Would be easy if the inbox message had the checksum or that the s3inbox added the checksum to the DB. - file, err := app.Inbox.NewFileReader(message.FilePath) + file, err := app.Inbox.NewFileReader(helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { switch { case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): - log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + log.Errorf("Failed to open file to ingest, file-id: %s, inbox path: %s, reason: (%s)", fileID, message.FilePath, err.Error()) jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { - log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) + log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ @@ -264,7 +268,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } @@ -279,7 +283,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig inboxChecksum := sha256.New() _, err = io.Copy(inboxChecksum, file) if err != nil { - log.Errorf("failed to calculate cheksum for file: %s, reason: %s", fileID, err.Error()) + log.Errorf("failed to calculate the checksum for file, file-id: %s, reason: %s", fileID, err.Error()) return "nack" } @@ -297,20 +301,20 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig archivedMsg, _ := json.Marshal(&msg) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } m, _ := json.Marshal(message) if err = app.DB.UpdateFileEventLog(fileInfo.Path, "enabled", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", correlationID) + log.Errorf("failed to set ingestion status for file from message, file-id: %s", fileID) return "nack" } if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } @@ -318,28 +322,28 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig return "ack" } case "": - // Catch all for inboxes that doesn't update the DB - log.Warnln("ingesting unregisterd file") + // Catch all for implementations that don't update the DB, e.g. for those not using S3inbox or sftpInbox + log.Infof("registering file, correlation-id: %s", correlationID) fileID, err = app.DB.RegisterFile(message.FilePath, message.User) if err != nil { - log.Errorf("InsertFile failed, reason: (%s)", err.Error()) + log.Errorf("failed to register file, correlation-id: %s, reason: (%s)", correlationID, err.Error()) return "nack" } case "uploaded": fileID, err = app.DB.GetFileID(correlationID) if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) + log.Errorf("failed to get ID for file, correlation-id: %s, reason: %s", correlationID, err.Error()) return "nack" } default: - log.Warnf("unsupported file status: %s", status) + log.Warnf("unsupported file status: %s, correlation-id: %s", status, correlationID) return "reject" } - file, err := app.Inbox.NewFileReader(message.FilePath) + file, err := app.Inbox.NewFileReader(helper.UnanonymizeFilepath(message.FilePath, message.User)) if err != nil { switch { case (strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:")): @@ -347,7 +351,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { - log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) + log.Errorf("failed to set error status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. fileError := broker.InfoError{ @@ -357,36 +361,36 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } return "ack" default: - log.Errorf("unexpected error when opening file for reading: %s", err.Error()) + log.Errorf("unexpected error when opening file for reading, file-id: %s, filepath: %s, reason: %s", fileID, message.FilePath, err.Error()) return "nack" } } - fileSize, err := app.Inbox.GetFileSize(message.FilePath, false) + fileSize, err := app.Inbox.GetFileSize(helper.UnanonymizeFilepath(message.FilePath, message.User), false) if err != nil { - log.Errorf("Failed to get file size of file to ingest, reason: (%s)", err.Error()) + log.Errorf("Failed to get file size of file to ingest, file-id: %s, filepath: %s, reason: (%s)", fileID, message.FilePath, err.Error()) // Since reading the file worked, this should eventually succeed so it is ok to requeue. return "nack" } dest, err := app.Archive.NewFileWriter(fileID) if err != nil { - log.Errorf("Failed to create archive file, reason: (%s)", err.Error()) + log.Errorf("Failed to create archive file, file-id: %s, reason: (%s)", fileID, err.Error()) // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. return "nack" } m, _ := json.Marshal(message) if err = app.DB.UpdateFileEventLog(fileID, "submitted", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", correlationID) + log.Errorf("failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock @@ -402,7 +406,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig for bytesRead < fileSize { i, _ := io.ReadFull(file, readBuffer) if i == 0 { - log.Errorln("readBuffer returned 0 bytes, this should not happen") + log.Errorf("readBuffer returned 0 bytes, this should not happen, file-id: %s", fileID) return "reject" } @@ -415,7 +419,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig h := bytes.NewReader(readBuffer) if _, err = io.Copy(hash, h); err != nil { - log.Errorf("Copy to hash failed while reading file, reason: (%s)", err.Error()) + log.Errorf("Copy to hash failed while reading file, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } @@ -433,15 +437,15 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig break } - log.Warnf("Decryption failed with key, trying next key. Reason: (%s)", err.Error()) + log.Warnf("Decryption failed with key, trying next key. file-id: %s, reason: (%s)", fileID, err.Error()) } // Check if decryption was successful with any key if privateKey == nil { - log.Errorf("All keys failed to decrypt the submitted file") + log.Errorf("All keys failed to decrypt the submitted file, file-id: %s", fileID) m, _ := json.Marshal(message) if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { - log.Errorf("Failed to set ingestion status for file from message: %v", correlationID) + log.Errorf("Failed to set ingestion status for file from message, file-id: %s, reason: %s", fileID, err.Error()) } // Send the message to an error queue so it can be analyzed. @@ -452,7 +456,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig } body, _ := json.Marshal(fileError) if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: %v", err) } return "ack" @@ -464,20 +468,20 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig keyhash := hex.EncodeToString(publicKey[:]) err = app.DB.SetKeyHash(keyhash, fileID) if err != nil { - log.Errorf("Key hash %s could not be set for fileID %s: (%s)", keyhash, fileID, err.Error()) + log.Errorf("Key hash %s could not be set for file, file-id: %s, reason: (%s)", keyhash, fileID, err.Error()) return "nack" } log.Debugln("store header") if err := app.DB.StoreHeader(header, fileID); err != nil { - log.Errorf("StoreHeader failed, reason: (%s)", err.Error()) + log.Errorf("StoreHeader failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for header read, reason: %v)", err.Error()) + log.Errorf("Failed to write to read buffer for header read, file-id: %s, reason: %v)", fileID, err.Error()) return "nack" } @@ -485,7 +489,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // Strip header from buffer h := make([]byte, len(header)) if _, err = byteBuf.Read(h); err != nil { - log.Errorf("Failed to strip header from buffer, reason: (%s)", err.Error()) + log.Errorf("Failed to strip header from buffer, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } @@ -494,7 +498,7 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig readBuffer = readBuffer[:i] } if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for full read, reason: (%s)", err.Error()) + log.Errorf("Failed to write to read buffer for full read, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } @@ -502,12 +506,12 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig // Write data to file if _, err = byteBuf.WriteTo(dest); err != nil { - log.Errorf("Failed to write to archive file, reason: (%s)", err.Error()) + log.Errorf("Failed to write to archive file, file-id: %s, reason: (%s)", fileID, err.Error()) _ = file.Close() _ = dest.Close() if err := app.Archive.RemoveFile(fileID); err != nil { - log.Errorf("error when removing uncompleted file: %s", fileID) + log.Errorf("error when removing uncompleted file, file-id: %s, reason: %s", fileID, err.Error()) } return "nack" @@ -524,38 +528,38 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig fileInfo.UploadedChecksum = fmt.Sprintf("%x", hash.Sum(nil)) fileInfo.Size, err = app.Archive.GetFileSize(fileID, true) if err != nil { - log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) + log.Errorf("Couldn't get file size from archive, file-id: %s, reason: %v)", fileID, err.Error()) return "nack" } - log.Debugf("Wrote archived file (corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", correlationID, message.User, message.FilePath, fileID, fileInfo.Size) + log.Debugf("Wrote archived file (file-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", fileID, message.User, message.FilePath, fileID, fileInfo.Size) status, err = app.DB.GetFileStatus(correlationID) if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) + log.Errorf("failed to get file status, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping ingestion", correlationID) + log.Infof("file is disabled, stopping ingestion, file-id: %s", fileID) return "ack" } if err := app.DB.SetArchived(fileInfo, fileID); err != nil { - log.Errorf("SetArchived failed, reason: (%s)", err.Error()) + log.Errorf("SetArchived failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if err := app.DB.UpdateFileEventLog(fileID, "archived", correlationID, "ingest", "{}", string(m)); err != nil { - log.Errorf("failed to set event log status for file: %s", correlationID) + log.Errorf("failed to set event log status for file, file-id: %s, reason: %s", fileID, err.Error()) return "nack" } - log.Debugf("File marked as archived (corr-id: %s, user: %s, filepath: %s, archivepath: %s)", correlationID, message.User, message.FilePath, fileID) + log.Debugf("File marked as archived (file-id: %s, user: %s, filepath: %s)", fileID, message.User, message.FilePath) // Send message to archived msg := schema.IngestionVerification{ @@ -571,14 +575,14 @@ func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrig err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + log.Errorf("Validation of outgoing message failed, file-id: %s, reason: (%s)", fileID, err.Error()) return "nack" } if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: %v", err) return "reject" } diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index bd647a2e93..ead55be340 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -169,13 +169,19 @@ type TestSuite struct { filePath string pubKeyList [][32]byte ingest Ingest + tempDir string + UserName string } func (ts *TestSuite) SetupSuite() { + var err error viper.Set("log.level", "debug") - tempDir := ts.T().TempDir() - keyFile1 := fmt.Sprintf("%s/c4gh1.key", tempDir) - keyFile2 := fmt.Sprintf("%s/c4gh2.key", tempDir) + ts.tempDir, err = os.MkdirTemp("", "c4gh-keys") + if err != nil { + ts.FailNow("Failed to create temp directory") + } + keyFile1 := fmt.Sprintf("%s/c4gh1.key", ts.tempDir) + keyFile2 := fmt.Sprintf("%s/c4gh2.key", ts.tempDir) publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test") if err != nil { @@ -232,11 +238,14 @@ func (ts *TestSuite) SetupSuite() { if err := ts.ingest.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), "the test key"); err != nil { ts.FailNow("failed to register the public key") } + + ts.UserName = "test-ingest" } func (ts *TestSuite) TearDownSuite() { _ = os.RemoveAll(ts.ingest.Conf.Archive.Posix.Location) _ = os.RemoveAll(ts.ingest.Conf.Inbox.Posix.Location) + _ = os.RemoveAll(ts.tempDir) } func (ts *TestSuite) SetupTest() { @@ -251,7 +260,13 @@ func (ts *TestSuite) SetupTest() { ts.FailNow("failed to create temp folder") } - f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") + // Ensure a folder with the user name exists + err = os.Mkdir(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName), 0750) + if err != nil { + ts.FailNow("failed to create temp folder") + } + + f, err := os.CreateTemp(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName), "") if err != nil { ts.FailNow("failed to create test file") } @@ -295,6 +310,11 @@ func (ts *TestSuite) SetupTest() { if err != nil { ts.FailNow("failed to setup inbox backend") } + + viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ + {FilePath: filepath.Join(ts.tempDir, "c4gh1.key"), Passphrase: "test"}, + {FilePath: filepath.Join(ts.tempDir, "c4gh2.key"), Passphrase: "test"}, + }) } func (ts *TestSuite) TestTryDecrypt_wrongFile() { tempDir := ts.T().TempDir() @@ -345,6 +365,7 @@ func (ts *TestSuite) TestTryDecrypt() { privateKeys, err := config.GetC4GHprivateKeys() assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(privateKeys)) for i, key := range privateKeys { header, err := tryDecrypt(key, buf) @@ -353,7 +374,7 @@ func (ts *TestSuite) TestTryDecrypt() { assert.NoError(ts.T(), err) assert.NotNil(ts.T(), header) default: - assert.Contains(ts.T(), err.Error(), "could not find matching public key heade") + assert.Contains(ts.T(), err.Error(), "could not find matching public key header") assert.Nil(ts.T(), header) } } @@ -404,38 +425,36 @@ func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { // messages of type `ingest` func (ts *TestSuite) TestIngestFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } func (ts *TestSuite) TestIngestFile_secondTime() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -444,30 +463,28 @@ func (ts *TestSuite) TestIngestFile_secondTime() { assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) } func (ts *TestSuite) TestIngestFile_unknownInboxType() { - userName := "test-ingest-unknown" message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(uuid.New().String(), message)) } func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -480,19 +497,18 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -518,7 +534,7 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName, ts.filePath)) if err != nil { ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } @@ -552,19 +568,18 @@ func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -584,19 +599,18 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -620,19 +634,18 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { } func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { // prepare the DB entries - userName := "test-ingest" - fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, ts.UserName) assert.NoError(ts.T(), err, "failed to register file in database") corrID := uuid.New().String() - if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, ts.UserName, "{}", "{}"); err != nil { ts.Fail("failed to update file event log") } message := schema.IngestionTrigger{ Type: "ingest", FilePath: ts.filePath, - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) @@ -675,7 +688,7 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { ts.FailNow("failed to create private c4gh key") } - outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.UserName, ts.filePath)) if err != nil { ts.FailNowf("failed to create encrypted test file: %s", err.Error()) } @@ -711,15 +724,47 @@ func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { } func (ts *TestSuite) TestIngestFile_missingFile() { // prepare the DB entries - userName := "test-ingest" corrID := uuid.New().String() basepath := filepath.Dir(ts.filePath) message := schema.IngestionTrigger{ Type: "ingest", FilePath: fmt.Sprintf("%s/missing.file.c4gh", basepath), - User: userName, + User: ts.UserName, } assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) } +func (ts *TestSuite) TestDetectMisingC4GHKeys() { + viper.Set("c4gh.privateKeys", "") + privateKeys, err := config.GetC4GHprivateKeys() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 0, len(privateKeys)) +} + +func (ts *TestSuite) TestRegisterC4ghKey_newDeployment() { + _, err := ts.ingest.DB.DB.Exec("TRUNCATE sda.encryption_keys CASCADE;") + assert.NoError(ts.T(), err) + + privateKeys, err := config.GetC4GHprivateKeys() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(privateKeys)) + + assert.NoError(ts.T(), ts.ingest.registerC4GHKey()) + + kh, err := ts.ingest.DB.ListKeyHashes() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(kh)) +} + +func (ts *TestSuite) TestRegisterC4ghKey_existingEntry() { + privateKeys, err := config.GetC4GHprivateKeys() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(privateKeys)) + + assert.NoError(ts.T(), ts.ingest.registerC4GHKey()) + + kh, err := ts.ingest.DB.ListKeyHashes() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 1, len(kh)) +} diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 1113f4a874..ee58fc8cc2 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -10,6 +10,7 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" @@ -92,7 +93,7 @@ func main() { case "mapping": log.Debug("Mapping type operation, mapping files to dataset") if err := db.MapFilesToDataset(mappings.DatasetID, mappings.AccessionIDs); err != nil { - log.Errorf("failed to map files to dataset, reason: %v", err) + log.Errorf("failed to map files to dataset, dataset-id: %s, reason: %v", mappings.DatasetID, err) // Nack message so the server gets notified that something is wrong and requeue the message if err := delivered.Nack(false, true); err != nil { @@ -104,13 +105,14 @@ func main() { for _, aID := range mappings.AccessionIDs { log.Debugf("Mapped file to dataset (corr-id: %s, datasetid: %s, accessionid: %s)", delivered.CorrelationId, mappings.DatasetID, aID) - filePath, err := db.GetInboxPath(aID) + fileInfo, err := db.GetFileInfoFromAccessionID(aID) if err != nil { - log.Errorf("failed to get inbox path for file with stable ID: %s", aID) + log.Errorf("failed to get file info for file with stable ID: %s", aID) } - err = inbox.RemoveFile(filePath) + + err = inbox.RemoveFile(helper.UnanonymizeFilepath(fileInfo.FilePath, fileInfo.User)) if err != nil { - log.Errorf("Remove file from inbox failed, reason: %v", err) + log.Errorf("Remove file from inbox %s failed, reason: %v", fileInfo.FilePath, err) } } @@ -147,6 +149,8 @@ func main() { continue } + default: + log.Errorf("unknown mapping type, %s", mappings.Type) } if err := delivered.Ack(false); err != nil { diff --git a/sda/cmd/notify/notify.go b/sda/cmd/notify/notify.go index 8e85ca3ac2..32a3153464 100644 --- a/sda/cmd/notify/notify.go +++ b/sda/cmd/notify/notify.go @@ -4,7 +4,6 @@ package main import ( "encoding/base64" "encoding/json" - "errors" "fmt" "net/smtp" "strconv" @@ -72,7 +71,7 @@ func main() { log.Errorf("Failed to send email, error %v", err) if e := d.Nack(false, false); e != nil { - log.Errorf("Failed to Nack message (corr-id: %s, errror: %v) ", d.CorrelationId, e) + log.Errorf("Failed to Nack message, error: %v) ", e) } continue @@ -156,7 +155,7 @@ func validator(queue, schemaPath string, delivery amqp091.Delivery) error { } return nil + default: + return fmt.Errorf("unknown queue, %s", queue) } - - return errors.New("Error") } diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index 5c847b6542..457ac838a6 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -200,6 +200,10 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * continue } + default: + log.Errorln("unknown raouting key") + + continue } } } diff --git a/sda/cmd/reencrypt/reencrypt.go b/sda/cmd/reencrypt/reencrypt.go index 5c25e721a9..0f954c3fb5 100644 --- a/sda/cmd/reencrypt/reencrypt.go +++ b/sda/cmd/reencrypt/reencrypt.go @@ -33,7 +33,7 @@ import ( // server struct is used to implement reencrypt.ReEncryptServer. type server struct { re.UnimplementedReencryptServer - c4ghPrivateKey *[32]byte + c4ghPrivateKeyList []*[32]byte } // hServer struct is used to implement the proxy grpc health.HealthServer. @@ -90,14 +90,14 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r newReaderPublicKeyList := [][chacha20poly1305.KeySize]byte{} newReaderPublicKeyList = append(newReaderPublicKeyList, newReaderPublicKey) - log.Debugf("crypt4ghkey: %v", *s.c4ghPrivateKey) - - newheader, err := headers.ReEncryptHeader(in.GetOldheader(), *s.c4ghPrivateKey, newReaderPublicKeyList, extraHeaderPackets...) - if err != nil { - return nil, status.Error(400, err.Error()) + for _, key := range s.c4ghPrivateKeyList { + newheader, err := headers.ReEncryptHeader(in.GetOldheader(), *key, newReaderPublicKeyList, extraHeaderPackets...) + if err == nil { + return &re.ReencryptResponse{Header: newheader}, nil + } } - return &re.ReencryptResponse{Header: newheader}, nil + return nil, status.Error(400, err.Error()) } // Check implements the healthgrpc.HealthServer Check method for the proxy grpc Health server. @@ -218,7 +218,7 @@ func main() { } s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: conf.ReEncrypt.Crypt4GHKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: conf.ReEncrypt.C4ghPrivateKeyList}) reflection.Register(s) // Add health service for reencrypt server diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 71da4b588f..7792174540 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -33,7 +33,7 @@ type ReEncryptTests struct { FileData []byte KeyPath string FileHeader []byte - PrivateKey *[32]byte + PrivateKeyList []*[32]byte UserPrivateKey [32]byte UserPublicKey [32]byte UserPubKeyString string @@ -72,10 +72,11 @@ func (ts *ReEncryptTests) SetupTest() { } ts.UserPubKeyString = base64.StdEncoding.EncodeToString(buf.Bytes()) - viper.Set("c4gh.filepath", ts.KeyPath+"/c4gh.key") - viper.Set("c4gh.passphrase", "test") + viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ + {FilePath: ts.KeyPath + "/c4gh.key", Passphrase: "test"}, + }) - ts.PrivateKey, err = config.GetC4GHKey() + ts.PrivateKeyList, err = config.GetC4GHprivateKeys() if err != nil { ts.T().FailNow() } @@ -97,7 +98,7 @@ func (ts *ReEncryptTests) TestReencryptHeader() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) if err := s.Serve(lis); err != nil { ts.T().Fail() } @@ -139,7 +140,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_DataEditList() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) if err := s.Serve(lis); err != nil { ts.T().Fail() } @@ -213,7 +214,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_BadPubKey() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) _ = s.Serve(lis) }() @@ -243,7 +244,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_NoHeader() { go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) _ = s.Serve(lis) }() @@ -298,7 +299,7 @@ func (ts *ReEncryptTests) TestReencryptHeader_TLS() { ) opts := []grpc.ServerOption{grpc.Creds(serverCreds)} s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKeyList: ts.PrivateKeyList}) if err := s.Serve(lis); err != nil { ts.T().Fail() } diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index ebf18b8e4b..0604ec6883 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -22,6 +22,7 @@ import ( "github.com/minio/minio-go/v6/pkg/signer" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" @@ -150,8 +151,9 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw username := token.Subject() rawFilepath := strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) + anonymizedFilepath := helper.AnonymizeFilepath(rawFilepath, username) - filepath, err := formatUploadFilePath(rawFilepath) + filepath, err := formatUploadFilePath(anonymizedFilepath) if err != nil { reportError(http.StatusNotAcceptable, err.Error(), w) @@ -192,7 +194,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") - message, err := p.CreateMessageFromRequest(r, token) + message, err := p.CreateMessageFromRequest(r, token, anonymizedFilepath) if err != nil { p.internalServerError(w, r, err.Error()) @@ -295,6 +297,7 @@ func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { } } + log.Debugf("Sending message with id %s", p.fileIDs[r.URL.Path]) if err := p.messenger.SendMessage(p.fileIDs[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { return fmt.Errorf("error when sending message to broker: %v", err) } @@ -382,18 +385,24 @@ func (p *Proxy) prependBucketToHostPath(r *http.Request) error { case strings.Contains(r.URL.String(), "?location") || strings.Contains(r.URL.String(), "&prefix"): r.URL.Path = "/" + bucket + "/" log.Debug("new Path: ", r.URL.Path) + default: + r.URL.Path = "/" + bucket + "/" } case http.MethodPost: r.URL.Path = "/" + bucket + r.URL.Path log.Debug("new Path: ", r.URL.Path) case http.MethodPut: r.URL.Path = "/" + bucket + r.URL.Path - log.Debug("new Path: ", r.URL.Path) + log.Info("new PUT Path: ", r.URL.Path) case http.MethodDelete: if strings.Contains(r.URL.String(), "?uploadId") { // abort multipart upload r.URL.Path = "/" + bucket + r.URL.Path } + default: + log.Errorf("unknown request: %s", r.Method) + + return errors.New("unknown request method") } log.Infof("User: %v, Request type %v, Path: %v", username, r.Method, r.URL.Path) @@ -482,7 +491,7 @@ func (p *Proxy) detectRequestType(r *http.Request) S3RequestType { // CreateMessageFromRequest is a function that can take a http request and // figure out the correct rabbitmq message to send from it. -func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token) (Event, error) { +func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token, anonymizedFilepath string) (Event, error) { event := Event{} checksum := Checksum{} var err error @@ -494,7 +503,8 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token) (Eve // Case for simple upload event.Operation = "upload" - event.Filepath = strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) + event.Filepath = anonymizedFilepath + event.Username = claims.Subject() checksum.Type = "sha256" event.Checksum = []any{checksum} diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 313e389661..13e901b58a 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -377,7 +377,8 @@ func (s *ProxyTests) TestServeHTTP_allowed() { // Put file works w = httptest.NewRecorder() - r.Method = "PUT" + r, err = http.NewRequest("PUT", "/dummy/file", nil) + assert.NoError(s.T(), err) s.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" proxy.allowedResponse(w, r, s.token) assert.Equal(s.T(), 200, w.Result().StatusCode) @@ -467,17 +468,18 @@ func (s *ProxyTests) TestMessageFormatting() { r.Header.Set("x-amz-content-sha256", "checksum") claims := jwt.New() - assert.NoError(s.T(), claims.Set("sub", "user@host.domain")) + user := "user@host.domain" + assert.NoError(s.T(), claims.Set("sub", user)) // start proxy that denies everything proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" - msg, err := proxy.CreateMessageFromRequest(r, claims) + msg, err := proxy.CreateMessageFromRequest(r, claims, "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), int64(1234), msg.Filesize) - assert.Equal(s.T(), "user/new_file.txt", msg.Filepath) + assert.Equal(s.T(), "new_file.txt", msg.Filepath) assert.Equal(s.T(), "user@host.domain", msg.Username) c, _ := json.Marshal(msg.Checksum[0]) @@ -488,10 +490,11 @@ func (s *ProxyTests) TestMessageFormatting() { // Test single shot upload r.Method = "PUT" - msg, err = proxy.CreateMessageFromRequest(r, jwt.New()) + msg, err = proxy.CreateMessageFromRequest(r, jwt.New(), "new_file.txt") assert.Nil(s.T(), err) assert.IsType(s.T(), Event{}, msg) assert.Equal(s.T(), "upload", msg.Operation) + assert.Equal(s.T(), "new_file.txt", msg.Filepath) } func (s *ProxyTests) TestDatabaseConnection() { @@ -506,7 +509,7 @@ func (s *ProxyTests) TestDatabaseConnection() { // PUT a file into the system filename := "/dummy/db-test-file" - + anonymFilename := "db-test-file" stringReader := strings.NewReader("a brand new string") r, _ := http.NewRequest("PUT", filename, stringReader) w := httptest.NewRecorder() @@ -525,7 +528,7 @@ func (s *ProxyTests) TestDatabaseConnection() { // Check that the file is in the database var fileID string query := "SELECT id FROM sda.files WHERE submission_file_path = $1;" - err = db.DB.QueryRow(query, filename[1:]).Scan(&fileID) + err = db.DB.QueryRow(query, anonymFilename).Scan(&fileID) assert.Nil(s.T(), err, "Failed to query database") assert.NotNil(s.T(), fileID, "File not found in database") diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 8dcf42e3e8..7d71051a70 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -96,7 +96,7 @@ func main() { err := schema.ValidateJSON(fmt.Sprintf("%s/dataset-mapping.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (dataset-mapping) failed, reason: (%s)", err.Error()) + log.Errorf("validation of incoming message (dataset-mapping) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Message validation failed in sync service", @@ -106,7 +106,7 @@ func main() { body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: (%v)", err) } if err := delivered.Ack(false); err != nil { log.Errorf("failed to Ack message, reason: (%s)", err.Error()) @@ -129,7 +129,7 @@ func main() { for _, aID := range message.AccessionIDs { if err := syncFiles(aID); err != nil { - log.Errorf("failed to sync archived file %s, reason: (%s)", aID, err.Error()) + log.Errorf("failed to sync archived file: accession-id: %s, reason: (%s)", aID, err.Error()) if err := delivered.Nack(false, false); err != nil { log.Errorf("failed to nack following GetFileSize error message") } @@ -165,17 +165,17 @@ func syncFiles(stableID string) error { log.Debugf("syncing file %s", stableID) inboxPath, err := db.GetInboxPath(stableID) if err != nil { - return fmt.Errorf("failed to get inbox path for file with stable ID: %s", stableID) + return fmt.Errorf("failed to get inbox path for file with stable ID: %s, reason: %v", stableID, err) } archivePath, err := db.GetArchivePath(stableID) if err != nil { - return fmt.Errorf("failed to get archive path for file with stable ID: %s", stableID) + return fmt.Errorf("failed to get archive path for file with stable ID: %s, reason: %v", stableID, err) } fileSize, err := archive.GetFileSize(archivePath, false) if err != nil { - return err + return fmt.Errorf("failed to get filesize for file with archivepath: %s, reason: %v", archivePath, err) } file, err := archive.NewFileReader(archivePath) diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index 908a2546a6..572ac6c5ce 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -41,8 +41,8 @@ func main() { log.Fatal(err) } archiveKeyList, err := config.GetC4GHprivateKeys() - if err != nil { - log.Fatal(err) + if err != nil || len(archiveKeyList) == 0 { + log.Fatal("no C4GH private keys configured") } defer mq.Channel.Close() @@ -74,7 +74,7 @@ func main() { log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), delivered.Body) if err != nil { - log.Errorf("validation of incoming message (ingestion-verifiation) failed, reason: (%s)", err.Error()) + log.Errorf("validation of incoming message (ingestion-verification) failed, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Message validation failed", @@ -84,10 +84,10 @@ func main() { body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + log.Errorf("failed to publish message, reason: %v", err) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) + log.Errorf("Failed to Ack message, reason: %v", err) } // Restart on new message @@ -104,7 +104,7 @@ func main() { // If the file has been canceled by the uploader, don't spend time working on it. status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) + log.Errorf("failed to get file status, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Getheader failed", @@ -124,7 +124,7 @@ func main() { continue } if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with correlation-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } @@ -134,7 +134,7 @@ func main() { header, err := db.GetHeader(message.FileID) if err != nil { - log.Errorf("GetHeader failed for file with ID: %v, readon: %v", message.FileID, err.Error()) + log.Errorf("GetHeader failed for file with ID: %v, reason: %v", message.FileID, err.Error()) if err := delivered.Ack(false); err != nil { log.Errorf("Failed to nack following getheader error message") } @@ -158,11 +158,11 @@ func main() { var file database.FileInfo file.Size, err = archive.GetFileSize(message.ArchivePath, false) if err != nil { //nolint:nestif - log.Errorf("Failed to get archived file size, reson: (%s)", err.Error()) + log.Errorf("Failed to get archived file size, file-id: %s, archive-path: %s, reason: (%s)", message.FileID, message.ArchivePath, err.Error()) if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", string(jsonMsg), string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) + log.Errorf("failed to set ingestion status for file from message, file-id: %v", message.FileID) } } @@ -187,7 +187,7 @@ func main() { archiveFileHash := sha256.New() f, err := archive.NewFileReader(message.ArchivePath) if err != nil { - log.Errorf("Failed to open archived file, reson: %v ", err.Error()) + log.Errorf("Failed to open archived file, file-id: %s, reason: %v ", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Failed to open archived file", @@ -215,7 +215,7 @@ func main() { } if key == nil { - log.Errorf("no matching key found for file: %s.", message.ArchivePath) + log.Errorf("no matching key found for file, file-id: %s, archive-path: %s", message.FileID, message.ArchivePath) continue } @@ -223,7 +223,7 @@ func main() { mr := io.MultiReader(bytes.NewReader(header), io.TeeReader(f, archiveFileHash)) c4ghr, err := streaming.NewCrypt4GHReader(mr, *key, nil) if err != nil { - log.Errorf("failed to open c4gh decryptor stream, reson: %s", err.Error()) + log.Errorf("failed to open c4gh decryptor stream, file-id: %s, archive-path: %s, reason: %s", message.FileID, message.ArchivePath, err.Error()) continue } @@ -233,7 +233,7 @@ func main() { stream := io.TeeReader(c4ghr, md5hash) if file.DecryptedSize, err = io.Copy(sha256hash, stream); err != nil { - log.Errorf("failed to copy decrypted data, reson: (%s)", err.Error()) + log.Errorf("failed to copy decrypted data, file-id: %s, reason: (%s)", message.FileID, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ @@ -244,11 +244,11 @@ func main() { body, _ := json.Marshal(infoErrorMessage) if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("Failed to publish error message: (%s)", err.Error()) + log.Errorf("Failed to publish error message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message: (%s)", err.Error()) + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } continue @@ -263,7 +263,7 @@ func main() { case message.ReVerify: decrypted, err := db.GetDecryptedChecksum(message.FileID) if err != nil { - log.Errorf("failed to get unencrypted checksum for file: %s, reson: %s", message.FilePath, err.Error()) + log.Errorf("failed to get unencrypted checksum for file, file-id: %s, reason: %s", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } @@ -272,9 +272,9 @@ func main() { } if file.DecryptedChecksum != decrypted { - log.Errorf("encrypted checksum don't match for file: %s", message.FilePath) + log.Errorf("encrypted checksum don't match for file, file-id: %s", message.FileID) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"decrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, reason: (%v)", err) + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%v)", err) } @@ -282,16 +282,16 @@ func main() { continue } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message: (%s)", err.Error()) + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } continue } if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { - log.Errorf("encrypted checksum don't match for file: %s, expected %s, got %s", message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) + log.Errorf("encrypted checksum mismatch for file, file-id: %s, filepath: %s, expected: %s, got: %s", message.FileID, message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { - log.Errorf("set status ready failed, reason: (%v)", err) + log.Errorf("set status ready failed, file-id: %s, reason: (%v)", message.FileID, err) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%v)", err) } @@ -301,7 +301,7 @@ func main() { } if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to ack message: (%s)", err.Error()) + log.Errorf("Failed to ack message, reason: (%s)", err.Error()) } continue @@ -318,14 +318,14 @@ func main() { verifiedMessage, _ := json.Marshal(&c) err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-accession-request.json", conf.Broker.SchemasPath), verifiedMessage) if err != nil { - log.Errorf("Validation of outgoing (ingestion-accession-request) failed, reason: (%s)", err.Error()) + log.Errorf("Validation of outgoing (ingestion-accession-request) failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) // Logging is in ValidateJSON so just restart on new message continue } status, err := db.GetFileStatus(delivered.CorrelationId) if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) + log.Errorf("failed to get file status, correlation-id: %s, reason: (%s)", delivered.CorrelationId, err.Error()) // Send the message to an error queue so it can be analyzed. infoErrorMessage := broker.InfoError{ Error: "Getheader failed", @@ -346,7 +346,7 @@ func main() { } if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) + log.Infof("file with correlation-id: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } @@ -356,7 +356,7 @@ func main() { fileInfo, err := db.GetFileInfo(message.FileID) if err != nil { - log.Errorf("failed to get info for file: %s", message.FileID) + log.Errorf("failed to get info for file, file-id: %s", message.FileID) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } @@ -366,7 +366,7 @@ func main() { if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { if err := db.SetVerified(file, message.FileID); err != nil { - log.Errorf("SetVerified failed, reason: (%s)", err.Error()) + log.Errorf("SetVerified failed, file-id: %s, reason: (%s)", message.FileID, err.Error()) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } @@ -374,11 +374,11 @@ func main() { continue } } else { - log.Infof("file with correlation id: %s is already verified", delivered.CorrelationId) + log.Infof("file is already verified, file-id: %s", message.FileID) } if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { - log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) + log.Errorf("failed to set event log status for file, file-id: %s", message.FileID) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } @@ -398,6 +398,7 @@ func main() { log.Errorf("failed to Ack message, reason: (%s)", err.Error()) } } + log.Infof("Successfully verified the file, file-id: %s, filepath: %s", message.FileID, message.FilePath) } }() diff --git a/sda/config_local.yaml b/sda/config_local.yaml index 8355143fdb..d2eb9f0885 100644 --- a/sda/config_local.yaml +++ b/sda/config_local.yaml @@ -3,6 +3,11 @@ log: level: "debug" api: rbacFile: ../.github/integration/sda/rbac.json + port: 8090 + +grpc: + host: "localhost" + port: 50051 archive: type: s3 @@ -86,7 +91,7 @@ server: cert: "" key: "" jwtpubkeypath: "/tmp/shared/keys/pub/" - jwtpubkeyurl: "http://oidc:8080/jwk" + jwtpubkeyurl: "http://localhost:8800/oidc/jwk" sync: api: diff --git a/sda/go.mod b/sda/go.mod index d41bb2cefc..27e056de21 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -1,22 +1,20 @@ module github.com/neicnordic/sensitive-data-archive -go 1.24.0 - -toolchain go1.24.1 +go 1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.36.3 - github.com/aws/aws-sdk-go-v2/config v1.29.14 - github.com/aws/aws-sdk-go-v2/credentials v1.17.67 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 - github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 - github.com/aws/smithy-go v1.22.3 - github.com/casbin/casbin/v2 v2.105.0 - github.com/coreos/go-oidc/v3 v3.14.1 - github.com/gin-gonic/gin v1.10.0 + github.com/aws/aws-sdk-go-v2 v1.39.0 + github.com/aws/aws-sdk-go-v2/config v1.31.7 + github.com/aws/aws-sdk-go-v2/credentials v1.18.11 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5 + github.com/aws/aws-sdk-go-v2/service/s3 v1.88.0 + github.com/aws/smithy-go v1.23.0 + github.com/casbin/casbin/v2 v2.122.0 + github.com/coreos/go-oidc/v3 v3.15.0 + github.com/gin-gonic/gin v1.10.1 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 - github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 + github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c github.com/kataras/iris/v12 v12.2.11 github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.10.9 @@ -31,85 +29,84 @@ require ( github.com/rabbitmq/amqp091-go v1.10.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.20.1 - github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.38.0 - golang.org/x/oauth2 v0.30.0 - google.golang.org/grpc v1.72.0 - google.golang.org/protobuf v1.36.6 + github.com/spf13/viper v1.21.0 + github.com/stretchr/testify v1.11.1 + golang.org/x/crypto v0.42.0 + golang.org/x/oauth2 v0.31.0 + google.golang.org/grpc v1.75.0 + google.golang.org/protobuf v1.36.8 ) require ( dario.cat/mergo v1.0.1 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect - github.com/BurntSushi/toml v1.4.0 // indirect + github.com/BurntSushi/toml v1.5.0 // indirect github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 // indirect - github.com/CloudyKit/jet/v6 v6.2.0 // indirect + github.com/CloudyKit/jet/v6 v6.3.1 // indirect github.com/Joker/jade v1.1.3 // indirect github.com/Microsoft/go-winio v0.6.2 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect - github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect - github.com/andybalholm/brotli v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b // indirect + github.com/andybalholm/brotli v1.1.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.29.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.38.3 // indirect github.com/aymerick/douceur v0.2.0 // indirect - github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect - github.com/bytedance/sonic v1.12.2 // indirect - github.com/bytedance/sonic/loader v0.2.0 // indirect - github.com/casbin/govaluate v1.3.0 // indirect + github.com/bmatcuk/doublestar/v4 v4.8.1 // indirect + github.com/bytedance/sonic v1.13.2 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/casbin/govaluate v1.6.0 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect - github.com/cloudwego/iasm v0.2.0 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/containerd/continuity v0.4.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/docker/cli v27.4.1+incompatible // indirect - github.com/docker/docker v27.2.1+incompatible // indirect + github.com/docker/docker v28.0.0+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect - github.com/fsnotify/fsnotify v1.8.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.5 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/fsnotify/fsnotify v1.9.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-jose/go-jose/v3 v3.0.4 // indirect - github.com/go-jose/go-jose/v4 v4.0.5 // indirect + github.com/go-jose/go-jose/v4 v4.1.1 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.22.1 // indirect - github.com/go-viper/mapstructure/v2 v2.2.1 // indirect - github.com/goccy/go-json v0.10.3 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect + github.com/go-viper/mapstructure/v2 v2.4.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt/v5 v5.2.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kataras/blocks v0.0.8 // indirect - github.com/kataras/golog v0.1.12 // indirect - github.com/kataras/pio v0.0.13 // indirect + github.com/kataras/blocks v0.0.11 // indirect + github.com/kataras/golog v0.1.13 // indirect + github.com/kataras/pio v0.0.14 // indirect github.com/kataras/sitemap v0.0.6 // indirect github.com/kataras/tunnel v0.0.4 // indirect - github.com/klauspost/compress v1.17.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.8 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lestrrat-go/blackmagic v1.0.3 // indirect @@ -118,7 +115,7 @@ require ( github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect github.com/mailgun/raymond/v2 v2.0.48 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/mailru/easyjson v0.9.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -130,20 +127,20 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opencontainers/runc v1.2.3 // indirect - github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.7.0 // indirect + github.com/sagikazarmark/locafero v0.11.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/segmentio/asm v1.2.0 // indirect - github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.12.0 // indirect - github.com/spf13/cast v1.7.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 // indirect + github.com/spf13/afero v1.15.0 // indirect + github.com/spf13/cast v1.10.0 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/subosito/gotenv v1.6.0 // indirect - github.com/tdewolff/minify/v2 v2.20.37 // indirect - github.com/tdewolff/parse/v2 v2.7.15 // indirect + github.com/tdewolff/minify/v2 v2.23.6 // indirect + github.com/tdewolff/parse/v2 v2.8.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect @@ -153,15 +150,14 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yosssi/ace v0.0.5 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/arch v0.10.0 // indirect - golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sys v0.33.0 // indirect - golang.org/x/text v0.25.0 // indirect - golang.org/x/time v0.8.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect - gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/arch v0.17.0 // indirect + golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect + golang.org/x/net v0.43.0 // indirect + golang.org/x/sys v0.36.0 // indirect + golang.org/x/text v0.29.0 // indirect + golang.org/x/time v0.11.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/sda/go.sum b/sda/go.sum index 6933adbf9a..6e98f3a961 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -4,12 +4,12 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= -github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= +github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53 h1:sR+/8Yb4slttB4vD+b9btVEnWgL3Q00OBTzVT8B9C0c= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= -github.com/CloudyKit/jet/v6 v6.2.0 h1:EpcZ6SR9n28BUGtNJSvlBqf90IpjeFr36Tizxhn/oME= -github.com/CloudyKit/jet/v6 v6.2.0/go.mod h1:d3ypHeIRNo2+XyqnGA8s+aphtcVpjP5hPwP/Lzo7Ro4= +github.com/CloudyKit/jet/v6 v6.3.1 h1:6IAo5Cx21xrHVaR8zzXN5gJatKV/wO7Nf6bfCnCSbUw= +github.com/CloudyKit/jet/v6 v6.3.1/go.mod h1:lf8ksdNsxZt7/yH/3n4vJQWA9RUq4wpaHtArHhGVMOw= github.com/Joker/hpp v1.0.0 h1:65+iuJYdRXv/XyN62C1uEmmOx3432rNG/rKlX6V7Kkc= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= github.com/Joker/jade v1.1.3 h1:Qbeh12Vq6BxURXT1qZBRHsDxeURB8ztcL6f3EXSGeHk= @@ -18,75 +18,76 @@ github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERo github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= -github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 h1:gjbp60h8IZQbN/TpDaYJedWbbD1h1aDPEwWnYWaDaUY= -github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= +github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b h1:3QuwXO2TnDssBmmNKfRlO3ur9zOupJRdM7evq61Oi84= +github.com/Shopify/goreferrer v0.0.0-20250513162709-b78e2829e40b/go.mod h1:NYezi6wtnJtBm5btoprXc5SvAdqH0XTXWnUup0MptAI= github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= -github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= -github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= -github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= -github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= -github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= +github.com/aws/aws-sdk-go-v2 v1.39.0 h1:xm5WV/2L4emMRmMjHFykqiA4M/ra0DJVSWUkDyBjbg4= +github.com/aws/aws-sdk-go-v2 v1.39.0/go.mod h1:sDioUELIUO9Znk23YVmIk86/9DOpkbyyVb1i/gUNFXY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1 h1:i8p8P4diljCr60PpJp6qZXNlgX4m2yQFpYk+9ZT+J4E= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.1/go.mod h1:ddqbooRZYNoJ2dsTwOty16rM+/Aqmk/GOXrK8cg7V00= +github.com/aws/aws-sdk-go-v2/config v1.31.7 h1:zS1O6hr6t0nZdBCMFc/c9OyZFyLhXhf/B2IZ9Y0lRQE= +github.com/aws/aws-sdk-go-v2/config v1.31.7/go.mod h1:GpHmi1PQDdL5pP4JaB00pU0ek4EXVcYH7IkjkUadQmM= +github.com/aws/aws-sdk-go-v2/credentials v1.18.11 h1:1Fnb+7Dk96/VYx/uYfzk5sU2V0b0y2RWZROiMZCN/Io= +github.com/aws/aws-sdk-go-v2/credentials v1.18.11/go.mod h1:iuvn9v10dkxU4sDgtTXGWY0MrtkEcmkUmjv4clxhuTc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7 h1:Is2tPmieqGS2edBnmOJIbdvOA6Op+rRpaYR60iBAwXM= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.7/go.mod h1:F1i5V5421EGci570yABvpIXgRIBPb5JM+lSkHF6Dq5w= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5 h1:fSuJX/VBJKufwJG/szWgUdRJVyRiEQDDXNh/6NPrTBg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.5/go.mod h1:LvN0noQuST+3Su55Wl++BkITpptnfN9g6Ohkv4zs9To= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7 h1:UCxq0X9O3xrlENdKf1r9eRJoKz/b0AfGkpp3a7FPlhg= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.7/go.mod h1:rHRoJUNUASj5Z/0eqI4w32vKvC7atoWR0jC+IkmVH8k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7 h1:Y6DTZUn7ZUC4th9FMBbo8LVE+1fyq3ofw+tRwkUd3PY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.7/go.mod h1:x3XE6vMnU9QvHN/Wrx2s44kwzV2o2g5x/siw4ZUJ9g8= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= -github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= -github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7 h1:BszAktdUo2xlzmYHjWMq70DqJ7cROM8iBd3f6hrpuMQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.7/go.mod h1:XJ1yHki/P7ZPuG4fd3f0Pg/dSGA2cTQBCLw82MH2H48= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1 h1:oegbebPEMA/1Jny7kvwejowCaHz1FWZAQ94WXFNCyTM= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.1/go.mod h1:kemo5Myr9ac0U9JfSjMo9yHLtw+pECEHsFtJ9tqCEI8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7 h1:zmZ8qvtE9chfhBPuKB2aQFxW5F/rpwXUgmcVCgQzqRw= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.8.7/go.mod h1:vVYfbpd2l+pKqlSIDIOgouxNsGu5il9uDp0ooWb0jys= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7 h1:mLgc5QIgOy26qyh5bvW+nDoAppxgn3J2WV3m9ewq7+8= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.7/go.mod h1:wXb/eQnqt8mDQIQTTmcw58B5mYGxzLGZGK8PWNFZ0BA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7 h1:u3VbDKUCWarWiU+aIUK4gjTr/wQFXV17y3hgNno9fcA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.7/go.mod h1:/OuMQwhSyRapYxq6ZNpPer8juGNrB4P5Oz8bZ2cgjQE= +github.com/aws/aws-sdk-go-v2/service/s3 v1.88.0 h1:k5JXPr+2SrPDwM3PdygZUenn0lVPLa3KOs7cCYqinFs= +github.com/aws/aws-sdk-go-v2/service/s3 v1.88.0/go.mod h1:xajPTguLoeQMAOE44AAP2RQoUhF8ey1g5IFHARv71po= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.2 h1:rcoTaYOhGE/zfxE1uR6X5fvj+uKkqeCNRE0rBbiQM34= +github.com/aws/aws-sdk-go-v2/service/sso v1.29.2/go.mod h1:Ql6jE9kyyWI5JHn+61UT/Y5Z0oyVJGmgmJbZD5g4unY= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3 h1:BSIfeFtU9tlSt8vEYS7KzurMoAuYzYPWhcZiMtxVf2M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.34.3/go.mod h1:XclEty74bsGBCr1s0VSaA11hQ4ZidK4viWK7rRfO88I= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.3 h1:yEiZ0ztgji2GsCb/6uQSITXcGdtmWMfLRys0jJFiUkc= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.3/go.mod h1:Z+Gd23v97pX9zK97+tX4ppAgqCt3Z2dIXB02CtBncK8= +github.com/aws/smithy-go v1.23.0 h1:8n6I3gXzWJB2DxBDnfxgBaSX6oe0d/t10qGz7OKqMCE= +github.com/aws/smithy-go v1.23.0/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= -github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= github.com/bmatcuk/doublestar/v4 v4.6.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= -github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= -github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= +github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= -github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/casbin/casbin/v2 v2.105.0 h1:dLj5P6pLApBRat9SADGiLxLZjiDPvA1bsPkyV4PGx6I= -github.com/casbin/casbin/v2 v2.105.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= -github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/casbin/casbin/v2 v2.122.0 h1:T960HruD6W3JZmoYQjKAu0YBLUL++5l7LormNJIcmkc= +github.com/casbin/casbin/v2 v2.122.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= +github.com/casbin/govaluate v1.6.0 h1:HXWFa4di61X4iZrWLJdpot/v1UkKUG+pPmcu7wWRxrI= +github.com/casbin/govaluate v1.6.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= -github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= -github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= -github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= -github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= -github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= +github.com/coreos/go-oidc/v3 v3.15.0 h1:R6Oz8Z4bqWR7VFQ+sPSvZPQv4x8M+sJkDO5ojgwlyAg= +github.com/coreos/go-oidc/v3 v3.15.0/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -99,35 +100,35 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvw github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= -github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= -github.com/docker/docker v27.2.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.0.0+incompatible h1:Olh0KS820sJ7nPsBKChVhk5pzqcwDR15fumfAd/p9hM= +github.com/docker/docker v28.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0Hw= github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= -github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= -github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= -github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= +github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= +github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= +github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= +github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= +github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= +github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= +github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= -github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-jose/go-jose/v4 v4.1.1 h1:JYhSgy4mXXzAdF3nUx3ygx347LRXJRrpgyU3adRmkAI= +github.com/go-jose/go-jose/v4 v4.1.1/go.mod h1:BdsZGqgdO3b6tTc6LSE56wcDbMMLuPsw5d4ZD5f94kA= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -136,16 +137,16 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= -github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= +github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= -github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= -github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= @@ -154,13 +155,14 @@ github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 h1:ZPy+2XJ8u0bB3sNFi+I72gMEMS7MTg7aZCCXPOjV8iw= -github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b h1:EY/KpStFl60qA17CptGXhwfZ+k1sFNJIUNR8DdbcuUk= +github.com/gomarkdown/markdown v0.0.0-20250311123330-531bef5e742b/go.mod h1:JDGcbDT52eL4fju3sZ4TeHGsQwhG9nbDV21aMyhwPoA= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -173,16 +175,16 @@ github.com/gorilla/css v1.0.1 h1:ntNaBIghp6JmvWnxbZKANoLyuXTPZ4cAMlo6RyhlbO8= github.com/gorilla/css v1.0.1/go.mod h1:BvnYkspnSzMmwRK+b8/xgNPLiIuNZr6vbZBTPQ2A3b0= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= github.com/iris-contrib/httpexpect/v2 v2.15.2/go.mod h1:JLDgIqnFy5loDSUv1OA2j0mb6p/rDhiCqigP22Uq9xE= -github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 h1:BuEwEv6gPS9pSX0+O5FIxxKUoRziXBFdWI1S4ivKQtY= -github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72/go.mod h1:DQJ0KlNPppOfMC+0x0ADeFQk0WmQMVU9rJQzFY4nUfA= +github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c h1:7gyBIX75xKttBJo35lyVLUb/OQh7jY2wcEIVUygmsL8= +github.com/iris-contrib/middleware/cors v0.0.0-20250207234507-372f6828ef8c/go.mod h1:SvRi2dE8S9rgUvgDrGg7hpb/1gt9zVnvLnWjMQJUONo= github.com/iris-contrib/schema v0.0.6 h1:CPSBLyx2e91H2yJzPuhGuifVRnZBBJ3pCOMbOvPZaTw= github.com/iris-contrib/schema v0.0.6/go.mod h1:iYszG0IOsuIsfzjymw1kMzTL8YQcCWlm65f3wX8J5iA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -191,31 +193,30 @@ github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/kataras/blocks v0.0.8 h1:MrpVhoFTCR2v1iOOfGng5VJSILKeZZI+7NGfxEh3SUM= -github.com/kataras/blocks v0.0.8/go.mod h1:9Jm5zx6BB+06NwA+OhTbHW1xkMOYxahnqTN5DveZ2Yg= -github.com/kataras/golog v0.1.12 h1:Bu7I/G4ilJlbfzjmU39O9N+2uO1pBcMK045fzZ4ytNg= -github.com/kataras/golog v0.1.12/go.mod h1:wrGSbOiBqbQSQznleVNX4epWM8rl9SJ/rmEacl0yqy4= +github.com/kataras/blocks v0.0.11 h1:JJdYW0AUaJKLx5kEWs/oRVCvKVXo+6CAAeaVAiJf7wE= +github.com/kataras/blocks v0.0.11/go.mod h1:b4UySrJySEOq6drKH9U3bOpMI+dRH148mayYfS3RFb8= +github.com/kataras/golog v0.1.13 h1:bGbPglTdCutekqwOUf8L1jq3tZ5ADG9gfPBd5p5SzKA= +github.com/kataras/golog v0.1.13/go.mod h1:oQmzBTCv/35TetBosjJl/k+LPdlJEblaTupkNwJlwj8= github.com/kataras/iris/v12 v12.2.11 h1:sGgo43rMPfzDft8rjVhPs6L3qDJy3TbBrMD/zGL1pzk= github.com/kataras/iris/v12 v12.2.11/go.mod h1:uMAeX8OqG9vqdhyrIPv8Lajo/wXTtAF43wchP9WHt2w= -github.com/kataras/pio v0.0.13 h1:x0rXVX0fviDTXOOLOmr4MUxOabu1InVSTu5itF8CXCM= -github.com/kataras/pio v0.0.13/go.mod h1:k3HNuSw+eJ8Pm2lA4lRhg3DiCjVgHlP8hmXApSej3oM= +github.com/kataras/pio v0.0.14 h1:VGBHOmhwrMMrZeuRqoSfOrFwG+v1JxQge8N50DhmRYQ= +github.com/kataras/pio v0.0.14/go.mod h1:ZIlcw5+5Zyb/kOlU7X4uosZ8dbnXmA4GcGKt1XyyTY0= github.com/kataras/sitemap v0.0.6 h1:w71CRMMKYMJh6LR2wTgnk5hSgjVNB9KL60n5e2KHvLY= github.com/kataras/sitemap v0.0.6/go.mod h1:dW4dOCNs896OR1HmG+dMLdT7JjDk7mYBzoIRwuj5jA4= github.com/kataras/tunnel v0.0.4 h1:sCAqWuJV7nPzGrlb0os3j49lk2JhILT0rID38NHNLpA= github.com/kataras/tunnel v0.0.4/go.mod h1:9FkU4LaeifdMWqZu7o20ojmW4B7hdhv2CMLwfnHGpYw= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= -github.com/klauspost/cpuid/v2 v2.2.8/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -240,8 +241,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -286,8 +287,8 @@ github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4 github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= +github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= @@ -301,8 +302,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= -github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= +github.com/sagikazarmark/locafero v0.11.0 h1:1iurJgmM9G3PA/I+wWYIOw/5SyBtxapeHDcg+AAIFXc= +github.com/sagikazarmark/locafero v0.11.0/go.mod h1:nVIGvgyzw595SUSUE6tvCp3YYTeHs15MvlmU87WwIik= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -311,24 +312,24 @@ github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiy github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= -github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= -github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= -github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= -github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= -github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= -github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8 h1:+jumHNA0Wrelhe64i8F6HNlS8pkoyMv5sreGx2Ry5Rw= +github.com/sourcegraph/conc v0.3.1-0.20240121214520-5f936abd7ae8/go.mod h1:3n1Cwaq1E1/1lhQhtRK2ts/ZwZEhjcQeJQ1RuC6Q/8U= +github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I= +github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU= +github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -341,17 +342,16 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= -github.com/tdewolff/minify/v2 v2.20.37 h1:Q97cx4STXCh1dlWDlNHZniE8BJ2EBL0+2b0n92BJQhw= -github.com/tdewolff/minify/v2 v2.20.37/go.mod h1:L1VYef/jwKw6Wwyk5A+T0mBjjn3mMPgmjjA688RNsxU= -github.com/tdewolff/parse/v2 v2.7.15 h1:hysDXtdGZIRF5UZXwpfn3ZWRbm+ru4l53/ajBRGpCTw= -github.com/tdewolff/parse/v2 v2.7.15/go.mod h1:3FbJWZp3XT9OWVN3Hmfp0p/a08v4h8J9W1aghka0soA= -github.com/tdewolff/test v1.0.11-0.20231101010635-f1265d231d52/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE= -github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739 h1:IkjBCtQOOjIn03u/dMQK9g+Iw9ewps4mCl1nB8Sscbo= -github.com/tdewolff/test v1.0.11-0.20240106005702-7de5f7df4739/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= +github.com/tdewolff/minify/v2 v2.23.6 h1:s0ktOnrlqKfhignkv9M/rh9wCfRlIPiIpjPOq3xUJ84= +github.com/tdewolff/minify/v2 v2.23.6/go.mod h1:VW3ISUd3gDOZuQ/jwZr4sCzsuX+Qvsx87FDMjk6Rvno= +github.com/tdewolff/parse/v2 v2.8.1 h1:J5GSHru6o3jF1uLlEKVXkDxxcVx6yzOlIVIotK4w2po= +github.com/tdewolff/parse/v2 v2.8.1/go.mod h1:Hwlni2tiVNKyzR1o6nUs4FOF07URA+JLBLd6dlIXYqo= +github.com/tdewolff/test v1.0.11 h1:FdLbwQVHxqG16SlkGveC0JVyrJN62COWTRyUFzfbtBE= +github.com/tdewolff/test v1.0.11/go.mod h1:XPuWBzvdUzhCuxWO1ojpXsyzsA5bFoS3tO/Q3kFuTG8= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= @@ -369,6 +369,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0 h1:6fRhSjgLCkTD3JnJxvaJ4Sj+TYblw757bqYgZaOq5ZY= github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= github.com/yosssi/ace v0.0.5 h1:tUkIP/BLdKqrlrPwcmH0shwEEhTRHoGnc1wFIWmaBUA= @@ -383,22 +385,22 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= -go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= -go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= -go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= -go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= -go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= -golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/arch v0.17.0 h1:4O3dfLzd+lQewptAHqjewQZQDyEdejz3VwgeYwkZneU= +golang.org/x/arch v0.17.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -408,10 +410,10 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= -golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= -golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= +golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI= +golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI= +golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= @@ -435,10 +437,10 @@ golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= -golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= +golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= +golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/oauth2 v0.31.0 h1:8Fq0yVZLh4j4YA47vHKFTa9Ew5XIrCP8LC6UeNZnLxo= +golang.org/x/oauth2 v0.31.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -469,8 +471,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= -golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= +golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -480,8 +482,8 @@ golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= -golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= -golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= +golang.org/x/term v0.35.0 h1:bZBVKBudEyhRcajGcNc3jIfWPqV4y/Kt2XcoigOWtDQ= +golang.org/x/term v0.35.0/go.mod h1:TPGtkTLesOwf2DE8CgVYiZinHAOuy5AYUYT1lENIZnA= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -492,10 +494,10 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= -golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= -golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= -golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk= +golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -511,12 +513,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= -google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= -google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7 h1:pFyd6EwwL2TqFf8emdthzeX+gZE1ElRq3iM8pui4KBY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250707201910-8d1bb00bc6a7/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.75.0 h1:+TW+dqTd2Biwe6KKfhE5JpiYIBWq865PhKGSXiivqt4= +google.golang.org/grpc v1.75.0/go.mod h1:JtPAzKiq4v1xcAB2hydNlWI2RnF85XXcV0mhKXr2ecQ= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 27de641f69..941c2b2658 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -15,6 +15,7 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/viper" + "google.golang.org/grpc/credentials" ) const ( @@ -51,9 +52,22 @@ type Config struct { Auth AuthConf } +type Grpc struct { + CACert string + ClientCert string + ClientKey string + ClientCreds credentials.TransportCredentials + ServerCert string + ServerKey string + Host string + Port int + Timeout int +} + type ReEncConfig struct { - APIConf - Crypt4GHKey *[32]byte + Grpc + C4ghPrivateKeyList []*[32]byte + Timeout int } type Sync struct { @@ -74,16 +88,18 @@ type SyncAPIConf struct { } type APIConf struct { - RBACpolicy []byte - CACert string - ServerCert string - ServerKey string - Host string - Port int - Session SessionConfig - DB *database.SDAdb - MQ *broker.AMQPBroker - INBOX storage.Backend + RBACpolicy []byte + CACert string + ServerCert string + ServerKey string + Host string + Port int + Session SessionConfig + DB *database.SDAdb + MQ *broker.AMQPBroker + INBOX storage.Backend + Grpc Grpc + AuditLogger *log.Logger } type SessionConfig struct { @@ -168,7 +184,7 @@ func NewConfig(app string) (*Config, error) { if viper.IsSet("configPath") { cp := viper.GetString("configPath") - log.Infof("configPath: %s", cp) + log.Debugf("configPath: %s", cp) if !strings.HasSuffix(cp, "/") { cp += "/" } @@ -177,21 +193,21 @@ func NewConfig(app string) (*Config, error) { if viper.IsSet("configFile") { viper.SetConfigFile(viper.GetString("configFile")) } - log.Infoln("reading config") + log.Debugf("Attempting to read configuration...") if err := viper.ReadInConfig(); err != nil { - log.Infoln(err.Error()) + log.Debugf("Error reading config : %v", err) if _, ok := err.(viper.ConfigFileNotFoundError); !ok { - log.Infoln("ReadInConfig Error") + log.Warnf("ReadInConfig Error : %v", err) return nil, err } - log.Infoln("No config file found, using ENVs only") + log.Debugf("No config file found, using ENVs only") } if viper.IsSet("log.format") { if viper.GetString("log.format") == "json" { log.SetFormatter(&log.JSONFormatter{}) - log.Info("The logs format is set to JSON") + log.Debugf("The logs format is set to JSON") } } @@ -199,7 +215,7 @@ func NewConfig(app string) (*Config, error) { stringLevel := viper.GetString("log.level") intLevel, err := log.ParseLevel(stringLevel) if err != nil { - log.Infof("Log level '%s' not supported, setting to 'trace'", stringLevel) + log.Debugf("Log level '%s' not supported, setting to 'trace'", stringLevel) intLevel = log.TraceLevel } log.SetLevel(intLevel) @@ -219,6 +235,7 @@ func NewConfig(app string) (*Config, error) { "db.user", "db.password", "db.database", + "grpc.host", } switch viper.GetString("inbox.type") { case S3: @@ -303,6 +320,8 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = append(requiredConfVars, []string{"archive.url", "archive.accesskey", "archive.secretkey", "archive.bucket"}...) case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) + default: + log.Warnln(" archive not configured, backup will not be performed.") } switch viper.GetString("backup.type") { @@ -310,6 +329,8 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = append(requiredConfVars, []string{"backup.url", "backup.accesskey", "backup.secretkey", "backup.bucket"}...) case POSIX: requiredConfVars = append(requiredConfVars, []string{"backup.location"}...) + default: + log.Warnln(" backup destination not configured, backup will not be performed.") } case "intercept": requiredConfVars = []string{ @@ -339,6 +360,8 @@ func NewConfig(app string) (*Config, error) { requiredConfVars = append(requiredConfVars, []string{"inbox.url", "inbox.accesskey", "inbox.secretkey", "inbox.bucket"}...) case POSIX: requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) + default: + return nil, errors.New("inbox.type not set") } case "notify": requiredConfVars = []string{ @@ -364,8 +387,7 @@ func NewConfig(app string) (*Config, error) { } case "reencrypt": requiredConfVars = []string{ - "c4gh.filepath", - "c4gh.passphrase", + "c4gh.privateKeys", } case "s3inbox": requiredConfVars = []string{ @@ -493,6 +515,11 @@ func NewConfig(app string) (*Config, error) { return nil, err } c.configSchemas() + + c.API.Grpc, err = configReEncryptClient() + if err != nil { + return nil, err + } case "auth": c.Auth.Cega.AuthURL = viper.GetString("auth.cega.authUrl") c.Auth.Cega.ID = viper.GetString("auth.cega.id") @@ -650,7 +677,9 @@ func NewConfig(app string) (*Config, error) { } c.configArchive() - c.configSync() + if err := c.configSync(); err != nil { + return nil, err + } c.configSchemas() case "sync-api": if err := c.configBroker(); err != nil { @@ -677,12 +706,13 @@ func NewConfig(app string) (*Config, error) { } c.configSchemas() + default: + return nil, errors.New("unknown app name") } return c, nil } -// configDatabase provides configuration for the database func (c *Config) configAPI() error { c.apiDefaults() api := APIConf{} @@ -698,6 +728,12 @@ func (c *Config) configAPI() error { api.ServerKey = viper.GetString("api.serverKey") api.ServerCert = viper.GetString("api.serverCert") api.CACert = viper.GetString("api.CACert") + if viper.GetBool("api.audit") { + api.AuditLogger = log.New() + api.AuditLogger.SetFormatter(&log.JSONFormatter{}) + api.AuditLogger.SetOutput(os.Stdout) + api.AuditLogger.SetLevel(log.InfoLevel) + } c.API = api @@ -712,6 +748,7 @@ func (c *Config) apiDefaults() { viper.SetDefault("api.session.secure", true) viper.SetDefault("api.session.httponly", true) viper.SetDefault("api.session.name", "api_session_key") + viper.SetDefault("api.audit", true) } // configArchive provides configuration for the archive storage @@ -900,10 +937,65 @@ func (c *Config) configOrchestrator() { } } -func (c *Config) configReEncryptServer() (err error) { - if viper.IsSet("grpc.host") { - c.ReEncrypt.Host = viper.GetString("grpc.host") +func configReEncryptClient() (Grpc, error) { + var grpc Grpc + grpc.Host = viper.GetString("grpc.host") + grpc.Port = 50051 + if viper.IsSet("grpc.port") { + grpc.Port = viper.GetInt("grpc.port") + } + + grpc.Timeout = 30 + if viper.IsSet("grpc.timeout") { + grpc.Timeout = viper.GetInt("grpc.timeout") } + + if viper.IsSet("grpc.clientcert") && viper.IsSet("grpc.clientkey") { + grpc.Port = 50443 + switch { + case viper.IsSet("grpc.cacert"): + cacertByte, err := os.ReadFile(viper.GetString("grpc.cacert")) + if err != nil { + return Grpc{}, fmt.Errorf("failed to read CA certificate file, reason: %s", err.Error()) + } + + caCert := x509.NewCertPool() + if !caCert.AppendCertsFromPEM(cacertByte) { + return Grpc{}, errors.New("failed to append CA certificate to cert pool") + } + + certs, err := tls.LoadX509KeyPair(viper.GetString("grpc.clientcert"), viper.GetString("grpc.clientkey")) + if err != nil { + return Grpc{}, errors.New("Failed to load client key pair for reencrypt") + } + + grpc.ClientCreds = credentials.NewTLS( + &tls.Config{ + Certificates: []tls.Certificate{certs}, + MinVersion: tls.VersionTLS13, + RootCAs: caCert, + }, + ) + default: + certs, err := tls.LoadX509KeyPair(viper.GetString("grpc.clientcert"), viper.GetString("grpc.clientkey")) + if err != nil { + return Grpc{}, errors.New("Failed to load client key pair for reencrypt") + } + + grpc.ClientCreds = credentials.NewTLS( + &tls.Config{ + Certificates: []tls.Certificate{certs}, + MinVersion: tls.VersionTLS13, + }, + ) + } + } + + return grpc, nil +} + +func (c *Config) configReEncryptServer() (err error) { + c.ReEncrypt.Port = 50051 if viper.IsSet("grpc.port") { c.ReEncrypt.Port = viper.GetInt("grpc.port") } @@ -921,10 +1013,13 @@ func (c *Config) configReEncryptServer() (err error) { c.ReEncrypt.Port = 50443 } - c.ReEncrypt.Crypt4GHKey, err = GetC4GHKey() + c.ReEncrypt.C4ghPrivateKeyList, err = GetC4GHprivateKeys() if err != nil { return err } + if len(c.ReEncrypt.C4ghPrivateKeyList) == 0 { + return errors.New("no C4GH private keys configured") + } return nil } @@ -1009,7 +1104,7 @@ func (c *Config) configSMTP() { } // configSync provides configuration for the sync destination storage -func (c *Config) configSync() { +func (c *Config) configSync() error { switch viper.GetString("sync.destination.type") { case S3: c.Sync.Destination.Type = S3 @@ -1020,6 +1115,8 @@ func (c *Config) configSync() { case POSIX: c.Sync.Destination.Type = POSIX c.Sync.Destination.Posix.Location = viper.GetString("sync.destination.location") + default: + return errors.New("sync.destination.type not set") } c.Sync.RemoteHost = viper.GetString("sync.remote.host") @@ -1029,6 +1126,8 @@ func (c *Config) configSync() { c.Sync.RemotePassword = viper.GetString("sync.remote.password") c.Sync.RemoteUser = viper.GetString("sync.remote.user") c.Sync.CenterPrefix = viper.GetString("sync.centerPrefix") + + return nil } // configSync provides configuration for the outgoing sync settings diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index 4c33844ff2..4cde11881e 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -59,6 +59,7 @@ func (ts *ConfigTestSuite) SetupTest() { viper.Set("db.user", "test") viper.Set("db.password", "test") viper.Set("db.database", "test") + viper.Set("grpc.host", "reencrypt") viper.Set("inbox.url", "testurl") viper.Set("inbox.accesskey", "testaccess") viper.Set("inbox.secretkey", "testsecret") @@ -226,7 +227,7 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { assert.Error(ts.T(), err) assert.Nil(ts.T(), config) - // testing deafult values + // testing default values ts.SetupTest() config, err = NewConfig("api") assert.NotNil(ts.T(), config) @@ -240,6 +241,7 @@ func (ts *ConfigTestSuite) TestAPIConfiguration() { assert.Equal(ts.T(), -1*time.Second, config.API.Session.Expiration) rbac, _ := os.ReadFile(viper.GetString("api.rbacFile")) assert.Equal(ts.T(), rbac, config.API.RBACpolicy) + assert.Equal(ts.T(), "reencrypt", config.API.Grpc.Host) viper.Reset() ts.SetupTest() @@ -466,11 +468,13 @@ func (ts *ConfigTestSuite) TestConfigReEncryptServer() { ts.T().FailNow() } - viper.Set("c4gh.filepath", keyPath+"/c4gh.key") - viper.Set("c4gh.passphrase", "test") + viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ + {FilePath: keyPath + "/c4gh.key", Passphrase: "test"}, + }) config, err := NewConfig("reencrypt") assert.NoError(ts.T(), err) assert.Equal(ts.T(), 50051, config.ReEncrypt.Port) + assert.Equal(ts.T(), 1, len(config.ReEncrypt.C4ghPrivateKeyList)) viper.Set("grpc.CACert", certPath+"/ca.crt") viper.Set("grpc.serverCert", certPath+"/tls.crt") @@ -481,6 +485,49 @@ func (ts *ConfigTestSuite) TestConfigReEncryptServer() { assert.Equal(ts.T(), certPath+"/tls.crt", config.ReEncrypt.ServerCert) } +func (ts *ConfigTestSuite) TestConfigReEncryptServer_multipleKeys() { + keyPath, _ := os.MkdirTemp("", "key") + defer os.RemoveAll(keyPath) + keyFile1 := keyPath + "/c4gh1.key" + keyFile2 := keyPath + "/c4gh2.key" + + _, err := helper.CreatePrivateKeyFile(keyFile1, "test") + assert.NoError(ts.T(), err) + _, err = helper.CreatePrivateKeyFile(keyFile2, "test") + assert.NoError(ts.T(), err) + + viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ + {FilePath: keyFile1, Passphrase: "test"}, + {FilePath: keyFile2, Passphrase: "test"}, + }) + + config, err := NewConfig("reencrypt") + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, len(config.ReEncrypt.C4ghPrivateKeyList)) +} +func (ts *ConfigTestSuite) TestConfigReEncryptServer_noKeys() { + viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{}) + _, err := NewConfig("reencrypt") + assert.Error(ts.T(), err) +} + +func (ts *ConfigTestSuite) TestConfigReEncryptClient() { + ts.SetupTest() + conf, err := configReEncryptClient() + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "reencrypt", conf.Host) + assert.Nil(ts.T(), conf.ClientCreds) +} + +func (ts *ConfigTestSuite) TestConfigReEncryptClient_withTLS() { + viper.Set("grpc.CACert", certPath+"/ca.crt") + viper.Set("grpc.clientCert", certPath+"/tls.crt") + viper.Set("grpc.clientKey", certPath+"/tls.key") + conf, err := configReEncryptClient() + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), conf.ClientCreds) + assert.Equal(ts.T(), 50443, conf.Port) +} func (ts *ConfigTestSuite) TestConfigAuth_CEGA() { ts.SetupTest() diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index 66be75d5c6..40372c8d98 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -84,11 +84,11 @@ func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFilePath = "SELECT submission_file_path from sda.files where " + - "submission_user= $1 and id = $2 " + - "AND EXISTS (SELECT 1 FROM " + - "(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) " + - "as subquery WHERE event = 'uploaded')" + const getFilePath = `SELECT submission_file_path from sda.files where +submission_user= $1 and id = $2 +AND EXISTS (SELECT 1 FROM +(SELECT event from sda.file_event_log where file_id = $2 order by started_at desc limit 1) +as subquery WHERE event = 'uploaded');` var filePath string err := db.QueryRow(getFilePath, submissionUser, fileID).Scan(&filePath) @@ -122,12 +122,12 @@ func (dbs *SDAdb) getFileIDByUserPathAndStatus(submissionUser, filePath, status dbs.checkAndReconnectIfNeeded() db := dbs.DB - const getFileID = "SELECT id from sda.files " + - "WHERE submission_user=$1 and submission_file_path =$2 and stable_id IS null " + - "AND EXISTS (SELECT 1 FROM " + - "(SELECT event from sda.file_event_log JOIN sda.files ON sda.files.id=sda.file_event_log.file_id " + - "WHERE submission_user=$1 and submission_file_path =$2 order by started_at desc limit 1) " + - "AS subquery WHERE event = $3)" + const getFileID = `SELECT id from sda.files +WHERE submission_user=$1 and submission_file_path =$2 and stable_id IS null +AND EXISTS (SELECT 1 FROM +(SELECT event from sda.file_event_log JOIN sda.files ON sda.files.id=sda.file_event_log.file_id +WHERE submission_user=$1 and submission_file_path =$2 order by started_at desc limit 1) +AS subquery WHERE event = $3);` var fileID string err := db.QueryRow(getFileID, submissionUser, filePath, status).Scan(&fileID) @@ -224,9 +224,10 @@ func (dbs *SDAdb) setArchived(file FileInfo, fileID string) error { } log.Debugf("checksum: %s", file.UploadedChecksum) - const addChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + - "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source)" + - "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + const addChecksum = `INSERT INTO sda.checksums(file_id, checksum, type, source) +VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source) +ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;` + if _, err := db.Exec(addChecksum, fileID, file.UploadedChecksum, "SHA256"); err != nil { return fmt.Errorf("addChecksum error: %s", err.Error()) } @@ -318,16 +319,18 @@ func (dbs *SDAdb) setVerified(file FileInfo, fileID string) error { return fmt.Errorf("setVerified error: %s", err.Error()) } - const addArchiveChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + - "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source)" + - "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + const addArchiveChecksum = `INSERT INTO sda.checksums(file_id, checksum, type, source) +VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source) +ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;` + if _, err := dbs.DB.Exec(addArchiveChecksum, fileID, file.ArchiveChecksum, "SHA256"); err != nil { return fmt.Errorf("addArchiveChecksum error: %s", err.Error()) } - const addUnencryptedChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + - "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source)" + - "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + const addUnencryptedChecksum = `INSERT INTO sda.checksums(file_id, checksum, type, source) +VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source) +ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;` + if _, err := dbs.DB.Exec(addUnencryptedChecksum, fileID, file.DecryptedChecksum, "SHA256"); err != nil { return fmt.Errorf("addUnencryptedChecksum error: %s", err.Error()) } @@ -571,9 +574,10 @@ func (dbs *SDAdb) getFileInfo(id string) (FileInfo, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB const getFileID = "SELECT archive_file_path, archive_file_size from sda.files where id = $1;" - const checkSum = "SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, " + - "MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted, " + - "MAX(checksum) FILTER(where source = 'UPLOADED') as Uploaded from sda.checksums where file_id = $1;" + const checkSum = `SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, +MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted, +MAX(checksum) FILTER(where source = 'UPLOADED') as Uploaded from sda.checksums where file_id = $1;` + var info FileInfo if err := db.QueryRow(getFileID, id).Scan(&info.Path, &info.Size); err != nil { return FileInfo{}, err @@ -607,6 +611,39 @@ func (dbs *SDAdb) GetHeaderForStableID(stableID string) ([]byte, error) { return header, nil } +// GetFileInfoFromAccessionID retrieves the file information needed for mapping +func (dbs *SDAdb) GetFileInfoFromAccessionID(accessionID string) (SyncData, error) { + var ( + s SyncData + err error + ) + + for count := 1; count <= RetryTimes; count++ { + s, err = dbs.getFileInfoFromAccessionID(accessionID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(3, float64(count))) * time.Second) + } + + return s, err +} + +// getFileInfoFromAccessionID is the actual function performing work for GetFileInfoFromAccessionID +func (dbs *SDAdb) getFileInfoFromAccessionID(accessionID string) (SyncData, error) { + dbs.checkAndReconnectIfNeeded() + + const query = "SELECT submission_user, submission_file_path from sda.files WHERE stable_id = $1;" + var data SyncData + if err := dbs.DB.QueryRow(query, accessionID).Scan(&data.User, &data.FilePath); err != nil { + log.Warnf("Error while searching for id %s: %v", accessionID, err) + + return SyncData{}, err + } + + return data, nil +} + // GetSyncData retrieves the file information needed to sync a dataset func (dbs *SDAdb) GetSyncData(accessionID string) (SyncData, error) { var ( @@ -704,14 +741,14 @@ func (dbs *SDAdb) getArchivePath(stableID string) (string, error) { } // GetUserFiles retrieves all the files a user submitted -func (dbs *SDAdb) GetUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) GetUserFiles(userID, pathPrefix string, allData bool) ([]*SubmissionFileInfo, error) { var err error files := []*SubmissionFileInfo{} // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { - files, err = dbs.getUserFiles(userID, allData) + files, err = dbs.getUserFiles(userID, pathPrefix, allData) if err == nil { break } @@ -722,19 +759,20 @@ func (dbs *SDAdb) GetUserFiles(userID string, allData bool) ([]*SubmissionFileIn } // getUserFiles is the actual function performing work for GetUserFiles -func (dbs *SDAdb) getUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) getUserFiles(userID, pathPrefix string, allData bool) ([]*SubmissionFileInfo, error) { dbs.checkAndReconnectIfNeeded() files := []*SubmissionFileInfo{} db := dbs.DB // select all files (that are not part of a dataset) of the user, each one annotated with its latest event - const query = "SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f " + - "LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " + - "AND f.id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id); " + const query = `SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f +LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id +WHERE f.submission_user = $1 and f.submission_file_path LIKE $2 +AND NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id);` // nolint:rowserrcheck - rows, err := db.Query(query, userID) + rows, err := db.Query(query, userID, fmt.Sprintf("%s%%", pathPrefix)) if err != nil { return nil, err } @@ -783,9 +821,10 @@ func (dbs *SDAdb) GetCorrID(user, path, accession string) (string, error) { func (dbs *SDAdb) getCorrID(user, path, accession string) (string, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "SELECT DISTINCT correlation_id FROM sda.file_event_log e " + - "RIGHT JOIN sda.files f ON e.file_id = f.id " + - "WHERE f.submission_file_path = $1 AND f.submission_user = $2 AND COALESCE(f.stable_id, '') = $3;" + const query = `SELECT DISTINCT correlation_id FROM sda.file_event_log e +RIGHT JOIN sda.files f ON e.file_id = f.id +WHERE f.submission_file_path = $1 AND f.submission_user = $2 AND COALESCE(f.stable_id, '') = $3;` + rows, err := db.Query(query, path, user, accession) if err != nil { return "", err @@ -815,7 +854,7 @@ func (dbs *SDAdb) ListActiveUsers() ([]string, error) { db := dbs.DB var users []string - rows, err := db.Query("SELECT DISTINCT submission_user FROM sda.files WHERE id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id) ORDER BY submission_user ASC;") + rows, err := db.Query("SELECT DISTINCT submission_user FROM sda.files f WHERE NOT EXISTS (SELECT 1 FROM sda.file_dataset d WHERE f.id = d.file_id) ORDER BY submission_user ASC;") if err != nil { return nil, err } @@ -1004,19 +1043,19 @@ func (dbs *SDAdb) ListUserDatasets(submissionUser string) ([]DatasetInfo, error) db := dbs.DB query := `SELECT dataset_id,event,event_date FROM sda.dataset_event_log WHERE - (dataset_id, event_date) IN ( - SELECT dataset_id,max(event_date) FROM sda.dataset_event_log WHERE - dataset_id IN ( - SELECT stable_id FROM sda.datasets WHERE - id IN ( - SELECT DISTINCT dataset_id FROM sda.file_dataset WHERE - file_id IN ( - SELECT id FROM sda.files WHERE submission_user = $1 AND stable_id IS NOT NULL - ) - ) +(dataset_id, event_date) IN ( + SELECT dataset_id,max(event_date) FROM sda.dataset_event_log WHERE + dataset_id IN ( + SELECT stable_id FROM sda.datasets WHERE + id IN ( + SELECT DISTINCT dataset_id FROM sda.file_dataset WHERE + file_id IN ( + SELECT id FROM sda.files WHERE submission_user = $1 AND stable_id IS NOT NULL ) - GROUP BY dataset_id - );` + ) + ) + GROUP BY dataset_id +);` rows, err := db.Query(query, submissionUser) if err != nil { @@ -1058,9 +1097,9 @@ func (dbs *SDAdb) updateUserInfo(userID, name, email string, groups []string) er dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "INSERT INTO sda.userinfo(id, name, email, groups) VALUES($1, $2, $3, $4)" + - "ON CONFLICT (id)" + - "DO UPDATE SET name = excluded.name, email = excluded.email, groups = excluded.groups;" + const query = `INSERT INTO sda.userinfo(id, name, email, groups) VALUES($1, $2, $3, $4) +ON CONFLICT (id) +DO UPDATE SET name = excluded.name, email = excluded.email, groups = excluded.groups;` result, err := db.Exec(query, userID, name, email, pq.Array(groups)) if err != nil { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index 369220aea3..bbdbc73572 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -458,9 +458,14 @@ func (suite *DatabaseTests) TestGetUserFiles() { assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) testCases := 5 testUser := "GetFilesUser" + sub := "submission_a" - for i := 0; i < testCases; i++ { - fileID, err := db.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", testUser, i), testUser) + for i := range testCases { + if i == 2 { + sub = "submission_b" + } + + fileID, err := db.RegisterFile(fmt.Sprintf("%v/%s/TestGetUserFiles-00%d.c4gh", testUser, sub, i), testUser) assert.NoError(suite.T(), err, "failed to register file in database") err = db.UpdateFileEventLog(fileID, "uploaded", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") @@ -469,11 +474,11 @@ func (suite *DatabaseTests) TestGetUserFiles() { err = db.UpdateFileEventLog(fileID, "ready", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") } - filelist, err := db.GetUserFiles("unknownuser", true) + filelist, err := db.GetUserFiles("unknownuser", "", true) assert.NoError(suite.T(), err, "failed to get (empty) file list of unknown user") assert.Empty(suite.T(), filelist, "file list of unknown user is not empty") - filelist, err = db.GetUserFiles(testUser, true) + filelist, err = db.GetUserFiles(testUser, "", true) assert.NoError(suite.T(), err, "failed to get file list") assert.Equal(suite.T(), testCases, len(filelist), "file list is of incorrect length") @@ -481,6 +486,10 @@ func (suite *DatabaseTests) TestGetUserFiles() { assert.Equal(suite.T(), "ready", fileInfo.Status, "incorrect file status") assert.Contains(suite.T(), fileInfo.AccessionID, "stableID-00", "incorrect file accession ID") } + + filteredFilelist, err := db.GetUserFiles(testUser, fmt.Sprintf("%s/submission_b", testUser), true) + assert.NoError(suite.T(), err, "failed to get file list") + assert.Equal(suite.T(), 3, len(filteredFilelist), "file list is of incorrect length") } func (suite *DatabaseTests) TestGetCorrID() { diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index e337768d83..08ad03e002 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "strings" "time" "github.com/lestrrat-go/jwx/v2/jwa" @@ -446,3 +447,15 @@ func CreatePrivateKeyFile(keyFile string, passphrase string) ([32]byte, error) { return publicKey, nil } + +func AnonymizeFilepath(fp string, username string) string { + return strings.ReplaceAll(fp, strings.Replace(username, "@", "_", 1)+"/", "") +} + +func UnanonymizeFilepath(fp string, username string) string { + if strings.HasPrefix(fp, strings.Replace(username, "@", "_", 1)) { + return fp + } + + return filepath.Join(strings.Replace(username, "@", "_", 1), fp) +} diff --git a/sda/internal/helper/helper_test.go b/sda/internal/helper/helper_test.go index 42f538817c..01c9227cea 100644 --- a/sda/internal/helper/helper_test.go +++ b/sda/internal/helper/helper_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "testing" "github.com/lestrrat-go/jwx/v2/jwk" @@ -151,3 +152,30 @@ func (ts *HelperTest) TestCreatePrivateKeyFile() { // Verify the public key is not empty assert.NotEqual(ts.T(), [32]byte{}, publicKey, "Public key should not be empty") } + +func (ts *HelperTest) TestAnonymizeFilepath() { + filePath := "main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := AnonymizeFilepath(filepath.Join(strings.Replace(userName, "@", "_", 1), filePath), userName) + assert.Equal(ts.T(), filePath, newPath) +} + +func (ts *HelperTest) TestAnonymizeFilepath_noUser() { + filePath := "main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := AnonymizeFilepath(filePath, userName) + assert.Equal(ts.T(), filePath, newPath) +} + +func (ts *HelperTest) TestUnanonymizeFilepath() { + filePath := "main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := UnanonymizeFilepath(filePath, userName) + assert.Equal(ts.T(), filepath.Join(strings.Replace(userName, "@", "_", 1), filePath), newPath) +} +func (ts *HelperTest) TestUnanonymizeFilepath_oldMessage() { + filePath := "test.user_demo.org/main_folder/sub_folder/file.name" + userName := "test.user@demo.org" + newPath := UnanonymizeFilepath(filePath, userName) + assert.Equal(ts.T(), filePath, newPath) +} diff --git a/tools/sda-admin b/tools/sda-admin index 6f6ebfdf53..41ae7f356d 100755 --- a/tools/sda-admin +++ b/tools/sda-admin @@ -167,8 +167,7 @@ upload () { # subdirectory of the user's S3 bucket. # if [ "$#" -gt 0 ]; then - "$SDA_CLI" upload \ - -config "$S3_CONFIG" \ + "$SDA_CLI" -config "$S3_CONFIG" upload \ ${target_dir+-targetDir "$target_dir"} \ "$@" fi