diff --git a/.github/integration/scripts/charts/dependencies.sh b/.github/integration/scripts/charts/dependencies.sh index f01365c514..1b885de96b 100644 --- a/.github/integration/scripts/charts/dependencies.sh +++ b/.github/integration/scripts/charts/dependencies.sh @@ -9,7 +9,7 @@ random-string() { } if [ "$1" == "local" ]; then - if [ ! "$(command crypt4gh)" ]; then + if [ ! "$(command crypt4gh --version)" ]; then echo "crypt4gh not installed, get it from here: https://github.com/neicnordic/crypt4gh/releases/latest" exit 1 elif [ "$(crypt4gh --version | cut -d ' ' -f1)" == "GA4GH" ]; then @@ -18,10 +18,20 @@ if [ "$1" == "local" ]; then exit 1 fi - if [ ! "$(command yq)" ]; then + if [ ! "$(command yq --version)" ]; then echo "yq not installed, get it from here: https://github.com/mikefarah/yq/releases/latest" exit 1 fi + + if [ ! "$(command jq --version)" ]; then + echo "jq not installed" + exit 1 + fi + + if [ ! "$(command xxd --version 2>&1)" ]; then + echo "xxd not installed" + exit 1 + fi else sudo curl --retry 100 -sL "https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64" -o /usr/bin/yq && sudo chmod +x /usr/bin/yq @@ -104,4 +114,4 @@ yq -i ' .releasetest.secrets.accessToken = strenv(TEST_TOKEN) ' "$values_file" -kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json" \ No newline at end of file +kubectl create secret generic api-rbac --from-file=".github/integration/sda/rbac.json" diff --git a/.github/integration/scripts/charts/deploy_charts.sh b/.github/integration/scripts/charts/deploy_charts.sh index 774ccc911c..204f096fba 100644 --- a/.github/integration/scripts/charts/deploy_charts.sh +++ b/.github/integration/scripts/charts/deploy_charts.sh @@ -36,6 +36,7 @@ if [ "$1" == "sda-mq" ]; then --set image.pullPolicy=IfNotPresent \ --set global.adminPassword="$ADMINPASS" \ --set global.adminUser=admin \ + --set global.ingress.hostName=broker.127.0.0.1.nip.io \ --set global.tls.enabled="$3" \ --set global.tls.clusterIssuer=cert-issuer \ --set persistence.enabled=false \ diff --git a/.github/integration/scripts/charts/values.yaml b/.github/integration/scripts/charts/values.yaml index 885b0f5c9b..7c119d3cde 100644 --- a/.github/integration/scripts/charts/values.yaml +++ b/.github/integration/scripts/charts/values.yaml @@ -1,12 +1,13 @@ global: schemaType: "isolated" ingress: - deploy: false + deploy: true hostName: - api: pipeline-sda-svc-api - auth: pipeline-sda-svc-auth - download: pipeline-sda-svc-download - s3Inbox: pipeline-sda-svc-inbox + api: api.127.0.0.1.nip.io + auth: auth.127.0.0.1.nip.io + download: download.127.0.0.1.nip.io + s3Inbox: inbox.127.0.0.1.nip.io + syncapi: sync-api.127.0.0.1.nip.io log: level: "debug" tls: diff --git a/.github/integration/scripts/make_certs.sh b/.github/integration/scripts/make_certs.sh index bde3bd1a05..7b0d66692c 100755 --- a/.github/integration/scripts/make_certs.sh +++ b/.github/integration/scripts/make_certs.sh @@ -34,6 +34,7 @@ openssl x509 -req -in "$out_dir/mq.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey # Create client certificate openssl req -config "$script_dir/ssl.cnf" -new -nodes -newkey rsa:4096 -keyout "$out_dir/client.key" -out "$out_dir/client.csr" -extensions client_cert -subj "/CN=admin" +openssl pkcs8 -topk8 -inform PEM -outform DER -in "$out_dir/client.key" -out "$out_dir/client.der" -nocrypt openssl x509 -req -in "$out_dir/client.csr" -days 1200 -CA "$out_dir/ca.crt" -CAkey "$out_dir/ca-key.pem" -set_serial 01 -out "$out_dir/client.crt" -extensions client_cert -extfile "$script_dir/ssl.cnf" if [ -n "$KEYSTORE_PASSWORD" ]; then @@ -68,7 +69,9 @@ chmod 600 /certs/*.key cp -p "$out_dir/ca.crt" /client_certs/ca.crt cp -p "$out_dir/client.crt" /client_certs/ cp -p "$out_dir/client.key" /client_certs/ +cp -p "$out_dir/client.der" /client_certs/ chmod 600 /client_certs/*.key +chmod 644 /client_certs/*.der # needed if testing locally mkdir -p /temp/certs diff --git a/.github/integration/scripts/make_sda_credentials.sh b/.github/integration/scripts/make_sda_credentials.sh index 9b305650e8..83bc347d0e 100644 --- a/.github/integration/scripts/make_sda_credentials.sh +++ b/.github/integration/scripts/make_sda_credentials.sh @@ -29,19 +29,58 @@ done mkdir -p /shared/keys/pub if [ ! -f "/shared/keys/jwt.key" ]; then echo "creating jwt key" - openssl ecparam -genkey -name prime256v1 -noout -out /shared/keys/jwt.key - openssl ec -in /shared/keys/jwt.key -outform PEM -pubout >/shared/keys/pub/jwt.pub + cat << 'EOF' > /shared/keys/jwt.key +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDhuZjxPmOGUIW1 +LhxzKfxkN+1aTbvI5w+AptqT33X+bWuzfjvhEodiNz0bBfQgJJpQ3TZ8J1IZpM2F +Tnzox+FGxKPe5T9Mgngzd4N6eByWVPXoNMk7IdmBXMdPZBFSyjMW4ba1MELCpiKV +05de4J5opRDwmHmyMqYJxBk78e3iiYYixVk+j1Ku+yFl4d2R29y2+O9PlZegJloe +8FGnKIGZApS/8t9iyCkXg8WbjSPzgYCTQKxn/E4lcGdTrAt/McKrWmAuppcr+rpP ++BInm3l5Zu/QiRSZcMb5O460ojP9eKnaUlDpGZv9CY5j4x4lq8vjU2kK77YXBO8I +2oxse5a5AgMBAAECggEABbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKP +RushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF+eqipSQXKghdKZCV +AsY8io0MmpXB1omDSygp7h3j52yHdayE2muav+VTAPOYn5QwG0/gGgVqYrR9x7CM +iTuyOIuGNO4Wlly4/5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI +3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq/77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsn +mccF9plVTzem7N/vo+T+hFRPUO9TZUao91mMV8iV5QKBgQD1nZbQW3NHdol0fXA8 +nw5JRkTLZx1zcZ5l36WVPkwCjJOyXQ2vWHm4lz7F81Rr8dQnMKLWMDKjrBT9Dbfs +xYK2bYxENS1W/n+0jOIaX/792DY9tfX7vvHU9yGSdoJE5os6DGCHYInOD0xnRmnl +3vS7gKv8miDwDzFsbjtDg6WfSwKBgQDrRLkmmfZCMcmLA02YSrErAlUseuyad7lY +HEJApXKfn262iHELlQa2zOBZpJGXIcHsNf1XGpMeU5pH+ILKE4Y5qbclq+AzFCcZ +nBFUfDeawmWdV5FJqNDd1L8Mb8aE+6q0Y5rNb3RL7A2ypH2ZeYKSGpHz3C7Rn5KW +voWAXRWriwKBgQCH4bxK3x0ivxiCgtcyIojDzwVGRnDLqmMIVzeDHqjsjBs2BTcJ +9/e3QK1w1BKzeWF2oPilaJrLY+tkqE9FxWtwQ6DjJ0xDIZ9DIuH/13X5t8EiWOWS +devSdzpyje+58JW78pcArk7u2hXZ2OHDU5qvlRsRL6/jP3SHWWCeFFnviwKBgGov +M02r0YygwfEfBYeFtp7Nx7lypZU2Eg4levWIdsp6f9KclEEA+u3IXD25XAiVMNw2 +pegJU3stioWPMSCZXUxrQAEdqOwE3XzehqfWBJaxxIEWQ7m2Gsb0PWIUlMnyeGJA +Tl8IPboCiVAmk5WQVREyMsuYhf0Qg23MAZ8k5CHvAoGBAJm55NQZVKAEDGd4a21q +TDcRddtPwwL2oP3qa0gbGk4YFRUCrX99hIejOTvQW1xf6vGxTd7E1QizvFse4yRz +ZRKyXIc7DCcdzOnpMrSd1+aXwZtRHLSw0EDS6PWeJZdjJYHxl2YpAmMdURdcGTrH +b6b/6vhU90+xL14CX7Awofp/ +-----END PRIVATE KEY----- +EOF + cat << 'EOF' > /shared/keys/pub/jwt.pub +-----BEGIN PUBLIC KEY----- +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4bmY8T5jhlCFtS4ccyn8 +ZDftWk27yOcPgKbak991/m1rs3474RKHYjc9GwX0ICSaUN02fCdSGaTNhU586Mfh +RsSj3uU/TIJ4M3eDengcllT16DTJOyHZgVzHT2QRUsozFuG2tTBCwqYildOXXuCe +aKUQ8Jh5sjKmCcQZO/Ht4omGIsVZPo9SrvshZeHdkdvctvjvT5WXoCZaHvBRpyiB +mQKUv/LfYsgpF4PFm40j84GAk0CsZ/xOJXBnU6wLfzHCq1pgLqaXK/q6T/gSJ5t5 +eWbv0IkUmXDG+TuOtKIz/Xip2lJQ6Rmb/QmOY+MeJavL41NpCu+2FwTvCNqMbHuW +uQIDAQAB +-----END PUBLIC KEY----- +EOF chmod 644 /shared/keys/pub/jwt.pub /shared/keys/jwt.key fi echo "creating token" -token="$(python /scripts/sign_jwt.py)" +python /scripts/sign_jwt.py testu@lifescience-ri.eu > "/shared/token" cat >/shared/s3cfg < + sh -c "pip install --upgrade pip && pip install aiohttp Authlib && python -u /mock_auth.py 0.0.0.0 8000" + + rabbitmq: + build: + context: ../../rabbitmq + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + container_name: rabbitmq + depends_on: + certfixer: + condition: service_completed_successfully + environment: + - RABBITMQ_SERVER_CACERT=/etc/rabbitmq/ssl/ca.crt + - RABBITMQ_SERVER_CERT=/etc/rabbitmq/ssl/mq.crt + - RABBITMQ_SERVER_KEY=/etc/rabbitmq/ssl/mq.key + - RABBITMQ_SERVER_VERIFY=verify_none + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms", + ] + interval: 5s + timeout: 20s + retries: 3 + restart: always + volumes: + - certs:/etc/rabbitmq/ssl/ + - rabbitmq_data:/var/lib/rabbitmq + ports: + - 5671:5671 + - 25671:15671 + + doa: + container_name: doa + build: + context: ../../sda-doa + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + certfixer: + condition: service_completed_successfully + ports: + - "8080:8080" + environment: + - SSL_MODE=verify-ca + - POSTGRES_USER=postgres + - POSTGRES_DB=sda + - POSTGRES_PASSWORD=rootpasswd + - DB_INSTANCE=postgres + - SSL_ENABLED=false + - BROKER_HOST=rabbitmq + - BROKER_VALIDATE=false + - OPENID_CONFIGURATION_URL=http://mockauth:8000/openid-configuration + - USERINFO_ENDPOINT_URL=http://mockauth:8000/userinfo + - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem + - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass + - OUTBOX_TYPE=POSIX + - OUTBOX_LOCATION=/outbox/%s/files/ + - SSL_ENABLED=false + - ROOT_CERT_PATH=/certs/ca.crt + - CERT_PATH=/certs/client.crt + - CERT_KEY=/certs/client.der + - BROKER_USERNAME=guest + + volumes: + - client_certs:/certs + - ../../sda-doa/test/body.enc:/test/body.enc + - test_file:/outbox + - encryption_files:/test + + integration_test: + container_name: integration_test + image: maven:3.9.9-eclipse-temurin-21 + profiles: [test] + volumes: + - ../../sda-doa/src:/sda-doa/src + - ../../sda-doa/pom.xml:/sda-doa/pom.xml + - ../../sda-doa/settings.xml:/root/.m2/settings.xml + - test_file:/sda-doa/outbox + - ./tests:/tests + - encryption_files:/test + - client_certs:/certs + + depends_on: + - doa + - mockauth + environment: + - OUTBOX_TYPE=POSIX + - DOA_URL=http://doa:8080 + - MOCKAUTH_URL=http://mockauth:8000 + - MINIO_HOST=outbox + + command: + - "/bin/sh" + - "/tests/run_scripts.sh" + - "/tests/doa" + +volumes: + certs: + client_certs: + rabbitmq_data: + test_file: + encryption_files: \ No newline at end of file diff --git a/.github/integration/sda-doa-s3-outbox.yml b/.github/integration/sda-doa-s3-outbox.yml new file mode 100644 index 0000000000..ada7422362 --- /dev/null +++ b/.github/integration/sda-doa-s3-outbox.yml @@ -0,0 +1,186 @@ +services: + certfixer: + command: + - /bin/sh + - /scripts/make_certs.sh + container_name: certfixer + image: alpine:latest + volumes: + - ./scripts:/scripts + - certs:/certs + - client_certs:/client_certs + - /tmp:/temp + + postgres: + build: + context: ../../postgresql + container_name: postgres + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres + depends_on: + certfixer: + condition: service_completed_successfully + ports: + - 5432:5432 + environment: + - LEGA_IN_PASSWORD=password + - LEGA_OUT_PASSWORD=password + - POSTGRES_PASSWORD=rootpasswd + - POSTGRES_SERVER_CACERT=/certs/ca.crt + - POSTGRES_SERVER_CERT=/certs/db.crt + - POSTGRES_SERVER_KEY=/certs/db.key + + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "lega_out" ] + interval: 5s + timeout: 20s + retries: 3 + volumes: + - certs:/certs + + mockauth: + container_name: mockauth + image: python:3.11-slim + ports: + - 8000:8000 + environment: + - ISSUER_URL=http://mockauth:8000 + volumes: + - ../../sda-doa/test/mock_auth.py:/mock_auth.py + - client_certs:/client_certs + command: > + sh -c "pip install --upgrade pip && pip install aiohttp Authlib && python -u /mock_auth.py 0.0.0.0 8000" + + rabbitmq: + build: + context: ../../rabbitmq + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq + container_name: rabbitmq + depends_on: + certfixer: + condition: service_completed_successfully + environment: + - RABBITMQ_SERVER_CACERT=/etc/rabbitmq/ssl/ca.crt + - RABBITMQ_SERVER_CERT=/etc/rabbitmq/ssl/mq.crt + - RABBITMQ_SERVER_KEY=/etc/rabbitmq/ssl/mq.key + - RABBITMQ_SERVER_VERIFY=verify_none + healthcheck: + test: + [ + "CMD", + "bash", + "-c", + "rabbitmq-diagnostics -q check_running && rabbitmq-diagnostics -q check_local_alarms", + ] + interval: 5s + timeout: 20s + retries: 3 + restart: always + volumes: + - certs:/etc/rabbitmq/ssl/ + - rabbitmq_data:/var/lib/rabbitmq + ports: + - 5671:5671 + - 25671:15671 + + outbox: + container_name: outbox + image: minio/minio + ports: + - 9000:9000 + environment: + - MINIO_ACCESS_KEY=minio + - MINIO_SECRET_KEY=miniostorage + command: server /data + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] + interval: 10s + timeout: 5s + retries: 5 + + init-bucket: + container_name: init-bucket + image: minio/mc + depends_on: + outbox: + condition: service_healthy + entrypoint: > + /bin/sh -c " + /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage; + /usr/bin/mc mb s3/lega; + exit 0; + " + + doa: + container_name: doa + build: + context: ../../sda-doa + image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-doa + depends_on: + postgres: + condition: service_healthy + rabbitmq: + condition: service_healthy + certfixer: + condition: service_completed_successfully + ports: + - "8080:8080" + environment: + - SSL_MODE=verify-ca + - POSTGRES_USER=postgres + - POSTGRES_DB=sda + - POSTGRES_PASSWORD=rootpasswd + - DB_INSTANCE=postgres + - SSL_ENABLED=false + - BROKER_HOST=rabbitmq + - BROKER_VALIDATE=false + - OPENID_CONFIGURATION_URL=http://mockauth:8000/openid-configuration + - USERINFO_ENDPOINT_URL=http://mockauth:8000/userinfo + - CRYPT4GH_PRIVATE_KEY_PATH=test/crypt4gh.sec.pem + - CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH=test/crypt4gh.pass + - OUTBOX_TYPE=S3 + - SSL_ENABLED=false + - ROOT_CERT_PATH=/certs/ca.crt + - CERT_PATH=/certs/client.crt + - CERT_KEY=/certs/client.der + - BROKER_USERNAME=guest + - S3_OUT_ENDPOINT=outbox + - S3_OUT_PORT=9000 + - S3_OUT_SECURE=false + + volumes: + - client_certs:/certs + - ../../sda-doa/test/body.enc:/test/body.enc + - encryption_files:/test + + integration_test: + container_name: integration_test + image: maven:3.9.9-eclipse-temurin-21 + profiles: [test] + volumes: + - ../../sda-doa/src:/sda-doa/src + - ../../sda-doa/pom.xml:/sda-doa/pom.xml + - ../../sda-doa/settings.xml:/root/.m2/settings.xml + - ./tests:/tests + - encryption_files:/test + - client_certs:/certs + + depends_on: + - doa + - mockauth + - init-bucket + environment: + - OUTBOX_TYPE=S3 + - DOA_URL=http://doa:8080 + - MOCKAUTH_URL=http://mockauth:8000 + - MINIO_HOST=outbox + + command: + - "/bin/sh" + - "/tests/run_scripts.sh" + - "/tests/doa" + +volumes: + certs: + client_certs: + rabbitmq_data: + encryption_files: diff --git a/.github/integration/sda-s3-integration.yml b/.github/integration/sda-s3-integration.yml index 9c65439515..642b3c39cc 100644 --- a/.github/integration/sda-s3-integration.yml +++ b/.github/integration/sda-s3-integration.yml @@ -83,14 +83,14 @@ services: context: ../../sda target: debug image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-s3inbox ] + command: [sda-s3inbox] container_name: s3inbox depends_on: credentials: condition: service_completed_successfully minio: condition: service_healthy - oidc: + mock-aai: condition: service_healthy postgres: condition: service_healthy @@ -102,6 +102,8 @@ services: - BROKER_ROUTINGKEY=inbox - DB_PASSWORD=inbox - DB_USER=inbox + extra_hosts: + - "localhost:host-gateway" restart: always volumes: - ./sda/config.yaml:/config.yaml @@ -112,7 +114,7 @@ services: ingest: image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-ingest ] + command: [sda-ingest] container_name: ingest depends_on: credentials: @@ -137,7 +139,7 @@ services: verify: image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-verify ] + command: [sda-verify] container_name: verify depends_on: credentials: @@ -162,7 +164,7 @@ services: finalize: image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-finalize ] + command: [sda-finalize] container_name: finalize depends_on: credentials: @@ -187,7 +189,7 @@ services: mapper: image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-mapper ] + command: [sda-mapper] container_name: mapper depends_on: credentials: @@ -209,42 +211,15 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared - oidc: - container_name: oidc - command: - - /bin/sh - - -c - - | - pip install --upgrade pip - pip install aiohttp Authlib joserfc requests - python -u /oidc.py - depends_on: - credentials: - condition: service_completed_successfully - extra_hosts: - - "localhost:host-gateway" - healthcheck: - test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] - interval: 10s - timeout: 2s - retries: 6 - image: python:3.11-slim - ports: - - "8080:8080" - restart: always - volumes: - - ./sda/oidc.py:/oidc.py - - shared:/shared - api: - command: [ sda-api ] + command: [sda-api] container_name: api depends_on: credentials: condition: service_completed_successfully postgres: condition: service_healthy - oidc: + mock-aai: condition: service_healthy rabbitmq: condition: service_healthy @@ -253,6 +228,8 @@ services: - BROKER_USER=api - DB_PASSWORD=api - DB_USER=api + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} ports: - "8090:8080" @@ -264,7 +241,7 @@ services: reencrypt: image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} - command: [ sda-reencrypt ] + command: [sda-reencrypt] container_name: reencrypt depends_on: credentials: @@ -282,13 +259,7 @@ services: credentials: condition: service_completed_successfully command: - [ - "python", - "/cega/users.py", - "0.0.0.0", - "8443", - "/shared/users.json" - ] + ["python", "/cega/users.py", "0.0.0.0", "8443", "/shared/users.json"] environment: - CEGA_USERS_PASSWORD=test - CEGA_USERS_USER=test @@ -300,7 +271,7 @@ services: - shared:/shared auth-cega: - command: [ sda-auth ] + command: [sda-auth] container_name: auth-cega depends_on: cega-nss: @@ -311,8 +282,13 @@ services: - AUTH_RESIGNJWT=true - AUTH_CEGA_ID=test - AUTH_CEGA_SECRET=test + - OIDC_REDIRECTURL=http://localhost:8888/oidc/login + - OIDC_ID=XC56EL11zz + - OIDC_SECRET=wHPVQaYXmdDHa - DB_PASSWORD=auth - DB_USER=auth + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} ports: - "8888:8080" @@ -321,32 +297,64 @@ services: - ./sda/config.yaml:/config.yaml - shared:/shared - auth-oidc: - command: [ sda-auth ] - container_name: auth-oidc + auth-aai: + command: [sda-auth] + container_name: auth-aai depends_on: - postgres: - condition: service_healthy - oidc: - condition: service_healthy + credentials: + condition: service_completed_successfully + mock-aai: + condition: service_started environment: - AUTH_RESIGNJWT=false - - OIDC_ID=XC56EL11xx - - OIDC_SECRET=wHPVQaYXmdDHg - - OIDC_PROVIDER=http://localhost:8080 - - OIDC_REDIRECTURL=http://localhost:8889/oidc/login - DB_PASSWORD=auth - DB_USER=auth extra_hosts: - "localhost:host-gateway" image: ghcr.io/biobanklab/sensitive-data-archive:PR${PR_NUMBER} ports: - - "8889:8080" + - "8801:8080" restart: always volumes: - ./sda/config.yaml:/config.yaml - shared:/shared + mock-aai: + container_name: ls-aai-mock + depends_on: + aai-db: + condition: service_healthy + environment: + - DOCKERHOST=localhost + extra_hosts: + - "localhost:host-gateway" + healthcheck: + test: + [ "CMD", "/bin/true" ] + interval: 10s + timeout: 2s + retries: 6 + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 + ports: + - "8800:8080" + volumes: + - "./sda/aai-mock:/etc/lsaai-mock" + aai-db: + container_name: ls-aai-db + environment: + MYSQL_ROOT_PASSWORD: "aaiPass" + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: "aai" + MYSQL_USER: "aai" + MYSQL_PASSWORD: "aaiPass" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 2s + retries: 6 + image: mysql/mysql-server:latest + volumes: + - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql integration_test: container_name: tester command: @@ -356,7 +364,7 @@ services: depends_on: auth-cega: condition: service_started - auth-oidc: + auth-aai: condition: service_started cega-nss: condition: service_started @@ -371,11 +379,11 @@ services: s3inbox: condition: service_started verify: - condition: service_started + condition: service_started api: - condition: service_started + condition: service_started reencrypt: - condition: service_started + condition: service_started extra_hosts: - "localhost:host-gateway" environment: diff --git a/.github/integration/sda-sync-integration.yml b/.github/integration/sda-sync-integration.yml index 5160ab9623..2af84742e1 100644 --- a/.github/integration/sda-sync-integration.yml +++ b/.github/integration/sda-sync-integration.yml @@ -89,7 +89,7 @@ services: condition: service_completed_successfully minio: condition: service_healthy - oidc: + mock-aai: condition: service_healthy postgres: condition: service_healthy @@ -101,6 +101,8 @@ services: - BROKER_ROUTINGKEY=inbox - DB_PASSWORD=inbox - DB_USER=inbox + extra_hosts: + - "localhost:host-gateway" restart: always volumes: - ./sda/config.yaml:/config.yaml @@ -251,30 +253,42 @@ services: volumes: - ./sda/config.yaml:/config.yaml - oidc: - container_name: oidc - command: - - /bin/sh - - -c - - | - pip install --upgrade pip - pip install aiohttp Authlib joserfc requests - python -u /oidc.py + mock-aai: + container_name: ls-aai-mock depends_on: - credentials: - condition: service_completed_successfully + aai-db: + condition: service_healthy + environment: + - DOCKERHOST=localhost + extra_hosts: + - "localhost:host-gateway" healthcheck: - test: ["CMD", "python3", "-c", 'import requests; print(requests.get(url = "http://localhost:8080/jwk").text)'] + test: + [ "CMD", "/bin/true" ] interval: 10s timeout: 2s retries: 6 - image: python:3.11-slim + image: registry.gitlab.ics.muni.cz:443/perun/deployment/proxyidp/proxyidp-public-docker-images/ls_aai_mock:2.5.2-broker2.1.10-tomcat9.0-jdk11 ports: - - "8080:8080" - restart: always + - "8800:8080" volumes: - - ./sda/oidc.py:/oidc.py - - shared:/shared + - "./sda/aai-mock:/etc/lsaai-mock" + aai-db: + container_name: ls-aai-db + environment: + MYSQL_ROOT_PASSWORD: "aaiPass" + MYSQL_ROOT_HOST: "%" + MYSQL_DATABASE: "aai" + MYSQL_USER: "aai" + MYSQL_PASSWORD: "aaiPass" + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost"] + interval: 10s + timeout: 2s + retries: 6 + image: mysql/mysql-server:latest + volumes: + - ./sda/aai-mock/aai-mock.sql:/docker-entrypoint-initdb.d/1.sql api: command: [ sda-api ] @@ -284,7 +298,7 @@ services: condition: service_completed_successfully postgres: condition: service_healthy - oidc: + mock-aai: condition: service_healthy rabbitmq: condition: service_healthy @@ -293,6 +307,8 @@ services: - BROKER_USER=api - DB_PASSWORD=api - DB_USER=api + extra_hosts: + - "localhost:host-gateway" image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER} ports: - "8090:8080" diff --git a/.github/integration/sda/aai-mock/aai-mock.sql b/.github/integration/sda/aai-mock/aai-mock.sql new file mode 100644 index 0000000000..ecc0ad5c24 --- /dev/null +++ b/.github/integration/sda/aai-mock/aai-mock.sql @@ -0,0 +1,206 @@ +CREATE TABLE IF NOT EXISTS access_token ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + token_value VARCHAR(4096), + expiration TIMESTAMP NULL, + token_type VARCHAR(256), + refresh_token_id BIGINT, + client_id VARCHAR(256) NOT NULL, + auth_holder_id BIGINT, + approved_site_id BIGINT +); + +CREATE TABLE IF NOT EXISTS authorization_code ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + code VARCHAR(256), + auth_holder_id BIGINT, + expiration TIMESTAMP NULL +); + +CREATE TABLE IF NOT EXISTS approved_site ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + user_id VARCHAR(256), + client_id VARCHAR(256), + creation_date TIMESTAMP NULL, + access_date TIMESTAMP NULL, + timeout_date TIMESTAMP NULL, + whitelisted_site_id BIGINT +); + +CREATE TABLE IF NOT EXISTS approved_site_scope ( + owner_id BIGINT, + scope VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS authentication_holder ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + user_auth_id BIGINT, + approved BOOLEAN, + redirect_uri VARCHAR(2048), + client_id VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_authority ( + owner_id BIGINT, + authority VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_resource_id ( + owner_id BIGINT, + resource_id VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_response_type ( + owner_id BIGINT, + response_type VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_extension ( + owner_id BIGINT, + extension VARCHAR(2048), + val VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_scope ( + owner_id BIGINT, + scope VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS authentication_holder_request_parameter ( + owner_id BIGINT, + param VARCHAR(2048), + val TEXT +); + +CREATE TABLE IF NOT EXISTS saved_user_auth ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + acr VARCHAR(1024), + auth_time BIGINT DEFAULT NULL, + name VARCHAR(1024), + authenticated BOOLEAN, + authentication_attributes TEXT +); + +CREATE TABLE IF NOT EXISTS saved_user_auth_authority ( + owner_id BIGINT, + authority VARCHAR(256) +); + +CREATE TABLE IF NOT EXISTS refresh_token ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + token_value VARCHAR(4096), + expiration TIMESTAMP NULL, + auth_holder_id BIGINT, + client_id VARCHAR(256) NOT NULL +); + +CREATE TABLE IF NOT EXISTS token_scope ( + owner_id BIGINT, + scope VARCHAR(2048) +); + +CREATE TABLE IF NOT EXISTS device_code ( + id BIGINT AUTO_INCREMENT PRIMARY KEY, + device_code VARCHAR(1024), + user_code VARCHAR(1024), + expiration TIMESTAMP NULL, + client_id VARCHAR(256), + approved BOOLEAN, + auth_holder_id BIGINT, + recorded_error TEXT DEFAULT NULL +); + +CREATE TABLE IF NOT EXISTS device_code_scope ( + owner_id BIGINT NOT NULL, + scope VARCHAR(256) NOT NULL +); + +CREATE TABLE IF NOT EXISTS device_code_request_parameter ( + owner_id BIGINT, + param VARCHAR(2048), + val VARCHAR(2048) +); + +alter table access_token + add constraint access_token_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete set null; + +alter table access_token + add constraint access_token_refresh_token_id_fk + foreign key (refresh_token_id) references refresh_token (id) + on update cascade on delete set null; + +alter table approved_site_scope + add constraint approved_site_scope_approved_site_id_fk + foreign key (owner_id) references approved_site (id) + on update cascade on delete cascade; + +alter table authentication_holder_authority + add constraint authentication_holder_authority_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_extension + add constraint authentication_holder_extension_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_request_parameter + add constraint auth_holder_request_parameter_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_resource_id + add constraint authentication_holder_resource_id_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder_response_type + add constraint authentication_holder_response_type_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authentication_holder + add constraint authentication_holder_saved_user_auth_id_fk + foreign key (user_auth_id) references saved_user_auth (id) + on update cascade on delete cascade; + +alter table authentication_holder_scope + add constraint authentication_holder_scope_authentication_holder_id_fk + foreign key (owner_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table authorization_code + add constraint authorization_code_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete cascade; + +alter table device_code + add constraint device_code_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete set null; + +alter table device_code_request_parameter + add constraint device_code_request_parameter_device_code_id_fk + foreign key (owner_id) references device_code (id) + on update cascade on delete cascade; + +alter table device_code_scope + add constraint device_code_scope_device_code_id_fk + foreign key (owner_id) references device_code (id) + on update cascade on delete cascade; + +alter table refresh_token + add constraint refresh_token_authentication_holder_id_fk + foreign key (auth_holder_id) references authentication_holder (id) + on update cascade on delete set null; + +alter table saved_user_auth_authority + add constraint saved_user_auth_authority_saved_user_auth_id_fk + foreign key (owner_id) references saved_user_auth (id) + on update cascade on delete cascade; + +alter table token_scope + add constraint token_scope_refresh_token_id_fk + foreign key (owner_id) references access_token (id) + on update cascade on delete cascade; \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/application.properties b/.github/integration/sda/aai-mock/application.properties new file mode 100644 index 0000000000..c5c30f8d1d --- /dev/null +++ b/.github/integration/sda/aai-mock/application.properties @@ -0,0 +1,12 @@ +main.oidc.issuer.url=http://${DOCKERHOST}:8800/oidc/ +web.baseURL=https://${DOCKERHOST}:8800/oidc + +# GA4GH broker +ga4gh.broker.url=http://${DOCKERHOST}:8800/ga4gh-broker/ +ga4gh.broker.username=broker-aai-user +ga4gh.broker.password=broker-aai-pass + +# DATABASE +jdbc.url=jdbc:mysql://ls-aai-db:3306/aai +jdbc.user=aai +jdbc.password=aaiPass \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/clients/aai-auth.yaml b/.github/integration/sda/aai-mock/clients/aai-auth.yaml new file mode 100644 index 0000000000..b6ec2ed326 --- /dev/null +++ b/.github/integration/sda/aai-mock/clients/aai-auth.yaml @@ -0,0 +1,8 @@ +client-name: "aai-auth" +client-id: "XC56EL11xx" +client-secret: "wHPVQaYXmdDHg" +redirect-uris: ["http://localhost:8801/oidc/login"] +token-endpoint-auth-method: "client_secret_basic" +scope: ["openid", "profile", "email", "ga4gh_passport_v1", "eduperson_entitlement"] +grant-types: ["authorization_code"] +post-logout-redirect-uris: ["http://localhost:8801/oidc/login"] \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/clients/cega-auth.yaml b/.github/integration/sda/aai-mock/clients/cega-auth.yaml new file mode 100644 index 0000000000..c0c28a73f4 --- /dev/null +++ b/.github/integration/sda/aai-mock/clients/cega-auth.yaml @@ -0,0 +1,8 @@ +client-name: "cega-auth" +client-id: "XC56EL11zz" +client-secret: "wHPVQaYXmdDHa" +redirect-uris: ["http://localhost:8888/oidc/login"] +token-endpoint-auth-method: "client_secret_basic" +scope: ["openid", "profile", "email", "ga4gh_passport_v1", "eduperson_entitlement"] +grant-types: ["authorization_code"] +post-logout-redirect-uris: ["http://localhost:8888/oidc/login"] \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/ga4gh-broker.sql b/.github/integration/sda/aai-mock/ga4gh-broker.sql new file mode 100644 index 0000000000..424907c502 --- /dev/null +++ b/.github/integration/sda/aai-mock/ga4gh-broker.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS visas( + id BIGINT AUTO_INCREMENT NOT NULL, + user_id VARCHAR(255) NOT NULL, + source VARCHAR(1024) NOT NULL, + linked_identity VARCHAR(1024) NOT NULL, + exp datetime NOT NULL, + jwt LONGTEXT NOT NULL, + CONSTRAINT pk_visas PRIMARY KEY (id) +); + +CREATE INDEX idx_visaentity_user_id_source ON visas (user_id, source); \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/ga4gh-broker/application.yaml b/.github/integration/sda/aai-mock/ga4gh-broker/application.yaml new file mode 100644 index 0000000000..47fa94f03c --- /dev/null +++ b/.github/integration/sda/aai-mock/ga4gh-broker/application.yaml @@ -0,0 +1,63 @@ +--- + +spring: + main: + allow-bean-definition-overriding: true + mvc: + pathmatch: + matching-strategy: ant_path_matcher + +logging: + level: + root: info + cz.muni.ics: debug + +mock-broker: + enabled: True + issuer: http://${DOCKERHOST}:8080/oidc/ + jku: http://${DOCKERHOST}:8800/ga4gh-broker/jwk + path-to-jwk-file: /etc/lsaai-mock/keys.jwks + visa-configs-path: /etc/lsaai-mock/ga4gh-broker/users/ + passport-repositories: [] +# Sample configuration for repository +# passport-repositories: +# - name: REMS-LOCAL-API +# url: http://rems-api.local/api/permissions/{user_id} +# jwks: http://rems-api.local/api/jwk +# headers: +# - header: x-rems-api-key +# value: api-key +# - header: x-rems-user-id +# value: user-id + +visa-writers: + enabled: False +# Uncomment to enable writing from visa sources +# enabled: True +# db: +# url: jdbc:mysql://localhost:3307/broker +# username: brokerUser +# password: brokerPass +# driver: "com.mysql.cj.jdbc.Driver" +# repositories: +# - name: Local REMS +# jwks: http://rems.local/api/jwk +# auth: +# api-key: +# api-key-header: X-visa-writer +# api-key-value: X-vis-writer-key + +passport-v1-0: + enabled: True + auth: + - username: "broker-aai-user" + password: "broker-aai-pass" + +passport-v1-2: + enabled: False +# Uncomment to enable token-exchange integration with AAI mock +# enabled: True +# auth: +# introspection-client-id: broker +# introspection-client-secret: broker-secret +# introspection-uri: http://localhost:8080/oidc/introspect \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/ga4gh-broker/users/test-user-visa.yaml b/.github/integration/sda/aai-mock/ga4gh-broker/users/test-user-visa.yaml new file mode 100644 index 0000000000..dbdcc3515e --- /dev/null +++ b/.github/integration/sda/aai-mock/ga4gh-broker/users/test-user-visa.yaml @@ -0,0 +1,34 @@ +--- +# Identifier of the user - has to match the value of 'sub' claim in AAI +userId: "testu@lifescience-ri.eu" +visas: + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "affiliate@dummy.uni" + type: "AffiliationAndRole" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "faculty@dummy.uni" + type: "AffiliationAndRole" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "https://lifescience-ri.eu/ls-login/ls-aai-aup.html" + type: "AcceptedTermsAndPolicies" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "https://doi.org/10.1038/s41431-018-0219-y" + type: "ResearcherStatus" + by: "system" + - source: https://dummy.uni/ + expires: 1893452400 + asserted: 1683049328 + value: "urn:sampledataset:gdi:testcase" + type: "ControlledAccessGrants" + by: "system" \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/keys.jwks b/.github/integration/sda/aai-mock/keys.jwks new file mode 100644 index 0000000000..e95becd288 --- /dev/null +++ b/.github/integration/sda/aai-mock/keys.jwks @@ -0,0 +1,18 @@ +{ + "keys": [ + { + "n": "4bmY8T5jhlCFtS4ccyn8ZDftWk27yOcPgKbak991_m1rs3474RKHYjc9GwX0ICSaUN02fCdSGaTNhU586MfhRsSj3uU_TIJ4M3eDengcllT16DTJOyHZgVzHT2QRUsozFuG2tTBCwqYildOXXuCeaKUQ8Jh5sjKmCcQZO_Ht4omGIsVZPo9SrvshZeHdkdvctvjvT5WXoCZaHvBRpyiBmQKUv_LfYsgpF4PFm40j84GAk0CsZ_xOJXBnU6wLfzHCq1pgLqaXK_q6T_gSJ5t5eWbv0IkUmXDG-TuOtKIz_Xip2lJQ6Rmb_QmOY-MeJavL41NpCu-2FwTvCNqMbHuWuQ", + "e": "AQAB", + "d": "BbwSX6anHqVzECxQurhJWj51gELTT4JXSXxztygJNmKPRushGFHBMMSYf9RB5IMpjH5iQPs6wb4HHqjk0YEqfwLF6wbF-eqipSQXKghdKZCVAsY8io0MmpXB1omDSygp7h3j52yHdayE2muav-VTAPOYn5QwG0_gGgVqYrR9x7CMiTuyOIuGNO4Wlly4_5RhLtSo0pal9AgBvX4crtVEwN8tPgqPVo9w71bSROt9EVNI3cZiFFrrapYiifckIGiPGQYQUd5ej9Mq_77Fa0fv0pk0ONQV8HwstQ5HY2WwJWsnmccF9plVTzem7N_vo-T-hFRPUO9TZUao91mMV8iV5Q", + "p": "9Z2W0FtzR3aJdH1wPJ8OSUZEy2cdc3GeZd-llT5MAoyTsl0Nr1h5uJc-xfNUa_HUJzCi1jAyo6wU_Q237MWCtm2MRDUtVv5_tIziGl_-_dg2PbX1-77x1PchknaCROaLOgxgh2CJzg9MZ0Zp5d70u4Cr_Jog8A8xbG47Q4Oln0s", + "q": "60S5Jpn2QjHJiwNNmEqxKwJVLHrsmne5WBxCQKVyn59utohxC5UGtszgWaSRlyHB7DX9VxqTHlOaR_iCyhOGOam3JavgMxQnGZwRVHw3msJlnVeRSajQ3dS_DG_GhPuqtGOazW90S-wNsqR9mXmCkhqR89wu0Z-Slr6FgF0Vq4s", + "dp": "h-G8St8dIr8YgoLXMiKIw88FRkZwy6pjCFc3gx6o7IwbNgU3Cff3t0CtcNQSs3lhdqD4pWiay2PrZKhPRcVrcEOg4ydMQyGfQyLh_9d1-bfBIljlknXr0nc6co3vufCVu_KXAK5O7toV2djhw1Oar5UbES-v4z90h1lgnhRZ74s", + "dq": "ai8zTavRjKDB8R8Fh4W2ns3HuXKllTYSDiV69Yh2ynp_0pyUQQD67chcPblcCJUw3Dal6AlTey2KhY8xIJldTGtAAR2o7ATdfN6Gp9YElrHEgRZDubYaxvQ9YhSUyfJ4YkBOXwg9ugKJUCaTlZBVETIyy5iF_RCDbcwBnyTkIe8", + "qi": "mbnk1BlUoAQMZ3hrbWpMNxF120_DAvag_eprSBsaThgVFQKtf32Eh6M5O9BbXF_q8bFN3sTVCLO8Wx7jJHNlErJchzsMJx3M6ekytJ3X5pfBm1EctLDQQNLo9Z4ll2MlgfGXZikCYx1RF1wZOsdvpv_q-FT3T7EvXgJfsDCh-n8", + "use": "sig", + "alg": "RS256", + "kty": "RSA", + "kid": "rsa1" + } + ] +} \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/userinfos/dummy-user.yaml b/.github/integration/sda/aai-mock/userinfos/dummy-user.yaml new file mode 100644 index 0000000000..ae0ef3b970 --- /dev/null +++ b/.github/integration/sda/aai-mock/userinfos/dummy-user.yaml @@ -0,0 +1,33 @@ +web-display-name: "Dummy User - submitter" + +sub: "test@dummy.org" + +name: "Dummy User" +preferred-username: "dummyu" +given-name: "Dummy" +family-name: "User" +middle-name: "" +nickname: "duus" +locale: "EN" + +email: "test@dummy.org" +email-verified: True + +country: "eu" + +eduperson-entitlement: ["urn:geant:lifescience-ri.eu:group:researchers#lifescience-ri.eu"] + +voperson-external-affiliation: ["unknown@orcid.org", "affiliate@dummy.uni", "faculty@dummy.uni"] + +eduperson-orcid: "https://orcid.org/0000-0000-XXXX-XXXX" + +schac-home-organization: ["dummy.uni", "orcid.org"] + +eduperson-scoped-affiliation: ["affiliate@lifescience-ri.eu", "member@lifescience-ri.eu"] + +voperson-current-external-affiliation: ["affiliate@dummy.uni", "faculty@dummy.uni"] + +authenticating-entity: "https://idp.dummy.uni/mock" + +acr: "https://refeds.org/profile/mfa" +authn-instant: "2023-01-01T10:15:30Z" \ No newline at end of file diff --git a/.github/integration/sda/aai-mock/userinfos/test-user.yaml b/.github/integration/sda/aai-mock/userinfos/test-user.yaml new file mode 100644 index 0000000000..54c2bd7454 --- /dev/null +++ b/.github/integration/sda/aai-mock/userinfos/test-user.yaml @@ -0,0 +1,33 @@ +web-display-name: "Test User - admin" + +sub: "testu@lifescience-ri.eu" + +name: "Test User" +preferred-username: "testu" +given-name: "Test" +family-name: "User" +middle-name: "dummy" +nickname: "tedu" +locale: "EN" + +email: "testu@dummy.uni" +email-verified: True + +country: "eu" + +eduperson-entitlement: ["urn:geant:lifescience-ri.eu:group:researchers#lifescience-ri.eu"] + +voperson-external-affiliation: ["unknown@orcid.org", "affiliate@dummy.uni", "faculty@dummy.uni"] + +eduperson-orcid: "https://orcid.org/0000-0000-XXXX-XXXX" + +schac-home-organization: ["dummy.uni", "orcid.org"] + +eduperson-scoped-affiliation: ["affiliate@lifescience-ri.eu", "member@lifescience-ri.eu"] + +voperson-current-external-affiliation: ["affiliate@dummy.uni", "faculty@dummy.uni"] + +authenticating-entity: "https://idp.dummy.uni/mock" + +acr: "https://refeds.org/profile/mfa" +authn-instant: "2023-05-01T10:15:30Z" \ No newline at end of file diff --git a/.github/integration/sda/config.yaml b/.github/integration/sda/config.yaml index 50dfe73a2b..c14fd719e1 100644 --- a/.github/integration/sda/config.yaml +++ b/.github/integration/sda/config.yaml @@ -23,7 +23,7 @@ auth: jwt: issuer: "https://auth:8888" privateKey: /shared/keys/jwt.key - signatureAlg: ES256 + signatureAlg: RS256 tokenTTL: 168 publicFile: "/shared/c4gh.pub.pem" resignJwt: @@ -77,17 +77,17 @@ c4gh: passphrase: "c4ghpass" oidc: - id: - jwkPath: /jwk - provider: http://oidc:8080 - redirectUrl: http://auth_oidc:8080/oidc/login - secret: + id: XC56EL11xx + jwkPath: jwk + provider: http://localhost:8800/oidc/ + redirectUrl: http://localhost:8801/oidc/login + secret: wHPVQaYXmdDHg server: cert: "" key: "" - jwtpubkeypath: "/shared/keys/pub/" - jwtpubkeyurl: "http://oidc:8080/jwk" + # jwtpubkeypath: "/shared/keys/pub/" + jwtpubkeyurl: "http://localhost:8800/oidc/jwk" sync: api: diff --git a/.github/integration/sda/rbac.json b/.github/integration/sda/rbac.json index a76d6c8498..7d37530658 100644 --- a/.github/integration/sda/rbac.json +++ b/.github/integration/sda/rbac.json @@ -12,8 +12,8 @@ }, { "role": "admin", - "path": "/dataset/verify/:dataset", - "action": "PUT" + "path": "/dataset/*", + "action": "(POST)|(PUT)" }, { "role": "submission", @@ -62,7 +62,7 @@ "rolebinding": "submission" }, { - "role": "requester@demo.org", + "role": "testu@lifescience-ri.eu", "rolebinding": "admin" } ] diff --git a/.github/integration/tests/doa/1.make_crypt4gh_files.sh b/.github/integration/tests/doa/1.make_crypt4gh_files.sh new file mode 100755 index 0000000000..568508b786 --- /dev/null +++ b/.github/integration/tests/doa/1.make_crypt4gh_files.sh @@ -0,0 +1,56 @@ +#!/bin/sh + +mkdir -p ./sda-doa/test/crypt4gh + +cat << EOF > ./sda-doa/test/crypt4gh/crypt4gh.pub.pem +-----BEGIN CRYPT4GH PUBLIC KEY----- +PQFOuEiuAOlghTGU0Z7u2EH5FwBtSlnukpiMAji/ml0= +-----END CRYPT4GH PUBLIC KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/crypt4gh.pub.pem + +cat << EOF > ./sda-doa/test/crypt4gh/my.pub.pem +-----BEGIN CRYPT4GH PUBLIC KEY----- +cUSN6pHzlIFgoclIfDSaDtUgAXRa+DUHBhTodeNL52w= +-----END CRYPT4GH PUBLIC KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/my.pub.pem + +cat << EOF > ./sda-doa/test/crypt4gh/my.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAOK6Q2g1KxcUELMt/RMhr9wARY2hhY2hhMjBf +cG9seTEzMDUAPL3l1Mt/LvDwD+yffT09Jog2AJO3uG0DaGAPDTPbOfTBKr/gWRik +PcF6893CkScij1pO3n9Ub8p1H4yLAQ== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/my.sec.pem + +cat << EOF > ./sda-doa/test/crypt4gh/crypt4gh.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf +cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J +kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./sda-doa/test/crypt4gh/crypt4gh.sec.pem + + +cat << EOF > ./test/crypt4gh.sec.pem +-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY----- +YzRnaC12MQAGc2NyeXB0ABQAAAAAr3MTvNgHj/z6U02GqdILFwARY2hhY2hhMjBf +cG9seTEzMDUAPNzNGWSc7hWSxjwfuQJt2haq0/eyvoFjQXsvp+RCvXSEVgqlO58J +kgjKQgpRb9qm09AGhYU4tbXg7pyCRg== +-----END CRYPT4GH ENCRYPTED PRIVATE KEY----- +EOF + +chmod 444 ./test/crypt4gh.sec.pem + +printf "password" > ./test/crypt4gh.pass + +chmod 444 ./test/crypt4gh.pass + +echo "CRYPT4GH files created successfully" \ No newline at end of file diff --git a/.github/integration/tests/doa/2.insert_test_data.sh b/.github/integration/tests/doa/2.insert_test_data.sh new file mode 100755 index 0000000000..0bcc28df28 --- /dev/null +++ b/.github/integration/tests/doa/2.insert_test_data.sh @@ -0,0 +1,37 @@ +#!/bin/sh + +export PGPASSWORD="password" +export PGSSLMODE="verify-ca" +export PGSSLCERT="/certs/client.crt" +export PGSSLKEY="/certs/client.key" +export PGSSLROOTCERT="/certs/ca.crt" + +if ! psql -h postgres -U lega_in -d sda -c "SELECT local_ega.insert_file('body.enc', 'requester@elixir-europe.org');"; then + echo "Insert File failed" +fi + +if ! psql -h postgres -U lega_in -d sda -c "UPDATE local_ega.files SET header = '637279707434676801000000010000006c00000000000000aa7ad1bb4f93bf5e4fb3bc28a95bc4d80bf2fd8075e69eb2ee15e0a4f08f1d78ab98c8fd9b50e675f71311936e8d0c6f73538962b836355d5d4371a12eae46addb43518b5236fb9554249710a473026f34b264a61d2ba52ed11abc1efa1d3478fa40a710' +WHERE id = 1;"; then + echo "Set Header For The File failed" +fi + +if ! psql -h postgres -U lega_in -d sda -c "UPDATE local_ega.files SET archive_path = 'test/body.enc', status = 'READY', stable_id = 'EGAF00000000014' WHERE id = 1;"; then + echo "Set File Data failed" +fi + +if ! psql -h postgres -U lega_out -d sda -c "INSERT INTO local_ega_ebi.filedataset(file_id, dataset_stable_id) values(1, 'EGAD00010000919');"; then + echo "Insert Dataset failed" +fi + +if ! psql -h postgres -U lega_out -d sda -c "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'registered', '{\"type\": \"mapping\"}');"; then + echo "Insert Event Log RELEASED failed" +fi + +if ! psql -h postgres -U lega_out -d sda -c "INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('EGAD00010000919', 'released', '{\"type\": \"release\"}');"; then + echo "Insert Event Log RELEASE failed" +fi + +export PGPASSWORD="rootpasswd" +if ! psql -h postgres -U postgres -d sda -c "INSERT INTO sda.dataset_references(dataset_id, reference_id, reference_scheme) values('1', 'GDI-NO-10001','GDI');"; then + echo "Insert Dataset Reference failed" +fi diff --git a/.github/integration/tests/doa/3.setup_outbox.sh b/.github/integration/tests/doa/3.setup_outbox.sh new file mode 100644 index 0000000000..90c7b9f16e --- /dev/null +++ b/.github/integration/tests/doa/3.setup_outbox.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd sda-doa || exit +mkdir -p outbox && chmod -R 750 outbox && chown -R nobody outbox \ No newline at end of file diff --git a/.github/integration/tests/doa/4.run_tests.sh b/.github/integration/tests/doa/4.run_tests.sh new file mode 100644 index 0000000000..3c2f20e6c3 --- /dev/null +++ b/.github/integration/tests/doa/4.run_tests.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd sda-doa || exit +mvn test \ No newline at end of file diff --git a/.github/integration/tests/postgres/20_inbox_queries.sh b/.github/integration/tests/postgres/20_inbox_queries.sh index e79abfa98b..f2e8d51bee 100644 --- a/.github/integration/tests/postgres/20_inbox_queries.sh +++ b/.github/integration/tests/postgres/20_inbox_queries.sh @@ -10,6 +10,17 @@ for host in migrate postgres; do exit 1 fi + newFileID=$(psql -U inbox -h "$host" -d sda -At -c "SELECT sda.register_file('inbox/test-file.c4gh', 'other-user');") + if [ -z "$newFileID" ]; then + echo "register_file failed" + exit 1 + fi + + if [ "$fileID" = "$newFileID" ]; then + echo "File IDs should not be the same" + exit 1 + fi + resp=$(psql -U inbox -h "$host" -d sda -At -c "INSERT INTO sda.file_event_log(file_id, event, user_id, message) VALUES('$fileID', 'uploaded', 'test-user', '{\"uploaded\": \"message\"}');") if [ "$resp" != "INSERT 0 1" ]; then echo "Mark file as uploaded failed" diff --git a/.github/integration/tests/sda/10_upload_test.sh b/.github/integration/tests/sda/10_upload_test.sh index 6400454542..8c045f66ea 100644 --- a/.github/integration/tests/sda/10_upload_test.sh +++ b/.github/integration/tests/sda/10_upload_test.sh @@ -33,13 +33,9 @@ done ## reupload a file under a different name s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/NB12878.bam.c4gh -## reupload a file with the same name -s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ - - echo "waiting for upload to complete" RETRY_TIMES=0 -until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 6 ]; do +until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 5 ]; do echo "waiting for upload to complete" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then @@ -49,50 +45,33 @@ until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messag sleep 2 done -num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") -if [ "$num_rows" -ne 5 ]; then - echo "database queries for register_files failed, expected 5 got $num_rows" - exit 1 -fi - -num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") -if [ "$num_log_rows" -ne 12 ]; then - echo "database queries for file_event_logs failed, expected 12 got $num_log_rows" - exit 1 -fi - -## test with token from OIDC service -echo "testing with OIDC token" -newToken=$(curl http://oidc:8080/tokens | jq '.[0]') -cp s3cfg oidc_s3cfg -sed -i "s/access_token=.*/access_token=$newToken/" oidc_s3cfg - -s3cmd -c oidc_s3cfg put NA12878.bam.c4gh s3://requester_demo.org/data/file1.c4gh +## reupload a file with the same name +s3cmd -c s3cfg put NA12878.bam.c4gh s3://test_dummy.org/ -## verify that messages exists in MQ -echo "waiting for upload to complete" +## expect 2 new messages, one for deletion of the overwritten file, one for the new upload +echo "waiting for re-upload to complete" RETRY_TIMES=0 until [ "$(curl -s -k -u guest:guest $URI/api/queues/sda/inbox | jq -r '."messages_ready"')" -eq 7 ]; do - echo "waiting for upload to complete" + echo "waiting for re-upload to complete" RETRY_TIMES=$((RETRY_TIMES + 1)) if [ "$RETRY_TIMES" -eq 30 ]; then - echo "::error::Time out while waiting for upload to complete" + echo "::error::Time out while waiting for re-upload to complete" exit 1 fi sleep 2 done + num_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.files;") -if [ "$num_rows" -ne 6 ]; then - echo "database queries for register_files failed, expected 6 got $num_rows" +if [ "$num_rows" -ne 5 ]; then + echo "database queries for register_files failed, expected 5 got $num_rows" exit 1 fi num_log_rows=$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(*) from sda.file_event_log;") -if [ "$num_log_rows" -ne 14 ]; then - echo "database queries for file_event_logs failed, expected 14 got $num_log_rows" +if [ "$num_log_rows" -ne 12 ]; then + echo "database queries for file_event_logs failed, expected 12 got $num_log_rows" exit 1 fi - echo "files uploaded successfully" diff --git a/.github/integration/tests/sda/11_api_test.sh b/.github/integration/tests/sda/11_api_test.sh index c2226e501d..05a79f7bff 100644 --- a/.github/integration/tests/sda/11_api_test.sh +++ b/.github/integration/tests/sda/11_api_test.sh @@ -2,8 +2,8 @@ set -e # Test the API files endpoint -token="$(curl -s http://oidc:8080/tokens | jq -r '.[0]')" -response="$(curl -s -k -L "http://api:8080/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')" +token="$(cat /shared/token)" +response="$(curl -s -k -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq -r 'sort_by(.inboxPath)|.[-1].fileStatus')" if [ "$response" != "uploaded" ]; then echo "API returned incorrect value, expected ready got: $response" exit 1 diff --git a/.github/integration/tests/sda/40_mapper_test.sh b/.github/integration/tests/sda/40_mapper_test.sh index 8be6d0c2bd..49cb486d71 100644 --- a/.github/integration/tests/sda/40_mapper_test.sh +++ b/.github/integration/tests/sda/40_mapper_test.sh @@ -199,7 +199,7 @@ done ## Use API to list the datasets -token="$(curl http://oidc:8080/tokens | jq -r '.[0]')" +token="$(cat /shared/token)" resp="$(curl -s -k -L -H "Authorization: Bearer $token" -X GET "http://api:8080/datasets/list" | jq '. | length')" if [ "$resp" -ne 2 ]; then echo "Error when listing key hash, expected 2 entries got: $resp" diff --git a/.github/integration/tests/sda/60_api_admin_test.sh b/.github/integration/tests/sda/60_api_admin_test.sh index cecf5a86c9..3e2bc143bc 100644 --- a/.github/integration/tests/sda/60_api_admin_test.sh +++ b/.github/integration/tests/sda/60_api_admin_test.sh @@ -2,7 +2,7 @@ set -e cd shared || true -token="$(curl http://oidc:8080/tokens | jq -r '.[0]')" +token="$(cat /shared/token)" # Upload a file and make sure it's listed result="$(curl -sk -L "http://api:8080/users/test@dummy.org/files" -H "Authorization: Bearer $token" | jq '. | length')" if [ "$result" -ne 2 ]; then diff --git a/.github/integration/tests/sda/92_handle_file_errors.sh b/.github/integration/tests/sda/92_handle_file_errors.sh index f715ebc90e..bc28ef3f35 100644 --- a/.github/integration/tests/sda/92_handle_file_errors.sh +++ b/.github/integration/tests/sda/92_handle_file_errors.sh @@ -40,7 +40,7 @@ encrypted_checksums=$( --arg md5 "$(echo "aa" | md5sum | cut -d ' ' -f1)" \ '$ARGS.named|to_entries|map(with_entries(select(.key=="key").key="type"))' ) - + bad_file_payload=$( jq -r -c -n \ --arg type ingest \ @@ -72,7 +72,6 @@ if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/ exit 1 fi - missing_file_payload=$( jq -r -c -n \ --arg type ingest \ @@ -82,6 +81,16 @@ missing_file_payload=$( '$ARGS.named|@base64' ) +CORRID=${CORRID//a/b} +properties=$( + jq -c -n \ + --argjson delivery_mode 2 \ + --arg correlation_id "$CORRID" \ + --arg content_encoding UTF-8 \ + --arg content_type application/json \ + '$ARGS.named' +) + missing_file=$( jq -c -n \ --arg vhost sda \ @@ -99,7 +108,6 @@ curl -s -u guest:guest 'http://rabbitmq:15672/api/exchanges/sda/sda/publish' \ sleep 10 - if [ $((stream_size++)) -eq "$(curl -s -u guest:guest http://rabbitmq:15672/api/queues/sda/error_stream | jq '.messages_ready')" ]; then echo "missing file not moved to error" exit 1 diff --git a/.github/workflows/build_pr_container.yaml b/.github/workflows/build_pr_container.yaml index 09c57cc5c7..1938e9b19e 100644 --- a/.github/workflows/build_pr_container.yaml +++ b/.github/workflows/build_pr_container.yaml @@ -117,7 +117,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on postgres - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@0.30.0 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false @@ -135,7 +135,7 @@ jobs: category: postgres - name: Run Trivy vulnerability scanner on rabbitmq - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@0.30.0 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false @@ -190,7 +190,7 @@ jobs: org.opencontainers.image.revision=${{ github.sha }} - name: Run Trivy vulnerability scanner on sftp-inbox - uses: aquasecurity/trivy-action@0.29.0 + uses: aquasecurity/trivy-action@0.30.0 env: TRIVY_SKIP_DB_UPDATE: false TRIVY_SKIP_JAVA_DB_UPDATE: false @@ -207,6 +207,47 @@ jobs: sarif_file: 'inbox-results.sarif' category: sftp-inbox + + - name: create maven settings.xml + uses: s4u/maven-settings-action@v3.1.0 + with: + servers: | + [{ + "id":"github-fega-norway", + "username": "${{github.actor}}", + "password": "${{ secrets.GITHUB_TOKEN }}" + }] + + - name: Copy settings.xml to sda-doa root + shell: bash + run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml + + - name: Build container for sda-doa + uses: docker/build-push-action@v6 + with: + context: ./sda-doa + push: true + tags: | + ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-doa + ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:PR${{ github.event.number }}-doa + labels: | + org.opencontainers.image.source=${{ github.event.repository.clone_url }} + org.opencontainers.image.created=$(date -u +'%Y-%m-%dT%H:%M:%SZ') + org.opencontainers.image.revision=${{ github.sha }} + + - name: Run Trivy vulnerability scanner on doa + uses: aquasecurity/trivy-action@0.30.0 + env: + TRIVY_SKIP_DB_UPDATE: true + TRIVY_SKIP_JAVA_DB_UPDATE: true + with: + image-ref: ghcr.io/${{ steps.string-github-repo.outputs.lowercase }}:sha-${{ github.sha }}-doa + format: "sarif" + hide-progress: true + ignore-unfixed: true + output: 'inbox-results.sarif' + severity: "CRITICAL,HIGH" + rabbitmq: needs: - build_go_images @@ -284,6 +325,35 @@ jobs: run: docker compose -f .github/integration/sda-sync-integration.yml run integration_test if: steps.changes.outputs.sync == 'true' || steps.changes.outputs.sync-api == 'true' + sda-doa: + needs: + - build_java_images + - build_server_images + runs-on: ubuntu-latest + strategy: + matrix: + storage: [s3, posix] + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: create maven settings.xml + uses: s4u/maven-settings-action@v3.1.0 + with: + servers: | + [{ + "id":"github-fega-norway", + "username": "${{github.actor}}", + "password": "${{ secrets.GITHUB_TOKEN }}" + }] + + - name: Copy settings.xml to sda-doa root + shell: bash + run: cp /home/runner/.m2/settings.xml ./sda-doa/settings.xml + + - name: Test sda-doa for ${{ matrix.storage }} storage + run: docker compose -f .github/integration/sda-doa-${{ matrix.storage }}-outbox.yml run integration_test + chart: needs: - build_go_images diff --git a/.github/workflows/code-linter.yaml b/.github/workflows/code-linter.yaml index 515c0b8512..56be2be57b 100644 --- a/.github/workflows/code-linter.yaml +++ b/.github/workflows/code-linter.yaml @@ -8,7 +8,7 @@ on: - '**/go.*' env: - go-version: '1.23' + go-version: '1.24' jobs: lint_download: @@ -27,9 +27,9 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.1.1 + uses: golangci/golangci-lint-action@v8.0.0 with: - args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -e G401,G501,G107,G115 --timeout 5m + args: --timeout 5m working-directory: sda-download lint_sda: @@ -48,9 +48,9 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.1.1 + uses: golangci/golangci-lint-action@v8.0.0 with: - args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m + args: --timeout 5m working-directory: sda lint_sda_admin: @@ -69,7 +69,7 @@ jobs: uses: actions/checkout@v4 - name: Run golangci-lint - uses: golangci/golangci-lint-action@v6.1.1 + uses: golangci/golangci-lint-action@v8.0.0 with: - args: -E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,rowserrcheck -e G401,G501,G107,G115 --timeout 5m + args: --timeout 5m working-directory: sda-admin diff --git a/.github/workflows/publish_charts.yml b/.github/workflows/publish_charts.yml index ba1793e375..e44bd6609e 100644 --- a/.github/workflows/publish_charts.yml +++ b/.github/workflows/publish_charts.yml @@ -29,6 +29,6 @@ jobs: uses: azure/setup-helm@v4 - name: Run chart-releaser - uses: helm/chart-releaser-action@v1.6.0 + uses: helm/chart-releaser-action@v1.7.0 env: CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/publish_container.yml b/.github/workflows/publish_container.yml index ceb02b181c..a60d76bae7 100644 --- a/.github/workflows/publish_container.yml +++ b/.github/workflows/publish_container.yml @@ -29,11 +29,12 @@ jobs: fetch-depth: '0' - name: Bump version and push tag id: bump_tag - uses: anothrNick/github-tag-action@1.71.0 + uses: anothrNick/github-tag-action@1.73.0 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} WITH_V: true DEFAULT_BUMP: patch + TAG_CONTEXT: branch push_to_registry: needs: tag_release @@ -56,7 +57,7 @@ jobs: - name: Pull PR containers run : | - for t in -download -postgres -rabbitmq -sftp-inbox; do + for t in -download -postgres -rabbitmq -sftp-inbox -doa; do docker pull ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }}$t done docker pull ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }} @@ -86,6 +87,11 @@ jobs: docker tag ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }} ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }} docker push ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }} + - name: Retag PR image for doa + run: | + docker tag ghcr.io/biobanklab/sensitive-data-archive:PR${{ github.event.number }}-doa ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }}-doa + docker push ghcr.io/biobanklab/sensitive-data-archive:${{ needs.tag_release.outputs.tag }}-doa + bump_chart_version: needs: [tag_release, push_to_registry] if: ${{ needs.tag_release.outputs.tag != '' && needs.push_to_registry.result == 'success' }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ffb2dec24d..c488d2a90b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,7 +5,7 @@ on: branches-ignore: [main] env: - go-version: '1.23' + go-version: '1.24' jobs: @@ -60,7 +60,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.1.2 + uses: codecov/codecov-action@v5.4.2 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-download/coverage.txt @@ -95,7 +95,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.1.2 + uses: codecov/codecov-action@v5.4.2 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda/coverage.txt @@ -130,7 +130,7 @@ jobs: go test -v -coverprofile=coverage.txt -covermode=atomic ./... - name: Codecov - uses: codecov/codecov-action@v5.1.2 + uses: codecov/codecov-action@v5.4.2 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./sda-admin/coverage.txt diff --git a/.gitignore b/.gitignore index 3c3f6f5a29..e3ca9a9c37 100644 --- a/.gitignore +++ b/.gitignore @@ -189,3 +189,13 @@ buildNumber.properties # End of https://www.gitignore.io/api/java,maven,eclipse,intellij+all db/ + +# MacOS desktop service store +.DS_Store + +# crypt4gh key files +*.pub.pem +*.sec.pem + +# sda-admin binary +sda-admin/sda-admin \ No newline at end of file diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000000..c05076ac27 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,60 @@ +version: "2" +formatters: + enable: + - gofmt + - goimports +linters: + default: none + enable: + - bodyclose + - gocritic + - gosec + - govet + - nestif + - nlreturn + - revive + - rowserrcheck + - staticcheck + settings: + gosec: + excludes: + - G104 + - G302 + - G304 + - G401 + - G501 + revive: + enable-all-rules: true + rules: + - name: add-constant + disabled: true + - name: bare-return + disabled: true + - name: cognitive-complexity # disable for now + disabled: true + - name: confusing-naming # disable for now + disabled: true + - name: confusing-results + disabled: true + - name: cyclomatic # disable for now, requires refactoring + disabled: true + - name: exported + disabled: true + - name: flag-parameter # this one requires refactoring + disabled: true + - name: function-length # this one requires refactoring + disabled: true + - name: get-return # this triggers on function names + disabled: true + - name: line-length-limit + disabled: true + - name: max-public-structs + disabled: true + - name: package-comments + disabled: true + - name: unchecked-type-assertion # disable for now + disabled: true + - name: unused-parameter # disable for now + disabled: true + - name: unused-receiver # disable for now + disabled: true diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index be6bdc9991..5a3b15ea6e 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -27,14 +27,15 @@ If successful, the curl command should output the HTTP code `200`. You can further check the endpoint `/metadata/datasets` using: ```sh -token=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') curl -H "Authorization: Bearer $token" http://localhost:18080/metadata/datasets ``` -If successful, the curl command should output a JSON body containing: +where `token` can be obtained by logging to http://localhost:8801 with the dummy user. + +If successful, the curl command should output a JSON body with an empty list and with HTTP code `200`: ```json -["EGAD74900000101"] +[] ``` ## Running other SDA services with `go run` @@ -75,27 +76,60 @@ CONFIGFILE=config_local.yaml go run cmd/ingest/ingest.go - Check if the `ingest` service works as expected by following these steps -```sh -# create a test file -seq 10 > /tmp/t1.txt + - Create a test file -# update the s3cmd config file -sed -i '/host_/s/s3inbox:8000/localhost:18000/g' /tmp/shared/s3cfg + ```sh + seq 10 > /tmp/t1.txt + ``` -# upload /tmp/t1.txt to s3inbox by sda-cli -sda-cli -config /tmp/shared/s3cfg upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt + - Download the s3cmd config file (`s3cmd-inbox.conf`) by logging to http://localhost:8801 with the admin user. After that update the config file by running -# use sda-admin to check if t1.txt has been uploaded -export API_HOST=http://localhost:8090 -export ACCESS_TOKEN=$(curl -s -k http://localhost:8080/tokens | jq -r '.[0]') -sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'uploaded' + ```sh + sed -i '/host_/s/inbox:8000/localhost:18000/g' s3cmd-inbox.conf + sed -i 's/use_https = True/use_https = False/g' s3cmd-inbox.conf + ``` -# register the Crypt4GH key -curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add + - Upload the test file to S3inbox by `sda-cli` -# use sda-admin to ingest the file t1.txt -sda-admin file ingest -filepath test_dummy.org/t1.txt.c4gh -user test@dummy.org + ```sh + sda-cli -config s3cmd-inbox.conf upload -encrypt-with-key /tmp/shared/c4gh.pub.pem /tmp/t1.txt + ``` -# verify that t1.txt has been ingested using sda-admin -sda-admin file list -user test@dummy.org # file test_dummy.org/t1.txt.c4gh should have fileStatus 'verified' -``` + - Use `sda-admin` to check if the test file has been uploaded + + ```sh + export API_HOST=http://localhost:8090 + export ACCESS_TOKEN=$(grep access_token s3cmd-inbox.conf | awk '{print $3}') + sda-admin file list -user testu@lifescience-ri.eu + ``` + + The file `testu_lifescience-ri.eu/t1.txt.c4gh` should have fileStatus `uploaded` + + + - Register the Crypt4GH key + + ```sh + curl -H "Authorization: Bearer $ACCESS_TOKEN" -H "Content-Type: application/json" -X POST -d '{"pubkey": "'"$( base64 -w0 /tmp/shared/c4gh.pub.pem)"'", "description": "pubkey"}' http://localhost:8090/c4gh-keys/add + ``` + + If the key is already registered in the database, you may see the following error: + + ``` + {"error":"key hash already exists or no rows were updated","status":409} + ``` + + This is expected and can be safely ignored. + + - Ingest the file test file + + ```sh + sda-admin file ingest -filepath testu_lifescience-ri.eu/t1.txt.c4gh -user testu@lifescience-ri.eu + ``` + + - Verify that the test file has been ingested + + ```sh + sda-admin file list -user testu@lifescience-ri.eu + ``` + + The file `testu_lifescience-ri.eu/t1.txt.c4gh` should have fileStatus `verified` diff --git a/Makefile b/Makefile index 0e0c16104a..9f8282f92a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,3 @@ -# linter modules to be included/excluded -LINT_INCLUDE=-E bodyclose,gocritic,gofmt,gosec,govet,nestif,nlreturn,revive,rowserrcheck -LINT_EXCLUDE=-e G401,G501,G107 - help: @echo 'Welcome!' @echo '' @@ -36,6 +32,8 @@ build-sda-sftp-inbox: @cd sda-sftp-inbox && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-sftp-inbox . build-sda-admin: @cd sda-admin && go build +build-sda-doa: + @cd sda-doa && docker build -t ghcr.io/neicnordic/sensitive-data-archive:PR$$(date +%F)-doa . go-version-check: SHELL:=/bin/bash @@ -128,17 +126,35 @@ integrationtest-sda-sync-run: integrationtest-sda-sync-down: @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-sync-integration.yml down -v --remove-orphans +integrationtest-sda-doa-posix: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml down -v --remove-orphans +integrationtest-sda-doa-posix-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml run integration_test +integrationtest-sda-doa-posix-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-posix-outbox.yml down -v --remove-orphans + +integrationtest-sda-doa-s3: build-all + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml run integration_test + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml down -v --remove-orphans +integrationtest-sda-doa-s3-run: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml run integration_test +integrationtest-sda-doa-s3-down: + @PR_NUMBER=$$(date +%F) docker compose -f .github/integration/sda-doa-s3-outbox.yml down -v --remove-orphans + + + # lint go code lint-all: lint-sda lint-sda-download lint-sda-admin lint-sda: @echo 'Running golangci-lint in the `sda` folder' - @cd sda && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) + @cd sda && golangci-lint run lint-sda-download: @echo 'Running golangci-lint in the `sda-download` folder' - @cd sda-download && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) + @cd sda-download && golangci-lint run lint-sda-admin: @echo 'Running golangci-lint in the `sda-admin` folder' - @cd sda-admin && golangci-lint run $(LINT_INCLUDE) $(LINT_EXCLUDE) + @cd sda-admin && golangci-lint run # run static code tests test-all: test-sda test-sda-download test-sda-sftp-inbox @@ -170,7 +186,8 @@ k3d-version-check: echo "kubectl is missing";\ fi k3d-create-cluster: - @k3d cluster create + @k3d cluster create --k3s-arg "--disable=traefik@server:0" --port "80:80@loadbalancer" --port "443:443@loadbalancer"; \ + helm upgrade --install ingress-nginx ingress-nginx --repo https://kubernetes.github.io/ingress-nginx --namespace ingress-nginx --create-namespace k3d-delete-cluster: @k3d cluster delete k3d-deploy-dependencies: @@ -186,4 +203,4 @@ k3d-deploy-sda-s3: k3d-deploy-sda-posix: @bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$$(date +%F)" false posix k3d-cleanup-all-deployments: - @bash .github/integration/scripts/charts/cleanup.sh \ No newline at end of file + @bash .github/integration/scripts/charts/cleanup.sh diff --git a/README.md b/README.md index ac5a977d50..9a75c7f33b 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,14 @@ This will build all required images, bring up the services, run the integration ```sh make integrationtest-sda-sync ``` +- Integration test for SDA-DOA using POSIX as the storage backend: + ```sh + make integrationtest-sda-doa-posix + ``` +- Integration test for SDA-DOA using S3 as the storage backend: + ```sh + make integrationtest-sda-doa-s3 + ``` #### Running the integration tests without shutting down the services This will run the integration tests and keep the services running after the tests are finished. @@ -110,7 +118,14 @@ This will run the integration tests and keep the services running after the test ```sh make integrationtest-sda-sync-run ``` - +- Integration test for SDA-DOA using POSIX as the storage backend: + ```sh + make integrationtest-sda-doa-posix-run + ``` +- Integration test for SDA-DOA using S3 as the storage backend: + ```sh + make integrationtest-sda-doa-s3-run + ``` After that, you will need to shut down the services manually. - Shut down services for SDA using POSIX as the storage backend @@ -125,6 +140,14 @@ After that, you will need to shut down the services manually. ```sh make integrationtest-sda-sync-down ``` +- Shut down services for SDA-DOA using POSIX as the storage backend + ```sh + make integrationtest-sda-doa-posix-down + ``` +- Shut down services for SDA-DOA using S3 as the storage backend + ```sh + make integrationtest-sda-doa-s3-down + ``` ### Linting the Go code @@ -176,33 +199,34 @@ wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash ``` -Once installed a cluster named `test-cluster` can be created as such: +#### Install kubectl + +If `kubectl` is not installed, run the following command to download the latest stable version. (substitue `linux/amd64` with `darwin/arm64` if you are using a Mac). ```sh -k3d cluster create test-cluster +curl -sLO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" ``` -Or by using the `make k3d-create-cluster` command, you can create a cluster named `k3s-default`. +#### Create a cluster +Once installed a cluster can be created using the `make k3d-create-cluster` command, you can create a cluster named `k3s-default`. The new cluster's connection details will automatically be merged into your default kubeconfig and activated. The command below should show the created node. ```sh kubectl get nodes ``` -Removing the cluster can be done using the `make k3d-delete-cluster` command or as shown below if a specific name is used during creation. +The Nginx ingress controller is deployed and will bind to ports 80 and 443 of the host system. A deployed service with an ingress definition can then be targeted by setting the `Host: HOSTNAME` header for that service. ```sh -k3d cluster delete test-cluster +curl -H "Host: test" http://localhost/ ``` -#### Install kubectl +For testing ingress endpoints with other applications like a web browser, the hostname in the ingress definition should have the `.127.0.0.1.nip.io` ending in order to not have to modify the `/etc/hosts` file, ex. `app.127.0.0.1.nip.io`. -If `kubectl` is not installed, run the following command to download the latest stable version. (substitue `linux/amd64` with `darwin/arm64` if you are using a Mac). +#### Remove the cluster -```sh -curl -sLO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" -``` +Removing the cluster can be done using the `make k3d-delete-cluster` command or as shown below if a specific name is used during creation. ### Deploy the components @@ -220,37 +244,28 @@ Deployment of the charts can be done as describe below in more detail, or by usi #### Bootstrap the dependencies -This script requires [yq](https://github.com/mikefarah/yq/releases/latest) and the GO version of [crypt4gh](https://github.com/neicnordic/crypt4gh/releases/latest) +This script requires [yq](https://github.com/mikefarah/yq/releases/latest), the GO version of [crypt4gh](https://github.com/neicnordic/crypt4gh/releases/latest) as well as [xxd](https://manpages.org/xxd) and [jq](https://manpages.org/jq) to be installed. -```sh -bash .github/integration/scripts/charts/dependencies.sh local -``` +Bootstrap the dependencies with the command: `make k3d-deploy-dependencies`. #### Deploy the Sensitive Data Archive components -Start by building the required containers using the `make build-all` command, once that has completed the images can be imported to the cluster. +Start by building and importing the required containers using the `make k3d-import-images`. -```sh -bash .github/integration/scripts/charts/import_local_images.sh -``` +The Postgres and RabbitMQ Needs to be deployed first using the following commands: `make k3d-deploy-postgres` and `make k3d-deploy-rabbitmq`. -The Postgres and RabbitMQ Needs to be deployed first, the bool at the end specifies if TLS should be enabled or not for the deployes services. -Replace `sda-db` in the example below with the helmc hart that shuld be installed. (`sda-db` or `sda-mq`) +Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`), `make k3d-deploy-sda-posix` or `make k3d-deploy-sda-s3`. -```sh -bash .github/integration/scripts/charts/deploy_charts.sh sda-db "$(date +%F)" false -``` +#### Testing with ingress -Once the DB and MQ are installed the SDA stack can be installed, here the desired storage backend needs to specified as well (`posix` or `s3`) +Once everything is deployed it is posible to interact with the services using the following hostnames: -```sh -bash .github/integration/scripts/charts/deploy_charts.sh sda-svc "$(date +%F)" false s3 -``` +- api.127.0.0.1.nip.io +- auth.127.0.0.1.nip.io +- broker.127.0.0.1.nip.io +- download.127.0.0.1.nip.io +- inbox.127.0.0.1.nip.io #### Cleanup all deployed components -Once the testing is concluded all deployed components can be removed. - -```sh -bash .github/integration/scripts/charts/cleanup.sh -``` +Once the testing is concluded all deployed components can be removed with the command `make k3d-cleanup-all-deployments` diff --git a/charts/sda-svc/templates/api-deploy.yaml b/charts/sda-svc/templates/api-deploy.yaml index 5d0486baff..ae607d6011 100644 --- a/charts/sda-svc/templates/api-deploy.yaml +++ b/charts/sda-svc/templates/api-deploy.yaml @@ -189,6 +189,8 @@ spec: - name: LOG_LEVEL value: {{ .Values.global.log.level | quote }} {{- end }} + - name: SCHEMA_TYPE + value: {{ default "federated" .Values.global.schemaType }} {{- if .Values.global.api.jwtPubKeyName }} - name: SERVER_JWTPUBKEYPATH value: {{ include "jwtPath" . }} diff --git a/charts/sda-svc/templates/download-deploy.yaml b/charts/sda-svc/templates/download-deploy.yaml index f7207afdc6..9b1a2134eb 100644 --- a/charts/sda-svc/templates/download-deploy.yaml +++ b/charts/sda-svc/templates/download-deploy.yaml @@ -94,7 +94,7 @@ spec: value: "{{ .Values.global.archive.volumePath }}" {{- end }} - name: OIDC_CONFIGURATION_URL - value: "{{ .Values.global.oidc.provider }}/.well-known/openid-configuration" + value: "{{ .Values.global.oidc.provider | trimSuffix "/" }}/.well-known/openid-configuration" {{- if .Values.global.download.trusted.iss }} - name: OIDC_TRUSTED_ISS value: {{ include "trustedIssPath" . }}/{{ default "iss.json" .Values.global.download.trusted.configFile }} diff --git a/charts/sda-svc/templates/s3-inbox-deploy.yaml b/charts/sda-svc/templates/s3-inbox-deploy.yaml index e71e077d38..bf97b7faa4 100644 --- a/charts/sda-svc/templates/s3-inbox-deploy.yaml +++ b/charts/sda-svc/templates/s3-inbox-deploy.yaml @@ -190,14 +190,16 @@ spec: - name: SERVER_KEY value: {{ include "tlsPath" . }}/tls.key {{- end }} - {{- if.Values.global.auth.jwtPub }} + {{- if .Values.global.auth.jwtPub }} - name: SERVER_JWTPUBKEYPATH value: {{ include "jwtPath" . }} - {{- end }} - {{- if not .Values.global.auth.resignJwt }} + {{- end }} + {{- if .Values.global.auth.resignJwt }} + {{- $_ := required "The signing pub key is required for s3inbox when auth resigns tokens" .Values.global.auth.jwtPub }} + {{- else }} - name: SERVER_JWTPUBKEYURL value: {{ .Values.global.oidc.provider }}{{ .Values.global.oidc.jwkPath }} - {{- end }} + {{- end }} {{- if .Values.global.log.format }} - name: LOG_FORMAT value: {{ .Values.global.log.format | quote }} diff --git a/postgresql/initdb.d/01_main.sql b/postgresql/initdb.d/01_main.sql index a7cacf3ae4..fc931b49e3 100644 --- a/postgresql/initdb.d/01_main.sql +++ b/postgresql/initdb.d/01_main.sql @@ -74,7 +74,7 @@ CREATE TABLE files ( created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(), last_modified TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(), - CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path) + CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user) ); -- The user info is used by auth to be able to link users to their name and email diff --git a/postgresql/migratedb.d/17.sql b/postgresql/migratedb.d/17.sql new file mode 100644 index 0000000000..6ac20f256c --- /dev/null +++ b/postgresql/migratedb.d/17.sql @@ -0,0 +1,21 @@ +DO +$$ +DECLARE +-- The version we know how to do migration from, at the end of a successful migration +-- we will no longer be at this version. + sourcever INTEGER := 16; + changes VARCHAR := 'Add submission user to constraint'; +BEGIN + IF (select max(version) from sda.dbschema_version) = sourcever then + RAISE NOTICE 'Doing migration from schema version % to %', sourcever, sourcever+1; + RAISE NOTICE 'Changes: %', changes; + INSERT INTO sda.dbschema_version VALUES(sourcever+1, now(), changes); + + ALTER TABLE sda.files DROP CONSTRAINT unique_ingested; + ALTER TABLE sda.files ADD CONSTRAINT unique_ingested UNIQUE(submission_file_path, archive_file_path, submission_user); + + ELSE + RAISE NOTICE 'Schema migration from % to % does not apply now, skipping', sourcever, sourcever+1; + END IF; +END +$$ \ No newline at end of file diff --git a/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go b/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go index c3a6f61cb1..591e8accc3 100644 --- a/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go +++ b/sda-admin/c4ghkeyhash/c4ghKeyHash_test.go @@ -40,41 +40,41 @@ func TestC4gh(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupSuite() { - suite.tempFolder = "/tmp/keys/" - err := os.MkdirAll(suite.tempFolder, 0750) +func (ts *TestSuite) SetupSuite() { + ts.tempFolder = "/tmp/keys/" + err := os.MkdirAll(ts.tempFolder, 0750) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } pub, _, err := keys.GenerateKeyPair() if err != nil { - suite.T().FailNow() + ts.T().FailNow() } buf := new(bytes.Buffer) if err := keys.WriteCrypt4GHX25519PublicKey(buf, pub); err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.b64String = base64.StdEncoding.EncodeToString(buf.Bytes()) + ts.b64String = base64.StdEncoding.EncodeToString(buf.Bytes()) - pubKeyFile, err := os.Create(suite.tempFolder + "/pub.key") + pubKeyFile, err := os.Create(ts.tempFolder + "/pub.key") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.pubkeyPath = suite.tempFolder + "/pub.key" + ts.pubkeyPath = ts.tempFolder + "/pub.key" _, err = pubKeyFile.Write(buf.Bytes()) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } } -func (suite *TestSuite) TearDownSuite() { - os.RemoveAll(suite.tempFolder) +func (ts *TestSuite) TearDownSuite() { + os.RemoveAll(ts.tempFolder) } -func (suite *TestSuite) TestAdd() { +func (ts *TestSuite) TestAdd() { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest @@ -84,21 +84,21 @@ func (suite *TestSuite) TestAdd() { token := "test-token" payload := C4ghPubKey{ - PubKey: suite.b64String, + PubKey: ts.b64String, Description: "test description", } jsonBody, err := json.Marshal(payload) if err != nil { - suite.T().Fail() + ts.T().Fail() } mockHelpers.On("PostRequest", expectedURL, token, jsonBody).Return([]byte(`{}`), nil) - assert.NoError(suite.T(), Add("http://example.com", token, suite.pubkeyPath, "test description")) - mockHelpers.AssertExpectations(suite.T()) + assert.NoError(ts.T(), Add("http://example.com", token, ts.pubkeyPath, "test description")) + mockHelpers.AssertExpectations(ts.T()) } -func (suite *TestSuite) TestDeprecate() { +func (ts *TestSuite) TestDeprecate() { mockHelpers := new(MockHelpers) originalFunc := helpers.PostRequest helpers.PostRequest = mockHelpers.PostRequest @@ -108,11 +108,11 @@ func (suite *TestSuite) TestDeprecate() { token := "test-token" mockHelpers.On("PostRequest", expectedURL, token, []byte(`{}`)).Return([]byte(`{}`), nil) - assert.NoError(suite.T(), Deprecate("http://example.com", token, "6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc507")) - mockHelpers.AssertExpectations(suite.T()) + assert.NoError(ts.T(), Deprecate("http://example.com", token, "6af1407abc74656b8913a7d323c4bfd30bf7c8ca359f74ae35357acef29dc507")) + mockHelpers.AssertExpectations(ts.T()) } -func (suite *TestSuite) TestList() { +func (ts *TestSuite) TestList() { mockHelpers := new(MockHelpers) mockHelpers.On("GetResponseBody", "http://example.com/c4gh-keys/list", "test-token").Return([]byte(`[{"hash":"cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23","description":"this is a test key","created_at":"2009-11-10 23:00:00","deprecated_at":""}]`), nil) originalFunc := helpers.GetResponseBody @@ -120,6 +120,6 @@ func (suite *TestSuite) TestList() { helpers.GetResponseBody = mockHelpers.GetResponseBody err := List("http://example.com", "test-token") - assert.NoError(suite.T(), err) - mockHelpers.AssertExpectations(suite.T()) + assert.NoError(ts.T(), err) + mockHelpers.AssertExpectations(ts.T()) } diff --git a/sda-admin/go.mod b/sda-admin/go.mod index 90cea71e00..bb272c942d 100644 --- a/sda-admin/go.mod +++ b/sda-admin/go.mod @@ -1,6 +1,6 @@ module github.com/neicnordic/sensitive-data-archive/sda-admin -go 1.23.1 +go 1.24.0 require ( github.com/neicnordic/crypt4gh v1.12.0 @@ -14,7 +14,7 @@ require ( github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.2 // indirect - golang.org/x/crypto v0.31.0 // indirect - golang.org/x/sys v0.28.0 // indirect + golang.org/x/crypto v0.35.0 // indirect + golang.org/x/sys v0.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-admin/go.sum b/sda-admin/go.sum index 1ed0c599f2..b244f10f24 100644 --- a/sda-admin/go.sum +++ b/sda-admin/go.sum @@ -14,12 +14,12 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs= +golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ= +golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= +golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU= +golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/sda-admin/helpers/helpers.go b/sda-admin/helpers/helpers.go index 31b0289cc7..352ee75b6b 100644 --- a/sda-admin/helpers/helpers.go +++ b/sda-admin/helpers/helpers.go @@ -87,7 +87,6 @@ func CheckValidChars(filename string) error { re := regexp.MustCompile(`[\\<>"\|\x00-\x1F\x7F\!\*\'\(\)\;\:\@\&\=\+\$\,\?\%\#\[\]]`) disallowedChars := re.FindAllString(filename, -1) if disallowedChars != nil { - return fmt.Errorf( "filepath '%v' contains disallowed characters: %+v", filename, diff --git a/sda-admin/main.go b/sda-admin/main.go index 560874f3ba..62490f6aaf 100644 --- a/sda-admin/main.go +++ b/sda-admin/main.go @@ -21,7 +21,7 @@ var ( ) // Command-line usage -const usage = `Usage: sda-admin [-uri URI] [-token TOKEN] [options] +const usage = `usage: sda-admin [-uri URI] [-token TOKEN] [options] Commands: user list List all users. @@ -42,7 +42,7 @@ Global Options: Additional Commands: version Show the version of sda-admin. help Show this help message. - -h, -help Show this help message.` + -h, -help Show this help message` var userUsage = `List Users: Usage: sda-admin user list @@ -182,14 +182,14 @@ func parseFlagsAndEnv() error { if apiURI == "" { apiURI = os.Getenv("API_HOST") if apiURI == "" { - return fmt.Errorf("error: either -uri must be provided or API_HOST environment variable must be set") + return errors.New("error: either -uri must be provided or API_HOST environment variable must be set") } } if token == "" { token = os.Getenv("ACCESS_TOKEN") if token == "" { - return fmt.Errorf("error: either -token must be provided or ACCESS_TOKEN environment variable must be set") + return errors.New("error: either -token must be provided or ACCESS_TOKEN environment variable must be set") } } diff --git a/sda-doa/Dockerfile b/sda-doa/Dockerfile index b534cd27b9..10ae01974d 100644 --- a/sda-doa/Dockerfile +++ b/sda-doa/Dockerfile @@ -1,4 +1,4 @@ -FROM maven:3-eclipse-temurin-21-alpine as builder +FROM maven:3-eclipse-temurin-21-alpine AS builder COPY pom.xml . diff --git a/sda-doa/README.md b/sda-doa/README.md index 3bfdb667e9..c7e61ed41a 100644 --- a/sda-doa/README.md +++ b/sda-doa/README.md @@ -1,6 +1,12 @@ -# NeIC sda-doa Data Out API -![Java CI](https://github.com/neicnordic/sda-doa/workflows/Java%20CI/badge.svg) +# Overview +The Data Out API (DOA) is a service that provides an interface for exporting datasets and files from the FEGA archive. It provides secure access to datasets and files, ensuring proper authorization and encryption. +## Installation + +1. Navigate to github packages +2. Select the package [sensitive-data-archive](https://github.com/orgs/neicnordic/packages/container/package/sensitive-data-archive) +3. Find the image you want to install (tag format: `vX.X.XXX-doa`) +4. Since the image runs as the _nobody_ user, if your outbox is posix you may need to adjust the ownership and permissions of the mounted volume to ensure the container can access the files. ## Configuration @@ -30,7 +36,7 @@ Environment variables used: | CERT_KEY | /etc/ega/ssl/client.key | Path to the client key for database connectivity | | POSTGRES_USER | lega_out | Database username | | POSTGRES_PASSWORD | | Database password | -| S3_ENDPOINT | archive | S3 server hostname | +| S3_ENDPOINT | vault | S3 server hostname | | S3_PORT | 443 | S3 server port | | S3_ACCESS_KEY | minio | S3 access key | | S3_SECRET_KEY | miniostorage | S3 secret key | @@ -53,54 +59,3 @@ Environment variables used: | VISA_PUBLIC_KEY_PATH | /etc/ega/jwt/visa.pem | Path to the public key for visas JWT validation | | CRYPT4GH_PRIVATE_KEY_PATH | /etc/ega/crypt4gh/key.pem | Path to the Crypt4GH private key | | CRYPT4GH_PRIVATE_KEY_PASSWORD_PATH | /etc/ega/crypt4gh/key.pass | Path to the Crypt4GH private key passphrase | -| LOGSTASH_HOST | | Hostname of the Logstash instance (if any) | -| LOGSTASH_PORT | | Port of the Logstash instance (if any) | - -If `LOGSTASH_HOST` or `LOGSTASH_PORT` is empty, Logstash logging will not be enabled. - -In addition, environment variables can be used to configure log level for different packages. Package loggers can be configured using corresponding package names, for example, to turn of logs of Spring, one can set environment variable `LOGGING_LEVEL_ORG_SPRINGFRAMEWORK=OFF`, or to set DOA's own logs to debug: `LOGGING_LEVEL_NO_UIO_IFI=DEBUG`, etc. - -## Sample Docker Swarm entry - -``` -... - doa: - image: neicnordic/sda-doa:latest - ports: - - 443:8080 - deploy: - restart_policy: - condition: on-failure - delay: 5s - window: 120s - environment: - - S3_PORT - - KEYSTORE_PASSWORD=${SERVER_CERT_PASSWORD} - - DB_INSTANCE=${DB_HOST} - - POSTGRES_DB=${DB_DATABASE_NAME} - - POSTGRES_PASSWORD=${DB_LEGA_OUT_PASSWORD} - - S3_ACCESS_KEY=${MINIO_ACCESS_KEY} - - S3_SECRET_KEY=${MINIO_SECRET_KEY} - - LOGSTASH_HOST - - LOGSTASH_PORT - secrets: - - source: rootCA.pem - target: /etc/ega/ssl/CA.cert - - source: rootCA.pem - target: /etc/ssl/certs/ca-certificates.crt - - source: server.p12 - target: /etc/ega/ssl/server.cert - - source: client.pem - target: /etc/ega/ssl/client.cert - - source: client-key.der - target: /etc/ega/ssl/client.key - - source: jwt.pub.pem - target: /etc/ega/jwt/passport.pem - - source: jwt.pub.pem - target: /etc/ega/jwt/visa.pem - - source: ega.sec.pem - target: /etc/ega/crypt4gh/key.pem - - source: ega.sec.pass - target: /etc/ega/crypt4gh/key.pass -... -``` diff --git a/sda-doa/docker-compose-posix-outbox.yml b/sda-doa/docker-compose-posix-outbox.yml deleted file mode 100644 index 3a593186bf..0000000000 --- a/sda-doa/docker-compose-posix-outbox.yml +++ /dev/null @@ -1,79 +0,0 @@ -version: '3.3' - -services: - - db: - build: - context: ../../postgresql - container_name: postgres - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres - ports: - - 5432:5432 - environment: - - LEGA_IN_PASSWORD=password - - LEGA_OUT_PASSWORD=password - - POSTGRES_SERVER_CERT=/etc/ega/pg.cert - - POSTGRES_SERVER_KEY=/etc/ega/pg.key - - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert - - PG_VERIFY_PEER=1 - - POSTGRES_PASSWORD=rootpasswd - secrets: - - source: server.pem - target: /etc/ega/pg.cert - uid: '70' - gid: '70' - mode: 0600 - - source: server-key.pem - target: /etc/ega/pg.key - uid: '70' - gid: '70' - mode: 0600 - - source: rootCA.pem - target: /etc/ega/CA.cert - uid: '70' - gid: '70' - mode: 0600 - volumes: - - db:/ega - - mockauth: - image: cscfi/beacon-python - ports: - - 8000:8000 - volumes: - - ./test/mock_auth.py:/mock_auth.py - entrypoint: ["python", "/mock_auth.py", "0.0.0.0", "8000"] - - private-mq: - build: - context: ../../rabbitmq - container_name: rabbitmq - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-rabbitmq - ports: - - 5671:5671 - - 15671:15671 - environment: - - SSL_VERIFY=verify_peer - - SSL_FAIL_IF_NO_PEER_CERT=false - - SSL_DEPTH=2 - - USER_NAME=admin - - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - - VIRTUAL_HOST=sda - secrets: - - source: server.pem - target: /etc/rabbitmq/ssl.cert - - source: server-key.pem - target: /etc/rabbitmq/ssl.key - - source: rootCA.pem - target: /etc/rabbitmq/CA.cert - -volumes: - db: - -secrets: - rootCA.pem: - file: test/rootCA.pem - server.pem: - file: test/localhost.pem - server-key.pem: - file: test/localhost-key.pem diff --git a/sda-doa/docker-compose-s3-outbox.yml b/sda-doa/docker-compose-s3-outbox.yml deleted file mode 100644 index a659a4c910..0000000000 --- a/sda-doa/docker-compose-s3-outbox.yml +++ /dev/null @@ -1,93 +0,0 @@ -version: '3.3' - -services: - - db: - image: ghcr.io/neicnordic/sensitive-data-archive:PR${PR_NUMBER}-postgres - ports: - - 5432:5432 - environment: - - LEGA_IN_PASSWORD=password - - LEGA_OUT_PASSWORD=password - - POSTGRES_SERVER_CERT=/etc/ega/pg.cert - - POSTGRES_SERVER_KEY=/etc/ega/pg.key - - POSTGRES_SERVER_CACERT=/etc/ega/CA.cert - - PG_VERIFY_PEER=1 - - POSTGRES_PASSWORD=rootpasswd - secrets: - - source: server.pem - target: /etc/ega/pg.cert - uid: '70' - gid: '70' - mode: 0600 - - source: server-key.pem - target: /etc/ega/pg.key - uid: '70' - gid: '70' - mode: 0600 - - source: rootCA.pem - target: /etc/ega/CA.cert - uid: '70' - gid: '70' - mode: 0600 - volumes: - - db:/ega - - mockauth: - image: cscfi/beacon-python - ports: - - 8000:8000 - volumes: - - ./test/mock_auth.py:/mock_auth.py - entrypoint: ["python", "/mock_auth.py", "0.0.0.0", "8000"] - - private-mq: - image: uiobmi/localega-broker-private:latest - ports: - - 5671:5671 - - 15671:15671 - environment: - - SSL_VERIFY=verify_peer - - SSL_FAIL_IF_NO_PEER_CERT=false - - SSL_DEPTH=2 - - USER_NAME=admin - - PASSWORD_HASH=4tHURqDiZzypw0NTvoHhpn8/MMgONWonWxgRZ4NXgR8nZRBz - - VIRTUAL_HOST=sda - secrets: - - source: server.pem - target: /etc/rabbitmq/ssl.cert - - source: server-key.pem - target: /etc/rabbitmq/ssl.key - - source: rootCA.pem - target: /etc/rabbitmq/CA.cert - - outbox: - image: minio/minio - ports: - - 9000:9000 - environment: - - MINIO_ACCESS_KEY=minio - - MINIO_SECRET_KEY=miniostorage - command: server /data - - createbucket: - image: minio/mc - depends_on: - - s3 - entrypoint: > - /bin/sh -c " - /usr/bin/mc config host add s3 http://outbox:9000 minio miniostorage; - /usr/bin/mc mb s3/lega; - exit 0; - " - -volumes: - db: - -secrets: - rootCA.pem: - file: test/rootCA.pem - server.pem: - file: test/localhost.pem - server-key.pem: - file: test/localhost-key.pem diff --git a/sda-doa/pom.xml b/sda-doa/pom.xml index 3df3a678c2..5b42709df8 100644 --- a/sda-doa/pom.xml +++ b/sda-doa/pom.xml @@ -5,7 +5,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.1 + 3.4.5 no.uio.ifi @@ -21,7 +21,7 @@ com.google.code.gson gson - 2.11.0 + 2.13.1 org.springframework.boot @@ -84,12 +84,12 @@ io.minio minio - 8.5.15 + 8.5.17 net.logstash.logback logstash-logback-encoder - 8.0 + 8.1 org.springframework.boot @@ -107,16 +107,10 @@ - - org.junit.jupiter - junit-jupiter-api - 5.11.4 - test - no.elixir crypt4gh - 3.0.5 + 3.0.24 org.slf4j @@ -127,7 +121,7 @@ no.elixir clearinghouse - 2.0.6 + 2.0.18 org.slf4j @@ -138,7 +132,7 @@ org.yaml snakeyaml - 2.3 + 2.4 diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java index 9a1859059e..ff43e6eff0 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/dto/ExportRequest.java @@ -12,6 +12,7 @@ @Data public class ExportRequest { + @ToString.Exclude private String jwtToken; private String datasetId; private String fileId; diff --git a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java index 0e6565adc6..e22b223c7e 100644 --- a/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java +++ b/sda-doa/src/main/java/no/uio/ifi/localega/doa/services/AAIService.java @@ -67,7 +67,7 @@ public Collection getDatasetIds(String accessToken) { .collect(Collectors.toList()); if (CollectionUtils.isEmpty(controlledAccessGrantsVisas)) { - log.info("Unauthorized access attempt: user doesn't have any valid visas. Access token: {}", accessToken); + log.info("Unauthorized access attempt: user doesn't have any valid visas."); } String subject = controlledAccessGrantsVisas.stream().findFirst().orElseThrow(RuntimeException::new).getSub(); diff --git a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java index 2ece458ab2..824e655b08 100644 --- a/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java +++ b/sda-doa/src/test/java/no/uio/ifi/localega/doa/LocalEGADOAApplicationTests.java @@ -31,10 +31,6 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.security.PrivateKey; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.PreparedStatement; -import java.util.Properties; import java.util.UUID; @Slf4j @@ -43,48 +39,20 @@ class LocalEGADOAApplicationTests { private static String validToken; private static String invalidToken; private static String validVisaToken; + private static String doaUrl; + private static String mockauthUrl; + private static String minioHost; + private static String outboxType; @SneakyThrows @BeforeAll public static void setup() { - String url = String.format("jdbc:postgresql://%s:%s/%s", "localhost", "5432", "sda"); - Properties props = new Properties(); -// props.setProperty("user", "lega_in"); //will be used when lega_in user is set in db again - props.setProperty("user", "postgres"); - props.setProperty("password", "rootpasswd"); - props.setProperty("ssl", "true"); - props.setProperty("application_name", "LocalEGA"); - props.setProperty("sslmode", "verify-full"); - props.setProperty("sslrootcert", new File("test/rootCA.pem").getAbsolutePath()); - props.setProperty("sslcert", new File("test/localhost-client.pem").getAbsolutePath()); - props.setProperty("sslkey", new File("test/localhost-client-key.der").getAbsolutePath()); - Connection connection = DriverManager.getConnection(url, props); - PreparedStatement file = connection.prepareStatement("SELECT local_ega.insert_file('body.enc','requester@elixir-europe.org');"); - file.executeQuery(); - PreparedStatement header = connection.prepareStatement("UPDATE local_ega.files SET header = '637279707434676801000000010000006c00000000000000aa7ad1bb4f93bf5e4fb3bc28a95bc4d80bf2fd8075e69eb2ee15e0a4f08f1d78ab98c8fd9b50e675f71311936e8d0c6f73538962b836355d5d4371a12eae46addb43518b5236fb9554249710a473026f34b264a61d2ba52ed11abc1efa1d3478fa40a710' WHERE id = 1;"); - header.executeUpdate(); - PreparedStatement finalize = connection.prepareStatement("UPDATE local_ega.files SET archive_path = 'test/body.enc', status = 'READY', stable_id = 'EGAF00000000014' WHERE id = 1;"); - finalize.executeUpdate(); - connection.close(); - -// props.setProperty("user", "lega_out"); //will be used when lega_out user is set in db again - connection = DriverManager.getConnection(url, props); - PreparedStatement dataset = connection.prepareStatement("INSERT INTO local_ega_ebi.filedataset(file_id, dataset_stable_id) values(1, 'EGAD00010000919');"); - dataset.executeUpdate(); - - PreparedStatement dataset_event_registered = connection.prepareStatement(prepareInsertQueryDatasetEvent("EGAD00010000919", "registered", "mapping")); - dataset_event_registered.executeUpdate(); + doaUrl = System.getenv("DOA_URL"); + mockauthUrl = System.getenv("MOCKAUTH_URL"); + minioHost = System.getenv("MINIO_HOST"); + outboxType = System.getenv("OUTBOX_TYPE"); - Thread.sleep(1000 * 3); - - PreparedStatement dataset_event_released = connection.prepareStatement(prepareInsertQueryDatasetEvent("EGAD00010000919", "released", "release")); - dataset_event_released.executeUpdate(); - - PreparedStatement datasetReferenceInsert = connection.prepareStatement("INSERT INTO sda.dataset_references(dataset_id, reference_id, reference_scheme) values('1', 'GDI-NO-10001','GDI');"); - datasetReferenceInsert.executeUpdate(); - connection.close(); - - JSONArray tokens = Unirest.get("http://localhost:8000/tokens").asJson().getBody().getArray(); + JSONArray tokens = Unirest.get(mockauthUrl + "/tokens").asJson().getBody().getArray(); validToken = tokens.getString(0); invalidToken = tokens.getString(1); validVisaToken = tokens.getString(2); @@ -93,7 +61,7 @@ public static void setup() { @SneakyThrows @AfterEach public void tearDown() { - File exportFolder = new File("requester@elixir-europe.org"); + File exportFolder = new File("outbox/requester@elixir-europe.org"); if (exportFolder.exists() && exportFolder.isDirectory()) { FileUtils.deleteDirectory(exportFolder); } @@ -101,19 +69,19 @@ public void tearDown() { @Test void testMetadataDatasetsNoToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets").asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataDatasetsInvalidToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataDatasetsValidToken() { - HttpResponse response = Unirest.get("http://localhost:8080/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/metadata/datasets").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); JSONArray datasets = response.getBody().getArray(); @@ -123,26 +91,26 @@ void testMetadataDatasetsValidToken() { @Test void testMetadataFilesNoToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000919/files").asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000919/files").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataFilesInvalidToken() { - int status = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testMetadataFilesValidTokenInvalidDataset() { - HttpResponse response = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000920/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000920/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.FORBIDDEN.value(), status); } @Test void testMetadataFilesValidTokenValidDataset() { - HttpResponse response = Unirest.get("http://localhost:8080/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/metadata/datasets/EGAD00010000919/files").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("[{\"fileId\":\"EGAF00000000014\",\"datasetId\":\"EGAD00010000919\",\"displayFileName\":\"body.enc\",\"fileName\":\"test/body.enc\",\"fileSize\":null,\"unencryptedChecksum\":null,\"unencryptedChecksumType\":null,\"decryptedFileSize\":null,\"decryptedFileChecksum\":null,\"decryptedFileChecksumType\":null,\"fileStatus\":\"READY\"}]", response.getBody().toString()); @@ -150,26 +118,26 @@ void testMetadataFilesValidTokenValidDataset() { @Test void testStreamingNoToken() { - int status = Unirest.get("http://localhost:8080/files/EGAF00000000014").asJson().getStatus(); + int status = Unirest.get(doaUrl + "/files/EGAF00000000014").asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testStreamingInvalidToken() { - int status = Unirest.get("http://localhost:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); + int status = Unirest.get(doaUrl + "/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + invalidToken).asJson().getStatus(); Assertions.assertEquals(HttpStatus.UNAUTHORIZED.value(), status); } @Test void testStreamingValidTokenInvalidFile() { - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000015").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); + HttpResponse response = Unirest.get(doaUrl + "/files/EGAF00000000015").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asJson(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.FORBIDDEN.value(), status); } @Test void testStreamingValidTokenValidFileFullPlain() { - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); + HttpResponse response = Unirest.get(doaUrl + "/files/EGAF00000000014").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(response.getBody())); @@ -177,7 +145,7 @@ void testStreamingValidTokenValidFileFullPlain() { @Test void testStreamingValidTokenValidFileRangePlain() { - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); + HttpResponse response = Unirest.get(doaUrl + "/files/EGAF00000000014?startCoordinate=100&endCoordinate=200").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); Assertions.assertEquals("09fbeae7cce2cd410b471b0a1a265fb53dc54c66c4c7c3111b8b9b95ac0e956f", DigestUtils.sha256Hex(response.getBody())); @@ -186,9 +154,9 @@ void testStreamingValidTokenValidFileRangePlain() { @SneakyThrows @Test void testStreamingValidTokenValidFileFullEncrypted() { - String publicKey = Files.readString(new File("test/crypt4gh.pub.pem").toPath()); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh.sec.pem"), "password".toCharArray()); - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); + String publicKey = Files.readString(new File("test/crypt4gh/crypt4gh.pub.pem").toPath()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/crypt4gh.sec.pem"), "password".toCharArray()); + HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014?destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(response.getBody()); @@ -201,9 +169,9 @@ void testStreamingValidTokenValidFileFullEncrypted() { @SneakyThrows @Test void testStreamingValidTokenValidFileRangeEncrypted() { - String publicKey = Files.readString(new File("test/crypt4gh.pub.pem").toPath()); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh.sec.pem"), "password".toCharArray()); - HttpResponse response = Unirest.get("http://localhost:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200&destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); + String publicKey = Files.readString(new File("test/crypt4gh/crypt4gh.pub.pem").toPath()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/crypt4gh.sec.pem"), "password".toCharArray()); + HttpResponse response = Unirest.get("http://doa:8080/files/EGAF00000000014?startCoordinate=100&endCoordinate=200&destinationFormat=crypt4gh").header(HttpHeaders.AUTHORIZATION, "Bearer " + validToken).header("Public-Key", publicKey).asBytes(); int status = response.getStatus(); Assertions.assertEquals(HttpStatus.OK.value(), status); try (ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(response.getBody()); @@ -216,13 +184,13 @@ void testStreamingValidTokenValidFileRangeEncrypted() { @SneakyThrows @Test void testPOSIXExportRequestFileValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } export("EGAF00000000014", false, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -232,13 +200,13 @@ void testPOSIXExportRequestFileValidToken() { @SneakyThrows @Test void testPOSIXExportRequestDatasetValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } export("EGAD00010000919", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -248,12 +216,12 @@ void testPOSIXExportRequestDatasetValidToken() { @SneakyThrows @Test void testS3ExportRequestFileValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } export("EGAF00000000014", false, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -264,12 +232,12 @@ void testS3ExportRequestFileValidToken() { @SneakyThrows @Test void testS3ExportRequestDatasetValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } export("EGAD00010000919", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -280,12 +248,12 @@ void testS3ExportRequestDatasetValidToken() { @SneakyThrows @Test void testS3ExportRequestReferenceValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } export("GDI-NO-10001", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -296,13 +264,13 @@ void testS3ExportRequestReferenceValidToken() { @SneakyThrows @Test void testPOSIXExportRequestReferenceValidToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } export("GDI-NO-10001", true, validToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -312,13 +280,13 @@ void testPOSIXExportRequestReferenceValidToken() { @SneakyThrows @Test void testPOSIXExportRequestFileValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } export("EGAF00000000014", false, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -328,13 +296,13 @@ void testPOSIXExportRequestFileValidVisaToken() { @SneakyThrows @Test void testPOSIXExportRequestDatasetValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } export("EGAD00010000919", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -344,12 +312,12 @@ void testPOSIXExportRequestDatasetValidVisaToken() { @SneakyThrows @Test void testS3ExportRequestFileValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } export("EGAF00000000014", false, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -360,12 +328,12 @@ void testS3ExportRequestFileValidVisaToken() { @SneakyThrows @Test void testS3ExportRequestDatasetValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } export("EGAD00010000919", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -376,13 +344,13 @@ void testS3ExportRequestDatasetValidVisaToken() { @SneakyThrows @Test void testPOSIXExportRequestReferenceValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("S3")) { + if (outboxType.equals("S3")) { Assertions.assertTrue(true); return; } export("GDI-NO-10001", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); - try (InputStream byteArrayInputStream = new FileInputStream("requester@elixir-europe.org/files/body.enc"); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); + try (InputStream byteArrayInputStream = new FileInputStream("outbox/requester@elixir-europe.org/files/body.enc"); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); Assertions.assertEquals("2aef808fb42fa7b1ba76cb16644773f9902a3fdc2569e8fdc049f38280c4577e", DigestUtils.sha256Hex(bytes)); @@ -392,12 +360,12 @@ void testPOSIXExportRequestReferenceValidVisaToken() { @SneakyThrows @Test void testS3ExportRequestReferenceValidVisaToken() { - if (System.getenv("OUTBOX_TYPE").equals("POSIX")) { + if (outboxType.equals("POSIX")) { Assertions.assertTrue(true); return; } export("GDI-NO-10001", true, validVisaToken); - PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/my.sec.pem"), "passw0rd".toCharArray()); + PrivateKey privateKey = KeyUtils.getInstance().readPrivateKey(new File("test/crypt4gh/my.sec.pem"), "passw0rd".toCharArray()); try (InputStream byteArrayInputStream = getMinioClient().getObject(GetObjectArgs.builder().bucket("lega").object("requester@elixir-europe.org/body.enc").build()); Crypt4GHInputStream crypt4GHInputStream = new Crypt4GHInputStream(byteArrayInputStream, privateKey)) { byte[] bytes = IOUtils.toByteArray(crypt4GHInputStream); @@ -406,7 +374,7 @@ void testS3ExportRequestReferenceValidVisaToken() { } @SneakyThrows void export(String id, boolean dataset, String token) { - String mqConnectionString = "amqps://admin:guest@localhost:5671/sda"; + String mqConnectionString = "amqps://guest:guest@rabbitmq:5671/sda"; ConnectionFactory factory = new ConnectionFactory(); factory.setUri(mqConnectionString); com.rabbitmq.client.Connection connectionFactory = factory.newConnection(); @@ -427,7 +395,7 @@ void export(String id, boolean dataset, String token) { token, dataset ? "datasetId" : "fileId", id, - FileUtils.readFileToString(new File("test/my.pub.pem"), Charset.defaultCharset())); + FileUtils.readFileToString(new File("test/crypt4gh/my.pub.pem"), Charset.defaultCharset())); channel.basicPublish("", "exportRequests", properties, @@ -439,11 +407,7 @@ void export(String id, boolean dataset, String token) { } MinioClient getMinioClient() { - return MinioClient.builder().endpoint("localhost", 9000, false).region("us-west-1").credentials("minio", "miniostorage").build(); + return MinioClient.builder().endpoint(minioHost, 9000, false).region("us-west-1").credentials("minio", "miniostorage").build(); } - - private static String prepareInsertQueryDatasetEvent(String datasetId, String event, String msgType) { - return String.format("INSERT INTO sda.dataset_event_log(dataset_id, event, message) VALUES('%s','%s','{\"type\": \"%s\"}');", datasetId, event, msgType); - } } diff --git a/sda-doa/test/mock_auth.py b/sda-doa/test/mock_auth.py index 9e90ad1aa5..a1eea5d351 100644 --- a/sda-doa/test/mock_auth.py +++ b/sda-doa/test/mock_auth.py @@ -1,14 +1,16 @@ """Mock OAUTH2 aiohttp.web server.""" from aiohttp import web -from authlib.jose import jwt, jwk +from authlib.jose import jwt, JsonWebKey from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa +import os def generate_token(): """Generate RSA Key pair to be used to sign token and the JWT Token itself.""" + issuer_url = os.getenv('ISSUER_URL', 'http://localhost:8000') private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend()) public_key = private_key.public_key().public_bytes(encoding=serialization.Encoding.PEM, format=serialization.PublicFormat.SubjectPublicKeyInfo) @@ -19,7 +21,7 @@ def generate_token(): # See available claims here: https://www.iana.org/assignments/jwt/jwt.xhtml # the important claim is the "authorities" header = { - "jku": "http://localhost:8000/jwk", + "jku": f"{issuer_url}/jwk", "kid": "rsa1", "alg": "RS256", "typ": "JWT" @@ -29,14 +31,14 @@ def generate_token(): "aud": ["aud2", "aud3"], "azp": "azp", "scope": "openid ga4gh_passport_v1", - "iss": "http://localhost:8000/", + "iss": f"{issuer_url}/", "exp": 9999999999, "iat": 1561621913, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102" } empty_payload = { "sub": "requester@elixir-europe.org", - "iss": "http://localhost:8000/", + "iss": f"{issuer_url}/", "exp": 99999999999, "iat": 1547794655, "jti": "6ad7aa42-3e9c-4833-bd16-765cb80c2102" @@ -44,7 +46,7 @@ def generate_token(): # Craft 4 passports, 2 for bona fide status and 2 for dataset permissions # passport for bona fide: terms passport_terms = { - "iss": "http://localhost:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "AcceptedTermsAndPolicies", @@ -59,7 +61,7 @@ def generate_token(): } # passport for bona fide: status passport_status = { - "iss": "http://localhost:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ResearcherStatus", @@ -74,7 +76,7 @@ def generate_token(): } # passport for dataset permissions 1 passport_dataset1 = { - "iss": "http://localhost:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -89,7 +91,7 @@ def generate_token(): } # passport for dataset permissions 2 passport_dataset2 = { - "iss": "http://localhost:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -103,7 +105,7 @@ def generate_token(): "jti": "9fa600d6-4148-47c1-b708-36c4ba2e980e" } passport_dataset_gdi = { - "iss": "http://129.177.177.134:8000/", + "iss": f"{issuer_url}/", "sub": "requester@elixir-europe.org", "ga4gh_visa_v1": { "type": "ControlledAccessGrants", @@ -116,15 +118,15 @@ def generate_token(): "exp": 99999999999, "jti": "2b322848-506b-492c-914f-47f9da967cdd" } - public_jwk = jwk.dumps(public_key, kty='RSA') - private_jwk = jwk.dumps(pem, kty='RSA') - dataset_encoded = jwt.encode(header, dataset_payload, private_jwk).decode('utf-8') - empty_encoded = jwt.encode(header, empty_payload, private_jwk).decode('utf-8') - passport_terms_encoded = jwt.encode(header, passport_terms, private_jwk).decode('utf-8') - passport_status_encoded = jwt.encode(header, passport_status, private_jwk).decode('utf-8') - passport_dataset1_encoded = jwt.encode(header, passport_dataset1, private_jwk).decode('utf-8') - passport_dataset2_encoded = jwt.encode(header, passport_dataset2, private_jwk).decode('utf-8') - passport_dataset_gdi_encoded = jwt.encode(header, passport_dataset_gdi, private_jwk).decode('utf-8') + public_jwk = JsonWebKey.import_key(public_key, {'kty': 'RSA', 'kid': 'rsa1', 'use': 'sig'}).as_dict() + private_key_obj = JsonWebKey.import_key(pem, {'kty': 'RSA', 'kid': 'rsa1', 'use': 'sig'}) + dataset_encoded = jwt.encode(header, dataset_payload, private_key_obj).decode('utf-8') + empty_encoded = jwt.encode(header, empty_payload, private_key_obj).decode('utf-8') + passport_terms_encoded = jwt.encode(header, passport_terms, private_key_obj).decode('utf-8') + passport_status_encoded = jwt.encode(header, passport_status, private_key_obj).decode('utf-8') + passport_dataset1_encoded = jwt.encode(header, passport_dataset1, private_key_obj).decode('utf-8') + passport_dataset2_encoded = jwt.encode(header, passport_dataset2, private_key_obj).decode('utf-8') + passport_dataset_gdi_encoded = jwt.encode(header, passport_dataset_gdi, private_key_obj).decode('utf-8') return (public_jwk, dataset_encoded, empty_encoded, passport_terms_encoded, passport_status_encoded, passport_dataset1_encoded, passport_dataset2_encoded, passport_dataset_gdi_encoded) @@ -167,8 +169,9 @@ async def userinfo(request): async def openid_configuration(request): """Mock ELIXIR AAI OpenID configuration endpoint.""" + issuer_url = os.getenv('ISSUER_URL', 'http://localhost:8000') data = { - "jwks_uri": "http://localhost:8000/jwk" + "jwks_uri": f"{issuer_url}/jwk" } return web.json_response(data) diff --git a/sda-download/.github/workflows/test.yml b/sda-download/.github/workflows/test.yml index 8fe495e021..047f9154c0 100644 --- a/sda-download/.github/workflows/test.yml +++ b/sda-download/.github/workflows/test.yml @@ -8,7 +8,7 @@ on: - "schemas/**" env: - go-version: '1.23' + go-version: '1.24' jobs: build: diff --git a/sda-download/Dockerfile b/sda-download/Dockerfile index b9ceec0366..30f9453531 100644 --- a/sda-download/Dockerfile +++ b/sda-download/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.23-alpine AS builder +FROM golang:1.24-alpine AS builder ENV GOPATH=$PWD ENV CGO_ENABLED=0 diff --git a/sda-download/api/api.go b/sda-download/api/api.go index 4cbcb67714..ae1a4c8dde 100644 --- a/sda-download/api/api.go +++ b/sda-download/api/api.go @@ -2,8 +2,10 @@ package api import ( "crypto/tls" + "encoding/json" "fmt" "net/http" + "strings" "time" "github.com/gin-gonic/gin" @@ -31,12 +33,36 @@ func healthResponse(c *gin.Context) { func Setup() *http.Server { // Set up routing log.Info("(2/5) Registering endpoint handlers") - + if log.GetLevel() != log.DebugLevel { + gin.SetMode(gin.ReleaseMode) + } router := gin.New() - router.Use( - gin.LoggerWithWriter(gin.DefaultWriter, "/health"), - gin.Recovery(), - ) + if log.GetLevel() == log.DebugLevel { + router.Use(gin.LoggerWithConfig( + gin.LoggerConfig{ + Formatter: func(params gin.LogFormatterParams) string { + s, _ := json.Marshal(map[string]any{ + "level": "debug", + "method": params.Method, + "path": params.Path, + "remote_addr": params.ClientIP, + "status_code": params.StatusCode, + "time": params.TimeStamp.Format(time.RFC3339), + }) + + return string(s) + "\n" + }, + + Skip: func(c *gin.Context) bool { + // skip logging HEAD requests to / and all requests to /health + // (HEAD request to health are redirected to path "") + return (c.Request.Method == "HEAD" && strings.Trim(c.FullPath(), "/") == "") || + c.FullPath() == "/health" + }, + Output: gin.DefaultWriter, + }, + )) + } router.HandleMethodNotAllowed = true diff --git a/sda-download/api/api_test.go b/sda-download/api/api_test.go index 6d7ab59750..d52fef9361 100644 --- a/sda-download/api/api_test.go +++ b/sda-download/api/api_test.go @@ -7,7 +7,6 @@ import ( ) func TestSetup(t *testing.T) { - // Create web server app config.Config.App.Host = "localhost" config.Config.App.Port = 8080 diff --git a/sda-download/api/middleware/middleware.go b/sda-download/api/middleware/middleware.go index e63f00048e..de6a9846d4 100644 --- a/sda-download/api/middleware/middleware.go +++ b/sda-download/api/middleware/middleware.go @@ -20,7 +20,6 @@ const requestContextKey = "requestContextKey" // The datasets are stored into a session cache for subsequent requests, and also // to the current request context for use in the endpoints. func TokenMiddleware() gin.HandlerFunc { - return func(c *gin.Context) { // Check if dataset permissions are cached to session sessionCookie, err := c.Cookie(config.Config.Session.Name) @@ -84,7 +83,6 @@ func TokenMiddleware() gin.HandlerFunc { // Forward request to the next endpoint handler c.Next() } - } // GetCacheFromContext is a helper function that endpoints can use to get data diff --git a/sda-download/api/middleware/middleware_test.go b/sda-download/api/middleware/middleware_test.go index 302568c2a1..3aec599b3f 100644 --- a/sda-download/api/middleware/middleware_test.go +++ b/sda-download/api/middleware/middleware_test.go @@ -23,7 +23,6 @@ const token string = "token" func testEndpoint(_ *gin.Context) {} func TestTokenMiddleware_Fail_GetToken(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken @@ -60,11 +59,9 @@ func TestTokenMiddleware_Fail_GetToken(t *testing.T) { // Return mock functions to originals auth.GetToken = originalGetToken - } func TestTokenMiddleware_Fail_GetVisas(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken originalGetVisas := auth.GetVisas @@ -106,11 +103,9 @@ func TestTokenMiddleware_Fail_GetVisas(t *testing.T) { // Return mock functions to originals auth.GetToken = originalGetToken auth.GetVisas = originalGetVisas - } func TestTokenMiddleware_Fail_GetPermissions(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken originalGetVisas := auth.GetVisas @@ -149,11 +144,9 @@ func TestTokenMiddleware_Fail_GetPermissions(t *testing.T) { auth.GetToken = originalGetToken auth.GetVisas = originalGetVisas auth.GetPermissions = originalGetPermissions - } func TestTokenMiddleware_Success_NoCache(t *testing.T) { - // Save original to-be-mocked functions originalGetToken := auth.GetToken originalGetVisas := auth.GetVisas @@ -215,11 +208,9 @@ func TestTokenMiddleware_Success_NoCache(t *testing.T) { auth.GetVisas = originalGetVisas auth.GetPermissions = originalGetPermissions session.NewSessionKey = originalNewSessionKey - } func TestTokenMiddleware_Success_FromCache(t *testing.T) { - // Save original to-be-mocked functions originalGetCache := session.Get @@ -275,7 +266,6 @@ func TestTokenMiddleware_Success_FromCache(t *testing.T) { // Return mock functions to originals session.Get = originalGetCache - } func TestStoreDatasets(t *testing.T) { @@ -294,11 +284,9 @@ func TestStoreDatasets(t *testing.T) { if !reflect.DeepEqual(datasets, storedDatasets) { t.Errorf("TestStoreDatasets failed, got %s, expected %s", storedDatasets, datasets) } - } func TestGetDatasets(t *testing.T) { - // Get a request context for testing if data is saved w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) @@ -314,5 +302,4 @@ func TestGetDatasets(t *testing.T) { if !reflect.DeepEqual(datasets, storedDatasets) { t.Errorf("TestStoreDatasets failed, got %s, expected %s", storedDatasets, datasets) } - } diff --git a/sda-download/api/s3/s3.go b/sda-download/api/s3/s3.go index 4bea88bbc4..35cb03667e 100644 --- a/sda-download/api/s3/s3.go +++ b/sda-download/api/s3/s3.go @@ -259,7 +259,6 @@ func GetEcnryptedObject(c *gin.Context) { // which can include slashes, so finding the separation between filename and // dataset name is done by comparing to accessible datasets. func parseParams(c *gin.Context) *gin.Context { - // When doing list requests from s3cmd, the tool will assume that the first // slash delimits the bucket, and the rest is the prefix. We use this to // "restore" the path in case the prefix param is set. @@ -319,7 +318,6 @@ func Download(c *gin.Context) { // S3 request types are described here: // https://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html switch { - case strings.Contains(c.Request.URL.String(), "?location"): GetBucketLocation(c) diff --git a/sda-download/api/s3/s3_test.go b/sda-download/api/s3/s3_test.go index c490c65d29..5c05a05af9 100644 --- a/sda-download/api/s3/s3_test.go +++ b/sda-download/api/s3/s3_test.go @@ -23,17 +23,16 @@ type S3TestSuite struct { Mock sqlmock.Sqlmock } -func (suite *S3TestSuite) SetupTest() { - +func (ts *S3TestSuite) SetupTest() { var err error var db *sql.DB // create mock database testConnInfo := "host=localhost port=5432 user=user password=pass dbname=db sslmode=disable" - db, suite.Mock, err = sqlmock.New() + db, ts.Mock, err = sqlmock.New() if err != nil { - suite.T().Fatalf("error '%s' when creating mock database connection", err) + ts.T().Fatalf("error '%s' when creating mock database connection", err) } database.DB = &database.SQLdb{DB: db, ConnInfo: testConnInfo} @@ -53,15 +52,14 @@ func (suite *S3TestSuite) SetupTest() { } } -func (suite *S3TestSuite) TearDownTest() { +func (ts *S3TestSuite) TearDownTest() { } func TestS3TestSuite(t *testing.T) { suite.Run(t, new(S3TestSuite)) } -func (suite *S3TestSuite) TestGetBucketLocation() { - +func (ts *S3TestSuite) TestGetBucketLocation() { // Send a request through the middleware to get datasets w := httptest.NewRecorder() @@ -71,24 +69,23 @@ func (suite *S3TestSuite) TestGetBucketLocation() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := "\nus-west-2" - assert.Equal(suite.T(), expected, string(body), "Wrong location from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong location from S3") } -func (suite *S3TestSuite) TestListBuckets() { - +func (ts *S3TestSuite) TestListBuckets() { // Setup a mock database to handle queries query := `SELECT stable_id, created_at FROM sda.datasets WHERE stable_id = \$1` - suite.Mock.ExpectQuery(query).WithArgs("dataset1"). + ts.Mock.ExpectQuery(query).WithArgs("dataset1"). WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "nyss")) - suite.Mock.ExpectQuery(query).WithArgs("dataset10"). + ts.Mock.ExpectQuery(query).WithArgs("dataset10"). WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "nyligen")) - suite.Mock.ExpectQuery(query).WithArgs("https://url/dataset"). + ts.Mock.ExpectQuery(query).WithArgs("https://url/dataset"). WillReturnRows(sqlmock.NewRows([]string{"stable_id", "created_at"}).AddRow("dataset1", "snart")) // Send a request through the middleware to get datasets @@ -100,7 +97,7 @@ func (suite *S3TestSuite) TestListBuckets() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := xml.Header + @@ -110,14 +107,13 @@ func (suite *S3TestSuite) TestListBuckets() { "snartdataset1" + "" - assert.Equal(suite.T(), expected, string(body), "Wrong bucket list from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong bucket list from S3") - err = suite.Mock.ExpectationsWereMet() - assert.Nilf(suite.T(), err, "there were unfulfilled expectations: %s", err) + err = ts.Mock.ExpectationsWereMet() + assert.Nilf(ts.T(), err, "there were unfulfilled expectations: %s", err) } -func (suite *S3TestSuite) TestListByPrefix() { - +func (ts *S3TestSuite) TestListByPrefix() { // Setup a mock database to handle queries fileInfo := &database.FileInfo{ FileID: "file1", @@ -154,7 +150,7 @@ func (suite *S3TestSuite) TestListByPrefix() { LEFT JOIN \(SELECT file_id, checksum, type FROM sda.checksums WHERE source = 'UNENCRYPTED'\) sha ON files.id = sha.file_id WHERE datasets.stable_id = \$1; ` - suite.Mock.ExpectQuery(query). + ts.Mock.ExpectQuery(query). WithArgs("dataset1"). WillReturnRows(sqlmock.NewRows([]string{"file_id", "dataset_id", "display_file_name", "user_id", "file_path", "file_size", @@ -176,7 +172,7 @@ func (suite *S3TestSuite) TestListByPrefix() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := xml.Header + @@ -188,14 +184,13 @@ func (suite *S3TestSuite) TestListByPrefix() { "dataset1" + "" - assert.Equal(suite.T(), expected, string(body), "Wrong object list from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong object list from S3") - err = suite.Mock.ExpectationsWereMet() - assert.Nilf(suite.T(), err, "there were unfulfilled expectations: %s", err) + err = ts.Mock.ExpectationsWereMet() + assert.Nilf(ts.T(), err, "there were unfulfilled expectations: %s", err) } -func (suite *S3TestSuite) TestListObjects() { - +func (ts *S3TestSuite) TestListObjects() { // Setup a mock database to handlequeries fileInfo := &database.FileInfo{ FileID: "file1", @@ -232,7 +227,7 @@ func (suite *S3TestSuite) TestListObjects() { LEFT JOIN \(SELECT file_id, checksum, type FROM sda.checksums WHERE source = 'UNENCRYPTED'\) sha ON files.id = sha.file_id WHERE datasets.stable_id = \$1; ` - suite.Mock.ExpectQuery(query). + ts.Mock.ExpectQuery(query). WithArgs("dataset1"). WillReturnRows(sqlmock.NewRows([]string{"file_id", "dataset_id", "display_file_name", "user_id", "file_path", "file_size", @@ -253,7 +248,7 @@ func (suite *S3TestSuite) TestListObjects() { response := w.Result() body, err := io.ReadAll(response.Body) - assert.Nil(suite.T(), err, "failed to parse body from location response") + assert.Nil(ts.T(), err, "failed to parse body from location response") defer response.Body.Close() expected := xml.Header + @@ -265,14 +260,13 @@ func (suite *S3TestSuite) TestListObjects() { "dataset1" + "" - assert.Equal(suite.T(), expected, string(body), "Wrong object list from S3") + assert.Equal(ts.T(), expected, string(body), "Wrong object list from S3") - err = suite.Mock.ExpectationsWereMet() - assert.Nilf(suite.T(), err, "there were unfulfilled expectations: %s", err) + err = ts.Mock.ExpectationsWereMet() + assert.Nilf(ts.T(), err, "there were unfulfilled expectations: %s", err) } -func (suite *S3TestSuite) TestParseParams() { - +func (ts *S3TestSuite) TestParseParams() { type paramTest struct { Path string Dataset string @@ -291,13 +285,12 @@ func (suite *S3TestSuite) TestParseParams() { } for _, params := range testParams { - // response function to check parameter parsing testParseParams := func(c *gin.Context) { parseParams(c) - assert.Equal(suite.T(), params.Dataset, c.Param("dataset"), "Failed to parse dataset name") - assert.Equal(suite.T(), params.Filename, c.Param("filename"), "Failed to parse file name") + assert.Equal(ts.T(), params.Dataset, c.Param("dataset"), "Failed to parse dataset name") + assert.Equal(ts.T(), params.Filename, c.Param("filename"), "Failed to parse file name") c.AbortWithStatus(http.StatusAccepted) } @@ -310,10 +303,8 @@ func (suite *S3TestSuite) TestParseParams() { router.ServeHTTP(w, httptest.NewRequest("GET", params.Path, nil)) response := w.Result() - defer response.Body.Close() - - assert.Equal(suite.T(), http.StatusAccepted, response.StatusCode, "Request failed") + response.Body.Close() + assert.Equal(ts.T(), http.StatusAccepted, response.StatusCode, "Request failed") } - } diff --git a/sda-download/api/sda/sda.go b/sda-download/api/sda/sda.go index e531606c0d..53a08ae73d 100644 --- a/sda-download/api/sda/sda.go +++ b/sda-download/api/sda/sda.go @@ -103,7 +103,6 @@ func reencryptHeader(oldHeader []byte, reencKey string) ([]byte, error) { return nil, err } - log.Debugf("Response from the reencrypt service: %v", res) return res.Header, nil } @@ -136,7 +135,6 @@ func find(datasetID string, datasets []string) bool { // getFiles returns files belonging to a dataset var getFiles = func(datasetID string, ctx *gin.Context) ([]*database.FileInfo, int, error) { - // Retrieve dataset list from request context // generated by the authentication middleware cache := middleware.GetCacheFromContext(ctx) @@ -161,7 +159,6 @@ var getFiles = func(datasetID string, ctx *gin.Context) ([]*database.FileInfo, i // Files serves a list of files belonging to a dataset func Files(c *gin.Context) { - // get dataset parameter dataset := c.Param("dataset") @@ -290,7 +287,7 @@ func Download(c *gin.Context) { return } - wholeFile := true + wholeFile := true //nolint:staticcheck if start != 0 || end != 0 { wholeFile = false } @@ -348,7 +345,6 @@ func Download(c *gin.Context) { } if c.Request.Method == http.MethodHead { - // Create headers for htsget, containing size of the crypt4gh header reencKey := c.GetHeader("Client-Public-Key") headerSize := bytes.NewReader(fileDetails.Header).Size() @@ -383,7 +379,6 @@ func Download(c *gin.Context) { } log.Debugf("Public key from the request header = %v", reencKey) - log.Debugf("old c4gh file header = %v\n", fileDetails.Header) newHeader, err := reencryptHeader(fileDetails.Header, reencKey) if err != nil { log.Errorf("Failed to reencrypt the file header, reason: %v", err) @@ -391,7 +386,6 @@ func Download(c *gin.Context) { return } - log.Debugf("Reencrypted c4gh file header = %v", newHeader) newHr := bytes.NewReader(newHeader) @@ -469,17 +463,14 @@ func Download(c *gin.Context) { var adjustSeekPos = func(fileStream io.ReadSeeker, start, end int64) (int64, int64, error) { if start != 0 { - // We don't want to read from start, skip ahead to where we should be _, err := fileStream.Seek(start, 0) if err != nil { - return 0, 0, fmt.Errorf("error occurred while finding sending start: %v", err) } // adjust end to reflect that the file start has been moved end -= start start = 0 - } return start, end, nil @@ -487,7 +478,6 @@ var adjustSeekPos = func(fileStream io.ReadSeeker, start, end int64) (int64, int // used from: https://github.com/neicnordic/crypt4gh/blob/master/examples/reader/main.go#L48C1-L113C1 var sendStream = func(reader io.Reader, writer http.ResponseWriter, start, end int64) error { - // Calculate how much we should read (if given) togo := end - start @@ -549,7 +539,7 @@ var calculateCoords = func(start, end int64, htsget_range string, fileDetails *d return 0, 0, err } if a > b { - return 0, 0, fmt.Errorf("endCoordinate must be greater than startCoordinate") + return 0, 0, errors.New("endCoordinate must be greater than startCoordinate") } // Byte ranges are inclusive; +1 so that the last byte is included @@ -575,5 +565,4 @@ var calculateCoords = func(start, end int64, htsget_range string, fileDetails *d } return start, headlength.Size() + bodyEnd, nil - } diff --git a/sda-download/api/sda/sda_test.go b/sda-download/api/sda/sda_test.go index 3b2a06a673..087ca86f3c 100644 --- a/sda-download/api/sda/sda_test.go +++ b/sda-download/api/sda/sda_test.go @@ -35,7 +35,6 @@ import ( ) func TestDatasets(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext @@ -72,11 +71,9 @@ func TestDatasets(t *testing.T) { // Return mock functions to originals middleware.GetCacheFromContext = originalGetCacheFromContext - } func TestFind_Found(t *testing.T) { - // Test case datasets := []string{"dataset1", "dataset2", "dataset3"} @@ -89,11 +86,9 @@ func TestFind_Found(t *testing.T) { if found != expectedFound { t.Errorf("TestFind_Found failed, got %t expected %t", found, expectedFound) } - } func TestFind_NotFound(t *testing.T) { - // Test case datasets := []string{"dataset1", "dataset2", "dataset3"} @@ -106,11 +101,9 @@ func TestFind_NotFound(t *testing.T) { if found != expectedFound { t.Errorf("TestFind_Found failed, got %t expected %t", found, expectedFound) } - } func TestGetFiles_Fail_Database(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext originalGetFilesDB := database.GetFiles @@ -147,11 +140,9 @@ func TestGetFiles_Fail_Database(t *testing.T) { // Return mock functions to originals middleware.GetCacheFromContext = originalGetCacheFromContext database.GetFiles = originalGetFilesDB - } func TestGetFiles_Fail_NotFound(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext @@ -186,7 +177,6 @@ func TestGetFiles_Fail_NotFound(t *testing.T) { } func TestGetFiles_Success(t *testing.T) { - // Save original to-be-mocked functions originalGetCacheFromContext := middleware.GetCacheFromContext originalGetFilesDB := database.GetFiles @@ -229,11 +219,9 @@ func TestGetFiles_Success(t *testing.T) { // Return mock functions to originals middleware.GetCacheFromContext = originalGetCacheFromContext database.GetFiles = originalGetFilesDB - } func TestFiles_Fail(t *testing.T) { - // Save original to-be-mocked functions originalGetFiles := getFiles @@ -272,11 +260,9 @@ func TestFiles_Fail(t *testing.T) { // Return mock functions to originals getFiles = originalGetFiles - } func TestFiles_Success(t *testing.T) { - // Save original to-be-mocked functions originalGetFiles := getFiles @@ -335,11 +321,9 @@ func TestFiles_Success(t *testing.T) { // Return mock functions to originals getFiles = originalGetFiles - } func TestDownload_Fail_UnencryptedDownloadNotAllowed(t *testing.T) { - // Save original to-be-mocked config originalServeUnencryptedDataTrigger := config.Config.C4GH.PublicKeyB64 config.Config.C4GH.PublicKeyB64 = "" @@ -396,7 +380,6 @@ func TestDownload_Fail_UnencryptedDownloadNotAllowed(t *testing.T) { } func TestDownload_Fail_FileNotFound(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -445,11 +428,9 @@ func TestDownload_Fail_FileNotFound(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - } func TestDownload_Fail_NoPermissions(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -504,11 +485,9 @@ func TestDownload_Fail_NoPermissions(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - } func TestDownload_Fail_GetFile(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -569,11 +548,9 @@ func TestDownload_Fail_GetFile(t *testing.T) { config.Config.C4GH.PrivateKey = originalC4ghPrivateKeyFilepath viper.Set("c4gh.transientKeyPath", "") viper.Set("c4gh.transientPassphrase", "") - } func TestDownload_Fail_OpenFile(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) @@ -742,7 +719,6 @@ func (f *fakeGRPC) ServeHTTP(w http.ResponseWriter, r *http.Request) { } func TestDownload_Whole_Range_Encrypted(t *testing.T) { - privateKeyFilePath, err := GenerateTestC4ghKey(t) assert.NoError(t, err) diff --git a/sda-download/cmd/main.go b/sda-download/cmd/main.go index 2c4b2c347b..11734022df 100644 --- a/sda-download/cmd/main.go +++ b/sda-download/cmd/main.go @@ -26,7 +26,7 @@ func init() { // Set middleware // nolint:gocritic // this nolint can be removed, if you have more than one middlewares available - switch conf.App.Middleware { + switch conf.App.Middleware { //nolint:revive default: api.SelectedMiddleware = middleware.TokenMiddleware } diff --git a/sda-download/dev_utils/config-notls_local.yaml b/sda-download/dev_utils/config-notls_local.yaml index d425077a8f..57cd74d5d4 100644 --- a/sda-download/dev_utils/config-notls_local.yaml +++ b/sda-download/dev_utils/config-notls_local.yaml @@ -32,4 +32,4 @@ db: oidc: # oidc configuration API must have values for "userinfo_endpoint" and "jwks_uri" configuration: - url: "http://localhost:8080/.well-known/openid-configuration" + url: "http://localhost:8800/oidc/.well-known/openid-configuration" diff --git a/sda-download/go.mod b/sda-download/go.mod index 042125200d..73eae28d65 100644 --- a/sda-download/go.mod +++ b/sda-download/go.mod @@ -1,24 +1,26 @@ module github.com/neicnordic/sda-download -go 1.23.1 +go 1.24.0 + +toolchain go1.24.1 require ( github.com/DATA-DOG/go-sqlmock v1.5.2 - github.com/aws/aws-sdk-go v1.55.5 + github.com/aws/aws-sdk-go v1.55.7 github.com/dgraph-io/ristretto v1.0.0 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/johannesboyne/gofakes3 v0.0.0-20240914150247-80d9d7b1b250 - github.com/lestrrat-go/jwx/v2 v2.1.3 + github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.10.9 - github.com/neicnordic/crypt4gh v1.13.0 + github.com/neicnordic/crypt4gh v1.14.0 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.19.0 + github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.32.0 + golang.org/x/crypto v0.38.0 golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 - google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.36.2 + google.golang.org/grpc v1.72.0 + google.golang.org/protobuf v1.36.6 ) require ( @@ -30,53 +32,49 @@ require ( github.com/cloudwego/iasm v0.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.1 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.3 // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lestrrat-go/blackmagic v1.0.2 // indirect + github.com/lestrrat-go/blackmagic v1.0.3 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/shabbyrobe/gocovmerge v0.0.0-20230507112040-c3350d9342df // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect golang.org/x/tools v0.25.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect - gopkg.in/ini.v1 v1.67.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/sda-download/go.sum b/sda-download/go.sum index ccffa3e18c..e4383bafcb 100644 --- a/sda-download/go.sum +++ b/sda-download/go.sum @@ -3,8 +3,8 @@ filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU= github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU= github.com/aws/aws-sdk-go v1.44.256/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= -github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/aws/aws-sdk-go v1.55.7 h1:UJrkFq7es5CShfBwlWAC8DA077vp8PyVbQd3lqLiztE= +github.com/aws/aws-sdk-go v1.55.7/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= @@ -24,8 +24,8 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= github.com/dgraph-io/ristretto v1.0.0 h1:SYG07bONKMlFDUYu5pEu3DGAh8c2OFNzKm6G9J4Si84= github.com/dgraph-io/ristretto v1.0.0/go.mod h1:jTi2FiYEhQ1NsMmA7DeBykizjOuY88NhKBkepyu1jPc= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= @@ -34,8 +34,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= @@ -54,6 +54,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27HYW8P9FDk5PbgA= github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -63,8 +65,6 @@ github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -84,33 +84,29 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= -github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= +github.com/lestrrat-go/blackmagic v1.0.3 h1:94HXkVLxkZO9vJI/w2u1T0DAoprShFd13xtnSINtDWs= +github.com/lestrrat-go/blackmagic v1.0.3/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.3 h1:Ud4lb2QuxRClYAmRleF50KrbKIoM1TddXgBrneT5/Jo= -github.com/lestrrat-go/jwx/v2 v2.1.3/go.mod h1:q6uFgbgZfEmQrfJfrCo90QcQOcXFMfbI/fO0NqRtvZo= +github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVfecA= +github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.13.0 h1:NbSAPx1+zFpG6a8GCVwW80y/TGHfGdXJF/zqQKqlHZ8= -github.com/neicnordic/crypt4gh v1.13.0/go.mod h1:lfNIrhlcQrSf5awgCaW+poCsRBlvKOrNjR3CBvXU5Ek= +github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U= +github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -122,10 +118,8 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500/go.mod h1:+njLrG5wSeoG4Ds61rFgEzKvenR2UHbjMoDHsczxly0= @@ -136,14 +130,14 @@ github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVs github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -164,24 +158,26 @@ github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65E github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= golang.org/x/arch v0.10.0 h1:S3huipmSclq3PJMNe76NGwkBR504WFkQ5dhzWzP8ZW8= golang.org/x/arch v0.10.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= @@ -193,8 +189,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -209,23 +205,23 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= -golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190829051458-42f498d34c4d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -235,17 +231,15 @@ golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/tools v0.25.0 h1:oFU9pkj/iJgs+0DT+VMHrx+oBKs/LJMV+Uvg78sl+fE= golang.org/x/tools v0.25.0/go.mod h1:/vtpO8WL1N9cQC3FN5zPqb//fRXskFHbLKk4OW1Q7rg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= -google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= -gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/sda-download/internal/config/config.go b/sda-download/internal/config/config.go index af468ee1d7..2e9a7129c0 100644 --- a/sda-download/internal/config/config.go +++ b/sda-download/internal/config/config.go @@ -171,11 +171,10 @@ func NewConfig() (*Map, error) { } if err := viper.ReadInConfig(); err != nil { - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - log.Infoln("No config file found, using ENVs only") - } else { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { return nil, err } + log.Infoln("No config file found, using ENVs only") } requiredConfVars := []string{ "db.host", "db.user", "db.password", "db.database", "oidc.configuration.url", "grpc.host", @@ -380,7 +379,6 @@ func (c *Map) appConfig() error { var err error if viper.GetString("c4gh.transientKeyPath") != "" { - if !viper.IsSet("c4gh.transientPassphrase") { return errors.New("c4gh.transientPassphrase is not set") } @@ -434,7 +432,7 @@ func (c *Map) configDatabase() error { } if db.SslMode == "verify-full" { // Since verify-full is specified, these are required. - if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { + if !viper.IsSet("db.clientCert") && !viper.IsSet("db.clientKey") { return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") } } @@ -475,12 +473,12 @@ func readTrustedIssuers(filePath string) ([]TrustedISS, error) { } func constructWhitelist(obj []TrustedISS) *jwk.MapWhitelist { - keys := make(map[string]bool) + keyMap := make(map[string]bool) wl := jwk.NewMapWhitelist() for _, value := range obj { - if _, ok := keys[value.JKU]; !ok { - keys[value.JKU] = true + if _, ok := keyMap[value.JKU]; !ok { + keyMap[value.JKU] = true wl.Add(value.JKU) } } diff --git a/sda-download/internal/config/config_test.go b/sda-download/internal/config/config_test.go index d493c683ac..e656301ded 100644 --- a/sda-download/internal/config/config_test.go +++ b/sda-download/internal/config/config_test.go @@ -23,7 +23,7 @@ type TestSuite struct { suite.Suite } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("db.host", "test") viper.Set("db.user", "test") viper.Set("db.password", "test") @@ -33,7 +33,7 @@ func (suite *TestSuite) SetupTest() { viper.Set("oidc.configuration.url", "test") } -func (suite *TestSuite) TearDownTest() { +func (ts *TestSuite) TearDownTest() { viper.Reset() } @@ -41,39 +41,38 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) TestConfigFile() { +func (ts *TestSuite) TestConfigFile() { viper.Set("configFile", "test") config, err := NewConfig() - assert.Nil(suite.T(), config) - assert.Error(suite.T(), err) - assert.Equal(suite.T(), "test", viper.ConfigFileUsed()) + assert.Nil(ts.T(), config) + assert.Error(ts.T(), err) + assert.Equal(ts.T(), "test", viper.ConfigFileUsed()) } -func (suite *TestSuite) TestMissingRequiredConfVar() { +func (ts *TestSuite) TestMissingRequiredConfVar() { for _, requiredConfVar := range requiredConfVars { requiredConfVarValue := viper.Get(requiredConfVar) viper.Set(requiredConfVar, nil) expectedError := fmt.Errorf("%s not set", requiredConfVar) config, err := NewConfig() - assert.Nil(suite.T(), config) - if assert.Error(suite.T(), err) { - assert.Equal(suite.T(), expectedError, err) + assert.Nil(ts.T(), config) + if assert.Error(ts.T(), err) { + assert.Equal(ts.T(), expectedError, err) } viper.Set(requiredConfVar, requiredConfVarValue) } } -func (suite *TestSuite) TestAppConfig() { - +func (ts *TestSuite) TestAppConfig() { // Generate a crypth4gh private key file _, privateKey, err := keys.GenerateKeyPair() - assert.NoError(suite.T(), err) - tempDir := suite.T().TempDir() + assert.NoError(ts.T(), err) + tempDir := ts.T().TempDir() defer os.RemoveAll(tempDir) privateKeyFile, err := os.Create(fmt.Sprintf("%s/c4fg.key", tempDir)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = keys.WriteCrypt4GHX25519PrivateKey(privateKeyFile, privateKey, []byte("password")) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Test fail on missing middleware viper.Set("app.host", "test") @@ -87,7 +86,7 @@ func (suite *TestSuite) TestAppConfig() { c := &Map{} err = c.appConfig() - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") viper.Reset() viper.Set("app.host", "test") @@ -101,44 +100,42 @@ func (suite *TestSuite) TestAppConfig() { c = &Map{} err = c.appConfig() - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "test", c.App.Host) - assert.Equal(suite.T(), 1234, c.App.Port) - assert.Equal(suite.T(), "test", c.App.ServerCert) - assert.Equal(suite.T(), "test", c.App.ServerKey) - assert.NotEmpty(suite.T(), c.C4GH.PrivateKey) - assert.NotEmpty(suite.T(), c.C4GH.PublicKeyB64) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "test", c.App.Host) + assert.Equal(ts.T(), 1234, c.App.Port) + assert.Equal(ts.T(), "test", c.App.ServerCert) + assert.Equal(ts.T(), "test", c.App.ServerKey) + assert.NotEmpty(ts.T(), c.C4GH.PrivateKey) + assert.NotEmpty(ts.T(), c.C4GH.PublicKeyB64) // Check the private key that was loaded by checking the derived public key publicKey, err := base64.StdEncoding.DecodeString(c.C4GH.PublicKeyB64) - assert.Nilf(suite.T(), err, "Incorrect public c4gh key generated (error in base64 encoding)") + assert.Nilf(ts.T(), err, "Incorrect public c4gh key generated (error in base64 encoding)") _, err = keys.ReadPublicKey(bytes.NewReader(publicKey)) - assert.Nilf(suite.T(), err, "Incorrect public c4gh key generated (bad key)") + assert.Nilf(ts.T(), err, "Incorrect public c4gh key generated (bad key)") // Check false c4gh key viper.Set("c4gh.transientKeyPath", "some/nonexistent.key") err = c.appConfig() - assert.ErrorContains(suite.T(), err, "no such file or directory") + assert.ErrorContains(ts.T(), err, "no such file or directory") // Check false c4gh key viper.Set("c4gh.transientKeyPath", privateKeyFile.Name()) viper.Set("c4gh.transientPassphrase", "blablabla") err = c.appConfig() - assert.ErrorContains(suite.T(), err, "chacha20poly1305: message authentication failed") + assert.ErrorContains(ts.T(), err, "chacha20poly1305: message authentication failed") } -func (suite *TestSuite) TestArchiveConfig() { +func (ts *TestSuite) TestArchiveConfig() { viper.Set("archive.type", POSIX) viper.Set("archive.location", "/test") c := &Map{} c.configArchive() - assert.Equal(suite.T(), "/test", c.Archive.Posix.Location) - + assert.Equal(ts.T(), "/test", c.Archive.Posix.Location) } -func (suite *TestSuite) TestSessionConfig() { - +func (ts *TestSuite) TestSessionConfig() { viper.Set("session.expiration", 3600) viper.Set("session.domain", "test") viper.Set("session.secure", false) @@ -148,26 +145,24 @@ func (suite *TestSuite) TestSessionConfig() { c := &Map{} c.sessionConfig() - assert.Equal(suite.T(), time.Duration(3600*time.Second), c.Session.Expiration) - assert.Equal(suite.T(), "test", c.Session.Domain) - assert.Equal(suite.T(), false, c.Session.Secure) - assert.Equal(suite.T(), false, c.Session.HTTPOnly) - + assert.Equal(ts.T(), time.Duration(3600*time.Second), c.Session.Expiration) + assert.Equal(ts.T(), "test", c.Session.Domain) + assert.Equal(ts.T(), false, c.Session.Secure) + assert.Equal(ts.T(), false, c.Session.HTTPOnly) } -func (suite *TestSuite) TestDatabaseConfig() { - +func (ts *TestSuite) TestDatabaseConfig() { // Test error on missing SSL vars viper.Set("db.sslmode", "verify-full") c := &Map{} err := c.configDatabase() - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") // Test no error on SSL disabled viper.Set("db.sslmode", "disable") c = &Map{} err = c.configDatabase() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Test pass on SSL vars set viper.Set("db.host", "test") @@ -182,30 +177,28 @@ func (suite *TestSuite) TestDatabaseConfig() { c = &Map{} err = c.configDatabase() - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "test", c.DB.Host) - assert.Equal(suite.T(), 1234, c.DB.Port) - assert.Equal(suite.T(), "test", c.DB.User) - assert.Equal(suite.T(), "test", c.DB.Password) - assert.Equal(suite.T(), "test", c.DB.Database) - assert.Equal(suite.T(), "test", c.DB.CACert) - assert.Equal(suite.T(), "test", c.DB.ClientCert) - assert.Equal(suite.T(), "test", c.DB.ClientKey) - + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "test", c.DB.Host) + assert.Equal(ts.T(), 1234, c.DB.Port) + assert.Equal(ts.T(), "test", c.DB.User) + assert.Equal(ts.T(), "test", c.DB.Password) + assert.Equal(ts.T(), "test", c.DB.Database) + assert.Equal(ts.T(), "test", c.DB.CACert) + assert.Equal(ts.T(), "test", c.DB.ClientCert) + assert.Equal(ts.T(), "test", c.DB.ClientKey) } -func (suite *TestSuite) TestOIDC() { - +func (ts *TestSuite) TestOIDC() { // Test wrong file viper.Set("oidc.trusted.iss", "../../iss.json") c := &Map{} err := c.configureOIDC() - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") viper.Set("oidc.trusted.iss", "../../dev_utils/iss.json") c = &Map{} err = c.configureOIDC() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Test pass OIDC config viper.Set("oidc.trusted.iss", "../../dev_utils/iss.json") @@ -217,31 +210,29 @@ func (suite *TestSuite) TestOIDC() { whitelist := jwk.NewMapWhitelist().Add("https://mockauth:8000/idp/profile/oidc/keyset") c = &Map{} err = c.configureOIDC() - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "test", c.OIDC.ConfigurationURL) - assert.Equal(suite.T(), "test", c.OIDC.CACert) - assert.Equal(suite.T(), trustedList, c.OIDC.TrustedList) - assert.Equal(suite.T(), whitelist, c.OIDC.Whitelist) - + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "test", c.OIDC.ConfigurationURL) + assert.Equal(ts.T(), "test", c.OIDC.CACert) + assert.Equal(ts.T(), trustedList, c.OIDC.TrustedList) + assert.Equal(ts.T(), whitelist, c.OIDC.Whitelist) } -func (suite *TestSuite) TestConfigReencrypt() { - tempDir := suite.T().TempDir() +func (ts *TestSuite) TestConfigReencrypt() { + tempDir := ts.T().TempDir() c := &Map{} viper.Set("grpc.host", "localhost") - assert.NoError(suite.T(), c.configReencrypt()) - assert.Equal(suite.T(), 50051, c.Reencrypt.Port) + assert.NoError(ts.T(), c.configReencrypt()) + assert.Equal(ts.T(), 50051, c.Reencrypt.Port) // fail if set file doesn't exists viper.Set("grpc.clientcert", "/tmp/abracadabra") - assert.ErrorContains(suite.T(), c.configReencrypt(), "no such file or directory") + assert.ErrorContains(ts.T(), c.configReencrypt(), "no such file or directory") // any existing flle will make it pass viper.Set("grpc.clientcert", "config_test.go") - assert.NoError(suite.T(), c.configReencrypt()) + assert.NoError(ts.T(), c.configReencrypt()) // it will fail if certificate is set to a folder viper.Set("grpc.clientcert", tempDir) - assert.ErrorContains(suite.T(), c.configReencrypt(), "is a folder") - + assert.ErrorContains(ts.T(), c.configReencrypt(), "is a folder") } diff --git a/sda-download/internal/database/database.go b/sda-download/internal/database/database.go index b3470ddf1e..ec46a393db 100644 --- a/sda-download/internal/database/database.go +++ b/sda-download/internal/database/database.go @@ -65,10 +65,10 @@ func sanitizeString(str string) string { } // NewDB creates a new DB connection -func NewDB(config config.DatabaseConfig) (*SQLdb, error) { - connInfo := buildConnInfo(config) +func NewDB(conf config.DatabaseConfig) (*SQLdb, error) { + connInfo := buildConnInfo(conf) - log.Debugf("Connecting to DB %s:%d on database: %s with user: %s", config.Host, config.Port, config.Database, config.User) + log.Debugf("Connecting to DB %s:%d on database: %s with user: %s", conf.Host, conf.Port, conf.Database, conf.User) db, err := sqlOpen("postgres", connInfo) if err != nil { log.Errorf("failed to connect to database, %s", err) @@ -88,24 +88,24 @@ func NewDB(config config.DatabaseConfig) (*SQLdb, error) { } // buildConnInfo builds a connection string for the database -func buildConnInfo(config config.DatabaseConfig) string { +func buildConnInfo(conf config.DatabaseConfig) string { connInfo := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s", - config.Host, config.Port, config.User, config.Password, config.Database, config.SslMode) + conf.Host, conf.Port, conf.User, conf.Password, conf.Database, conf.SslMode) - if config.SslMode == "disable" { + if conf.SslMode == "disable" { return connInfo } - if config.CACert != "" { - connInfo += fmt.Sprintf(" sslrootcert=%s", config.CACert) + if conf.CACert != "" { + connInfo += fmt.Sprintf(" sslrootcert=%s", conf.CACert) } - if config.ClientCert != "" { - connInfo += fmt.Sprintf(" sslcert=%s", config.ClientCert) + if conf.ClientCert != "" { + connInfo += fmt.Sprintf(" sslcert=%s", conf.ClientCert) } - if config.ClientKey != "" { - connInfo += fmt.Sprintf(" sslkey=%s", config.ClientKey) + if conf.ClientKey != "" { + connInfo += fmt.Sprintf(" sslkey=%s", conf.ClientKey) } return connInfo @@ -127,7 +127,6 @@ func (dbs *SQLdb) checkAndReconnectIfNeeded() { log.Debugln("Reconnecting to DB") dbs.DB, _ = sqlOpen("postgres", dbs.ConnInfo) } - } // GetFiles retrieves the file details @@ -135,7 +134,7 @@ var GetFiles = func(datasetID string) ([]*FileInfo, error) { var ( r []*FileInfo = nil err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { @@ -212,7 +211,6 @@ func (dbs *SQLdb) getFiles(datasetID string) ([]*FileInfo, error) { // Iterate rows for rows.Next() { - // Read rows into struct fi := &FileInfo{} err := rows.Scan(&fi.FileID, &fi.DatasetID, &fi.DisplayFileName, @@ -253,9 +251,9 @@ func (dbs *SQLdb) getFiles(datasetID string) ([]*FileInfo, error) { // CheckDataset checks if dataset name exists var CheckDataset = func(dataset string) (bool, error) { var ( - r bool = false + r = false err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { @@ -293,7 +291,7 @@ var GetDatasetInfo = func(datasetID string) (*DatasetInfo, error) { var ( d *DatasetInfo = nil err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { @@ -411,9 +409,9 @@ func (dbs *SQLdb) getDatasetFileInfo(datasetID, filePath string) (*FileInfo, err // CheckFilePermission checks if user has permissions to access the dataset the file is a part of var CheckFilePermission = func(fileID string) (string, error) { var ( - r string = "" - err error = nil - count int = 0 + r = "" + err error = nil + count = 0 ) for count < dbRetryTimes { @@ -469,7 +467,7 @@ var GetFile = func(fileID string) (*FileDownload, error) { var ( r *FileDownload = nil err error = nil - count int = 0 + count = 0 ) for count < dbRetryTimes { r, err = DB.getFile(fileID) diff --git a/sda-download/internal/database/database_test.go b/sda-download/internal/database/database_test.go index 6b9922282a..6f18c2ca18 100644 --- a/sda-download/internal/database/database_test.go +++ b/sda-download/internal/database/database_test.go @@ -42,7 +42,6 @@ func TestMain(m *testing.M) { } func TestBuildConnInfo(t *testing.T) { - s := buildConnInfo(testPgconf) assert.Equalf(t, s, testConnInfo, "Bad string for verify-full: '%s' while expecting '%s'", s, testConnInfo) @@ -55,31 +54,28 @@ func TestBuildConnInfo(t *testing.T) { assert.Equalf(t, s, "host=localhost port=42 user=user password=password dbname=database sslmode=disable", "Bad string for disable: %s", s) - } // testLogFatalf -func testLogFatalf(f string, args ...interface{}) { +func testLogFatalf(f string, args ...any) { s := fmt.Sprintf(f, args...) panic(s) } func TestCheckAndReconnect(t *testing.T) { - db, mock, _ := sqlmock.New(sqlmock.MonitorPingsOption(true)) - mock.ExpectPing().WillReturnError(fmt.Errorf("ping fail for testing bad conn")) + mock.ExpectPing().WillReturnError(errors.New("ping fail for testing bad conn")) err := CatchPanicCheckAndReconnect(SQLdb{db, ""}) assert.Error(t, err, "Should have received error from checkAndReconnectOnNeeded fataling") - } func CatchPanicCheckAndReconnect(db SQLdb) (err error) { defer func() { r := recover() if r != nil { - err = fmt.Errorf("Caught panic") + err = errors.New("Caught panic") } }() @@ -96,7 +92,7 @@ func CatchNewDBPanic() (err error) { defer func() { r := recover() if r != nil { - err = fmt.Errorf("Caught panic") + err = errors.New("Caught panic") } }() @@ -106,7 +102,6 @@ func CatchNewDBPanic() (err error) { } func TestNewDB(t *testing.T) { - // Test failure first sqlOpen = func(_ string, _ string) (*sql.DB, error) { @@ -141,7 +136,7 @@ func TestNewDB(t *testing.T) { return db, nil } - mock.ExpectPing().WillReturnError(fmt.Errorf("ping fail for testing")) + mock.ExpectPing().WillReturnError(errors.New("ping fail for testing")) err = CatchNewDBPanic() @@ -162,7 +157,6 @@ func TestNewDB(t *testing.T) { err = mock.ExpectationsWereMet() assert.Nilf(t, err, "there were unfulfilled expectations: %s", err) - } // Helper function for "simple" sql tests @@ -187,7 +181,6 @@ func sqlTesterHelper(t *testing.T, f func(sqlmock.Sqlmock, *SQLdb) error) error func TestClose(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - mock.ExpectClose() testDb.Close() @@ -199,7 +192,6 @@ func TestClose(t *testing.T) { func TestCheckFilePermission(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := "dataset1" query := ` SELECT datasets.stable_id FROM sda.file_dataset @@ -230,7 +222,6 @@ func TestCheckFilePermission(t *testing.T) { func TestCheckDataset(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := true query := `SELECT stable_id FROM sda.datasets WHERE stable_id = \$1` mock.ExpectQuery(query). @@ -256,7 +247,6 @@ func TestCheckDataset(t *testing.T) { func TestGetDatasetInfo(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := &DatasetInfo{ DatasetID: "dataset1", CreatedAt: "now", @@ -285,7 +275,6 @@ func TestGetDatasetInfo(t *testing.T) { func TestGetDatasetFileInfo(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := &FileInfo{ FileID: "file1", DatasetID: "dataset1", @@ -355,7 +344,6 @@ func TestGetDatasetFileInfo(t *testing.T) { func TestGetFile(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := &FileDownload{ ArchivePath: "file.txt", ArchiveSize: 32, @@ -403,7 +391,6 @@ func TestGetFile(t *testing.T) { func TestGetFiles(t *testing.T) { r := sqlTesterHelper(t, func(mock sqlmock.Sqlmock, testDb *SQLdb) error { - expected := []*FileInfo{} fileInfo := &FileInfo{ FileID: "file1", diff --git a/sda-download/internal/session/session.go b/sda-download/internal/session/session.go index 83b40519b7..5b38d08baa 100644 --- a/sda-download/internal/session/session.go +++ b/sda-download/internal/session/session.go @@ -74,7 +74,6 @@ var NewSessionKey = func() string { var sessionKey string exists := true for exists { - // Generate the key key := uuid.New().String() sessionKey = key diff --git a/sda-download/internal/session/session_test.go b/sda-download/internal/session/session_test.go index b760eed186..ffccc3fe06 100644 --- a/sda-download/internal/session/session_test.go +++ b/sda-download/internal/session/session_test.go @@ -9,7 +9,6 @@ import ( ) func TestNewSessionKey(t *testing.T) { - // Initialise a cache for testing cache, _ := InitialiseSessionCache() SessionCache = cache @@ -24,11 +23,9 @@ func TestNewSessionKey(t *testing.T) { if len(key) != expectedLen { t.Errorf("TestNewSessionKey failed, expected key length %d but received %d", expectedLen, len(key)) } - } func TestGetSetCache_Found(t *testing.T) { - // Set expiration time config.Config.Session.Expiration = time.Duration(60 * time.Second) @@ -50,11 +47,9 @@ func TestGetSetCache_Found(t *testing.T) { if expectedExists != exists { t.Errorf("TestGetSetCache_Found failed, expected %t but received %t", expectedExists, exists) } - } func TestGetSetCache_NotFound(t *testing.T) { - // Set expiration time config.Config.Session.Expiration = time.Duration(60 * time.Second) @@ -76,5 +71,4 @@ func TestGetSetCache_NotFound(t *testing.T) { if expectedExists != exists { t.Errorf("TestGetSetCache_NotFound failed, expected %t but received %t", expectedExists, exists) } - } diff --git a/sda-download/internal/storage/seekable.go b/sda-download/internal/storage/seekable.go index 6c572f3661..845d150a70 100644 --- a/sda-download/internal/storage/seekable.go +++ b/sda-download/internal/storage/seekable.go @@ -3,6 +3,7 @@ package storage import ( "bytes" + "errors" "fmt" "io" "strings" @@ -14,7 +15,6 @@ import ( // NewFileReadSeeker returns an io.ReadSeeker instance func (pb *posixBackend) NewFileReadSeeker(filePath string) (io.ReadSeekCloser, error) { - reader, err := pb.NewFileReader(filePath) if err != nil { return nil, err @@ -22,7 +22,7 @@ func (pb *posixBackend) NewFileReadSeeker(filePath string) (io.ReadSeekCloser, e seeker, ok := reader.(io.ReadSeekCloser) if !ok { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, errors.New("invalid posixBackend") } return seeker, nil @@ -85,7 +85,6 @@ func (r *s3Reader) pruneCache() { // Prune the cache keepfrom := len(r.local) - 8 r.local = r.local[keepfrom:] - } func (r *s3Reader) prefetchSize() int64 { @@ -177,10 +176,10 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { switch whence { case io.SeekStart: if offset < 0 { - return r.currentOffset, fmt.Errorf("Invalid offset %v- can't be negative when seeking from start", offset) + return r.currentOffset, fmt.Errorf("invalid offset %v- can't be negative when seeking from start", offset) } if offset > r.objectSize { - return r.currentOffset, fmt.Errorf("Invalid offset %v - beyond end of object (size %v)", offset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset %v - beyond end of object (size %v)", offset, r.objectSize) } r.currentOffset = offset @@ -190,10 +189,10 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { case io.SeekCurrent: if r.currentOffset+offset < 0 { - return r.currentOffset, fmt.Errorf("Invalid offset %v from %v would be be before start", offset, r.currentOffset) + return r.currentOffset, fmt.Errorf("invalid offset %v from %v would be be before start", offset, r.currentOffset) } if offset > r.objectSize { - return r.currentOffset, fmt.Errorf("Invalid offset - %v from %v would end up beyond of object %v", offset, r.currentOffset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset - %v from %v would end up beyond of object %v", offset, r.currentOffset, r.objectSize) } r.currentOffset += offset @@ -203,10 +202,10 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { case io.SeekEnd: if r.objectSize+offset < 0 { - return r.currentOffset, fmt.Errorf("Invalid offset %v from end in %v bytes object, would be before file start", offset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset %v from end in %v bytes object, would be before file start", offset, r.objectSize) } if r.objectSize+offset > r.objectSize { - return r.currentOffset, fmt.Errorf("Invalid offset %v from end in %v bytes object", offset, r.objectSize) + return r.currentOffset, fmt.Errorf("invalid offset %v from end in %v bytes object", offset, r.objectSize) } r.currentOffset = r.objectSize + offset @@ -215,7 +214,7 @@ func (r *s3Reader) Seek(offset int64, whence int) (int64, error) { return r.currentOffset, nil } - return r.currentOffset, fmt.Errorf("Bad whence") + return r.currentOffset, errors.New("bad whence") } // removeFromOutstanding removes a prefetch from the list of outstanding prefetches once it's no longer active @@ -354,7 +353,7 @@ func (r *s3Reader) Read(dst []byte) (n int, err error) { responseRange := fmt.Sprintf("bytes %d-", r.currentOffset) if object.ContentRange == nil || !strings.HasPrefix(*object.ContentRange, responseRange) { - return 0, fmt.Errorf("Unexpected content range %v - expected prefix %v", object.ContentRange, responseRange) + return 0, fmt.Errorf("unexpected content range %v - expected prefix %v", object.ContentRange, responseRange) } b := bytes.Buffer{} @@ -383,7 +382,6 @@ type seekableMultiReader struct { // SeekableMultiReader constructs a multireader that supports seeking. Requires // all passed readers to be seekable func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { - r := make([]io.Reader, len(readers)) sizes := make([]int64, len(readers)) @@ -393,24 +391,22 @@ func SeekableMultiReader(readers ...io.Reader) (io.ReadSeeker, error) { for i, reader := range readers { seeker, ok := reader.(io.ReadSeeker) if !ok { - return nil, fmt.Errorf("Reader %d to SeekableMultiReader is not seekable", i) + return nil, fmt.Errorf("reader %d to SeekableMultiReader is not seekable", i) } size, err := seeker.Seek(0, io.SeekEnd) if err != nil { - return nil, fmt.Errorf("Size determination failed for reader %d to SeekableMultiReader: %v", i, err) + return nil, fmt.Errorf("size determination failed for reader %d to SeekableMultiReader: %v", i, err) } sizes[i] = size totalSize += size - } return &seekableMultiReader{r, sizes, 0, totalSize}, nil } func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { - switch whence { case io.SeekStart: r.currentOffset = offset @@ -418,21 +414,17 @@ func (r *seekableMultiReader) Seek(offset int64, whence int) (int64, error) { r.currentOffset += offset case io.SeekEnd: r.currentOffset = r.totalSize + offset - default: - return 0, fmt.Errorf("Unsupported whence") - + return 0, errors.New("unsupported whence") } return r.currentOffset, nil } func (r *seekableMultiReader) Read(dst []byte) (int, error) { - var readerStartAt int64 for i, reader := range r.readers { - if r.currentOffset < readerStartAt { // We want data from a previous reader (? HELP ?) readerStartAt += r.sizes[i] @@ -451,12 +443,12 @@ func (r *seekableMultiReader) Read(dst []byte) (int, error) { seekable, ok := reader.(io.ReadSeeker) if !ok { - return 0, fmt.Errorf("Expected seekable reader but changed") + return 0, errors.New("expected seekable reader but changed") } _, err := seekable.Seek(r.currentOffset-int64(readerStartAt), 0) if err != nil { - return 0, fmt.Errorf("Unexpected error while seeking: %v", err) + return 0, fmt.Errorf("unexpected error while seeking: %v", err) } n, err := seekable.Read(dst) diff --git a/sda-download/internal/storage/seekable_test.go b/sda-download/internal/storage/seekable_test.go index adbdf7b33a..7e9211a277 100644 --- a/sda-download/internal/storage/seekable_test.go +++ b/sda-download/internal/storage/seekable_test.go @@ -15,9 +15,7 @@ import ( ) func TestSeekableBackend(t *testing.T) { - for _, backendType := range []string{posixType, s3Type} { - testConf.Type = backendType backend, err := NewBackend(testConf) @@ -94,7 +92,6 @@ func TestSeekableBackend(t *testing.T) { _, err = seeker.Seek(0, 4) assert.NotNil(t, err, "Seek didn't fail when it should") - } offset, err := seeker.Seek(15, io.SeekStart) @@ -171,7 +168,6 @@ func TestSeekableBackend(t *testing.T) { } func TestS3SeekablePrefetchSize(t *testing.T) { - testConf.Type = s3Type chunkSize := testConf.S3.Chunksize testConf.S3.Chunksize = 5 * 1024 * 1024 @@ -297,7 +293,6 @@ func TestS3SeekableSpecial(t *testing.T) { } func TestSeekableMultiReader(t *testing.T) { - readers := make([]io.Reader, 10) for i := 0; i < 10; i++ { readers[i] = bytes.NewReader(writeData) diff --git a/sda-download/internal/storage/storage.go b/sda-download/internal/storage/storage.go index 358287e440..9c499b81e0 100644 --- a/sda-download/internal/storage/storage.go +++ b/sda-download/internal/storage/storage.go @@ -4,6 +4,7 @@ package storage import ( "crypto/tls" "crypto/x509" + "errors" "fmt" "io" "net/http" @@ -74,7 +75,7 @@ func newPosixBackend(config posixConf) (*posixBackend, error) { // NewFileReader returns an io.Reader instance func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if pb == nil { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -90,7 +91,7 @@ func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter returns an io.Writer instance func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if pb == nil { - return nil, fmt.Errorf("Invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) @@ -106,7 +107,7 @@ func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of the file func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { if pb == nil { - return 0, fmt.Errorf("Invalid posixBackend") + return 0, errors.New("invalid posixBackend") } stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -162,10 +163,9 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { if err != nil { if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && aerr.Code() != s3.ErrCodeBucketAlreadyExists { - log.Error("Unexpected issue while creating bucket", err) + log.Error("unexpected issue while creating bucket", err) } } } @@ -192,7 +192,7 @@ func newS3Backend(config S3Conf) (*s3Backend, error) { // NewFileReader returns an io.Reader instance func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { if sb == nil { - return nil, fmt.Errorf("Invalid s3Backend") + return nil, errors.New("invalid s3Backend") } r, err := sb.Client.GetObject(&s3.GetObjectInput{ @@ -226,12 +226,11 @@ func (sb *s3Backend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter uploads the contents of an io.Reader to a S3 bucket func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { if sb == nil { - return nil, fmt.Errorf("Invalid s3Backend") + return nil, errors.New("invalid s3Backend") } reader, writer := io.Pipe() go func() { - _, err := sb.Uploader.Upload(&s3manager.UploadInput{ Body: reader, Bucket: aws.String(sb.Bucket), @@ -250,7 +249,7 @@ func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { // GetFileSize returns the size of a specific object func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { if sb == nil { - return 0, fmt.Errorf("Invalid s3Backend") + return 0, errors.New("invalid s3Backend") } r, err := sb.Client.HeadObject(&s3.HeadObjectInput{ @@ -272,7 +271,6 @@ func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { Key: aws.String(filePath)}) time.Sleep(1 * time.Second) - } if err != nil { @@ -299,7 +297,7 @@ func transportConfigS3(config S3Conf) http.RoundTripper { if config.Cacert != "" { cacert, e := os.ReadFile(config.Cacert) // #nosec this file comes from our config if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) + log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main } if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { log.Debug("no certs appended, using system certs only") diff --git a/sda-download/internal/storage/storage_test.go b/sda-download/internal/storage/storage_test.go index 7de3720da8..6bd0921a35 100644 --- a/sda-download/internal/storage/storage_test.go +++ b/sda-download/internal/storage/storage_test.go @@ -80,7 +80,6 @@ func doCleanup() { cleanupFiles = cleanupFilesBack[0:0] } func TestNewBackend(t *testing.T) { - testConf.Type = posixType p, err := NewBackend(testConf) assert.Nil(t, err, "Backend posix failed") @@ -100,7 +99,6 @@ func TestNewBackend(t *testing.T) { } func TestMain(m *testing.M) { - err := setupFakeS3() if err != nil { @@ -113,7 +111,6 @@ func TestMain(m *testing.M) { os.Exit(ret) } func TestPosixBackend(t *testing.T) { - defer doCleanup() testConf.Type = posixType backend, err := NewBackend(testConf) @@ -187,7 +184,6 @@ func TestPosixBackend(t *testing.T) { assert.NotZero(t, buf.Len(), "Expected warning missing") buf.Reset() - } func setupFakeS3() (err error) { @@ -226,7 +222,6 @@ func setupFakeS3() (err error) { if err != nil { if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() != s3.ErrCodeBucketAlreadyOwnedByYou && aerr.Code() != s3.ErrCodeBucketAlreadyExists { log.Error("Unexpected issue while creating bucket: ", err) @@ -241,7 +236,6 @@ func setupFakeS3() (err error) { } func TestS3Fail(t *testing.T) { - testConf.Type = s3Type tmp := testConf.S3.URL @@ -296,7 +290,6 @@ func TestPOSIXFail(t *testing.T) { } func TestS3Backend(t *testing.T) { - testConf.Type = s3Type backend, err := NewBackend(testConf) assert.Nil(t, err, "Backend failed") @@ -360,5 +353,4 @@ func TestS3Backend(t *testing.T) { } log.SetOutput(os.Stdout) - } diff --git a/sda-download/pkg/auth/auth.go b/sda-download/pkg/auth/auth.go index 0862c17c4f..2c403deb15 100644 --- a/sda-download/pkg/auth/auth.go +++ b/sda-download/pkg/auth/auth.go @@ -30,12 +30,12 @@ type OIDCDetails struct { } // GetOIDCDetails requests OIDC configuration information -func GetOIDCDetails(url string) (OIDCDetails, error) { - log.Debugf("requesting OIDC config from %s", url) +func GetOIDCDetails(uri string) (OIDCDetails, error) { + log.Debugf("requesting OIDC config from %s", uri) // Prepare response body struct var u OIDCDetails // Do request - response, err := request.MakeRequest("GET", url, nil, nil) + response, err := request.MakeRequest("GET", uri, nil, nil) if err != nil { log.Errorf("request failed, %s", err) @@ -49,7 +49,7 @@ func GetOIDCDetails(url string) (OIDCDetails, error) { return u, err } defer response.Body.Close() - log.Debugf("received OIDC config %s from %s", u, url) + log.Debugf("received OIDC config %s from %s", u, uri) return u, nil } @@ -123,13 +123,12 @@ var GetToken = func(headers http.Header) (string, int, error) { // Check that header contains a token string var token string - if len(headerParts) == 2 { - token = headerParts[1] - } else { + if len(headerParts) != 2 { log.Debug("authorization check failed") return "", 400, errors.New("token string is missing from authorization header") } + token = headerParts[1] log.Debug("access token found") return token, 0, nil @@ -185,7 +184,6 @@ var GetPermissions = func(visas Visas) []string { // Iterate visas for _, v := range visas.Visa { - // Check that visa is of type ControlledAccessGrants if checkVisaType(v, "ControlledAccessGrants") { // Check that visa is valid and return visa token @@ -195,7 +193,6 @@ var GetPermissions = func(visas Visas) []string { datasets = getDatasets(verifiedVisa, datasets) } } - } log.Debugf("matched datasets: %s", datasets) @@ -204,7 +201,6 @@ var GetPermissions = func(visas Visas) []string { } func checkVisaType(visa string, visaType string) bool { - log.Debug("checking visa type") unknownToken, err := jwt.Parse([]byte(visa), jwt.WithVerify(false)) @@ -235,7 +231,6 @@ func checkVisaType(visa string, visaType string) bool { log.Debug("visa type check passed") return true - } func validateVisa(visa string) (jwt.Token, bool) { diff --git a/sda-download/pkg/auth/auth_test.go b/sda-download/pkg/auth/auth_test.go index 41f33ba4e5..76da2703bd 100644 --- a/sda-download/pkg/auth/auth_test.go +++ b/sda-download/pkg/auth/auth_test.go @@ -14,7 +14,6 @@ import ( ) func TestGetOIDCDetails_Fail_MakeRequest(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -46,7 +45,6 @@ func TestGetOIDCDetails_Fail_MakeRequest(t *testing.T) { } func TestGetOIDCDetails_Fail_JSONDecode(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -86,7 +84,6 @@ func TestGetOIDCDetails_Fail_JSONDecode(t *testing.T) { } func TestGetOIDCDetails_Success(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -125,7 +122,6 @@ func TestGetOIDCDetails_Success(t *testing.T) { } func TestGetToken_Fail_EmptyHeader(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "") @@ -145,11 +141,9 @@ func TestGetToken_Fail_EmptyHeader(t *testing.T) { if err.Error() != expectedError { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected %s, received %s", expectedError, err.Error()) } - } func TestGetToken_X_Amz_Security_token(t *testing.T) { - // Expected results expectedToken := "token" expectedCode := 0 @@ -168,11 +162,9 @@ func TestGetToken_X_Amz_Security_token(t *testing.T) { if err != nil { t.Errorf("TestGetToken_X_Amz_Security_token failed, expected nil, received %v", err) } - } func TestGetToken_Fail_WrongScheme(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "Basic token") @@ -192,11 +184,9 @@ func TestGetToken_Fail_WrongScheme(t *testing.T) { if err.Error() != expectedError { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected %s, received %s", expectedError, err.Error()) } - } func TestGetToken_Fail_MissingToken(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "Bearer") @@ -216,11 +206,9 @@ func TestGetToken_Fail_MissingToken(t *testing.T) { if err.Error() != expectedError { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected %s, received %s", expectedError, err.Error()) } - } func TestGetToken_Success(t *testing.T) { - // Test case header := http.Header{} header.Add("Authorization", "Bearer token") @@ -239,11 +227,9 @@ func TestGetToken_Success(t *testing.T) { if err != nil { t.Errorf("TestGetToken_Fail_EmptyHeader failed, expected nil, received %v", err) } - } func TestGetVisas_Fail_MakeRequest(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -268,11 +254,9 @@ func TestGetVisas_Fail_MakeRequest(t *testing.T) { // Return mock functions to originals request.MakeRequest = originalMakeRequest - } func TestGetVisas_Fail_JSONDecode(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -305,11 +289,9 @@ func TestGetVisas_Fail_JSONDecode(t *testing.T) { // Return mock functions to originals request.MakeRequest = originalMakeRequest - } func TestGetVisas_Success(t *testing.T) { - // Save original to-be-mocked functions originalMakeRequest := request.MakeRequest @@ -342,11 +324,9 @@ func TestGetVisas_Success(t *testing.T) { // Return mock functions to originals request.MakeRequest = originalMakeRequest - } func TestValidateTrustedIss(t *testing.T) { - // this also tests checkIss trustedList := []config.TrustedISS([]config.TrustedISS{{ISS: "https://demo.example", JKU: "https://mockauth:8000/idp/profile/oidc/keyset"}, {ISS: "https://demo1.example", JKU: "https://mockauth:8000/idp/profile/oidc/keyset"}}) @@ -364,7 +344,6 @@ func TestValidateTrustedIss(t *testing.T) { } func TestValidateTrustedIssNoConfig(t *testing.T) { - ok := validateTrustedIss(nil, "https://demo.example", "https://mockauth:8000/idp/profile/oidc/keyset") assert.True(t, ok, "this should be true") diff --git a/sda-download/pkg/request/request.go b/sda-download/pkg/request/request.go index 5aba81fb7c..49cca9c61f 100644 --- a/sda-download/pkg/request/request.go +++ b/sda-download/pkg/request/request.go @@ -56,7 +56,7 @@ var HTTPNewRequest = http.NewRequest var MakeRequest = func(method string, url string, headers map[string]string, body []byte) (*http.Response, error) { var ( response *http.Response - count int = 0 + count = 0 ) // Build HTTP request diff --git a/sda-download/pkg/request/request_test.go b/sda-download/pkg/request/request_test.go index 9de6e7d060..d34c26d606 100644 --- a/sda-download/pkg/request/request_test.go +++ b/sda-download/pkg/request/request_test.go @@ -41,7 +41,6 @@ func TestInitialiseClient(t *testing.T) { } func TestMakeRequest_Fail_HTTPNewRequest(t *testing.T) { - // Save original to-be-mocked functions originalHTTPMakeRequest := HTTPNewRequest @@ -68,11 +67,9 @@ func TestMakeRequest_Fail_HTTPNewRequest(t *testing.T) { // Return mock functions to originals HTTPNewRequest = originalHTTPMakeRequest - } func TestMakeRequest_Fail_StatusCode(t *testing.T) { - // Create mock client client := newTestClient(func(_ *http.Request) *http.Response { return &http.Response{ @@ -116,11 +113,9 @@ func TestMakeRequest_Fail_StatusCode(t *testing.T) { // Return mock functions to originals HTTPNewRequest = originalHTTPMakeRequest - } func TestMakeRequest_Success(t *testing.T) { - // Create mock client client := newTestClient(func(_ *http.Request) *http.Response { return &http.Response{ @@ -167,5 +162,4 @@ func TestMakeRequest_Success(t *testing.T) { // Return mock functions to originals HTTPNewRequest = originalHTTPMakeRequest - } diff --git a/sda-sftp-inbox/pom.xml b/sda-sftp-inbox/pom.xml index b428127b05..e52ee1ebbd 100644 --- a/sda-sftp-inbox/pom.xml +++ b/sda-sftp-inbox/pom.xml @@ -14,7 +14,7 @@ org.springframework.boot spring-boot-starter-parent - 3.4.1 + 3.4.5 @@ -24,6 +24,18 @@ 19 + + + + org.junit + junit-bom + 5.12.2 + pom + import + + + + net.i2p.crypto @@ -86,32 +98,32 @@ org.apache.sshd sshd-sftp - 2.14.0 + 2.15.0 commons-io commons-io - 2.18.0 + 2.19.0 com.amazonaws aws-java-sdk-s3 - 1.12.780 + 1.12.783 com.google.guava guava - 33.4.0-jre + 33.4.8-jre net.logstash.logback logstash-logback-encoder - 8.0 + 8.1 org.bouncycastle bcprov-jdk18on - 1.79 + 1.80 @@ -149,7 +161,7 @@ org.yaml snakeyaml - 2.3 + 2.4 junit @@ -160,10 +172,8 @@ org.junit.vintage junit-vintage-engine - 5.11.4 test - @@ -174,12 +184,12 @@ maven-surefire-plugin - 3.5.2 + 3.5.3 org.apache.maven.plugins maven-failsafe-plugin - 3.5.2 + 3.5.3 diff --git a/sda/Dockerfile b/sda/Dockerfile index c0d360e877..c3415e736e 100644 --- a/sda/Dockerfile +++ b/sda/Dockerfile @@ -1,4 +1,4 @@ -FROM "golang:${GOLANG_VERSION:-1.23}-bullseye" AS Build +FROM "golang:${GOLANG_VERSION:-1.24}-bullseye" AS Build ENV GO111MODULE=on ENV GOPATH=$PWD ENV CGO_ENABLED=0 diff --git a/sda/cmd/api/api.go b/sda/cmd/api/api.go index 338b9cf007..b57981b1a7 100644 --- a/sda/cmd/api/api.go +++ b/sda/cmd/api/api.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "math" "net/http" @@ -83,15 +84,14 @@ func main() { log.Fatalln(err) } } - } -func setup(config *config.Config) *http.Server { - model, _ := model.NewModelFromString(jsonadapter.Model) - e, err := casbin.NewEnforcer(model, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) +func setup(conf *config.Config) *http.Server { + m, _ := model.NewModelFromString(jsonadapter.Model) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) if err != nil { shutdown() - log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) + log.Fatalf("error when setting up RBAC enforcer, reason %s", err.Error()) // nolint # FIXME Fatal should only be called from main } r := gin.Default() @@ -117,7 +117,7 @@ func setup(config *config.Config) *http.Server { cfg := &tls.Config{MinVersion: tls.VersionTLS12} srv := &http.Server{ - Addr: config.API.Host + ":" + fmt.Sprint(config.API.Port), + Addr: conf.API.Host + ":" + fmt.Sprint(conf.API.Port), Handler: r, TLSConfig: cfg, TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), @@ -175,8 +175,8 @@ func readinessResponse(c *gin.Context) { } } - if DBRes := checkDB(Conf.API.DB, 5*time.Millisecond); DBRes != nil { - log.Debugf("DB connection error :%v", DBRes) + if dbRes := checkDB(Conf.API.DB, 5*time.Millisecond); dbRes != nil { + log.Debugf("DB connection error :%v", dbRes) Conf.API.DB.Reconnect() statusCode = http.StatusServiceUnavailable } @@ -184,14 +184,14 @@ func readinessResponse(c *gin.Context) { c.JSON(statusCode, "") } -func checkDB(database *database.SDAdb, timeout time.Duration) error { +func checkDB(db *database.SDAdb, timeout time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() - if database.DB == nil { - return fmt.Errorf("database is nil") + if db.DB == nil { + return errors.New("database is nil") } - return database.DB.PingContext(ctx) + return db.DB.PingContext(ctx) } func rbac(e *casbin.Enforcer) gin.HandlerFunc { @@ -216,7 +216,7 @@ func rbac(e *casbin.Enforcer) gin.HandlerFunc { return } - log.Debugln("authoriozed") + log.Debugln("authorized") } } @@ -232,7 +232,7 @@ func getFiles(c *gin.Context) { return } - files, err := Conf.API.DB.GetUserFiles(token.Subject()) + files, err := Conf.API.DB.GetUserFiles(token.Subject(), false) if err != nil { // something went wrong with querying or parsing rows c.JSON(502, err.Error()) @@ -268,10 +268,9 @@ func ingestFile(c *gin.Context) { corrID, err := Conf.API.DB.GetCorrID(ingest.User, ingest.FilePath, "") if err != nil { - switch { - case corrID == "": + if corrID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - default: + } else { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) } @@ -320,8 +319,8 @@ func deleteFile(c *gin.Context) { return } - var RetryTimes = 5 - for count := 1; count <= RetryTimes; count++ { + var retryTimes = 5 + for count := 1; count <= retryTimes; count++ { err = inbox.RemoveFile(filePath) if err == nil { break @@ -361,10 +360,9 @@ func setAccession(c *gin.Context) { corrID, err := Conf.API.DB.GetCorrID(accession.User, accession.FilePath, "") if err != nil { - switch { - case corrID == "": + if corrID == "" { c.AbortWithStatusJSON(http.StatusBadRequest, err.Error()) - default: + } else { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) } @@ -544,7 +542,7 @@ func listUserFiles(c *gin.Context) { username = strings.TrimPrefix(username, "/") username = strings.TrimSuffix(username, "/files") log.Debugln(username) - files, err := Conf.API.DB.GetUserFiles(username) + files, err := Conf.API.DB.GetUserFiles(username, true) if err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, err.Error()) diff --git a/sda/cmd/api/api.md b/sda/cmd/api/api.md index abccbb8c62..0501f3c583 100644 --- a/sda/cmd/api/api.md +++ b/sda/cmd/api/api.md @@ -121,7 +121,7 @@ Admin endpoints are only available to a set of whitelisted users specified in th Example: ```bash - curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01"}' https://HOSTNAME/dataset/create + curl -H "Authorization: Bearer $token" -H "Content-Type: application/json" -X POST -d '{"accession_ids": ["my-id-01", "my-id-02"], "dataset_id": "my-dataset-01", "user": "user@example.org"}' https://HOSTNAME/dataset/create ``` - `/dataset/release/*dataset` diff --git a/sda/cmd/api/api_test.go b/sda/cmd/api/api_test.go index 946d019d22..c8a38b4387 100644 --- a/sda/cmd/api/api_test.go +++ b/sda/cmd/api/api_test.go @@ -145,12 +145,12 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to rabbitmq: %s", err) } - RSAPath, _ := os.MkdirTemp("", "RSA") - if err := helper.CreateRSAkeys(RSAPath, RSAPath); err != nil { + rsaPath, _ := os.MkdirTemp("", "RSA") + if err := helper.CreateRSAkeys(rsaPath, rsaPath); err != nil { log.Panic("Failed to create RSA keys") } - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { log.Panic("Failed to create EC keys") } @@ -179,10 +179,10 @@ func TestMain(m *testing.M) { } OIDCport, _ = strconv.Atoi(oidc.GetPort("8080/tcp")) - OIDCHostAndPort := oidc.GetHostPort("8080/tcp") + oidcHostAndPort := oidc.GetHostPort("8080/tcp") client = http.Client{Timeout: 5 * time.Second} - req, err = http.NewRequest(http.MethodGet, "http://"+OIDCHostAndPort+"/jwk", http.NoBody) + req, err = http.NewRequest(http.MethodGet, "http://"+oidcHostAndPort+"/jwk", http.NoBody) if err != nil { log.Panic(err) } @@ -223,8 +223,8 @@ func TestMain(m *testing.M) { log.Fatalf("Could not purge resource: %s", err) } // cleanup temp files - _ = os.RemoveAll(ECPath) - _ = os.RemoveAll(RSAPath) + _ = os.RemoveAll(ecPath) + _ = os.RemoveAll(rsaPath) os.Exit(code) } @@ -240,7 +240,7 @@ type TestSuite struct { User string } -func (suite *TestSuite) TestShutdown() { +func (s *TestSuite) TestShutdown() { Conf = &config.Config{} Conf.Broker = broker.MQConf{ Host: "localhost", @@ -251,7 +251,7 @@ func (suite *TestSuite) TestShutdown() { Vhost: "/sda", } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Database = database.DBConf{ Host: "localhost", @@ -262,20 +262,20 @@ func (suite *TestSuite) TestShutdown() { SslMode: "disable", } Conf.API.DB, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // make sure all conections are alive - assert.Equal(suite.T(), false, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), false, Conf.API.MQ.Connection.IsClosed()) - assert.Equal(suite.T(), nil, Conf.API.DB.DB.Ping()) + assert.Equal(s.T(), false, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), false, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), nil, Conf.API.DB.DB.Ping()) shutdown() - assert.Equal(suite.T(), true, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), true, Conf.API.MQ.Connection.IsClosed()) - assert.Equal(suite.T(), "sql: database is closed", Conf.API.DB.DB.Ping().Error()) + assert.Equal(s.T(), true, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), true, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), "sql: database is closed", Conf.API.DB.DB.Ping().Error()) } -func (suite *TestSuite) TestReadinessResponse() { +func (s *TestSuite) TestReadinessResponse() { gin.SetMode(gin.ReleaseMode) r := gin.Default() r.GET("/ready", readinessResponse) @@ -283,78 +283,78 @@ func (suite *TestSuite) TestReadinessResponse() { defer ts.Close() res, err := http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the connection to force a reconnection Conf.API.MQ.Connection.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the channel to force a reconneciton Conf.API.MQ.Channel.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close DB connection to force a reconnection Conf.API.DB.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() } // Initialise configuration and create jwt keys -func (suite *TestSuite) SetupSuite() { +func (s *TestSuite) SetupSuite() { log.SetLevel(log.DebugLevel) - suite.Path = "/tmp/keys/" - suite.KeyName = "example.demo" + s.Path = "/tmp/keys/" + s.KeyName = "example.demo" log.Print("Creating JWT keys for testing") - privpath, pubpath, err := helper.MakeFolder(suite.Path) - assert.NoError(suite.T(), err) - suite.PrivatePath = privpath - suite.PublicPath = pubpath + privpath, pubpath, err := helper.MakeFolder(s.Path) + assert.NoError(s.T(), err) + s.PrivatePath = privpath + s.PublicPath = pubpath err = helper.CreateRSAkeys(privpath, pubpath) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // Create a valid token for queries to the API - prKeyParsed, err := helper.ParsePrivateRSAKey(suite.PrivatePath, "/rsa") - assert.NoError(suite.T(), err) + prKeyParsed, err := helper.ParsePrivateRSAKey(s.PrivatePath, "/rsa") + assert.NoError(s.T(), err) token, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) - suite.Token = token + assert.NoError(s.T(), err) + s.Token = token user, ok := helper.DefaultTokenClaims["sub"].(string) - assert.True(suite.T(), ok) - suite.User = user + assert.True(s.T(), ok) + s.User = user c := &config.Config{} - ServerConf := config.ServerConfig{} - ServerConf.Jwtpubkeypath = suite.PublicPath - c.Server = ServerConf + serverConf := config.ServerConfig{} + serverConf.Jwtpubkeypath = s.PublicPath + c.Server = serverConf Conf = c @@ -368,7 +368,7 @@ func (suite *TestSuite) SetupSuite() { SslMode: "disable", } Conf.API.DB, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Broker = broker.MQConf{ Host: "localhost", @@ -379,9 +379,9 @@ func (suite *TestSuite) SetupSuite() { Vhost: "/sda", } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) - suite.RBAC = []byte(`{"policy":[{"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, + s.RBAC = []byte(`{"policy":[{"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, {"role":"submission","path":"/dataset/release/*dataset","action":"POST"}, {"role":"submission","path":"/file/ingest","action":"POST"}, @@ -392,10 +392,10 @@ func (suite *TestSuite) SetupSuite() { "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"admin"}]}`) } -func (suite *TestSuite) TearDownSuite() { - assert.NoError(suite.T(), os.RemoveAll(suite.Path)) +func (s *TestSuite) TearDownSuite() { + assert.NoError(s.T(), os.RemoveAll(s.Path)) } -func (suite *TestSuite) SetupTest() { +func (s *TestSuite) SetupTest() { Conf.Database = database.DBConf{ Host: "localhost", Port: dbPort, @@ -405,55 +405,56 @@ func (suite *TestSuite) SetupTest() { SslMode: "disable", } Conf.API.DB, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) _, err = Conf.API.DB.DB.Exec("TRUNCATE sda.files CASCADE") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Broker = broker.MQConf{ - Host: "localhost", - Port: mqPort, - User: "guest", - Password: "guest", - Exchange: "sda", - Vhost: "/sda", + Host: "localhost", + Port: mqPort, + User: "guest", + Password: "guest", + Exchange: "sda", + Vhost: "/sda", + SchemasPath: "../../schemas/isolated", } Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // purge the queue so that the test passes when all tests are run as well as when run standalone. client := http.Client{Timeout: 30 * time.Second} for _, queue := range []string{"accession", "archived", "ingest", "mappings", "verified"} { req, err := http.NewRequest(http.MethodDelete, "http://"+BrokerAPI+"/api/queues/sda/"+queue+"/contents", http.NoBody) - assert.NoError(suite.T(), err, "failed to generate query") + assert.NoError(s.T(), err, "failed to generate query") req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") res.Body.Close() } } -func (suite *TestSuite) TestDatabasePingCheck() { +func (s *TestSuite) TestDatabasePingCheck() { emptyDB := database.SDAdb{} - assert.Error(suite.T(), checkDB(&emptyDB, 1*time.Second), "nil DB should fail") + assert.Error(s.T(), checkDB(&emptyDB, 1*time.Second), "nil DB should fail") db, err := database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), checkDB(db, 1*time.Second), "ping should succeed") + assert.NoError(s.T(), err) + assert.NoError(s.T(), checkDB(db, 1*time.Second), "ping should succeed") } -func (suite *TestSuite) TestAPIGetFiles() { +func (s *TestSuite) TestAPIGetFiles() { gin.SetMode(gin.ReleaseMode) r := gin.Default() m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r.GET("/files", rbac(e), func(c *gin.Context) { @@ -464,85 +465,91 @@ func (suite *TestSuite) TestAPIGetFiles() { filesURL := ts.URL + "/files" client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("GET", filesURL, nil) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) // Test query when no files is in db resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() filesData := []database.SubmissionFileInfo{} err = json.NewDecoder(resp.Body).Decode(&filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 0) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 0) + assert.NoError(s.T(), err) // Insert a file and make sure it is listed - file1 := fmt.Sprintf("/%v/TestAPIGetFiles.c4gh", suite.User) - fileID, err := Conf.API.DB.RegisterFile(file1, suite.User) - assert.NoError(suite.T(), err, "failed to register file in database") + file1 := fmt.Sprintf("/%v/TestAPIGetFiles.c4gh", s.User) + fileID, err := Conf.API.DB.RegisterFile(file1, s.User) + assert.NoError(s.T(), err, "failed to register file in database") corrID := uuid.New().String() latestStatus := "uploaded" - err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, suite.User, "{}", "{}") - assert.NoError(suite.T(), err, "got (%v) when trying to update file status") + err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "got (%v) when trying to update file status") resp, err = client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() err = json.NewDecoder(resp.Body).Decode(&filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 1) - assert.Equal(suite.T(), filesData[0].Status, latestStatus) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 1) + assert.Equal(s.T(), filesData[0].Status, latestStatus) + assert.NoError(s.T(), err) // Update the file's status and make sure only the lastest status is listed + err = Conf.API.DB.SetAccessionID("stableID", fileID) + if err != nil { + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), "stableID", fileID) + } latestStatus = "ready" - err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, suite.User, "{}", "{}") - assert.NoError(suite.T(), err, "got (%v) when trying to update file status") + err = Conf.API.DB.UpdateFileEventLog(fileID, latestStatus, corrID, s.User, "{}", "{}") + assert.NoError(s.T(), err, "got (%v) when trying to update file status") resp, err = client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + assert.NoError(s.T(), err) + assert.NotContains(s.T(), string(data), "accessionID") - err = json.NewDecoder(resp.Body).Decode(&filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 1) - assert.Equal(suite.T(), filesData[0].Status, latestStatus) - - assert.NoError(suite.T(), err) + err = json.Unmarshal(data, &filesData) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 1) + assert.Equal(s.T(), filesData[0].Status, latestStatus) + assert.Empty(s.T(), filesData[0].AccessionID) // Insert a second file and make sure it is listed - file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", suite.User) - _, err = Conf.API.DB.RegisterFile(file2, suite.User) - assert.NoError(suite.T(), err, "failed to register file in database") + file2 := fmt.Sprintf("/%v/TestAPIGetFiles2.c4gh", s.User) + _, err = Conf.API.DB.RegisterFile(file2, s.User) + assert.NoError(s.T(), err, "failed to register file in database") resp, err = client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() err = json.NewDecoder(resp.Body).Decode(&filesData) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), len(filesData), 2) + assert.NoError(s.T(), err) + assert.Equal(s.T(), len(filesData), 2) for _, fileInfo := range filesData { switch fileInfo.InboxPath { case file1: - assert.Equal(suite.T(), fileInfo.Status, latestStatus) + assert.Equal(s.T(), fileInfo.Status, latestStatus) case file2: - assert.Equal(suite.T(), fileInfo.Status, "registered") + assert.Equal(s.T(), fileInfo.Status, "registered") } } - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) } func TestApiTestSuite(t *testing.T) { @@ -553,23 +560,23 @@ func testEndpoint(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) } -func (suite *TestSuite) TestRBAC() { +func (s *TestSuite) TestRBAC() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/c4gh-keys/list", nil) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/c4gh-keys/list", rbac(e), testEndpoint) @@ -578,30 +585,30 @@ func (suite *TestSuite) TestRBAC() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "ok") + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "ok") } -func (suite *TestSuite) TestRBAC_badUser() { +func (s *TestSuite) TestRBAC_badUser() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.API.RBACpolicy = []byte(`{"policy":[{"role":"admin","path":"/admin/*","action":"(GET)|(POST)|(PUT)"}], "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy","rolebinding":"submission"}]}`) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&Conf.API.RBACpolicy)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/admin/list-users", nil) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/admin/list-users", rbac(e), testEndpoint) @@ -609,21 +616,21 @@ func (suite *TestSuite) TestRBAC_badUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusUnauthorized, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) } -func (suite *TestSuite) TestRBAC_noToken() { +func (s *TestSuite) TestRBAC_noToken() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&[]byte{})) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() @@ -636,27 +643,27 @@ func (suite *TestSuite) TestRBAC_noToken() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusUnauthorized, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "no access token supplied") + assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "no access token supplied") } -func (suite *TestSuite) TestRBAC_emptyPolicy() { +func (s *TestSuite) TestRBAC_emptyPolicy() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&[]byte{})) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("GET", "/files", nil) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/files", rbac(e), testEndpoint) @@ -665,30 +672,30 @@ func (suite *TestSuite) TestRBAC_emptyPolicy() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusUnauthorized, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "not authorized") + assert.Equal(s.T(), http.StatusUnauthorized, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "not authorized") } -func (suite *TestSuite) TestIngestFile() { +func (s *TestSuite) TestIngestFile() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" type ingest struct { @@ -699,7 +706,7 @@ func (suite *TestSuite) TestIngestFile() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/ingest", bytes.NewBuffer(ingestMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/ingest", rbac(e), ingestFile) @@ -707,7 +714,7 @@ func (suite *TestSuite) TestIngestFile() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -715,29 +722,29 @@ func (suite *TestSuite) TestIngestFile() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/ingest", http.NoBody) req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestIngestFile_NoUser() { +func (s *TestSuite) TestIngestFile_NoUser() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -756,19 +763,19 @@ func (suite *TestSuite) TestIngestFile_NoUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (suite *TestSuite) TestIngestFile_WrongUser() { +func (s *TestSuite) TestIngestFile_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -788,21 +795,21 @@ func (suite *TestSuite) TestIngestFile_WrongUser() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "sql: no rows in result set") + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") } -func (suite *TestSuite) TestIngestFile_WrongFilePath() { +func (s *TestSuite) TestIngestFile_WrongFilePath() { user := "dummy" filePath := "/inbox/dummy/file10.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -814,7 +821,7 @@ func (suite *TestSuite) TestIngestFile_WrongFilePath() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest("POST", "/file/ingest", bytes.NewBuffer(ingestMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/ingest", ingestFile) @@ -823,52 +830,52 @@ func (suite *TestSuite) TestIngestFile_WrongFilePath() { okResponse := w.Result() defer okResponse.Body.Close() b, _ := io.ReadAll(okResponse.Body) - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) - assert.Contains(suite.T(), string(b), "sql: no rows in result set") + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Contains(s.T(), string(b), "sql: no rows in result set") } -func (suite *TestSuite) TestSetAccession() { +func (s *TestSuite) TestSetAccession() { user := "dummy" filePath := "/inbox/dummy/file11.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + err = Conf.API.DB.SetArchived(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + err = Conf.API.DB.SetVerified(fileInfo, fileID) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } type accession struct { @@ -881,7 +888,7 @@ func (suite *TestSuite) TestSetAccession() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) @@ -889,7 +896,7 @@ func (suite *TestSuite) TestSetAccession() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -897,31 +904,31 @@ func (suite *TestSuite) TestSetAccession() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/accession", http.NoBody) req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestSetAccession_WrongUser() { +func (s *TestSuite) TestSetAccession_WrongUser() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } type accession struct { @@ -934,7 +941,7 @@ func (suite *TestSuite) TestSetAccession_WrongUser() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) @@ -942,22 +949,22 @@ func (suite *TestSuite) TestSetAccession_WrongUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (suite *TestSuite) TestSetAccession_WrongFormat() { +func (s *TestSuite) TestSetAccession_WrongFormat() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/federated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } type accession struct { @@ -970,7 +977,7 @@ func (suite *TestSuite) TestSetAccession_WrongFormat() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/file/accession", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/accession", rbac(e), setAccession) @@ -978,62 +985,62 @@ func (suite *TestSuite) TestSetAccession_WrongFormat() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, okResponse.StatusCode) } -func (suite *TestSuite) TestCreateDataset() { +func (s *TestSuite) TestCreateDataset() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + err = Conf.API.DB.SetArchived(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + err = Conf.API.DB.SetVerified(fileInfo, fileID) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "dummy"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", rbac(e), createDataset) @@ -1041,7 +1048,7 @@ func (suite *TestSuite) TestCreateDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -1049,76 +1056,76 @@ func (suite *TestSuite) TestCreateDataset() { req, _ := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/mappings", http.NoBody) req.SetBasicAuth("guest", "guest") res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") - assert.NoError(suite.T(), json.Unmarshal(body, &data), "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to read response from broker") + assert.NoError(s.T(), json.Unmarshal(body, &data), "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestCreateDataset_BadFormat() { +func (s *TestSuite) TestCreateDataset_BadFormat() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + err = Conf.API.DB.SetArchived(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + err = Conf.API.DB.SetVerified(fileInfo, fileID) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}") - assert.NoError(suite.T(), err, "got (%v) when marking file as ready", err) + assert.NoError(s.T(), err, "got (%v) when marking file as ready", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/federated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "dummy"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", rbac(e), createDataset) @@ -1126,16 +1133,16 @@ func (suite *TestSuite) TestCreateDataset_BadFormat() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "does not match pattern") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "does not match pattern") } -func (suite *TestSuite) TestCreateDataset_MissingAccessionIDs() { +func (s *TestSuite) TestCreateDataset_MissingAccessionIDs() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" @@ -1143,7 +1150,7 @@ func (suite *TestSuite) TestCreateDataset_MissingAccessionIDs() { // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", createDataset) @@ -1151,23 +1158,23 @@ func (suite *TestSuite) TestCreateDataset_MissingAccessionIDs() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "at least one accessionID is reqired") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "at least one accessionID is reqired") } -func (suite *TestSuite) TestCreateDataset_WrongIDs() { +func (s *TestSuite) TestCreateDataset_WrongIDs() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "dummy"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", createDataset) @@ -1175,58 +1182,58 @@ func (suite *TestSuite) TestCreateDataset_WrongIDs() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "accession ID not found: ") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "accession ID not found: ") } -func (suite *TestSuite) TestCreateDataset_WrongUser() { +func (s *TestSuite) TestCreateDataset_WrongUser() { user := "dummy" filePath := "/inbox/dummy/file12.c4gh" fileID, err := Conf.API.DB.RegisterFile(filePath, user) - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") - assert.NoError(suite.T(), err, "failed to update satus of file in database") + assert.NoError(s.T(), err, "failed to update satus of file in database") encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), Size: 1000, Path: filePath, DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, } - err = Conf.API.DB.SetArchived(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") + err = Conf.API.DB.SetArchived(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Archived") - err = Conf.API.DB.SetVerified(fileInfo, fileID, fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + err = Conf.API.DB.SetVerified(fileInfo, fileID) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) err = Conf.API.DB.SetAccessionID("API:accession-id-11", fileID) - assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + assert.NoError(s.T(), err, "got (%v) when marking file as verified", err) gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" accessionMsg, _ := json.Marshal(dataset{AccessionIDs: []string{"API:accession-id-11"}, DatasetID: "API:dataset-01", User: "tester"}) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/create", bytes.NewBuffer(accessionMsg)) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/create", createDataset) @@ -1234,58 +1241,58 @@ func (suite *TestSuite) TestCreateDataset_WrongUser() { router.ServeHTTP(w, r) response := w.Result() body, err := io.ReadAll(response.Body) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) - assert.Contains(suite.T(), string(body), "accession ID owned by other user") + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) + assert.Contains(s.T(), string(body), "accession ID owned by other user") } -func (suite *TestSuite) TestReleaseDataset() { +func (s *TestSuite) TestReleaseDataset() { user := "TestReleaseDataset" for i := 0; i < 3; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } if err := Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_TestReleaseDataset_00", "accession_TestReleaseDataset_01", "accession_TestReleaseDataset_02"}); err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/API:dataset-01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1293,7 +1300,7 @@ func (suite *TestSuite) TestReleaseDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -1301,38 +1308,38 @@ func (suite *TestSuite) TestReleaseDataset() { req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestReleaseDataset_NoDataset() { +func (s *TestSuite) TestReleaseDataset_NoDataset() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1340,29 +1347,29 @@ func (suite *TestSuite) TestReleaseDataset_NoDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } -func (suite *TestSuite) TestReleaseDataset_BadDataset() { +func (s *TestSuite) TestReleaseDataset_BadDataset() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/non-existing", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1370,58 +1377,58 @@ func (suite *TestSuite) TestReleaseDataset_BadDataset() { router.ServeHTTP(w, r) response := w.Result() defer response.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, response.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, response.StatusCode) } -func (suite *TestSuite) TestReleaseDataset_DeprecatedDataset() { +func (s *TestSuite) TestReleaseDataset_DeprecatedDataset() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } } if err := Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}); err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("test-dataset-01", "deprecated", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/dataset/release/test-dataset-01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/dataset/release/*dataset", rbac(e), releaseDataset) @@ -1429,53 +1436,53 @@ func (suite *TestSuite) TestReleaseDataset_DeprecatedDataset() { router.ServeHTTP(w, r) response := w.Result() defer response.Body.Close() - assert.Equal(suite.T(), http.StatusBadRequest, response.StatusCode) + assert.Equal(s.T(), http.StatusBadRequest, response.StatusCode) } -func (suite *TestSuite) TestListActiveUsers() { +func (s *TestSuite) TestListActiveUsers() { testUsers := []string{"User-A", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 3; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), user) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } } err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_User-A_00", "accession_User-A_01", "accession_User-A_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/users", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/users", rbac(e), listActiveUsers) @@ -1483,58 +1490,58 @@ func (suite *TestSuite) TestListActiveUsers() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) var users []string err = json.NewDecoder(okResponse.Body).Decode(&users) - assert.NoError(suite.T(), err, "failed to list users from DB") - assert.Equal(suite.T(), []string{"User-B", "User-C"}, users) + assert.NoError(s.T(), err, "failed to list users from DB") + assert.Equal(s.T(), []string{"User-B", "User-C"}, users) } -func (suite *TestSuite) TestListUserFiles() { +func (s *TestSuite) TestListUserFiles() { testUsers := []string{"user_example.org", "User-B", "User-C"} for _, user := range testUsers { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/%v/TestGetUserFiles-00%d.c4gh", user, i), strings.ReplaceAll(user, "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } err = Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}") if err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } } err = Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/users/user@example.org/files", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/users/:username/files", rbac(e), listUserFiles) @@ -1542,26 +1549,27 @@ func (suite *TestSuite) TestListUserFiles() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) files := []database.SubmissionFileInfo{} err = json.NewDecoder(okResponse.Body).Decode(&files) - assert.NoError(suite.T(), err, "failed to list users from DB") - assert.Equal(suite.T(), 2, len(files)) + assert.NoError(s.T(), err, "failed to list users from DB") + assert.Equal(s.T(), 2, len(files)) + assert.Contains(s.T(), files[0].AccessionID, "accession_user_example.org_0") } -func (suite *TestSuite) TestAddC4ghHash() { +func (s *TestSuite) TestAddC4ghHash() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r := gin.Default() @@ -1570,7 +1578,7 @@ func (suite *TestSuite) TestAddC4ghHash() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Create a valid request body keyhash := schema.C4ghPubKey{ @@ -1578,38 +1586,38 @@ func (suite *TestSuite) TestAddC4ghHash() { Description: "Test key description", } body, err := json.Marshal(keyhash) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() // Isert pubkey again and expect error resp2, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusConflict, resp2.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusConflict, resp2.StatusCode) defer resp2.Body.Close() } -func (suite *TestSuite) TestAddC4ghHash_emptyJson() { +func (s *TestSuite) TestAddC4ghHash_emptyJson() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r := gin.Default() r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) @@ -1617,34 +1625,34 @@ func (suite *TestSuite) TestAddC4ghHash_emptyJson() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Create an invalid request body body := []byte(`{"invalid_json"}`) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() } -func (suite *TestSuite) TestAddC4ghHash_notBase64() { +func (s *TestSuite) TestAddC4ghHash_notBase64() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) m, err := model.NewModelFromString(jsonadapter.Model) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC model") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC model") } - e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&suite.RBAC)) + e, err := casbin.NewEnforcer(m, jsonadapter.NewAdapter(&s.RBAC)) if err != nil { - suite.T().Logf("failure: %v", err) - suite.FailNow("failed to setup RBAC enforcer") + s.T().Logf("failure: %v", err) + s.FailNow("failed to setup RBAC enforcer") } r := gin.Default() r.POST("/c4gh-keys/add", rbac(e), addC4ghHash) @@ -1652,24 +1660,24 @@ func (suite *TestSuite) TestAddC4ghHash_notBase64() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Create an invalid request body body := []byte(`{"pubkey": "asdkjsahfd=", "decription": ""}`) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/add", bytes.NewBuffer(body)) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) req.Header.Add("Content-Type", "application/json") resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() } -func (suite *TestSuite) TestListC4ghHashes() { - assert.NoError(suite.T(), Conf.API.DB.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", "this is a test key"), "failed to register key in database") +func (s *TestSuite) TestListC4ghHashes() { + assert.NoError(s.T(), Conf.API.DB.AddKeyHash("cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", "this is a test key"), "failed to register key in database") expectedResponse := database.C4ghKeyHash{ Hash: "cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", @@ -1679,7 +1687,7 @@ func (suite *TestSuite) TestListC4ghHashes() { } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) r := gin.Default() r.GET("/c4gh-keys/list", listC4ghHashes) @@ -1687,34 +1695,34 @@ func (suite *TestSuite) TestListC4ghHashes() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("GET", ts.URL+"/c4gh-keys/list", nil) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() hashes := []database.C4ghKeyHash{} err = json.NewDecoder(resp.Body).Decode(&hashes) - assert.NoError(suite.T(), err, "failed to list users from DB") + assert.NoError(s.T(), err, "failed to list users from DB") for n, h := range hashes { if h.Hash == "cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23" { - assert.Equal(suite.T(), expectedResponse, hashes[n]) + assert.Equal(s.T(), expectedResponse, hashes[n]) break } } } -func (suite *TestSuite) TestDeprecateC4ghHash() { - assert.NoError(suite.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd9991453839581fc69ee26e0daefde6a5d2660fc23", "this is a deprecation test key"), "failed to register key in database") +func (s *TestSuite) TestDeprecateC4ghHash() { + assert.NoError(s.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd9991453839581fc69ee26e0daefde6a5d2660fc23", "this is a deprecation test key"), "failed to register key in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) r := gin.Default() r.POST("/c4gh-keys/deprecate/*keyHash", deprecateC4ghHash) @@ -1722,29 +1730,29 @@ func (suite *TestSuite) TestDeprecateC4ghHash() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/deprecate/abc8f5cc8d936ce437a52cd9991453839581fc69ee26e0daefde6a5d2660fc23", http.NoBody) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, resp.StatusCode) defer resp.Body.Close() // a second time gives an error since the key is alreadu deprecated resp2, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp2.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp2.StatusCode) defer resp2.Body.Close() } -func (suite *TestSuite) TestDeprecateC4ghHash_wrongHash() { - assert.NoError(suite.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc99", "this is a deprecation test key"), "failed to register key in database") +func (s *TestSuite) TestDeprecateC4ghHash_wrongHash() { + assert.NoError(s.T(), Conf.API.DB.AddKeyHash("abc8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc99", "this is a deprecation test key"), "failed to register key in database") gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) r := gin.Default() r.POST("/c4gh-keys/deprecate/*keyHash", deprecateC4ghHash) @@ -1752,58 +1760,58 @@ func (suite *TestSuite) TestDeprecateC4ghHash_wrongHash() { defer ts.Close() client := &http.Client{} - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) req, err := http.NewRequest("POST", ts.URL+"/c4gh-keys/deprecate/xyz8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23", http.NoBody) - assert.NoError(suite.T(), err) - req.Header.Add("Authorization", "Bearer "+suite.Token) + assert.NoError(s.T(), err) + req.Header.Add("Authorization", "Bearer "+s.Token) resp, err := client.Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, resp.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, resp.StatusCode) defer resp.Body.Close() } -func (suite *TestSuite) TestListDatasets() { +func (s *TestSuite) TestListDatasets() { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/dummy/TestGetUserFiles-00%d.c4gh", i), "dummy") if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } stableID := fmt.Sprintf("accession_%s_0%d", "dummy", i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_dummy_00", "accession_dummy_01", "accession_dummy_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_dummy_03", "accession_dummy_04"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/datasets/list", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/datasets/list", listAllDatasets) @@ -1812,56 +1820,56 @@ func (suite *TestSuite) TestListDatasets() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) datasets := []database.DatasetInfo{} err = json.NewDecoder(okResponse.Body).Decode(&datasets) - assert.NoError(suite.T(), err, "failed to list datasets from DB") - assert.Equal(suite.T(), 2, len(datasets)) - assert.Equal(suite.T(), "released", datasets[1].Status) - assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) + assert.NoError(s.T(), err, "failed to list datasets from DB") + assert.Equal(s.T(), 2, len(datasets)) + assert.Equal(s.T(), "released", datasets[1].Status) + assert.Equal(s.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) } -func (suite *TestSuite) TestListUserDatasets() { +func (s *TestSuite) TestListUserDatasets() { for i := 0; i < 5; i++ { fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), strings.ReplaceAll("user_example.org", "_", "@")) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } stableID := fmt.Sprintf("accession_%s_0%d", "user_example.org", i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/datasets/list/user@example.org", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/datasets/list/:username", listUserDatasets) @@ -1869,56 +1877,56 @@ func (suite *TestSuite) TestListUserDatasets() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) datasets := []database.DatasetInfo{} err = json.NewDecoder(okResponse.Body).Decode(&datasets) - assert.NoError(suite.T(), err, "failed to list datasets from DB") - assert.Equal(suite.T(), 2, len(datasets)) - assert.Equal(suite.T(), "released", datasets[1].Status) - assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) + assert.NoError(s.T(), err, "failed to list datasets from DB") + assert.Equal(s.T(), 2, len(datasets)) + assert.Equal(s.T(), "released", datasets[1].Status) + assert.Equal(s.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) } -func (suite *TestSuite) TestListDatasetsAsUser() { +func (s *TestSuite) TestListDatasetsAsUser() { for i := 0; i < 5; i++ { - fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), suite.User) + fileID, err := Conf.API.DB.RegisterFile(fmt.Sprintf("/user_example.org/TestGetUserFiles-00%d.c4gh", i), s.User) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } stableID := fmt.Sprintf("accession_user_example.org_0%d", i) err = Conf.API.DB.SetAccessionID(stableID, fileID) if err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } } err = Conf.API.DB.MapFilesToDataset("API:dataset-01", []string{"accession_user_example.org_00", "accession_user_example.org_01", "accession_user_example.org_02"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-01", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } err = Conf.API.DB.MapFilesToDataset("API:dataset-02", []string{"accession_user_example.org_03", "accession_user_example.org_04"}) if err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "registered", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } if err := Conf.API.DB.UpdateDatasetEvent("API:dataset-02", "released", "{}"); err != nil { - suite.FailNow("failed to update dataset event") + s.FailNow("failed to update dataset event") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodGet, "/datasets", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.GET("/datasets", listDatasets) @@ -1926,72 +1934,72 @@ func (suite *TestSuite) TestListDatasetsAsUser() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) datasets := []database.DatasetInfo{} err = json.NewDecoder(okResponse.Body).Decode(&datasets) - assert.NoError(suite.T(), err, "failed to list datasets from DB") - assert.Equal(suite.T(), 2, len(datasets)) - assert.Equal(suite.T(), "released", datasets[1].Status) - assert.Equal(suite.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) + assert.NoError(s.T(), err, "failed to list datasets from DB") + assert.Equal(s.T(), 2, len(datasets)) + assert.Equal(s.T(), "released", datasets[1].Status) + assert.Equal(s.T(), "API:dataset-01|registered", fmt.Sprintf("%s|%s", datasets[0].DatasetID, datasets[0].Status)) } -func (suite *TestSuite) TestReVerifyFile() { +func (s *TestSuite) TestReVerifyFile() { user := "TestReVerify" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerify-00%d.c4gh", user, i) fileID, err := Conf.API.DB.RegisterFile(filePath, user) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}"); err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) if err != nil { - suite.FailNow("failed to calculate Checksum") + s.FailNow("failed to calculate Checksum") } decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) if err != nil { - suite.FailNow("failed to calculate DecryptedChecksum") + s.FailNow("failed to calculate DecryptedChecksum") } fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), - Size: 1000, - Path: filePath, + ArchiveChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, + Path: filePath, + Size: 1000, + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } - if err := Conf.API.DB.SetArchived(fileInfo, fileID, fileID); err != nil { - suite.FailNow("failed to mark file as Archived") + if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { + s.FailNow("failed to mark file as Archived") } - if err := Conf.API.DB.SetVerified(fileInfo, fileID, fileID); err != nil { - suite.FailNow("failed to mark file as Verified") + if err := Conf.API.DB.SetVerified(fileInfo, fileID); err != nil { + s.FailNow("failed to mark file as Verified") } stableID := fmt.Sprintf("accession_%s_0%d", user, i) if err := Conf.API.DB.SetAccessionID(stableID, fileID); err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}"); err != nil { - suite.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) + s.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) } } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) - Conf.Broker.SchemasPath = "../../schemas/isolated" + assert.NoError(s.T(), setupJwtAuth()) // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/file/verify/accession_TestReVerify_01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.PUT("/file/verify/:accession", reVerifyFile) @@ -1999,7 +2007,7 @@ func (suite *TestSuite) TestReVerifyFile() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the message shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -2007,27 +2015,27 @@ func (suite *TestSuite) TestReVerifyFile() { req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 1, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 1, data.MessagesReady) } -func (suite *TestSuite) TestReVerifyFile_wrongAccession() { +func (s *TestSuite) TestReVerifyFile_wrongAccession() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/file/verify/accession_TestReVerify_99", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.POST("/file/verify/:accession", reVerifyFile) @@ -2035,69 +2043,70 @@ func (suite *TestSuite) TestReVerifyFile_wrongAccession() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } -func (suite *TestSuite) TestReVerifyDataset() { +func (s *TestSuite) TestReVerifyDataset() { user := "TestReVerifyDataset" for i := 0; i < 3; i++ { filePath := fmt.Sprintf("/%v/TestReVerifyDataset-00%d.c4gh", user, i) fileID, err := Conf.API.DB.RegisterFile(filePath, user) if err != nil { - suite.FailNow("failed to register file in database") + s.FailNow("failed to register file in database") } if err := Conf.API.DB.UpdateFileEventLog(fileID, "uploaded", fileID, user, "{}", "{}"); err != nil { - suite.FailNow("failed to update satus of file in database") + s.FailNow("failed to update satus of file in database") } encSha := sha256.New() _, err = encSha.Write([]byte("Checksum")) if err != nil { - suite.FailNow("failed to calculate Checksum") + s.FailNow("failed to calculate Checksum") } decSha := sha256.New() _, err = decSha.Write([]byte("DecryptedChecksum")) if err != nil { - suite.FailNow("failed to calculate DecryptedChecksum") + s.FailNow("failed to calculate DecryptedChecksum") } fileInfo := database.FileInfo{ - Checksum: fmt.Sprintf("%x", encSha.Sum(nil)), - Size: 1000, - Path: filePath, + ArchiveChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), DecryptedChecksum: fmt.Sprintf("%x", decSha.Sum(nil)), DecryptedSize: 948, + Path: filePath, + Size: 1000, + UploadedChecksum: fmt.Sprintf("%x", encSha.Sum(nil)), } - if err := Conf.API.DB.SetArchived(fileInfo, fileID, fileID); err != nil { - suite.FailNow("failed to mark file as Archived") + if err := Conf.API.DB.SetArchived(fileInfo, fileID); err != nil { + s.FailNow("failed to mark file as Archived") } - if err := Conf.API.DB.SetVerified(fileInfo, fileID, fileID); err != nil { - suite.FailNow("failed to mark file as Verified") + if err := Conf.API.DB.SetVerified(fileInfo, fileID); err != nil { + s.FailNow("failed to mark file as Verified") } stableID := fmt.Sprintf("%s_0%d", user, i) if err := Conf.API.DB.SetAccessionID(stableID, fileID); err != nil { - suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) + s.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } if err := Conf.API.DB.UpdateFileEventLog(fileID, "ready", fileID, "finalize", "{}", "{}"); err != nil { - suite.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) + s.FailNowf("got (%s) when updating file status: %s", err.Error(), filePath) } } if err := Conf.API.DB.MapFilesToDataset("test-dataset-01", []string{"TestReVerifyDataset_00", "TestReVerifyDataset_01", "TestReVerifyDataset_02"}); err != nil { - suite.FailNow("failed to map files to dataset") + s.FailNow("failed to map files to dataset") } gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/dataset/verify/test-dataset-01", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.PUT("/dataset/verify/*dataset", reVerifyDataset) @@ -2105,7 +2114,7 @@ func (suite *TestSuite) TestReVerifyDataset() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusOK, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusOK, okResponse.StatusCode) // verify that the messages shows up in the queue time.Sleep(10 * time.Second) // this is needed to ensure we don't get any false negatives @@ -2113,27 +2122,27 @@ func (suite *TestSuite) TestReVerifyDataset() { req.SetBasicAuth("guest", "guest") client := http.Client{Timeout: 30 * time.Second} res, err := client.Do(req) - assert.NoError(suite.T(), err, "failed to query broker") + assert.NoError(s.T(), err, "failed to query broker") var data struct { MessagesReady int `json:"messages_ready"` } body, err := io.ReadAll(res.Body) res.Body.Close() - assert.NoError(suite.T(), err, "failed to read response from broker") + assert.NoError(s.T(), err, "failed to read response from broker") err = json.Unmarshal(body, &data) - assert.NoError(suite.T(), err, "failed to unmarshal response") - assert.Equal(suite.T(), 3, data.MessagesReady) + assert.NoError(s.T(), err, "failed to unmarshal response") + assert.Equal(s.T(), 3, data.MessagesReady) } -func (suite *TestSuite) TestReVerifyDataset_wrongDatasetName() { +func (s *TestSuite) TestReVerifyDataset_wrongDatasetName() { gin.SetMode(gin.ReleaseMode) - assert.NoError(suite.T(), setupJwtAuth()) + assert.NoError(s.T(), setupJwtAuth()) Conf.Broker.SchemasPath = "../../schemas/isolated" // Mock request and response holders w := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPut, "/dataset/verify/wrong_dataset", http.NoBody) - r.Header.Add("Authorization", "Bearer "+suite.Token) + r.Header.Add("Authorization", "Bearer "+s.Token) _, router := gin.CreateTestContext(w) router.PUT("/dataset/verify/*dataset", reVerifyDataset) @@ -2141,5 +2150,5 @@ func (suite *TestSuite) TestReVerifyDataset_wrongDatasetName() { router.ServeHTTP(w, r) okResponse := w.Result() defer okResponse.Body.Close() - assert.Equal(suite.T(), http.StatusNotFound, okResponse.StatusCode) + assert.Equal(s.T(), http.StatusNotFound, okResponse.StatusCode) } diff --git a/sda/cmd/api/swagger_v1.yml b/sda/cmd/api/swagger_v1.yml new file mode 100644 index 0000000000..f0efc433de --- /dev/null +++ b/sda/cmd/api/swagger_v1.yml @@ -0,0 +1,412 @@ +openapi: 3.0.3 +info: + title: SDA admin API + version: "1.0" + description: This is the admin API for the sensitive data archive. +paths: + /c4gh-keys/add: + post: + description: Registers an crypt4gh public key in the database + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/C4ghKeyAdd" + responses: + "200": + description: Successful operation + "400": + description: Bad payload + "401": + description: Authentication failure. + "409": + description: Key hash already exists in the database. + "500": + description: Internal application error. + /c4gh-keys/deprecate/{keyHash}: + post: + description: Deprecate a given key hash + parameters: + - in: path + name: keyHash + schema: + type: string + required: true + responses: + "200": + description: Successful operation + "400": + description: Bad key hash + "401": + description: Authentication failure + "500": + description: Internal application error + /c4gh-keys/list: + get: + description: Returns a list of all registered encryptions keys and their statuses. + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/C4ghKeysList" + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /dataset/create: + post: + description: Create a new dataset + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/DatasetCreate" + responses: + "200": + description: Successful operation + "400": + description: Bad request body content + "401": + description: Authentication failure + "500": + description: Internal application error + /dataset/release/{datasetID}: + post: + description: Release a dataset, so that it is accessible for downloading + parameters: + - in: path + name: datasetID + schema: + type: string + required: true + responses: + "200": + description: Successful operation + "400": + description: Bad dataset ID + "401": + description: Authentication failure + "500": + description: Internal application error + /dataset/verify/{datasetID}: + put: + description: Triggers reverification of all files in the dataset. + parameters: + - in: path + name: datasetID + schema: + type: string + required: true + responses: + "200": + description: Successful operation + "400": + description: Bad dataset ID + "401": + description: Authentication failure + "500": + description: Internal application error + /datasets: + get: + description: Lists datasets belonging to the calling userName + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatasetInfo" + description: Successful operation + "500": + description: Internal application error + /datasets/list: + get: + description: Lists all datasets with their status. + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatasetInfo" + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /datasets/list/{userName}: + get: + description: Lists all datasets with their status submitted by a specific user. + parameters: + - in: path + name: userName + schema: + type: string + required: true + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/DatasetInfo" + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /file/accession: + post: + description: Assigns accession ID to a given file. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/FileAccession" + responses: + "200": + description: Successful operation. + "400": + description: Bad payload + "401": + description: Authentication failure. + "500": + description: Internal application error. + /file/ingest: + post: + description: Trigger ingestion of a given file. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/FileIngest" + responses: + "200": + description: Successful operation. + "400": + description: Bad payload + "401": + description: Authentication failure. + "500": + description: Internal application error. + /file/{userName}/{fileID}: + delete: + description: Delete a file from the inbox + parameters: + - in: path + name: fileID + schema: + type: string + required: true + - in: path + name: userName + schema: + type: string + required: true + responses: + "200": + description: Successful operation. + "400": + description: No file Id supplied + "401": + description: Authentication failure. + "404": + description: Bad user file ID combination + "500": + description: Internal application error. + /file/verify/{accessionID}: + post: + description: Trigger re-verification of a file with a given accession ID. + parameters: + - in: path + name: accessionID + schema: + type: string + required: true + responses: + "200": + description: Successful operation. + "400": + description: Bad payload + "401": + description: Authentication failure. + "500": + description: Internal application error. + /files: + get: + description: List all files belonging to the calling user that is not part of a dataset. + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FileInfo" + description: Successful operation + "400": + description: Bad username + "401": + description: Authentication failure + "500": + description: Internal application error + /ready: + get: + description: Returns the status of the application. + responses: + "200": + content: + application/json: + example: + {} + description: The service is operational + "503": + content: + application/json: + example: + {} + description: Unhealthy service + security: [] + /users: + get: + description: Lists all users with ongoing submissions. + responses: + "200": + content: + application/json: + schema: + example: ["test.user@demo.org","demo.user@demo.org"] + type: array + items: + type: string + description: Successful operation + "401": + description: Authentication failure + "500": + description: Internal application error + /users/{userName}/files: + get: + description: List all files belonging to the given user that is not part of a dataset. + parameters: + - in: path + name: userName + schema: + type: string + required: true + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FileInfo" + description: Successful operation + "400": + description: Bad username + "401": + description: Authentication failure + "500": + description: Internal application error +components: + schemas: + C4ghKeyAdd: + type: object + properties: + description: + type: string + example: pubkey description + pubkey: + type: string + example: 19GT1JNQVRgIGNhbiBiZSBzZXQgdG8gYGpzb25gIHRvIGdldCBsb2dzIGluIEpTT04gZm9ybWF0= + C4ghKeysList: + type: object + properties: + hash: + type: string + example: cbd8f5cc8d936ce437a52cd7991453839581fc69ee26e0daefde6a5d2660fc23 + description: + type: string + example: This is an example + createdAt: + type: string + example: "2025-03-01T12:13:14.123Z" + deprecatedAt: + type: string + example: "2025-03-02T13:14:15.123Z" + DatasetCreate: + type: object + properties: + accession_ids: + example: ["zz-file-123456-asdfgh", "zz-file-123456-qwerty"] + type: array + items: + type: string + dataset_id: + type: string + example: zz-dataset-123456-asdfgh + user: + type: string + example: test.user@dummy.org + DatasetInfo: + type: object + properties: + DatasetId: + type: string + example: zz-dataset-123456-asdfgh + Status: + type: string + example: registered + Timestamp: + type: string + example: 2025-03-30T09:10:11.321Z + FileAccession: + type: object + properties: + accession_id: + type: string + example: zz-file-123456-zxcvbn + filepath: + type: string + example: uploads/file-1.c4gh + user: + type: string + example: test.user@dummy.org + FileIngest: + type: object + properties: + filepath: + type: string + example: uploads/file-1.c4gh + user: + type: string + example: test.user@dummy.org + FileInfo: + type: object + properties: + FileID: + type: string + example: e996e130-c08b-4b33-98d1-9aebbbf75850 + InboxPath: + type: string + example: uploads/file-001.c4gh + Status: + type: string + example: verified + CreatedAt: + type: string + example: "2025-03-02T13:14:15.123Z" + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT +security: + - bearerAuth: [] \ No newline at end of file diff --git a/sda/cmd/auth/cega.go b/sda/cmd/auth/cega.go index 54f2919bf5..2cd199bdbd 100644 --- a/sda/cmd/auth/cega.go +++ b/sda/cmd/auth/cega.go @@ -7,15 +7,9 @@ import ( "strings" "github.com/neicnordic/sensitive-data-archive/internal/config" - log "github.com/sirupsen/logrus" - bcrypt "golang.org/x/crypto/bcrypt" + "golang.org/x/crypto/bcrypt" ) -// EGALoginError is used to store message errors -type EGALoginError struct { - Reason string -} - // CegaUserResponse captures the response list type CegaUserResponse struct { PasswordHash string `json:"passwordHash"` @@ -49,7 +43,7 @@ func authenticateWithCEGA(conf config.CegaConfig, username string) (*http.Respon req, err := http.NewRequest("GET", fmt.Sprintf("%s/%s", strings.TrimSuffix(conf.AuthURL, "/"), username), payload) if err != nil { - log.Fatal(err) + return nil, err } req.Header.Add("Authorization", "Basic "+getb64Credentials(conf.ID, conf.Secret)) diff --git a/sda/cmd/auth/cega_test.go b/sda/cmd/auth/cega_test.go index 647296400a..cf22a36d6a 100644 --- a/sda/cmd/auth/cega_test.go +++ b/sda/cmd/auth/cega_test.go @@ -21,22 +21,22 @@ func TestCegaTestSuite(t *testing.T) { suite.Run(t, new(CegaTests)) } -func (suite *CegaTests) TestGetb64Credentials() { +func (ts *CegaTests) TestGetb64Credentials() { user := "testUser" password := "password" expected := base64.StdEncoding.EncodeToString([]byte(user + ":" + password)) - assert.Equal(suite.T(), expected, getb64Credentials(user, password), "base64 encoding of credentials failing") + assert.Equal(ts.T(), expected, getb64Credentials(user, password), "base64 encoding of credentials failing") } -func (suite *CegaTests) TestVerifyPassword() { +func (ts *CegaTests) TestVerifyPassword() { password := "password" hash, err := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) if err != nil { log.Error(err) } - assert.Equal(suite.T(), true, verifyPassword(password, string(hash)), "password hash verification failing on correct hash") - assert.Equal(suite.T(), false, verifyPassword(password, "wronghash"), "password hash verification returning true for wrong hash") + assert.Equal(ts.T(), true, verifyPassword(password, string(hash)), "password hash verification failing on correct hash") + assert.Equal(ts.T(), false, verifyPassword(password, "wronghash"), "password hash verification returning true for wrong hash") } diff --git a/sda/cmd/auth/frontend/static/custom.css b/sda/cmd/auth/frontend/static/custom.css index d442c23dda..6fa3a05358 100644 --- a/sda/cmd/auth/frontend/static/custom.css +++ b/sda/cmd/auth/frontend/static/custom.css @@ -7,7 +7,7 @@ } #loginbox1 { - max-width: 600px; + max-width: 800px; } #logintext { diff --git a/sda/cmd/auth/frontend/templates/ega.html b/sda/cmd/auth/frontend/templates/ega.html index 6e921f69a5..ac3ec3f0ab 100644 --- a/sda/cmd/auth/frontend/templates/ega.html +++ b/sda/cmd/auth/frontend/templates/ega.html @@ -31,18 +31,12 @@ Welcome, {{.User}}!

{{if .Token}} -

- Your access token is: -

- 
{{.Token}}
- {{end}} - {{if .ExpDate}} -

- Your access token expires (UTC): -

- 
{{.ExpDate}}
+

+ For uploading to the Inbox, your access token {{if .ExpDate}}(expires at {{.ExpDate}} UTC){{end}} is: +

+ 
{{.Token}}
{{end}} - Download inbox s3cmd credentials + Download credentials to upload to the Inbox Continue diff --git a/sda/cmd/auth/frontend/templates/oidc.html b/sda/cmd/auth/frontend/templates/oidc.html index 4d14a5afa2..0123887372 100644 --- a/sda/cmd/auth/frontend/templates/oidc.html +++ b/sda/cmd/auth/frontend/templates/oidc.html @@ -28,29 +28,24 @@

- Welcome, {{.User}}! + Welcome, {{.Fullname}}!

- {{if .Passport}} -

- Your visas are the following: -

- {{ range .Passport }} - 
{{.}}
- {{ end }} - {{ end }} - {{if .Token}} -

- Your access token is: -

- 
{{.Token}}
+ {{if (not .cegaID)}} + {{if .ResignedToken}} +

+ For uploading to the Inbox, your access token {{if .ExpDateResigned}}(expires at {{.ExpDateResigned}} UTC){{end}} is: +

+ 
{{.ResignedToken}}
+ {{end}} + Download credentials to upload to the Inbox {{end}} - {{if .ExpDate}} + {{if .RawToken}}

- Your access token expires (UTC): + For downloading from the Archive, your access token {{if .ExpDateRaw}}(expires at {{.ExpDateRaw}} UTC){{end}} is:

- 
{{.ExpDate}}
+ 
{{.RawToken}}
{{end}} - Download inbox s3cmd credentials + Download credentials to access the Archive Continue
diff --git a/sda/cmd/auth/info_test.go b/sda/cmd/auth/info_test.go index a39fb1f109..ae46fc5ad2 100644 --- a/sda/cmd/auth/info_test.go +++ b/sda/cmd/auth/info_test.go @@ -21,38 +21,37 @@ func TestInfoTestSuite(t *testing.T) { suite.Run(t, new(InfoTests)) } -func (suite *InfoTests) SetupTest() { - suite.TempDir, _ = os.MkdirTemp("", "key") +func (ts *InfoTests) SetupTest() { + ts.TempDir, _ = os.MkdirTemp("", "key") pub, _, err := keys.GenerateKeyPair() if err != nil { - suite.FailNowf("Filed to generate crypt4gh keypair", err.Error()) + ts.FailNowf("Filed to generate crypt4gh keypair", err.Error()) } buf := new(bytes.Buffer) if err := keys.WriteCrypt4GHX25519PublicKey(buf, pub); err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.pubKeyb64 = base64.StdEncoding.EncodeToString(buf.Bytes()) + ts.pubKeyb64 = base64.StdEncoding.EncodeToString(buf.Bytes()) - pubKeyFile, err := os.Create(suite.TempDir + "/pub.key") + pubKeyFile, err := os.Create(ts.TempDir + "/pub.key") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } _, err = pubKeyFile.Write(buf.Bytes()) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - } -func (suite *InfoTests) TestReadPublicKeyFile() { - pubKey, err := readPublicKeyFile(suite.TempDir + "/pub.key") - assert.NoError(suite.T(), err, "Reading public key from disk failed") - assert.Equal(suite.T(), suite.pubKeyb64, pubKey) +func (ts *InfoTests) TestReadPublicKeyFile() { + pubKey, err := readPublicKeyFile(ts.TempDir + "/pub.key") + assert.NoError(ts.T(), err, "Reading public key from disk failed") + assert.Equal(ts.T(), ts.pubKeyb64, pubKey) } -func (suite *InfoTests) TearDownTest() { - os.RemoveAll(suite.TempDir) +func (ts *InfoTests) TearDownTest() { + os.RemoveAll(ts.TempDir) } diff --git a/sda/cmd/auth/jwt.go b/sda/cmd/auth/jwt.go index b38e02c5f6..cc3a15521c 100644 --- a/sda/cmd/auth/jwt.go +++ b/sda/cmd/auth/jwt.go @@ -10,7 +10,7 @@ import ( "github.com/lestrrat-go/jwx/v2/jwt" ) -func generateJwtToken(tokenClaims map[string]interface{}, keyPath, alg string) (string, string, error) { +func generateJwtToken(tokenClaims map[string]any, keyPath, alg string) (string, string, error) { prKey, err := os.ReadFile(filepath.Clean(keyPath)) if err != nil { return "", "", err diff --git a/sda/cmd/auth/jwt_test.go b/sda/cmd/auth/jwt_test.go index 5d74b5f675..055df14b32 100644 --- a/sda/cmd/auth/jwt_test.go +++ b/sda/cmd/auth/jwt_test.go @@ -25,38 +25,38 @@ func TestJWTTestSuite(t *testing.T) { suite.Run(t, new(JWTTests)) } -func (suite *JWTTests) SetupTest() { +func (ts *JWTTests) SetupTest() { var err error - suite.TempDir, err = os.MkdirTemp(os.TempDir(), "jwt-test") - assert.NoError(suite.T(), err) + ts.TempDir, err = os.MkdirTemp(os.TempDir(), "jwt-test") + assert.NoError(ts.T(), err) ecKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) ecKeyBytes, err := jwk.EncodePEM(ecKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/ec", ecKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/ec", ecKeyBytes, 0600)) ecPubKeyBytes, err := jwk.EncodePEM(&ecKey.PublicKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/ec.pub", ecPubKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/ec.pub", ecPubKeyBytes, 0600)) rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) rsaKeyBytes, err := jwk.EncodePEM(rsaKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/rsa", rsaKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/rsa", rsaKeyBytes, 0600)) rsaPubKeyBytes, err := jwk.EncodePEM(&rsaKey.PublicKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), os.WriteFile(suite.TempDir+"/rsa.pub", rsaPubKeyBytes, 0600)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), os.WriteFile(ts.TempDir+"/rsa.pub", rsaPubKeyBytes, 0600)) } -func (suite *JWTTests) TearDownTest() { - os.RemoveAll(suite.TempDir) +func (ts *JWTTests) TearDownTest() { + os.RemoveAll(ts.TempDir) } -func (suite *JWTTests) TestGenerateJwtToken() { +func (ts *JWTTests) TestGenerateJwtToken() { type KeyAlgo struct { Algorithm string Keyfile string @@ -64,11 +64,11 @@ func (suite *JWTTests) TestGenerateJwtToken() { } algorithms := []KeyAlgo{ - {Algorithm: "RS256", Keyfile: suite.TempDir + "/rsa", Pubfile: suite.TempDir + "/rsa.pub"}, - {Algorithm: "ES256", Keyfile: suite.TempDir + "/ec", Pubfile: suite.TempDir + "/ec.pub"}, + {Algorithm: "RS256", Keyfile: ts.TempDir + "/rsa", Pubfile: ts.TempDir + "/rsa.pub"}, + {Algorithm: "ES256", Keyfile: ts.TempDir + "/ec", Pubfile: ts.TempDir + "/ec.pub"}, } - claims := map[string]interface{}{ + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(2 * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: "http://local.issuer", @@ -76,25 +76,25 @@ func (suite *JWTTests) TestGenerateJwtToken() { } for _, test := range algorithms { - ts, expiration, err := generateJwtToken(claims, test.Keyfile, test.Algorithm) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), ts) + t, expiration, err := generateJwtToken(claims, test.Keyfile, test.Algorithm) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), t) keyData, err := os.ReadFile(test.Pubfile) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) key, err := jwk.ParseKey(keyData, jwk.WithPEM(true)) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), jwk.AssignKeyID(key)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), jwk.AssignKeyID(key)) keySet := jwk.NewSet() - assert.NoError(suite.T(), keySet.AddKey(key)) + assert.NoError(ts.T(), keySet.AddKey(key)) - token, err := jwt.Parse([]byte(ts), jwt.WithKeySet(keySet, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "http://local.issuer", token.Issuer()) - assert.Equal(suite.T(), "test@foo.bar", token.Subject()) + token, err := jwt.Parse([]byte(t), jwt.WithKeySet(keySet, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "http://local.issuer", token.Issuer()) + assert.Equal(ts.T(), "test@foo.bar", token.Subject()) // check that the expiration string is a date _, err = time.Parse("2006-01-02 15:04:05", expiration) - assert.Nil(suite.T(), err, "Couldn't parse expiration date for jwt") + assert.Nil(ts.T(), err, "Couldn't parse expiration date for jwt") } } diff --git a/sda/cmd/auth/main.go b/sda/cmd/auth/main.go index b2b8c17eda..eb8fa6448c 100644 --- a/sda/cmd/auth/main.go +++ b/sda/cmd/auth/main.go @@ -27,8 +27,9 @@ type LoginOption struct { } type OIDCData struct { - S3Conf map[string]string - OIDCID OIDCIdentity + S3ConfInbox map[string]string + S3ConfDownload map[string]string + OIDCID OIDCIdentity } type AuthHandler struct { @@ -40,8 +41,10 @@ type AuthHandler struct { pubKey string } -func (auth AuthHandler) getInboxConfig(ctx iris.Context, authType string) { - +// getS3Config retrieves S3 config from session flash and serves it as a +// downloadable s3cmd file with the specified fileName. Redirects to home if +// config is missing. +func (auth AuthHandler) getS3Config(ctx iris.Context, authType string, fileName string) { log.Infoln(ctx.Request().URL.Path) s := sessions.Get(ctx) @@ -52,7 +55,8 @@ func (auth AuthHandler) getInboxConfig(ctx iris.Context, authType string) { return } s3cfmap := s3conf.(map[string]string) - ctx.ResponseWriter().Header().Set("Content-Disposition", "attachment; filename=s3cmd.conf") + ctx.ResponseWriter().Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fileName)) + s3c := "[default]\n" for k, v := range s3cfmap { @@ -70,7 +74,6 @@ func (auth AuthHandler) getInboxConfig(ctx iris.Context, authType string) { // getMain returns the index.html page func (auth AuthHandler) getMain(ctx iris.Context) { - ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) err := ctx.View("index.html") @@ -83,7 +86,6 @@ func (auth AuthHandler) getMain(ctx iris.Context) { // getLoginOptions returns the available login providers as JSON func (auth AuthHandler) getLoginOptions(ctx iris.Context) { - var response []LoginOption // Only add the OIDC option if it has both id and secret if auth.Config.OIDC.ID != "" && auth.Config.OIDC.Secret != "" { @@ -104,7 +106,6 @@ func (auth AuthHandler) getLoginOptions(ctx iris.Context) { // postEGA handles post requests for logging in using EGA func (auth AuthHandler) postEGA(ctx iris.Context) { - s := sessions.Get(ctx) userform := ctx.FormValues() @@ -114,24 +115,23 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { res, err := authenticateWithCEGA(auth.Config.Cega, username) if err != nil { - log.Error(err) + log.Errorf("No response from cega, error: %v", err) + res = &http.Response{ + Body: io.NopCloser(nil), + StatusCode: http.StatusInternalServerError, + } } - defer res.Body.Close() switch res.StatusCode { case 200: - if err != nil { - log.Error(err) - - return - } - var ur CegaUserResponse - err = json.NewDecoder(res.Body).Decode(&ur) + err := json.NewDecoder(res.Body).Decode(&ur) if err != nil { - log.Error("Failed to parse response: ", err) + log.Error("Failed to parse cega response: ", err) + s.SetFlash("message", "Problems connecting to EGA authentication server") + ctx.Redirect("/ega/login", iris.StatusSeeOther) return } @@ -142,7 +142,7 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { if ok { log.WithFields(log.Fields{"authType": "cega", "user": username}).Info("Valid password entered by user") - claims := map[string]interface{}{ + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(time.Duration(auth.Config.JwtTTL) * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: auth.Config.JwtIssuer, @@ -151,34 +151,47 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { token, expDate, err := generateJwtToken(claims, auth.Config.JwtPrivateKey, auth.Config.JwtSignatureAlg) if err != nil { log.Errorf("error when generating token: %v", err) + s.SetFlash("message", "Unexpected error, please try again.") + ctx.Redirect("/ega/login", iris.StatusSeeOther) + + return } s3conf := getS3ConfigMap(token, auth.Config.S3Inbox, username) s.SetFlash("ega", s3conf) + ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) ctx.ViewData("User", username) ctx.ViewData("Token", token) ctx.ViewData("ExpDate", expDate) - err = ctx.View("ega.html") - if err != nil { - log.Error("Failed to parse response: ", err) - return + if err != nil { + log.Error("Failed to create view: ", err) + + // Since the context has already started writing the response to + // the client, the resulting page will be ugly, but at least + // show an error message, and an oppertunity to log in again. + ctx.ViewData("Reason", "Unexpected error, please try again.") + err = ctx.View("loginform.html") + log.Error("Failed to create backup view: ", err) } - } else { log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Invalid password entered by user") s.SetFlash("message", "Provided credentials are not valid") ctx.Redirect("/ega/login", iris.StatusSeeOther) } - case 404: + case 500, 502, 503: log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Failed to authenticate user") s.SetFlash("message", "EGA authentication server could not be contacted") ctx.Redirect("/ega/login", iris.StatusSeeOther) + case 401: + log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Failed to authenticate service (auth_cega_id/secret)") + s.SetFlash("message", "Problems connecting to EGA authentication server") + ctx.Redirect("/ega/login", iris.StatusSeeOther) default: log.WithFields(log.Fields{"authType": "cega", "user": username}).Error("Failed to authenticate user") s.SetFlash("message", "Provided credentials are not valid") @@ -188,34 +201,22 @@ func (auth AuthHandler) postEGA(ctx iris.Context) { // getEGALogin returns the EGA login form func (auth AuthHandler) getEGALogin(ctx iris.Context) { - s := sessions.Get(ctx) - message := s.GetFlashString("message") - if message == "" { - ctx.ViewData("infoUrl", auth.Config.InfoURL) - ctx.ViewData("infoText", auth.Config.InfoText) - err := ctx.View("loginform.html") - if err != nil { - log.Error("Failed to return to login form: ", err) - - return - } - - return - } ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) - err := ctx.View("loginform.html", EGALoginError{Reason: message}) + message := s.GetFlashString("message") + if message != "" { + ctx.ViewData("Reason", message) + } + err := ctx.View("loginform.html") if err != nil { log.Error("Failed to view invalid credentials form: ", err) - - return } } // getEGAConf returns an s3config file for an oidc login func (auth AuthHandler) getEGAConf(ctx iris.Context) { - auth.getInboxConfig(ctx, "ega") + auth.getS3Config(ctx, "ega", "s3cmd-inbox.conf") } // getOIDC redirects to the oidc page defined in auth.Config @@ -264,48 +265,54 @@ func (auth AuthHandler) elixirLogin(ctx iris.Context) *OIDCData { return nil } - err = auth.Config.DB.UpdateUserInfo(idStruct.User, idStruct.Profile, idStruct.Email, idStruct.EdupersonEntitlement) + err = auth.Config.DB.UpdateUserInfo(idStruct.User, idStruct.Fullname, idStruct.Email, idStruct.EdupersonEntitlement) if err != nil { log.Warn("Could not log user info.") } if auth.Config.ResignJwt { - claims := map[string]interface{}{ + log.Debugf("Resigning token for user %s", idStruct.User) + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(time.Duration(auth.Config.JwtTTL) * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: auth.Config.JwtIssuer, - jwt.SubjectKey: idStruct.Profile, + jwt.SubjectKey: idStruct.User, } token, expDate, err := generateJwtToken(claims, auth.Config.JwtPrivateKey, auth.Config.JwtSignatureAlg) if err != nil { log.Errorf("error when generating token: %v", err) } - idStruct.Token = token - idStruct.ExpDate = expDate + idStruct.ResignedToken = token + idStruct.ExpDateResigned = expDate } log.WithFields(log.Fields{"authType": "oidc", "user": idStruct.User}).Infof("User was authenticated") - s3conf := getS3ConfigMap(idStruct.Token, auth.Config.S3Inbox, idStruct.User) + s3confInbox := getS3ConfigMap(idStruct.ResignedToken, auth.Config.S3Inbox, idStruct.User) + s3confDownload := getS3ConfigMap(idStruct.RawToken, auth.Config.S3Inbox, idStruct.User) - return &OIDCData{S3Conf: s3conf, OIDCID: idStruct} + return &OIDCData{S3ConfInbox: s3confInbox, S3ConfDownload: s3confDownload, OIDCID: idStruct} } // getOIDCLogin renders the `oidc.html` template to the given iris context func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { - oidcData := auth.elixirLogin(ctx) if oidcData == nil { return } s := sessions.Get(ctx) - s.SetFlash("oidc", oidcData.S3Conf) + s.SetFlash("oidcInbox", oidcData.S3ConfInbox) + s.SetFlash("oidcDownload", oidcData.S3ConfDownload) + ctx.ViewData("cegaID", auth.Config.Cega.ID) ctx.ViewData("infoUrl", auth.Config.InfoURL) ctx.ViewData("infoText", auth.Config.InfoText) ctx.ViewData("User", oidcData.OIDCID.User) + ctx.ViewData("Fullname", oidcData.OIDCID.Fullname) ctx.ViewData("Passport", oidcData.OIDCID.Passport) - ctx.ViewData("Token", oidcData.OIDCID.Token) - ctx.ViewData("ExpDate", oidcData.OIDCID.ExpDate) + ctx.ViewData("RawToken", oidcData.OIDCID.RawToken) + ctx.ViewData("ResignedToken", oidcData.OIDCID.ResignedToken) + ctx.ViewData("ExpDateRaw", oidcData.OIDCID.ExpDateRaw) + ctx.ViewData("ExpDateResigned", oidcData.OIDCID.ExpDateResigned) err := ctx.View("oidc.html") if err != nil { @@ -317,7 +324,6 @@ func (auth AuthHandler) getOIDCLogin(ctx iris.Context) { // getOIDCCORSLogin returns the oidc data as JSON to the given iris context func (auth AuthHandler) getOIDCCORSLogin(ctx iris.Context) { - oidcData := auth.elixirLogin(ctx) if oidcData == nil { return @@ -331,21 +337,24 @@ func (auth AuthHandler) getOIDCCORSLogin(ctx iris.Context) { } } -// getOIDCConf returns an s3config file for an oidc login -func (auth AuthHandler) getOIDCConf(ctx iris.Context) { - auth.getInboxConfig(ctx, "oidc") +// getOIDCConfInbox returns an s3config file for uploading to the Inbox +func (auth AuthHandler) getOIDCConfInbox(ctx iris.Context) { + auth.getS3Config(ctx, "oidcInbox", "s3cmd-inbox.conf") +} + +// getOIDCConfDownload returns an s3config file for downloading from the Archive +func (auth AuthHandler) getOIDCConfDownload(ctx iris.Context) { + auth.getS3Config(ctx, "oidcDownload", "s3cmd-download.conf") } // globalHeaders presets common response headers func globalHeaders(ctx iris.Context) { - ctx.ResponseWriter().Header().Set("X-Content-Type-Options", "nosniff") ctx.Next() } // addCSPheaders implements CSP and recommended complementary policies func addCSPheaders(ctx iris.Context) { - ctx.ResponseWriter().Header().Set("Content-Security-Policy", "default-src 'self';"+ "script-src-elem 'self';"+ "img-src 'self' data:;"+ @@ -358,9 +367,8 @@ func addCSPheaders(ctx iris.Context) { } func main() { - // Initialise config - config, err := config.NewConfig("auth") + conf, err := config.NewConfig("auth") if err != nil { log.Errorf("Failed to generate config, reason: %v", err) os.Exit(1) @@ -369,14 +377,14 @@ func main() { var oauth2Config oauth2.Config var provider *oidc.Provider - if config.Auth.OIDC.ID != "" && config.Auth.OIDC.Secret != "" { + if conf.Auth.OIDC.ID != "" && conf.Auth.OIDC.Secret != "" { // Initialise OIDC client - oauth2Config, provider = getOidcClient(config.Auth.OIDC) + oauth2Config, provider = getOidcClient(conf.Auth.OIDC) } // Create handler struct for the web server authHandler := AuthHandler{ - Config: config.Auth, + Config: conf.Auth, OAuth2Config: oauth2Config, OIDCProvider: provider, htmlDir: "./frontend/templates", @@ -390,12 +398,12 @@ func main() { // Start sessions handler in order to send flash messages sess := sessions.New(sessions.Config{Cookie: "_session_id", AllowReclaim: true}) - if config.Server.CORS.AllowOrigin != "" { + if conf.Server.CORS.AllowOrigin != "" { // Set CORS context corsContext := cors.New(cors.Options{ - AllowedOrigins: strings.Split(config.Server.CORS.AllowOrigin, ","), - AllowedMethods: strings.Split(config.Server.CORS.AllowMethods, ","), - AllowCredentials: config.Server.CORS.AllowCredentials, + AllowedOrigins: strings.Split(conf.Server.CORS.AllowOrigin, ","), + AllowedMethods: strings.Split(conf.Server.CORS.AllowMethods, ","), + AllowCredentials: conf.Server.CORS.AllowCredentials, }) app.Use(corsContext) } @@ -403,7 +411,7 @@ func main() { app.Use(sess.Handler()) // Connect to DB - authHandler.Config.DB, err = database.NewSDAdb(config.Database) + authHandler.Config.DB, err = database.NewSDAdb(conf.Database) if err != nil { log.Error(err) panic(err) @@ -427,7 +435,8 @@ func main() { // OIDC endpoints app.Get("/oidc", authHandler.getOIDC) - app.Get("/oidc/s3conf", authHandler.getOIDCConf) + app.Get("/oidc/s3conf-inbox", authHandler.getOIDCConfInbox) + app.Get("/oidc/s3conf-download", authHandler.getOIDCConfDownload) app.Get("/oidc/login", authHandler.getOIDCLogin) app.Get("/oidc/cors_login", authHandler.getOIDCCORSLogin) @@ -441,12 +450,10 @@ func main() { app.UseGlobal(globalHeaders) - if config.Server.Cert != "" && config.Server.Key != "" { - + if conf.Server.Cert != "" && conf.Server.Key != "" { log.Infoln("Serving content using https") - err = app.Run(iris.TLS("0.0.0.0:8080", config.Server.Cert, config.Server.Key)) + err = app.Run(iris.TLS("0.0.0.0:8080", conf.Server.Cert, conf.Server.Key)) } else { - log.Infoln("Serving content using http") server := &http.Server{ Addr: "0.0.0.0:8080", diff --git a/sda/cmd/auth/oidc.go b/sda/cmd/auth/oidc.go index 5f30858e79..96cf971eb9 100644 --- a/sda/cmd/auth/oidc.go +++ b/sda/cmd/auth/oidc.go @@ -18,11 +18,13 @@ import ( type OIDCIdentity struct { User string Passport []string - Token string - Profile string + RawToken string + ResignedToken string + Fullname string Email string EdupersonEntitlement []string - ExpDate string + ExpDateRaw string + ExpDateResigned string } // Configure an OpenID Connect aware OAuth2 client. @@ -30,7 +32,7 @@ func getOidcClient(conf config.OIDCConfig) (oauth2.Config, *oidc.Provider) { contx := context.Background() provider, err := oidc.NewProvider(contx, conf.Provider) if err != nil { - log.Fatal(err) + log.Fatal(err) // nolint # FIXME Fatal should only be called from main } oauth2Config := oauth2.Config{ @@ -92,7 +94,7 @@ func authenticateWithOidc(oauth2Config oauth2.Config, provider *oidc.Provider, c // Extract custom passports, name and email claims var claims struct { PassportClaim []string `json:"ga4gh_passport_v1"` - ProfileClaim string `json:"name"` + FullnameClaim string `json:"name"` EmailClaim string `json:"email"` EdupersonEntitlement []string `json:"eduperson_entitlement"` } @@ -104,12 +106,14 @@ func authenticateWithOidc(oauth2Config oauth2.Config, provider *oidc.Provider, c idStruct = OIDCIdentity{ User: userInfo.Subject, - Token: rawAccessToken, + RawToken: rawAccessToken, + ResignedToken: rawAccessToken, Passport: claims.PassportClaim, - Profile: claims.ProfileClaim, + Fullname: claims.FullnameClaim, Email: claims.EmailClaim, EdupersonEntitlement: claims.EdupersonEntitlement, - ExpDate: rawExpDate, + ExpDateRaw: rawExpDate, + ExpDateResigned: rawExpDate, } return idStruct, err diff --git a/sda/cmd/auth/oidc_test.go b/sda/cmd/auth/oidc_test.go index 6a3d395cca..500aa5d591 100644 --- a/sda/cmd/auth/oidc_test.go +++ b/sda/cmd/auth/oidc_test.go @@ -33,69 +33,68 @@ func TestOIDCTestSuite(t *testing.T) { suite.Run(t, new(OIDCTests)) } -func (suite *OIDCTests) SetupTest() { +func (ts *OIDCTests) SetupTest() { var err error - suite.mockServer, err = mockoidc.Run() - assert.NoError(suite.T(), err) + ts.mockServer, err = mockoidc.Run() + assert.NoError(ts.T(), err) // create an elixir config that has the needed endpoints set - suite.OIDCConfig = config.OIDCConfig{ - ID: suite.mockServer.ClientID, - Provider: suite.mockServer.Issuer(), + ts.OIDCConfig = config.OIDCConfig{ + ID: ts.mockServer.ClientID, + Provider: ts.mockServer.Issuer(), RedirectURL: "http://redirect", - Secret: suite.mockServer.ClientSecret, + Secret: ts.mockServer.ClientSecret, } } -func (suite *OIDCTests) TearDownTest() { - err := suite.mockServer.Shutdown() - assert.NoError(suite.T(), err) +func (ts *OIDCTests) TearDownTest() { + err := ts.mockServer.Shutdown() + assert.NoError(ts.T(), err) } -func (suite *OIDCTests) TestGetOidcClient() { - +func (ts *OIDCTests) TestGetOidcClient() { expectedEndpoint := oauth2.Endpoint{ - AuthURL: suite.mockServer.AuthorizationEndpoint(), - TokenURL: suite.mockServer.TokenEndpoint(), + AuthURL: ts.mockServer.AuthorizationEndpoint(), + TokenURL: ts.mockServer.TokenEndpoint(), AuthStyle: 0} - oauth2Config, provider := getOidcClient(suite.OIDCConfig) - assert.Equal(suite.T(), suite.mockServer.ClientID, oauth2Config.ClientID, "ClientID was modified when creating the oauth2Config") - assert.Equal(suite.T(), suite.mockServer.ClientSecret, oauth2Config.ClientSecret, "ClientSecret was modified when creating the oauth2Config") - assert.Equal(suite.T(), suite.OIDCConfig.RedirectURL, oauth2Config.RedirectURL, "RedirectURL was modified when creating the oauth2Config") - assert.Equal(suite.T(), expectedEndpoint, oauth2Config.Endpoint, "Issuer was modified when creating the oauth2Config") - assert.Equal(suite.T(), expectedEndpoint, provider.Endpoint(), "provider has the wrong endpoint") - assert.Equal(suite.T(), []string{"openid", "ga4gh_passport_v1 profile email eduperson_entitlement"}, oauth2Config.Scopes, "oauth2Config has the wrong scopes") + oauth2Config, provider := getOidcClient(ts.OIDCConfig) + assert.Equal(ts.T(), ts.mockServer.ClientID, oauth2Config.ClientID, "ClientID was modified when creating the oauth2Config") + assert.Equal(ts.T(), ts.mockServer.ClientSecret, oauth2Config.ClientSecret, "ClientSecret was modified when creating the oauth2Config") + assert.Equal(ts.T(), ts.OIDCConfig.RedirectURL, oauth2Config.RedirectURL, "RedirectURL was modified when creating the oauth2Config") + assert.Equal(ts.T(), expectedEndpoint, oauth2Config.Endpoint, "Issuer was modified when creating the oauth2Config") + assert.Equal(ts.T(), expectedEndpoint, provider.Endpoint(), "provider has the wrong endpoint") + assert.Equal(ts.T(), []string{"openid", "ga4gh_passport_v1 profile email eduperson_entitlement"}, oauth2Config.Scopes, "oauth2Config has the wrong scopes") } -func (suite *OIDCTests) TestAuthenticateWithOidc() { - +func (ts *OIDCTests) TestAuthenticateWithOidc() { // Create a code to authenticate - - session, err := suite.mockServer.SessionStore.NewSession( + session, err := ts.mockServer.SessionStore.NewSession( "openid email profile", "nonce", mockoidc.DefaultUser(), "", "") if err != nil { log.Error(err) } code := session.SessionID - jwkURL := suite.mockServer.JWKSEndpoint() + jwkURL := ts.mockServer.JWKSEndpoint() - oauth2Config, provider := getOidcClient(suite.OIDCConfig) + oauth2Config, provider := getOidcClient(ts.OIDCConfig) elixirIdentity, err := authenticateWithOidc(oauth2Config, provider, code, jwkURL) - assert.Nil(suite.T(), err, "Failed to authenticate with OIDC") - assert.NotEqual(suite.T(), "", elixirIdentity.Token, "Empty token returned from OIDC authentication") + assert.Nil(ts.T(), err, "Failed to authenticate with OIDC") + // Ensure both RawToken and ResignedToken are not empty + assert.NotEqual(ts.T(), "", elixirIdentity.RawToken, "Empty RawToken returned from OIDC authentication") + assert.NotEqual(ts.T(), "", elixirIdentity.ResignedToken, "Empty ResignedToken returned from OIDC authentication") } -func (suite *OIDCTests) TestValidateJwt() { - session, err := suite.mockServer.SessionStore.NewSession("openid email profile", "nonce", mockoidc.DefaultUser(), "", "") - assert.NoError(suite.T(), err) - oauth2Config, provider := getOidcClient(suite.OIDCConfig) - jwkURL := suite.mockServer.JWKSEndpoint() +func (ts *OIDCTests) TestValidateJwt() { + session, err := ts.mockServer.SessionStore.NewSession("openid email profile", "nonce", mockoidc.DefaultUser(), "", "") + assert.NoError(ts.T(), err) + oauth2Config, provider := getOidcClient(ts.OIDCConfig) + jwkURL := ts.mockServer.JWKSEndpoint() elixirIdentity, _ := authenticateWithOidc(oauth2Config, provider, session.SessionID, jwkURL) - elixirJWT := elixirIdentity.Token + elixirJWT := elixirIdentity.RawToken - claims := map[string]interface{}{ + claims := map[string]any{ jwt.ExpirationKey: time.Now().UTC().Add(2 * time.Hour), jwt.IssuedAtKey: time.Now().UTC(), jwt.IssuerKey: "http://local.issuer", @@ -103,88 +102,88 @@ func (suite *OIDCTests) TestValidateJwt() { } token := jwt.New() for key, value := range claims { - assert.NoError(suite.T(), token.Set(key, value)) + assert.NoError(ts.T(), token.Set(key, value)) } // key from mock server - suiteKey, err := jwk.FromRaw(suite.mockServer.Keypair.PrivateKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), suiteKey.Set(jwk.KeyIDKey, suite.mockServer.Keypair.Kid)) + suiteKey, err := jwk.FromRaw(ts.mockServer.Keypair.PrivateKey) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), suiteKey.Set(jwk.KeyIDKey, ts.mockServer.Keypair.Kid)) // Create HS256 test token mySigningKey := []byte("AllYourBase") testTokenHS256, err := jwt.Sign(token, jwt.WithKey(jwa.HS256, mySigningKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create RSA test token rsaRawKey, err := rsa.GenerateKey(rand.Reader, 2048) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) rsaKey, err := jwk.FromRaw(rsaRawKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), jwk.AssignKeyID(rsaKey)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), jwk.AssignKeyID(rsaKey)) testTokenRSA, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, rsaKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create ECDSA test token ecRawKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) ecKey, err := jwk.FromRaw(ecRawKey) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), jwk.AssignKeyID(ecKey)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), jwk.AssignKeyID(ecKey)) testTokenEC, err := jwt.Sign(token, jwt.WithKey(jwa.ES256, ecKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create expired RSA test token expiredTokenRSA := jwt.New() for key, value := range claims { - assert.NoError(suite.T(), token.Set(key, value)) + assert.NoError(ts.T(), token.Set(key, value)) } - assert.NoError(suite.T(), expiredTokenRSA.Set(jwt.ExpirationKey, time.Now().UTC().Add(-4*time.Hour))) + assert.NoError(ts.T(), expiredTokenRSA.Set(jwt.ExpirationKey, time.Now().UTC().Add(-4*time.Hour))) testExpiredTokenRSA, err := jwt.Sign(expiredTokenRSA, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // sanity check - _, expDate, err := validateToken(elixirJWT, suite.mockServer.JWKSEndpoint()) - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), expDate, elixirIdentity.ExpDate, "Returned wrong exp date but without returning errors") + _, expDate, err := validateToken(elixirJWT, ts.mockServer.JWKSEndpoint()) + assert.Nil(ts.T(), err) + assert.Equal(ts.T(), expDate, elixirIdentity.ExpDateRaw, "Returned wrong exp date but without returning errors") // Not a jwk url _, _, err = validateToken(elixirJWT, "http://some/jwk/endpoint") - assert.ErrorContains(suite.T(), err, "failed to fetch \"http://some/jwk/endpoint") + assert.ErrorContains(ts.T(), err, "failed to fetch \"http://some/jwk/endpoint") // correct private key, RSA oidcTokenRSA, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) - _, _, err = validateToken(string(oidcTokenRSA), suite.mockServer.JWKSEndpoint()) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) + _, _, err = validateToken(string(oidcTokenRSA), ts.mockServer.JWKSEndpoint()) + assert.NoError(ts.T(), err) // wrong signing method - _, _, err = validateToken(string(testTokenHS256), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: key provider 0 failed:") + _, _, err = validateToken(string(testTokenHS256), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: key provider 0 failed:") // wrong private key, RSA - _, _, err = validateToken(string(testTokenRSA), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: key provider 0 failed:") + _, _, err = validateToken(string(testTokenRSA), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: key provider 0 failed:") // wrong private key, ECDSA - _, _, err = validateToken(string(testTokenEC), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: key provider 0 failed:") + _, _, err = validateToken(string(testTokenEC), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: key provider 0 failed:") // expired token - _, _, err = validateToken(string(testExpiredTokenRSA), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: \"exp\" not satisfied") + _, _, err = validateToken(string(testExpiredTokenRSA), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: \"exp\" not satisfied") // check that we handle the case where token has no expiration date - assert.NoError(suite.T(), token.Set(jwt.ExpirationKey, 0)) + assert.NoError(ts.T(), token.Set(jwt.ExpirationKey, 0)) noExpiryToken, err := jwt.Sign(token, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) - _, _, err = validateToken(string(noExpiryToken), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: iitr between exp and iat is less than") + assert.NoError(ts.T(), err) + _, _, err = validateToken(string(noExpiryToken), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: iitr between exp and iat is less than") // check that we handle the case where token has no exp key - assert.NoError(suite.T(), token.Remove(jwt.ExpirationKey)) + assert.NoError(ts.T(), token.Remove(jwt.ExpirationKey)) noExpiryToken, err = jwt.Sign(token, jwt.WithKey(jwa.RS256, suiteKey)) - assert.NoError(suite.T(), err) - _, _, err = validateToken(string(noExpiryToken), suite.mockServer.JWKSEndpoint()) - assert.ErrorContains(suite.T(), err, "signed token not valid: \"exp\" not satisfied: required claim not found") + assert.NoError(ts.T(), err) + _, _, err = validateToken(string(noExpiryToken), ts.mockServer.JWKSEndpoint()) + assert.ErrorContains(ts.T(), err, "signed token not valid: \"exp\" not satisfied: required claim not found") } diff --git a/sda/cmd/auth/s3conf_test.go b/sda/cmd/auth/s3conf_test.go index 31a6e3de6a..040118617f 100644 --- a/sda/cmd/auth/s3conf_test.go +++ b/sda/cmd/auth/s3conf_test.go @@ -19,13 +19,12 @@ func TestS3ConfTestSuite(t *testing.T) { suite.Run(t, new(S3ConfTests)) } -func (suite *S3ConfTests) SetupTest() {} +func (ts *S3ConfTests) SetupTest() {} -func (suite *S3ConfTests) TearDownTest() {} +func (ts *S3ConfTests) TearDownTest() {} //nolint:goconst -func (suite *S3ConfTests) TestGetS3ConfigMap() { - +func (ts *S3ConfTests) TestGetS3ConfigMap() { // variable values token := "tokenvaluestring" inboxHost := "s3://inboxHost" @@ -45,19 +44,18 @@ func (suite *S3ConfTests) TestGetS3ConfigMap() { s3conf := getS3ConfigMap(token, inboxHost, user) - assert.Equal(suite.T(), user, s3conf["access_key"], fmt.Sprintf("access_key should be %v", user)) - assert.Equal(suite.T(), user, s3conf["secret_key"], fmt.Sprintf("secret_key should be %v", user)) - assert.Equal(suite.T(), token, s3conf["access_token"], fmt.Sprintf("access_token should be %v", token)) - assert.Equal(suite.T(), checkSslCertificate, s3conf["check_ssl_certificate"], fmt.Sprintf("check_ssl_certificate should be %v", checkSslCertificate)) - assert.Equal(suite.T(), checkSslHostname, s3conf["check_ssl_hostname"], fmt.Sprintf("check_ssl_hostname should be %v", checkSslHostname)) - assert.Equal(suite.T(), encoding, s3conf["encoding"], fmt.Sprintf("encoding should be %v", encoding)) - assert.Equal(suite.T(), encrypt, s3conf["encrypt"], fmt.Sprintf("encrypt should be %v", encrypt)) - assert.Equal(suite.T(), guessMimeType, s3conf["guess_mime_type"], fmt.Sprintf("guess_mime_type should be %v", guessMimeType)) - assert.Equal(suite.T(), inboxHost, s3conf["host_base"], fmt.Sprintf("host_base should be %v", inboxHost)) - assert.Equal(suite.T(), inboxHost, s3conf["host_bucket"], fmt.Sprintf("host_bucket should be %v", inboxHost)) - assert.Equal(suite.T(), humanReadableSizes, s3conf["human_readable_sizes"], fmt.Sprintf("human_readable_sizes should be %v", humanReadableSizes)) - assert.Equal(suite.T(), fmt.Sprintf("%v", chunkSize), s3conf["multipart_chunk_size_mb"], fmt.Sprintf("multipart_chunk_size_mb should be %v", chunkSize)) - assert.Equal(suite.T(), useHTTPS, s3conf["use_https"], fmt.Sprintf("use_https should be '%v'", useHTTPS)) - assert.Equal(suite.T(), fmt.Sprintf("%v", socketTimeout), s3conf["socket_timeout"], fmt.Sprintf("socket_timeout should be %v", socketTimeout)) - + assert.Equal(ts.T(), user, s3conf["access_key"], fmt.Sprintf("access_key should be %v", user)) + assert.Equal(ts.T(), user, s3conf["secret_key"], fmt.Sprintf("secret_key should be %v", user)) + assert.Equal(ts.T(), token, s3conf["access_token"], fmt.Sprintf("access_token should be %v", token)) + assert.Equal(ts.T(), checkSslCertificate, s3conf["check_ssl_certificate"], fmt.Sprintf("check_ssl_certificate should be %v", checkSslCertificate)) + assert.Equal(ts.T(), checkSslHostname, s3conf["check_ssl_hostname"], fmt.Sprintf("check_ssl_hostname should be %v", checkSslHostname)) + assert.Equal(ts.T(), encoding, s3conf["encoding"], fmt.Sprintf("encoding should be %v", encoding)) + assert.Equal(ts.T(), encrypt, s3conf["encrypt"], fmt.Sprintf("encrypt should be %v", encrypt)) + assert.Equal(ts.T(), guessMimeType, s3conf["guess_mime_type"], fmt.Sprintf("guess_mime_type should be %v", guessMimeType)) + assert.Equal(ts.T(), inboxHost, s3conf["host_base"], fmt.Sprintf("host_base should be %v", inboxHost)) + assert.Equal(ts.T(), inboxHost, s3conf["host_bucket"], fmt.Sprintf("host_bucket should be %v", inboxHost)) + assert.Equal(ts.T(), humanReadableSizes, s3conf["human_readable_sizes"], fmt.Sprintf("human_readable_sizes should be %v", humanReadableSizes)) + assert.Equal(ts.T(), fmt.Sprintf("%v", chunkSize), s3conf["multipart_chunk_size_mb"], fmt.Sprintf("multipart_chunk_size_mb should be %v", chunkSize)) + assert.Equal(ts.T(), useHTTPS, s3conf["use_https"], fmt.Sprintf("use_https should be '%v'", useHTTPS)) + assert.Equal(ts.T(), fmt.Sprintf("%v", socketTimeout), s3conf["socket_timeout"], fmt.Sprintf("socket_timeout should be %v", socketTimeout)) } diff --git a/sda/cmd/auth/swagger_v1.yml b/sda/cmd/auth/swagger_v1.yml new file mode 100644 index 0000000000..c0123d02bf --- /dev/null +++ b/sda/cmd/auth/swagger_v1.yml @@ -0,0 +1,33 @@ +openapi: 3.0.3 +info: + title: SDA auth frontend + version: "1.0" + description: This is the auth frontend for the sensitive data archive. +paths: + /info: + get: + description: Returns the info JSON + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/Info" + description: Successful operation +components: + schemas: + Info: + type: object + properties: + client_id: + example: 7daee13a-cd72-4b92-8a59-89c0e063b20f + type: string + inbox_uri: + example: inbox.demo.org + type: string + oidc_uri: + example: https://aai.provider.org/oidc + type: string + public_key: + example: LS0tLS1CRUdJTiBDUllQVDRHSCBQVUJMSUMgS0VZLS0t + type: string \ No newline at end of file diff --git a/sda/cmd/download/download.go b/sda/cmd/download/download.go new file mode 100644 index 0000000000..38dd16da61 --- /dev/null +++ b/sda/cmd/download/download.go @@ -0,0 +1,3 @@ +package main + +func main() {} diff --git a/sda/cmd/download/swagger_v1.yml b/sda/cmd/download/swagger_v1.yml new file mode 100644 index 0000000000..aa4d791d49 --- /dev/null +++ b/sda/cmd/download/swagger_v1.yml @@ -0,0 +1,256 @@ +openapi: 3.0.3 +info: + title: SDA file download API + version: "1.0" + description: This is the external data out API for the Sensitive Data Archive. Data is delivered encrypted with a user supplied crypth4gh public key. +paths: + /file: + get: + description: Returns the requested file to the user. Uses query parameters in order to handle files requested by the `filePath` entity. + parameters: + - in: header + description: Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key + name: public_key + required: true + schema: + type: string + - in: query + name: dataset + schema: + type: string + required: true + description: Dataset identifier + - in: query + name: fileId + schema: + type: string + required: false + description: Stable ID of the file to download. It is an error to supply both filePath and fileId + - in: query + name: filePath + schema: + type: string + required: false + description: File path of the file to download. It is an error to supply both filePath and fileId + - in: query + name: end + description: End of requested data in the form of the 0-index of the first octet not included in the response. That is, data delivered will have octet indices in the closed-open set [start,end). Defaults to end of file if not set + required: false + schema: + minimum: 0 + type: integer + - in: query + name: start + description: Start of the requested data. Defaults to start of file if not set + required: false + schema: + default: 0 + minimum: 0 + type: integer + responses: + "200": + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successful operation + "206": + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successful partial file delivery + "400": + description: Bad request ex. start larger than end + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "416": + description: Unsatisfiable range ex. start larger than the size of the file + "500": + description: Internal application error + security: + - bearerAuth: [] + /file/{fileId}: + get: + description: Returns the requested file to the user. + parameters: + - in: header + description: Public key used to re-encrypt the file (header) with. This should be supplied as the base64 encoding of the PEM (RFC 7468) encoding of the public key + name: public_key + required: true + schema: + type: string + - in: path + name: fileId + schema: + type: string + required: true + description: Stable ID of the file to download + - in: header + name: Range + required: false + schema: + type: string + description: 0-indexed byte range of the file that is to be delivered as per RFC7233. + example: "Range: bytes=20-200, 300-400" + responses: + "200": + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successful operation + "206": + content: + application/octet-stream: + schema: + type: string + format: binary + description: Successful partial file delivery + "400": + description: Bad request ex. start larger than end + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "416": + description: Unsatisfiable range ex. start larger than the size of the file + "500": + description: Internal application error + security: + - bearerAuth: [] + /info/datasets: + get: + description: Returns a list of the datasets the user has access to, an empty list is a valid response. + responses: + "200": + description: Successful operation + content: + application/json: + example: ["aa-dataset-123456-asdfgh", "bb-dataset-123456-asdfgh"] + schema: + type: array + "401": + description: Authentication failure + "500": + description: Internal application error + security: + - bearerAuth: [] + /info/dataset: + get: + description: Returns an array with metadata about the dataset + parameters: + - in: query + name: dataset + schema: + type: string + required: true + description: Dataset identifier + responses: + "200": + content: + application/json: + schema: + $ref: "#/components/schemas/DatasetInfo" + description: Successful operation + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "500": + description: Internal application error + security: + - bearerAuth: [] + /info/dataset/files: + get: + description: Returns an array with metadata about all files in a dataset + parameters: + - in: query + name: dataset + schema: + type: string + required: true + description: Dataset identifier + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/FileInfo" + description: Successful operation + "401": + description: Authentication failure + "403": + description: The user does not have access to the dataset + "500": + description: Internal application error + security: + - bearerAuth: [] + /health/ready: + get: + description: Returns the status of the application. + responses: + "200": + content: + application/json: + example: + { + "c4ghService": "responding", + "database": "responding", + "oidcServer": "responding", + "storageBackend": "responding", + } + description: The service is operational + "503": + description: Unhealthy service + /health/live: + get: + description: Returns 200 as long as the main thread is running + responses: + "200": + description: The service is operational +components: + schemas: + DatasetInfo: + type: object + properties: + date: + type: string + format: date-time + example: "2025-02-14T14:51:26.639Z" + files: + type: integer + format: int32 + example: 1234 + name: + type: string + example: aa-dataset-123456-asdfgh + size: + type: integer + format: int64 + example: 6597069766656 + FileInfo: + type: object + properties: + filePath: + type: string + example: samples/controls/sample1.cram.c4gh + size: + type: integer + format: int64 + example: 56623104 + stableId: + type: string + example: aa-file-123456-asdfgh + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: JWT diff --git a/sda/cmd/finalize/finalize.go b/sda/cmd/finalize/finalize.go index 3a2bd33cd0..df2096cac9 100644 --- a/sda/cmd/finalize/finalize.go +++ b/sda/cmd/finalize/finalize.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "errors" "fmt" "io" @@ -244,13 +245,13 @@ func backupFile(delivered amqp.Delivery) error { } // Get size on disk, will also give some time for the file to appear if it has not already - diskFileSize, err := archive.GetFileSize(filePath) + diskFileSize, err := archive.GetFileSize(filePath, false) if err != nil { return fmt.Errorf("failed to get size info for archived file, reason: %v", err) } if diskFileSize != int64(fileSize) { - return fmt.Errorf("file size in archive does not match database for archive file") + return errors.New("file size in archive does not match database for archive file") } file, err := archive.NewFileReader(filePath) diff --git a/sda/cmd/finalize/finalize_test.go b/sda/cmd/finalize/finalize_test.go index 0f8a2819d2..aea1d90847 100644 --- a/sda/cmd/finalize/finalize_test.go +++ b/sda/cmd/finalize/finalize_test.go @@ -15,6 +15,6 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } diff --git a/sda/cmd/ingest/ingest.go b/sda/cmd/ingest/ingest.go index cfd4bd62ba..6c66889fb6 100644 --- a/sda/cmd/ingest/ingest.go +++ b/sda/cmd/ingest/ingest.go @@ -28,7 +28,18 @@ import ( log "github.com/sirupsen/logrus" ) +type Ingest struct { + Archive storage.Backend + ArchiveKeyList []*[32]byte + Conf *config.Config + DB *database.SDAdb + Inbox storage.Backend + MQ *broker.AMQPBroker +} + func main() { + app := Ingest{} + var err error sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) // Create a function to handle panic and exit gracefully @@ -39,71 +50,71 @@ func main() { }() forever := make(chan bool) - conf, err := config.NewConfig("ingest") + + app.Conf, err = config.NewConfig("ingest") if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - mq, err := broker.NewMQ(conf.Broker) + app.MQ, err = broker.NewMQ(app.Conf.Broker) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - db, err := database.NewSDAdb(conf.Database) + app.DB, err = database.NewSDAdb(app.Conf.Database) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - if db.Version < 8 { + if app.DB.Version < 8 { log.Error("database schema v8 is required") sigc <- syscall.SIGINT panic(err) } - archiveKeyList, err := config.GetC4GHprivateKeys() + app.ArchiveKeyList, err = config.GetC4GHprivateKeys() if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - archive, err := storage.NewBackend(conf.Archive) + app.Archive, err = storage.NewBackend(app.Conf.Archive) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - inbox, err := storage.NewBackend(conf.Inbox) + app.Inbox, err = storage.NewBackend(app.Conf.Inbox) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - defer mq.Channel.Close() - defer mq.Connection.Close() - defer db.Close() + defer app.MQ.Channel.Close() + defer app.MQ.Connection.Close() + defer app.DB.Close() go func() { - connError := mq.ConnectionWatcher() + connError := app.MQ.ConnectionWatcher() log.Error(connError) forever <- false }() go func() { - connError := mq.ChannelWatcher() + connError := app.MQ.ChannelWatcher() log.Error(connError) forever <- false }() log.Info("starting ingest service") - var message schema.IngestionTrigger go func() { start := time.Now() for i := 1; i > 0; i++ { - h, err := db.ListKeyHashes() + h, err := app.DB.ListKeyHashes() if err != nil { log.Errorln(err.Error()) } @@ -119,14 +130,15 @@ func main() { log.Errorln("no crypt4gh key hash registered") } - messages, err := mq.GetMessages(conf.Broker.Queue) + messages, err := app.MQ.GetMessages(app.Conf.Broker.Queue) if err != nil { - log.Fatal(err) + log.Panic(err) } - mainWorkLoop: + for delivered := range messages { log.Debugf("received a message (corr-id: %s, message: %s)", delivered.CorrelationId, delivered.Body) - err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", conf.Broker.SchemasPath), delivered.Body) + message := schema.IngestionTrigger{} + err := schema.ValidateJSON(fmt.Sprintf("%s/ingestion-trigger.json", app.Conf.Broker.SchemasPath), delivered.Body) if err != nil { log.Errorf("validation of incoming message (ingestion-trigger) failed, reason: (%s)", err.Error()) // Send the message to an error queue so it can be analyzed. @@ -137,440 +149,445 @@ func main() { } body, _ := json.Marshal(infoErrorMessage) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { + if err := app.MQ.SendMessage(delivered.CorrelationId, app.Conf.Broker.Exchange, "error", body); err != nil { log.Errorf("failed to publish message, reason: (%s)", err.Error()) } if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } - // Restart on new message continue } // we unmarshal the message in the validation step so this is safe to do _ = json.Unmarshal(delivered.Body, &message) + log.Infof("Received work (corr-id: %s, filepath: %s, user: %s)", delivered.CorrelationId, message.FilePath, message.User) - log.Infof( - "Received work (corr-id: %s, filepath: %s, user: %s)", - delivered.CorrelationId, message.FilePath, message.User, - ) - + ackNack := "" switch message.Type { case "cancel": - fileUUID, err := db.GetFileID(delivered.CorrelationId) - if err != nil || fileUUID == "" { - log.Errorf("failed to get ID for file from message: %v", delivered.CorrelationId) - - if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - if err := db.UpdateFileEventLog(fileUUID, "disabled", delivered.CorrelationId, "ingest", "{}", string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) - if err = delivered.Nack(false, false); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue + ackNack = app.cancelFile(delivered.CorrelationId, message) + case "ingest": + ackNack = app.ingestFile(delivered.CorrelationId, message) + default: + log.Errorln("unexpected ingest message type") + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message for reason: (%s)", err.Error()) } + } + switch ackNack { + case "ack": if err := delivered.Ack(false); err != nil { log.Errorf("failed to ack message for reason: (%s)", err.Error()) } - - continue - case "ingest": - var fileID string - status, err := db.GetFileStatus(delivered.CorrelationId) - if err != nil && err.Error() != "sql: no rows in result set" { - log.Errorf("failed to get status for file, reason: (%s)", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue + case "nack": + if err = delivered.Nack(false, false); err != nil { + log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) } - - switch status { - case "disabled": - fileID, err := db.GetFileID(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - fileInfo, err := db.GetFileInfo(fileID) - if err != nil { - log.Errorf("failed to get info for file: %s", fileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - if err = db.UpdateFileEventLog(fileInfo.Path, "enabled", delivered.CorrelationId, "ingest", "{}", string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - - if fileInfo.Checksum != "" { - msg := schema.IngestionVerification{ - User: message.User, - FilePath: message.FilePath, - FileID: fileID, - ArchivePath: fileInfo.Path, - EncryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fileInfo.Checksum}, - }, - } - archivedMsg, _ := json.Marshal(&msg) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), archivedMsg) - if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) - - continue - } - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, archivedMsg); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - - continue - } - - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) - } - - continue - } - case "": - // Catch all for inboxes that doesn't update the DB - fileID, err = db.RegisterFile(message.FilePath, message.User) - if err != nil { - log.Errorf("InsertFile failed, reason: (%s)", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } - default: - fileID, err = db.GetFileID(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get ID for file, reason: %s", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } + default: + // will catch `reject`s, failures that should not be requeued. + if err := delivered.Reject(false); err != nil { + log.Errorf("failed to reject message for reason: (%s)", err.Error()) } + } + } + }() - file, err := inbox.NewFileReader(message.FilePath) - if err != nil { //nolint:nestif - log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) - if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") { - jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) - if err := db.UpdateFileEventLog(fileID, "error", delivered.CorrelationId, "ingest", string(jsonMsg), string(delivered.Body)); err != nil { - log.Errorf("failed to set error status for file from message: %v, reason: %s", delivered.CorrelationId, err.Error()) - } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to open file to ingest", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - if err = delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - // Restart on new message - continue - } + <-forever +} - fileSize, err := inbox.GetFileSize(message.FilePath) - if err != nil { - log.Errorf("Failed to get file size of file to ingest, reason: (%s)", err.Error()) - // Nack message so the server gets notified that something is wrong and requeue the message. - // Since reading the file worked, this should eventually succeed so it is ok to requeue. - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Failed to get file size of file to ingest", - Reason: err.Error(), - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err = mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - - // Restart on new message - continue - } +func (app *Ingest) cancelFile(correlationID string, message schema.IngestionTrigger) string { + fileUUID, err := app.DB.GetFileID(correlationID) + if err != nil { + log.Errorf("failed to get ID for file from message (correlationID: %s), reason: %s", correlationID, err.Error()) + if strings.Contains(err.Error(), "sql: no rows in result set") { + return "reject" + } - if err = db.UpdateFileEventLog(fileID, "submitted", delivered.CorrelationId, message.User, "{}", string(delivered.Body)); err != nil { - log.Errorf("failed to set ingestion status for file from message: %v", delivered.CorrelationId) - } + return "nack" + } - dest, err := archive.NewFileWriter(fileID) - if err != nil { - log.Errorf("Failed to create archive file, reason: (%s)", err.Error()) - // Nack message so the server gets notified that something is wrong and requeue the message. - // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileUUID, "disabled", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set event log status for file: %s", correlationID) - continue - } + return "nack" + } - // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock - var bufSize int - if bufSize = 4 * 1024 * 1024; conf.Inbox.S3.Chunksize > 4*1024*1024 { - bufSize = conf.Inbox.S3.Chunksize - } - readBuffer := make([]byte, bufSize) - hash := sha256.New() - var bytesRead int64 - var byteBuf bytes.Buffer - - for bytesRead < fileSize { - i, _ := io.ReadFull(file, readBuffer) - if i == 0 { - return - } - // truncate the readbuffer if the file is smaller than the buffer size - if i < len(readBuffer) { - readBuffer = readBuffer[:i] - } - - bytesRead += int64(i) - - h := bytes.NewReader(readBuffer) - if _, err = io.Copy(hash, h); err != nil { - log.Errorf("Copy to hash failed while reading file, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - //nolint:nestif - if bytesRead <= int64(len(readBuffer)) { - var privateKey *[32]byte - var header []byte - - // Iterate over the key list to try decryption - for _, key := range archiveKeyList { - header, err = tryDecrypt(key, readBuffer) - if err == nil { - privateKey = key - - break - } - log.Warnf("Decryption failed with key, trying next key. Reason: (%s)", err.Error()) - } - - // Check if decryption was successful with any key - if privateKey == nil { - log.Errorf("All keys failed to decrypt the submitted file") - if err := db.UpdateFileEventLog(fileID, "error", delivered.CorrelationId, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(delivered.Body)); err != nil { - log.Errorf("Failed to set ingestion status for file from message: %v", delivered.CorrelationId) - } - - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed to Ack message, reason: (%s)", err.Error()) - } - - // Send the message to an error queue so it can be analyzed. - fileError := broker.InfoError{ - Error: "Trying to decrypt the submitted file failed", - Reason: "Decryption failed with the available key(s)", - OriginalMessage: message, - } - body, _ := json.Marshal(fileError) - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, "error", body); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - // Proceed with the successful key - // Set the file's hex encoded public key - publicKey := keys.DerivePublicKey(*privateKey) - keyhash := hex.EncodeToString(publicKey[:]) - err = db.SetKeyHash(keyhash, fileID) - if err != nil { - log.Errorf("Key hash %s could not be set for fileID %s: (%s)", keyhash, fileID, err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - log.Debugln("store header") - if err := db.StoreHeader(header, fileID); err != nil { - log.Errorf("StoreHeader failed, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for header read, reason: %v)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - - // Strip header from buffer - h := make([]byte, len(header)) - if _, err = byteBuf.Read(h); err != nil { - log.Errorf("Failed to strip header from buffer, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - } else { - if i < len(readBuffer) { - readBuffer = readBuffer[:i] - } - if _, err = byteBuf.Write(readBuffer); err != nil { - log.Errorf("Failed to write to read buffer for full read, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - - continue mainWorkLoop - } - } - - // Write data to file - if _, err = byteBuf.WriteTo(dest); err != nil { - log.Errorf("Failed to write to archive file, reason: (%s)", err.Error()) - - continue mainWorkLoop - } - } + return "ack" +} - file.Close() - dest.Close() +func (app *Ingest) ingestFile(correlationID string, message schema.IngestionTrigger) string { + var fileID string + status, err := app.DB.GetFileStatus(correlationID) + if err != nil && err.Error() != "sql: no rows in result set" { + log.Errorf("failed to get status for file, reason: (%s)", err.Error()) - // At this point we should do checksum comparison, but that requires updating the AWS library + return "nack" + } - fileInfo := database.FileInfo{} - fileInfo.Path = fileID - fileInfo.Checksum = fmt.Sprintf("%x", hash.Sum(nil)) - fileInfo.Size, err = archive.GetFileSize(fileID) - if err != nil { - log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } + switch status { + case "disabled": + fileID, err = app.DB.GetFileID(correlationID) + if err != nil { + log.Errorf("failed to get ID for file, reason: %s", err.Error()) - continue - } + return "nack" + } - log.Debugf("Wrote archived file (corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", - delivered.CorrelationId, message.User, message.FilePath, fileID, fileInfo.Size) + fileInfo, err := app.DB.GetFileInfo(fileID) + if err != nil { + log.Errorf("failed to get info for file: %s, reason: %s", fileID, err.Error()) - status, err = db.GetFileStatus(delivered.CorrelationId) - if err != nil { - log.Errorf("failed to get file status, reason: (%s)", err.Error()) - if err = delivered.Nack(false, true); err != nil { - log.Errorf("Failed to Nack message, reason: (%s)", err.Error()) - } - } - if status == "disabled" { - log.Infof("file with correlation ID: %s is disabled, stopping ingestion", delivered.CorrelationId) - if err := delivered.Ack(false); err != nil { - log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) - } + return "nack" + } - continue + // What if the file in the inbox is different this time? + // Check uploaded checksum in the DB against the checksum of the file. + // Would be easy if the inbox message had the checksum or that the s3inbox added the checksum to the DB. + file, err := app.Inbox.NewFileReader(message.FilePath) + if err != nil { + switch { + case strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:"): + log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { + log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) } - - if err := db.SetArchived(fileInfo, fileID, delivered.CorrelationId); err != nil { - log.Errorf("SetArchived failed, reason: (%s)", err.Error()) + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to open file to ingest", + Reason: err.Error(), + OriginalMessage: message, } + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) - log.Debugf("File marked as archived (corr-id: %s, user: %s, filepath: %s, archivepath: %s)", - delivered.CorrelationId, message.User, message.FilePath, fileID) - - // Send message to archived - msg := schema.IngestionVerification{ - User: message.User, - FilePath: message.FilePath, - FileID: fileID, - ArchivePath: fileID, - EncryptedChecksums: []schema.Checksums{ - {Type: "sha256", Value: fmt.Sprintf("%x", hash.Sum(nil))}, - }, + return "reject" } - archivedMsg, _ := json.Marshal(&msg) - err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", conf.Broker.SchemasPath), archivedMsg) - if err != nil { - log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + return "ack" + default: + return "nack" + } + } + defer file.Close() + + inboxChecksum := sha256.New() + _, err = io.Copy(inboxChecksum, file) + if err != nil { + log.Errorf("failed to calculate cheksum for file: %s, reason: %s", fileID, err.Error()) + + return "nack" + } + + if fileInfo.UploadedChecksum == string(inboxChecksum.Sum(nil)) && fileInfo.ArchiveChecksum != "" { + msg := schema.IngestionVerification{ + User: message.User, + FilePath: message.FilePath, + FileID: fileID, + ArchivePath: fileInfo.Path, + EncryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fileInfo.ArchiveChecksum}, + }, + } + archivedMsg, _ := json.Marshal(&msg) + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) + if err != nil { + log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + + return "nack" + } + + m, _ := json.Marshal(message) + if err = app.DB.UpdateFileEventLog(fileInfo.Path, "enabled", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set ingestion status for file from message: %v", correlationID) + + return "nack" + } + + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + return "reject" + } + + return "ack" + } + case "": + // Catch all for inboxes that doesn't update the DB + log.Warnln("ingesting unregisterd file") + fileID, err = app.DB.RegisterFile(message.FilePath, message.User) + if err != nil { + log.Errorf("InsertFile failed, reason: (%s)", err.Error()) + + return "nack" + } + case "uploaded": + fileID, err = app.DB.GetFileID(correlationID) + if err != nil { + log.Errorf("failed to get ID for file, reason: %s", err.Error()) + + return "nack" + } + default: + log.Warnf("unsupported file status: %s", status) + + return "reject" + } + + file, err := app.Inbox.NewFileReader(message.FilePath) + if err != nil { + switch { + case (strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:")): + log.Errorf("Failed to open file to ingest reason: (%s)", err.Error()) + jsonMsg, _ := json.Marshal(map[string]string{"error": err.Error()}) + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", string(jsonMsg), string(m)); err != nil { + log.Errorf("failed to set error status for file from message: %v, reason: %s", correlationID, err.Error()) + } + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Failed to open file to ingest", + Reason: err.Error(), + OriginalMessage: message, + } + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + return "reject" + } + + return "ack" + default: + log.Errorf("unexpected error when opening file for reading: %s", err.Error()) + + return "nack" + } + } + + fileSize, err := app.Inbox.GetFileSize(message.FilePath, false) + if err != nil { + log.Errorf("Failed to get file size of file to ingest, reason: (%s)", err.Error()) + // Since reading the file worked, this should eventually succeed so it is ok to requeue. + return "nack" + } + + dest, err := app.Archive.NewFileWriter(fileID) + if err != nil { + log.Errorf("Failed to create archive file, reason: (%s)", err.Error()) + // NewFileWriter returns an error when the backend itself fails so this is reasonable to requeue. + return "nack" + } + + m, _ := json.Marshal(message) + if err = app.DB.UpdateFileEventLog(fileID, "submitted", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set ingestion status for file from message: %v", correlationID) + } + + // 4MiB readbuffer, this must be large enough that we get the entire header and the first 64KiB datablock + var bufSize int + if bufSize = 4 * 1024 * 1024; app.Conf.Inbox.S3.Chunksize > 4*1024*1024 { + bufSize = app.Conf.Inbox.S3.Chunksize + } + readBuffer := make([]byte, bufSize) + hash := sha256.New() + var bytesRead int64 + var byteBuf bytes.Buffer + + for bytesRead < fileSize { + i, _ := io.ReadFull(file, readBuffer) + if i == 0 { + log.Errorln("readBuffer returned 0 bytes, this should not happen") - continue + return "reject" + } + // truncate the readbuffer if the file is smaller than the buffer size + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } + + bytesRead += int64(i) + + h := bytes.NewReader(readBuffer) + if _, err = io.Copy(hash, h); err != nil { + log.Errorf("Copy to hash failed while reading file, reason: (%s)", err.Error()) + + return "nack" + } + + switch { + case bytesRead <= int64(len(readBuffer)): + var privateKey *[32]byte + var header []byte + + // Iterate over the key list to try decryption + for _, key := range app.ArchiveKeyList { + header, err = tryDecrypt(key, readBuffer) + if err == nil { + privateKey = key + + break } + log.Warnf("Decryption failed with key, trying next key. Reason: (%s)", err.Error()) + } - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, archivedMsg); err != nil { - // TODO fix resend mechanism - log.Errorf("failed to publish message, reason: (%s)", err.Error()) + // Check if decryption was successful with any key + if privateKey == nil { + log.Errorf("All keys failed to decrypt the submitted file") + m, _ := json.Marshal(message) + if err := app.DB.UpdateFileEventLog(fileID, "error", correlationID, "ingest", `{"error" : "Decryption failed with all available key(s)"}`, string(m)); err != nil { + log.Errorf("Failed to set ingestion status for file from message: %v", correlationID) + } - // Do not try to ACK message to make sure we have another go - continue + // Send the message to an error queue so it can be analyzed. + fileError := broker.InfoError{ + Error: "Trying to decrypt the submitted file failed", + Reason: "Decryption failed with the available key(s)", + OriginalMessage: message, } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + body, _ := json.Marshal(fileError) + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, "error", body); err != nil { + log.Errorf("failed to publish message, reason: (%s)", err.Error()) } + + return "ack" + } + + // Proceed with the successful key + // Set the file's hex encoded public key + publicKey := keys.DerivePublicKey(*privateKey) + keyhash := hex.EncodeToString(publicKey[:]) + err = app.DB.SetKeyHash(keyhash, fileID) + if err != nil { + log.Errorf("Key hash %s could not be set for fileID %s: (%s)", keyhash, fileID, err.Error()) + + return "nack" + } + + log.Debugln("store header") + if err := app.DB.StoreHeader(header, fileID); err != nil { + log.Errorf("StoreHeader failed, reason: (%s)", err.Error()) + + return "nack" + } + + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for header read, reason: %v)", err.Error()) + + return "nack" + } + + // Strip header from buffer + h := make([]byte, len(header)) + if _, err = byteBuf.Read(h); err != nil { + log.Errorf("Failed to strip header from buffer, reason: (%s)", err.Error()) + + return "nack" + } + default: + if i < len(readBuffer) { + readBuffer = readBuffer[:i] + } + if _, err = byteBuf.Write(readBuffer); err != nil { + log.Errorf("Failed to write to read buffer for full read, reason: (%s)", err.Error()) + + return "nack" } } - }() - <-forever + // Write data to file + if _, err = byteBuf.WriteTo(dest); err != nil { + log.Errorf("Failed to write to archive file, reason: (%s)", err.Error()) + + _ = file.Close() + _ = dest.Close() + if err := app.Archive.RemoveFile(fileID); err != nil { + log.Errorf("error when removing uncompleted file: %s", fileID) + } + + return "nack" + } + } + + file.Close() + dest.Close() + + // At this point we should do checksum comparison, but that requires updating the AWS library + + fileInfo := database.FileInfo{} + fileInfo.Path = fileID + fileInfo.UploadedChecksum = fmt.Sprintf("%x", hash.Sum(nil)) + fileInfo.Size, err = app.Archive.GetFileSize(fileID, true) + if err != nil { + log.Errorf("Couldn't get file size from archive, reason: %v)", err.Error()) + + return "nack" + } + + log.Debugf("Wrote archived file (corr-id: %s, user: %s, filepath: %s, archivepath: %s, archivedsize: %d)", correlationID, message.User, message.FilePath, fileID, fileInfo.Size) + + status, err = app.DB.GetFileStatus(correlationID) + if err != nil { + log.Errorf("failed to get file status, reason: (%s)", err.Error()) + + return "nack" + } + + if status == "disabled" { + log.Infof("file with correlation ID: %s is disabled, stopping ingestion", correlationID) + + return "ack" + } + + if err := app.DB.SetArchived(fileInfo, fileID); err != nil { + log.Errorf("SetArchived failed, reason: (%s)", err.Error()) + + return "nack" + } + + if err := app.DB.UpdateFileEventLog(fileID, "archived", correlationID, "ingest", "{}", string(m)); err != nil { + log.Errorf("failed to set event log status for file: %s", correlationID) + + return "nack" + } + log.Debugf("File marked as archived (corr-id: %s, user: %s, filepath: %s, archivepath: %s)", correlationID, message.User, message.FilePath, fileID) + + // Send message to archived + msg := schema.IngestionVerification{ + User: message.User, + FilePath: message.FilePath, + FileID: fileID, + ArchivePath: fileID, + EncryptedChecksums: []schema.Checksums{ + {Type: "sha256", Value: fmt.Sprintf("%x", hash.Sum(nil))}, + }, + } + archivedMsg, _ := json.Marshal(&msg) + + err = schema.ValidateJSON(fmt.Sprintf("%s/ingestion-verification.json", app.Conf.Broker.SchemasPath), archivedMsg) + if err != nil { + log.Errorf("Validation of outgoing message failed, reason: (%s)", err.Error()) + + return "nack" + } + + if err := app.MQ.SendMessage(correlationID, app.Conf.Broker.Exchange, app.Conf.Broker.RoutingKey, archivedMsg); err != nil { + // TODO fix resend mechanism + log.Errorf("failed to publish message, reason: (%s)", err.Error()) + + return "reject" + } + + return "ack" } // tryDecrypt tries to decrypt the start of buf. func tryDecrypt(key *[32]byte, buf []byte) ([]byte, error) { - log.Debugln("Try decrypting the first data block") a := bytes.NewReader(buf) b, err := streaming.NewCrypt4GHReader(a, *key, nil) @@ -578,7 +595,6 @@ func tryDecrypt(key *[32]byte, buf []byte) ([]byte, error) { log.Error(err) return nil, err - } _, err = b.ReadByte() if err != nil { diff --git a/sda/cmd/ingest/ingest_test.go b/sda/cmd/ingest/ingest_test.go index 48c55a3da0..bd647a2e93 100644 --- a/sda/cmd/ingest/ingest_test.go +++ b/sda/cmd/ingest/ingest_test.go @@ -1,126 +1,725 @@ package main import ( + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/hex" "fmt" "io" + "net/http" "os" + "path" + "path/filepath" + "runtime" + "strconv" "testing" + "time" + log "github.com/sirupsen/logrus" + + "github.com/google/uuid" "github.com/neicnordic/crypt4gh/keys" "github.com/neicnordic/crypt4gh/streaming" + "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" + "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" + "github.com/neicnordic/sensitive-data-archive/internal/schema" + "github.com/neicnordic/sensitive-data-archive/internal/storage" + "github.com/ory/dockertest/v3" + "github.com/ory/dockertest/v3/docker" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) -type TestSuite struct { - suite.Suite -} +var DBport, MQport int +var BrokerAPI string + +func TestMain(m *testing.M) { + if _, err := os.Stat("/.dockerenv"); err == nil { + m.Run() + } + _, b, _, _ := runtime.Caller(0) + rootDir := path.Join(path.Dir(b), "../../../") + + // uses a sensible default on windows (tcp/http) and linux/osx (socket) + pool, err := dockertest.NewPool("") + if err != nil { + log.Fatalf("Could not construct pool: %s", err) + } + + // uses pool to try to connect to Docker + err = pool.Client.Ping() + if err != nil { + log.Fatalf("Could not connect to Docker: %s", err) + } + + // pulls an image, creates a container based on it and runs it + postgres, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "postgres", + Tag: "15.2-alpine3.17", + Env: []string{ + "POSTGRES_PASSWORD=rootpasswd", + "POSTGRES_DB=sda", + }, + Mounts: []string{ + fmt.Sprintf("%s/postgresql/initdb.d:/docker-entrypoint-initdb.d", rootDir), + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + log.Fatalf("Could not start resource: %s", err) + } + + dbHostAndPort := postgres.GetHostPort("5432/tcp") + DBport, _ = strconv.Atoi(postgres.GetPort("5432/tcp")) + databaseURL := fmt.Sprintf("postgres://postgres:rootpasswd@%s/sda?sslmode=disable", dbHostAndPort) + + pool.MaxWait = 120 * time.Second + if err = pool.Retry(func() error { + db, err := sql.Open("postgres", databaseURL) + if err != nil { + log.Println(err) + + return err + } + + query := "SELECT MAX(version) FROM sda.dbschema_version;" + var dbVersion int + + return db.QueryRow(query).Scan(&dbVersion) + }); err != nil { + log.Fatalf("Could not connect to postgres: %s", err) + } + + // pulls an image, creates a container based on it and runs it + rabbitmq, err := pool.RunWithOptions(&dockertest.RunOptions{ + Repository: "ghcr.io/neicnordic/sensitive-data-archive", + Tag: "v0.3.89-rabbitmq", + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + log.Fatalf("Could not start resource: %s", err) + } + + MQport, _ = strconv.Atoi(rabbitmq.GetPort("5672/tcp")) + BrokerAPI = rabbitmq.GetHostPort("15672/tcp") + + client := http.Client{Timeout: 30 * time.Second} + req, err := http.NewRequest(http.MethodGet, "http://"+BrokerAPI+"/api/queues/sda/", http.NoBody) + if err != nil { + log.Fatal(err) + } + req.SetBasicAuth("guest", "guest") -var pubKeyList = [][32]byte{} + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + if err := pool.Retry(func() error { + res, err := client.Do(req) + if err != nil || res.StatusCode != 200 { + return err + } + res.Body.Close() + + return nil + }); err != nil { + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + if err := pool.Purge(rabbitmq); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + log.Fatalf("Could not connect to rabbitmq: %s", err) + } + + log.Println("starting tests") + code := m.Run() + + log.Println("tests completed") + if err := pool.Purge(postgres); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } + if err := pool.Purge(rabbitmq); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } -func TestConfigTestSuite(t *testing.T) { + os.Exit(code) +} + +func TestIngestTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { - viper.Set("log.level", "debug") - archive := suite.T().TempDir() - defer os.RemoveAll(archive) - viper.Set("archive.location", archive) +type TestSuite struct { + suite.Suite + filePath string + pubKeyList [][32]byte + ingest Ingest +} - tempDir := suite.T().TempDir() +func (ts *TestSuite) SetupSuite() { + viper.Set("log.level", "debug") + tempDir := ts.T().TempDir() keyFile1 := fmt.Sprintf("%s/c4gh1.key", tempDir) keyFile2 := fmt.Sprintf("%s/c4gh2.key", tempDir) publicKey, err := helper.CreatePrivateKeyFile(keyFile1, "test") - assert.NoError(suite.T(), err) + if err != nil { + ts.FailNow("Failed to create c4gh key") + } // Add only the first public key to the list - pubKeyList = append(pubKeyList, publicKey) + ts.pubKeyList = append(ts.pubKeyList, publicKey) _, err = helper.CreatePrivateKeyFile(keyFile2, "test") - assert.NoError(suite.T(), err) + if err != nil { + ts.FailNow("Failed to create c4gh key") + } viper.Set("c4gh.privateKeys", []config.C4GHprivateKeyConf{ {FilePath: keyFile1, Passphrase: "test"}, {FilePath: keyFile2, Passphrase: "test"}, }) + viper.Set("archive.type", "posix") + viper.Set("archive.location", "/tmp/") + viper.Set("broker.host", "localhost") + viper.Set("broker.port", MQport) + viper.Set("broker.user", "guest") + viper.Set("broker.password", "guest") + viper.Set("broker.queue", "ingest") + viper.Set("broker.routingkey", "verify") + viper.Set("broker.vhost", "sda") + viper.Set("db.host", "localhost") + viper.Set("db.port", DBport) + viper.Set("db.user", "postgres") + viper.Set("db.password", "rootpasswd") + viper.Set("db.database", "sda") + viper.Set("db.sslMode", "disable") + viper.Set("inbox.type", "posix") + viper.Set("inbox.location", "/tmp/") + viper.Set("schema.path", "../../schemas/isolated/") + + ts.ingest.Conf, err = config.NewConfig("ingest") + if err != nil { + ts.FailNowf("failed to init config: %s", err.Error()) + } + ts.ingest.DB, err = database.NewSDAdb(ts.ingest.Conf.Database) + if err != nil { + ts.FailNowf("failed to setup database connection: %s", err.Error()) + } + ts.ingest.MQ, err = broker.NewMQ(ts.ingest.Conf.Broker) + if err != nil { + ts.FailNowf("failed to setup rabbitMQ connection: %s", err.Error()) + } + ts.ingest.ArchiveKeyList, err = config.GetC4GHprivateKeys() + if err != nil { + ts.FailNow("no private keys configured") + } + + if err := ts.ingest.DB.AddKeyHash(hex.EncodeToString(publicKey[:]), "the test key"); err != nil { + ts.FailNow("failed to register the public key") + } +} + +func (ts *TestSuite) TearDownSuite() { + _ = os.RemoveAll(ts.ingest.Conf.Archive.Posix.Location) + _ = os.RemoveAll(ts.ingest.Conf.Inbox.Posix.Location) +} + +func (ts *TestSuite) SetupTest() { + var err error + ts.ingest.Conf.Archive.Posix.Location, err = os.MkdirTemp("", "archive") + if err != nil { + ts.FailNow("failed to create temp folder") + } + + ts.ingest.Conf.Inbox.Posix.Location, err = os.MkdirTemp("", "inbox") + if err != nil { + ts.FailNow("failed to create temp folder") + } + + f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") + if err != nil { + ts.FailNow("failed to create test file") + } + defer f.Close() + + _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + ts.FailNow("failed to write data to test file") + } + + outFileName := f.Name() + ".c4gh" + outFile, err := os.Create(outFileName) + if err != nil { + ts.FailNow("failed to create encrypted test file") + } + defer outFile.Close() + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + ts.FailNow("failed to create private c4gh key") + } + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(outFile, privateKey, ts.pubKeyList, nil) + if err != nil { + ts.FailNow("failed to create c4gh writer") + } + + _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + ts.FailNow("failed to write data to encrypted test file") + } + crypt4GHWriter.Close() + + ts.filePath = filepath.Base(outFileName) - viper.Set("broker.host", "test") - viper.Set("broker.port", 123) - viper.Set("broker.user", "test") - viper.Set("broker.password", "test") - viper.Set("broker.queue", "test") - viper.Set("broker.routingkey", "test") - viper.Set("db.host", "test") - viper.Set("db.port", 123) - viper.Set("db.user", "test") - viper.Set("db.password", "test") - viper.Set("db.database", "test") -} - -func (suite *TestSuite) TestTryDecrypt_wrongFile() { - tempDir := suite.T().TempDir() + ts.ingest.Archive, err = storage.NewBackend(ts.ingest.Conf.Archive) + if err != nil { + ts.FailNow("failed to setup archive backend") + } + ts.ingest.Inbox, err = storage.NewBackend(ts.ingest.Conf.Inbox) + if err != nil { + ts.FailNow("failed to setup inbox backend") + } +} +func (ts *TestSuite) TestTryDecrypt_wrongFile() { + tempDir := ts.T().TempDir() err := os.WriteFile(fmt.Sprintf("%s/dummy.file", tempDir), []byte("hello\ngo\n"), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) file, err := os.Open(fmt.Sprintf("%s/dummy.file", tempDir)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer file.Close() buf, err := io.ReadAll(file) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) privateKeys, err := config.GetC4GHprivateKeys() - assert.NoError(suite.T(), err) - assert.Len(suite.T(), privateKeys, 2) + assert.NoError(ts.T(), err) + assert.Len(ts.T(), privateKeys, 2) header, err := tryDecrypt(privateKeys[0], buf) - assert.Nil(suite.T(), header) - assert.EqualError(suite.T(), err, "not a Crypt4GH file") + assert.Nil(ts.T(), header) + assert.EqualError(ts.T(), err, "not a Crypt4GH file") } - -func (suite *TestSuite) TestTryDecrypt() { +func (ts *TestSuite) TestTryDecrypt() { _, signingKey, err := keys.GenerateKeyPair() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // encrypt test file - tempDir := suite.T().TempDir() + tempDir := ts.T().TempDir() unencryptedFile, err := os.CreateTemp(tempDir, "unencryptedFile-") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = os.WriteFile(unencryptedFile.Name(), []byte("content"), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) encryptedFile, err := os.CreateTemp(tempDir, "encryptedFile-") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) - crypt4GHWriter, err := streaming.NewCrypt4GHWriter(encryptedFile, signingKey, pubKeyList, nil) - assert.NoError(suite.T(), err) + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(encryptedFile, signingKey, ts.pubKeyList, nil) + assert.NoError(ts.T(), err) _, err = io.Copy(crypt4GHWriter, unencryptedFile) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) crypt4GHWriter.Close() file, err := os.Open(encryptedFile.Name()) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer file.Close() buf, err := io.ReadAll(file) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) privateKeys, err := config.GetC4GHprivateKeys() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) for i, key := range privateKeys { header, err := tryDecrypt(key, buf) - switch { - case i == 0: - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), header) + switch i { + case 0: + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), header) default: - assert.Contains(suite.T(), err.Error(), "could not find matching public key heade") - assert.Nil(suite.T(), header) + assert.Contains(ts.T(), err.Error(), "could not find matching public key heade") + assert.Nil(ts.T(), header) } } } + +// messages of type `cancel` +func (ts *TestSuite) TestCancelFile() { + // prepare the DB entries + userName := "test-cancel" + file1 := fmt.Sprintf("/%v/TestCancelMessage.c4gh", userName) + fileID, err := ts.ingest.DB.RegisterFile(file1, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "cancel", + FilePath: file1, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.cancelFile(corrID, message)) +} +func (ts *TestSuite) TestCancelFile_wrongCorrelationID() { + // prepare the DB entries + userName := "test-cancel" + file1 := fmt.Sprintf("/%v/TestCancelMessage_wrongCorrelationID.c4gh", userName) + fileID, err := ts.ingest.DB.RegisterFile(file1, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "cancel", + FilePath: file1, + User: userName, + } + + assert.Equal(ts.T(), "reject", ts.ingest.cancelFile(uuid.New().String(), message)) +} + +// messages of type `ingest` +func (ts *TestSuite) TestIngestFile() { + // prepare the DB entries + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) +} +func (ts *TestSuite) TestIngestFile_secondTime() { + // prepare the DB entries + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + // file is already in `archived` state + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) +} +func (ts *TestSuite) TestIngestFile_unknownInboxType() { + userName := "test-ingest-unknown" + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(uuid.New().String(), message)) +} +func (ts *TestSuite) TestIngestFile_reingestCancelledFile() { + // prepare the DB entries + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) +} +func (ts *TestSuite) TestIngestFile_reingestCancelledFileNewChecksum() { + // prepare the DB entries + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + // over write the encrypted file to generate new checksum + f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") + if err != nil { + ts.FailNow("failed to create test file") + } + defer f.Close() + + _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + ts.FailNow("failed to write data to test file") + } + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + ts.FailNow("failed to create private c4gh key") + } + + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) + if err != nil { + ts.FailNowf("failed to create encrypted test file: %s", err.Error()) + } + defer outFile.Close() + + sha256hash := sha256.New() + mr := io.MultiWriter(outFile, sha256hash) + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, ts.pubKeyList, nil) + if err != nil { + ts.FailNowf("failed to create c4gh writer: %s", err.Error()) + } + + _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + ts.FailNow("failed to write data to encrypted test file") + } + crypt4GHWriter.Close() + + // reingestion should work + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + // DB should have the new checksum + var dbChecksum string + const q = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" + if err := ts.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { + ts.FailNow("failed to get checksum from database") + } + + assert.Equal(ts.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) +} +func (ts *TestSuite) TestIngestFile_reingestVerifiedFile() { + // prepare the DB entries + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + // fake file verification + sha256hash := sha256.New() + var fi database.FileInfo + fi.ArchiveChecksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedSize = 10 * 1024 * 1024 + fi.Size = (10 * 1024 * 1024) + 456 + if err := ts.ingest.DB.SetVerified(fi, fileID); err != nil { + ts.Fail("failed to mark file as verified") + } + + assert.Equal(ts.T(), "reject", ts.ingest.ingestFile(corrID, message)) +} +func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFile() { + // prepare the DB entries + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + // fake file verification + sha256hash := sha256.New() + var fi database.FileInfo + fi.ArchiveChecksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedChecksum = hex.EncodeToString(sha256hash.Sum(nil)) + fi.DecryptedSize = 10 * 1024 * 1024 + fi.Size = (10 * 1024 * 1024) + 456 + if err := ts.ingest.DB.SetVerified(fi, fileID); err != nil { + ts.Fail("failed to mark file as verified") + } + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) +} +func (ts *TestSuite) TestIngestFile_reingestVerifiedCancelledFileNewChecksum() { + // prepare the DB entries + userName := "test-ingest" + fileID, err := ts.ingest.DB.RegisterFile(ts.filePath, userName) + assert.NoError(ts.T(), err, "failed to register file in database") + corrID := uuid.New().String() + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "uploaded", corrID, userName, "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: ts.filePath, + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + var firstDbChecksum string + const q1 = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" + if err := ts.ingest.DB.DB.QueryRow(q1, fileID).Scan(&firstDbChecksum); err != nil { + ts.FailNow("failed to get checksum from database") + } + + // fake file verification + verifiedSha256 := sha256.New() + var fi database.FileInfo + fi.ArchiveChecksum = hex.EncodeToString(verifiedSha256.Sum(nil)) + fi.DecryptedChecksum = hex.EncodeToString(verifiedSha256.Sum(nil)) + fi.DecryptedSize = 10 * 1024 * 1024 + fi.Size = (10 * 1024 * 1024) + 456 + if err := ts.ingest.DB.SetVerified(fi, fileID); err != nil { + ts.Fail("failed to mark file as verified") + } + + if err = ts.ingest.DB.UpdateFileEventLog(fileID, "disabled", corrID, "ingest", "{}", "{}"); err != nil { + ts.Fail("failed to update file event log") + } + + // over write the encrypted file to generate new checksum + f, err := os.CreateTemp(ts.ingest.Conf.Inbox.Posix.Location, "") + if err != nil { + ts.FailNow("failed to create test file") + } + defer f.Close() + + _, err = io.Copy(f, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + ts.FailNow("failed to write data to test file") + } + + _, privateKey, err := keys.GenerateKeyPair() + if err != nil { + ts.FailNow("failed to create private c4gh key") + } + + outFile, err := os.Create(path.Join(ts.ingest.Conf.Inbox.Posix.Location, ts.filePath)) + if err != nil { + ts.FailNowf("failed to create encrypted test file: %s", err.Error()) + } + defer outFile.Close() + + sha256hash := sha256.New() + mr := io.MultiWriter(outFile, sha256hash) + + crypt4GHWriter, err := streaming.NewCrypt4GHWriter(mr, privateKey, ts.pubKeyList, nil) + if err != nil { + ts.FailNowf("failed to create c4gh writer: %s", err.Error()) + } + + _, err = io.Copy(crypt4GHWriter, io.LimitReader(rand.Reader, 10*1024*1024)) + if err != nil { + ts.FailNow("failed to write data to encrypted test file") + } + crypt4GHWriter.Close() + + // reingestion should work + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) + + // DB should have the new checksum + var dbChecksum string + const q = "SELECT checksum from sda.checksums WHERE source = 'UPLOADED' and file_id = $1;" + if err := ts.ingest.DB.DB.QueryRow(q, fileID).Scan(&dbChecksum); err != nil { + ts.FailNow("failed to get checksum from database") + } + + assert.Equal(ts.T(), dbChecksum, hex.EncodeToString(sha256hash.Sum(nil))) + + assert.NotEqual(ts.T(), dbChecksum, firstDbChecksum) +} +func (ts *TestSuite) TestIngestFile_missingFile() { + // prepare the DB entries + userName := "test-ingest" + corrID := uuid.New().String() + basepath := filepath.Dir(ts.filePath) + + message := schema.IngestionTrigger{ + Type: "ingest", + FilePath: fmt.Sprintf("%s/missing.file.c4gh", basepath), + User: userName, + } + + assert.Equal(ts.T(), "ack", ts.ingest.ingestFile(corrID, message)) +} diff --git a/sda/cmd/intercept/intercept.go b/sda/cmd/intercept/intercept.go index 4d4a7f625d..3aa44f4109 100644 --- a/sda/cmd/intercept/intercept.go +++ b/sda/cmd/intercept/intercept.go @@ -118,7 +118,7 @@ func main() { // typeFromMessage returns the type value given a JSON structure for the message // supplied in body func typeFromMessage(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err diff --git a/sda/cmd/intercept/intercept_test.go b/sda/cmd/intercept/intercept_test.go index a591cea4fe..01e5cb8eef 100644 --- a/sda/cmd/intercept/intercept_test.go +++ b/sda/cmd/intercept/intercept_test.go @@ -17,7 +17,7 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } @@ -51,7 +51,7 @@ type missing struct { FilePath string `json:"filepath"` } -func (suite *TestSuite) TestMessageSelection_Accession() { +func (ts *TestSuite) TestMessageSelection_Accession() { msg := accession{ Type: "accession", User: "foo", @@ -65,11 +65,11 @@ func (suite *TestSuite) TestMessageSelection_Accession() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgType, msgAccession, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgType, msgAccession, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Cancel() { +func (ts *TestSuite) TestMessageSelection_Cancel() { msg := ingest{ Type: "cancel", User: "foo", @@ -79,11 +79,11 @@ func (suite *TestSuite) TestMessageSelection_Cancel() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgType, msgCancel, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgType, msgCancel, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Ingest() { +func (ts *TestSuite) TestMessageSelection_Ingest() { msg := ingest{ Type: "ingest", User: "foo", @@ -93,11 +93,11 @@ func (suite *TestSuite) TestMessageSelection_Ingest() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgIngest, msgType, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgIngest, msgType, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Mapping() { +func (ts *TestSuite) TestMessageSelection_Mapping() { msg := mapping{ Type: "mapping", DatasetID: "EGAD12345678900", @@ -109,11 +109,11 @@ func (suite *TestSuite) TestMessageSelection_Mapping() { msgType, err := typeFromMessage(message) - assert.Nil(suite.T(), err, "Unexpected error from typeFromMessage") - assert.Equal(suite.T(), msgMapping, msgType, "message type from message does not match expected") + assert.Nil(ts.T(), err, "Unexpected error from typeFromMessage") + assert.Equal(ts.T(), msgMapping, msgType, "message type from message does not match expected") } -func (suite *TestSuite) TestMessageSelection_Notype() { +func (ts *TestSuite) TestMessageSelection_Notype() { msg := missing{ User: "foo", FilePath: "/tmp/foo", @@ -122,6 +122,6 @@ func (suite *TestSuite) TestMessageSelection_Notype() { msgType, err := typeFromMessage(message) - assert.Error(suite.T(), err, "Unexpected lack of error from typeFromMessage") - assert.Equal(suite.T(), "", msgType, "message type from message does not match expected") + assert.Error(ts.T(), err, "Unexpected lack of error from typeFromMessage") + assert.Equal(ts.T(), "", msgType, "message type from message does not match expected") } diff --git a/sda/cmd/mapper/mapper.go b/sda/cmd/mapper/mapper.go index 86fc771488..1113f4a874 100644 --- a/sda/cmd/mapper/mapper.go +++ b/sda/cmd/mapper/mapper.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "errors" "fmt" "github.com/neicnordic/sensitive-data-archive/internal/broker" @@ -159,7 +160,7 @@ func main() { // schemaFromDatasetOperation returns the operation done with dataset supplied in body of the message func schemaFromDatasetOperation(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err @@ -167,12 +168,12 @@ func schemaFromDatasetOperation(body []byte) (string, error) { datasetMessageType, ok := message["type"] if !ok { - return "", fmt.Errorf("malformed message, dataset message type is missing") + return "", errors.New("malformed message, dataset message type is missing") } datasetOpsType, ok := datasetMessageType.(string) if !ok { - return "", fmt.Errorf("could not cast operation attribute to string") + return "", errors.New("could not cast operation attribute to string") } switch datasetOpsType { @@ -183,7 +184,6 @@ func schemaFromDatasetOperation(body []byte) (string, error) { case "deprecate": return "dataset-deprecate", nil default: - return "", fmt.Errorf("could not recognize mapping operation") + return "", errors.New("could not recognize mapping operation") } - } diff --git a/sda/cmd/mapper/mapper_test.go b/sda/cmd/mapper/mapper_test.go index 0f8a2819d2..aea1d90847 100644 --- a/sda/cmd/mapper/mapper_test.go +++ b/sda/cmd/mapper/mapper_test.go @@ -15,6 +15,6 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } diff --git a/sda/cmd/notify/notify.go b/sda/cmd/notify/notify.go index 0466e06833..8e85ca3ac2 100644 --- a/sda/cmd/notify/notify.go +++ b/sda/cmd/notify/notify.go @@ -4,6 +4,7 @@ package main import ( "encoding/base64" "encoding/json" + "errors" "fmt" "net/smtp" "strconv" @@ -93,7 +94,7 @@ func getUser(queue string, orgMsg []byte) string { _ = json.Unmarshal(orgMsg, ¬ify) orgMsg, _ := base64.StdEncoding.DecodeString(notify.OriginalMessage.(string)) - var message map[string]interface{} + var message map[string]any _ = json.Unmarshal(orgMsg, &message) return fmt.Sprint(message["user"]) @@ -157,5 +158,5 @@ func validator(queue, schemaPath string, delivery amqp091.Delivery) error { return nil } - return fmt.Errorf("Error") + return errors.New("Error") } diff --git a/sda/cmd/notify/notify_test.go b/sda/cmd/notify/notify_test.go index 4026b44188..2bb29d1b08 100644 --- a/sda/cmd/notify/notify_test.go +++ b/sda/cmd/notify/notify_test.go @@ -22,12 +22,11 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } func TestGetUser(t *testing.T) { - archivedMsg := schema.IngestionVerification{ User: "JohnDoe", FilePath: "path/to file", @@ -54,7 +53,6 @@ func TestGetUser(t *testing.T) { orgUser := getUser("error", infoErrorBytes) assert.Equal(t, "JohnDoe", orgUser) - } func TestSetSubject(t *testing.T) { diff --git a/sda/cmd/orchestrate/orchestrate.go b/sda/cmd/orchestrate/orchestrate.go index cc117af046..5c847b6542 100644 --- a/sda/cmd/orchestrate/orchestrate.go +++ b/sda/cmd/orchestrate/orchestrate.go @@ -9,7 +9,7 @@ import ( "os" "time" - uuid "github.com/google/uuid" + "github.com/google/uuid" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/schema" @@ -101,7 +101,7 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * messages, err := mq.GetMessages(queue) if err != nil { - log.Fatal(err) + log.Fatal(err) // nolint # FIXME Fatal should only be called from main } for delivered := range messages { log.Debugf("Received a message: %s", delivered.Body) @@ -137,7 +137,7 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * } var publishMsg []byte - var publishType interface{} + var publishType any routingSchema, err := schemaNameFromQueue(routingKey, nil, conf) @@ -201,7 +201,6 @@ func processQueue(mq *broker.AMQPBroker, queue string, routingKey string, conf * continue } } - } } @@ -232,7 +231,7 @@ func schemaNameFromQueue(queue string, body []byte, conf *config.Config) (string // schemaFromInboxOperation returns the operation done in inbox // supplied in body of the message func schemaFromInboxOperation(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err @@ -258,13 +257,12 @@ func schemaFromInboxOperation(body []byte) (string, error) { default: return "", errors.New("could not recognize inbox operation") } - } // schemaFromDatasetOperation returns the operation done with dataset // supplied in body of the message func schemaFromDatasetOperation(body []byte) (string, error) { - message := make(map[string]interface{}) + message := make(map[string]any) err := json.Unmarshal(body, &message) if err != nil { return "", err @@ -290,10 +288,9 @@ func schemaFromDatasetOperation(body []byte) (string, error) { default: return "", errors.New("could not recognize inbox operation") } - } -func ingestMessage(body []byte) ([]byte, interface{}) { +func ingestMessage(body []byte) ([]byte, any) { var message upload err := json.Unmarshal(body, &message) if err != nil { @@ -312,7 +309,7 @@ func ingestMessage(body []byte) ([]byte, interface{}) { return publish, new(trigger) } -func finalizeMessage(body []byte, conf *config.Config) ([]byte, interface{}) { +func finalizeMessage(body []byte, conf *config.Config) ([]byte, any) { var message request err := json.Unmarshal(body, &message) if err != nil { @@ -335,7 +332,7 @@ func finalizeMessage(body []byte, conf *config.Config) ([]byte, interface{}) { return publish, new(finalize) } -func mappingMessage(body []byte, conf *config.Config) ([]byte, interface{}) { +func mappingMessage(body []byte, conf *config.Config) ([]byte, any) { var message finalize if err := json.Unmarshal(body, &message); err != nil { return nil, nil @@ -355,7 +352,7 @@ func mappingMessage(body []byte, conf *config.Config) ([]byte, interface{}) { return publish, new(mapping) } -func releaseMessage(body []byte, conf *config.Config) ([]byte, interface{}) { +func releaseMessage(body []byte, conf *config.Config) ([]byte, any) { var message finalize if err := json.Unmarshal(body, &message); err != nil { return nil, nil @@ -374,7 +371,7 @@ func releaseMessage(body []byte, conf *config.Config) ([]byte, interface{}) { return publish, new(mapping) } -func validateMsg(delivered *amqp091.Delivery, mq *broker.AMQPBroker, routingKey string, routingSchema string, publishMsg []byte, publishType interface{}) error { +func validateMsg(delivered *amqp091.Delivery, mq *broker.AMQPBroker, routingKey string, routingSchema string, publishMsg []byte, publishType any) error { err := schema.ValidateJSON(fmt.Sprintf("%s/%s.json", mq.Conf.SchemasPath, routingSchema), delivered.Body) if err != nil { return err diff --git a/sda/cmd/reencrypt/reencrypt.go b/sda/cmd/reencrypt/reencrypt.go index 789a699aa1..5c25e721a9 100644 --- a/sda/cmd/reencrypt/reencrypt.go +++ b/sda/cmd/reencrypt/reencrypt.go @@ -68,7 +68,6 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r dataEditList := in.GetDataeditlist() if len(dataEditList) > 0 { // linter doesn't like checking for nil before len - // Check that G115: integer overflow conversion int -> uint32 is satisfied if len(dataEditList) > int(math.MaxUint32) { return nil, status.Error(400, "data edit list too long") @@ -106,7 +105,6 @@ func (s *server) ReencryptHeader(_ context.Context, in *re.ReencryptRequest) (*r // server status. The corresponding grpc health server serves as a proxy to the internal health // service of the reencrypt server so that k8s grpc probes can be used when TLS is enabled. func (p *hServer) Check(ctx context.Context, in *healthgrpc.HealthCheckRequest) (*healthgrpc.HealthCheckResponse, error) { - rpcCtx, rpcCancel := context.WithTimeout(ctx, time.Second*2) defer rpcCancel() diff --git a/sda/cmd/reencrypt/reencrypt_test.go b/sda/cmd/reencrypt/reencrypt_test.go index 134ce77c63..71da4b588f 100644 --- a/sda/cmd/reencrypt/reencrypt_test.go +++ b/sda/cmd/reencrypt/reencrypt_test.go @@ -43,64 +43,63 @@ func TestReEncryptTests(t *testing.T) { suite.Run(t, new(ReEncryptTests)) } -func (suite *ReEncryptTests) SetupTest() { +func (ts *ReEncryptTests) SetupTest() { var err error log.SetLevel(log.InfoLevel) repKey := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" - suite.KeyPath, _ = os.MkdirTemp("", "key") - if err := os.WriteFile(suite.KeyPath+"/c4gh.key", []byte(repKey), 0600); err != nil { - suite.T().FailNow() + ts.KeyPath, _ = os.MkdirTemp("", "key") + if err := os.WriteFile(ts.KeyPath+"/c4gh.key", []byte(repKey), 0600); err != nil { + ts.T().FailNow() } - suite.UserPublicKey, suite.UserPrivateKey, err = keys.GenerateKeyPair() + ts.UserPublicKey, ts.UserPrivateKey, err = keys.GenerateKeyPair() if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - key, err := os.Create(suite.KeyPath + "/new.key") + key, err := os.Create(ts.KeyPath + "/new.key") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - if err := keys.WriteCrypt4GHX25519PrivateKey(key, suite.UserPrivateKey, []byte("password")); err != nil { - suite.T().FailNow() + if err := keys.WriteCrypt4GHX25519PrivateKey(key, ts.UserPrivateKey, []byte("password")); err != nil { + ts.T().FailNow() } buf := new(bytes.Buffer) - if err := keys.WriteCrypt4GHX25519PublicKey(buf, suite.UserPublicKey); err != nil { - suite.T().FailNow() + if err := keys.WriteCrypt4GHX25519PublicKey(buf, ts.UserPublicKey); err != nil { + ts.T().FailNow() } - suite.UserPubKeyString = base64.StdEncoding.EncodeToString(buf.Bytes()) + ts.UserPubKeyString = base64.StdEncoding.EncodeToString(buf.Bytes()) - viper.Set("c4gh.filepath", suite.KeyPath+"/c4gh.key") + viper.Set("c4gh.filepath", ts.KeyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") - suite.PrivateKey, err = config.GetC4GHKey() + ts.PrivateKey, err = config.GetC4GHKey() if err != nil { - suite.T().FailNow() + ts.T().FailNow() } - suite.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") - suite.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") - + ts.FileHeader, _ = hex.DecodeString("637279707434676801000000010000006c000000000000007ca283608311dacfc32703a3cc9a2b445c9a417e036ba5943e233cfc65a1f81fdcc35036a584b3f95759114f584d1e81e8cf23a9b9d1e77b9e8f8a8ee8098c2a3e9270fe6872ef9d1c948caf8423efc7ce391081da0d52a49b1e6d0706f267d6140ff12b") + ts.FileData, _ = hex.DecodeString("e046718f01d52c626276ce5931e10afd99330c4679b3e2a43fdf18146e85bae8eaee83") } -func (suite *ReEncryptTests) TearDownTest() { - os.RemoveAll(suite.KeyPath) +func (ts *ReEncryptTests) TearDownTest() { + os.RemoveAll(ts.KeyPath) } -func (suite *ReEncryptTests) TestReencryptHeader() { +func (ts *ReEncryptTests) TestReencryptHeader() { lis, err := net.Listen("tcp", "localhost:50051") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) if err := s.Serve(lis); err != nil { - suite.T().Fail() + ts.T().Fail() } }() @@ -108,7 +107,7 @@ func (suite *ReEncryptTests) TestReencryptHeader() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50051", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -116,33 +115,33 @@ func (suite *ReEncryptTests) TestReencryptHeader() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString}) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString}) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr := bytes.NewReader(res.Header) - fileStream := io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err := streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err := io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "content", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) } -func (suite *ReEncryptTests) TestReencryptHeader_DataEditList() { +func (ts *ReEncryptTests) TestReencryptHeader_DataEditList() { lis, err := net.Listen("tcp", "localhost:50054") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) if err := s.Serve(lis); err != nil { - suite.T().Fail() + ts.T().Fail() } }() @@ -150,7 +149,7 @@ func (suite *ReEncryptTests) TestReencryptHeader_DataEditList() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50054", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -158,64 +157,63 @@ func (suite *ReEncryptTests) TestReencryptHeader_DataEditList() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString, Dataeditlist: []uint64{1, 2, 1, 2}}) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString, Dataeditlist: []uint64{1, 2, 1, 2}}) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr := bytes.NewReader(res.Header) - fileStream := io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err := streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err := io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "onen", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "onen", string(data)) hr = bytes.NewReader(res.Header) - header, err := headers.NewHeader(hr, suite.UserPrivateKey) - assert.NoError(suite.T(), err) + header, err := headers.NewHeader(hr, ts.UserPrivateKey) + assert.NoError(ts.T(), err) packet := header.GetDataEditListHeaderPacket() - assert.NotNilf(suite.T(), packet, "DataEditList HeaderPacket not found") - assert.Equal(suite.T(), uint32(4), packet.NumberLengths) - assert.Equal(suite.T(), uint64(1), packet.Lengths[0]) - assert.Equal(suite.T(), uint64(2), packet.Lengths[1]) - assert.Equal(suite.T(), uint64(1), packet.Lengths[2]) - assert.Equal(suite.T(), uint64(2), packet.Lengths[3]) - assert.Equal(suite.T(), int(4), len(packet.Lengths)) - - res, err = c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString, Dataeditlist: []uint64{}}) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + assert.NotNilf(ts.T(), packet, "DataEditList HeaderPacket not found") + assert.Equal(ts.T(), uint32(4), packet.NumberLengths) + assert.Equal(ts.T(), uint64(1), packet.Lengths[0]) + assert.Equal(ts.T(), uint64(2), packet.Lengths[1]) + assert.Equal(ts.T(), uint64(1), packet.Lengths[2]) + assert.Equal(ts.T(), uint64(2), packet.Lengths[3]) + assert.Equal(ts.T(), int(4), len(packet.Lengths)) + + res, err = c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString, Dataeditlist: []uint64{}}) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr = bytes.NewReader(res.Header) - fileStream = io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream = io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err = streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err = streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err = io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "content", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) hr = bytes.NewReader(res.Header) - header, err = headers.NewHeader(hr, suite.UserPrivateKey) - assert.NoError(suite.T(), err) + header, err = headers.NewHeader(hr, ts.UserPrivateKey) + assert.NoError(ts.T(), err) packet = header.GetDataEditListHeaderPacket() - assert.Nilf(suite.T(), packet, "DataEditList HeaderPacket found when not expected") - + assert.Nilf(ts.T(), packet, "DataEditList HeaderPacket found when not expected") } -func (suite *ReEncryptTests) TestReencryptHeader_BadPubKey() { +func (ts *ReEncryptTests) TestReencryptHeader_BadPubKey() { lis, err := net.Listen("tcp", "localhost:50052") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) _ = s.Serve(lis) }() @@ -223,7 +221,7 @@ func (suite *ReEncryptTests) TestReencryptHeader_BadPubKey() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50052", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -231,21 +229,21 @@ func (suite *ReEncryptTests) TestReencryptHeader_BadPubKey() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: "BadKey"}) - assert.Contains(suite.T(), err.Error(), "illegal base64 data") - assert.Nil(suite.T(), res) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: "BadKey"}) + assert.Contains(ts.T(), err.Error(), "illegal base64 data") + assert.Nil(ts.T(), res) } -func (suite *ReEncryptTests) TestReencryptHeader_NoHeader() { +func (ts *ReEncryptTests) TestReencryptHeader_NoHeader() { lis, err := net.Listen("tcp", "localhost:50053") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { var opts []grpc.ServerOption s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) _ = s.Serve(lis) }() @@ -253,7 +251,7 @@ func (suite *ReEncryptTests) TestReencryptHeader_NoHeader() { opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) conn, err := grpc.NewClient("localhost:50053", opts...) if err != nil { - suite.T().FailNow() + ts.T().FailNow() } defer conn.Close() @@ -261,32 +259,32 @@ func (suite *ReEncryptTests) TestReencryptHeader_NoHeader() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: make([]byte, 0), Publickey: suite.UserPubKeyString}) - assert.Contains(suite.T(), err.Error(), "no header received") - assert.Nil(suite.T(), res) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: make([]byte, 0), Publickey: ts.UserPubKeyString}) + assert.Contains(ts.T(), err.Error(), "no header received") + assert.Nil(ts.T(), res) } -func (suite *ReEncryptTests) TestReencryptHeader_TLS() { - certPath := suite.T().TempDir() +func (ts *ReEncryptTests) TestReencryptHeader_TLS() { + certPath := ts.T().TempDir() helper.MakeCerts(certPath) rootCAs := x509.NewCertPool() cacertFile, err := os.ReadFile(certPath + "/ca.crt") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } ok := rootCAs.AppendCertsFromPEM(cacertFile) if !ok { - suite.T().FailNow() + ts.T().FailNow() } certs, err := tls.LoadX509KeyPair(certPath+"/tls.crt", certPath+"/tls.key") if err != nil { - suite.T().Log(err.Error()) - suite.T().FailNow() + ts.T().Log(err.Error()) + ts.T().FailNow() } lis, err := net.Listen("tcp", "localhost:50443") if err != nil { - suite.T().FailNow() + ts.T().FailNow() } go func() { @@ -300,9 +298,9 @@ func (suite *ReEncryptTests) TestReencryptHeader_TLS() { ) opts := []grpc.ServerOption{grpc.Creds(serverCreds)} s := grpc.NewServer(opts...) - re.RegisterReencryptServer(s, &server{c4ghPrivateKey: suite.PrivateKey}) + re.RegisterReencryptServer(s, &server{c4ghPrivateKey: ts.PrivateKey}) if err := s.Serve(lis); err != nil { - suite.T().Fail() + ts.T().Fail() } }() @@ -315,8 +313,8 @@ func (suite *ReEncryptTests) TestReencryptHeader_TLS() { ) conn, err := grpc.NewClient("localhost:50443", grpc.WithTransportCredentials(clientCreds)) if err != nil { - suite.T().Log(err.Error()) - suite.T().FailNow() + ts.T().Log(err.Error()) + ts.T().FailNow() } defer conn.Close() @@ -324,22 +322,22 @@ func (suite *ReEncryptTests) TestReencryptHeader_TLS() { defer cancel() c := re.NewReencryptClient(conn) - res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: suite.FileHeader, Publickey: suite.UserPubKeyString}) + res, err := c.ReencryptHeader(ctx, &re.ReencryptRequest{Oldheader: ts.FileHeader, Publickey: ts.UserPubKeyString}) if err != nil { - suite.T().Log(err.Error()) - suite.T().FailNow() + ts.T().Log(err.Error()) + ts.T().FailNow() } - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), res) - assert.Equal(suite.T(), "crypt4gh", string(res.Header[:8])) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), res) + assert.Equal(ts.T(), "crypt4gh", string(res.Header[:8])) hr := bytes.NewReader(res.Header) - fileStream := io.MultiReader(hr, bytes.NewReader(suite.FileData)) + fileStream := io.MultiReader(hr, bytes.NewReader(ts.FileData)) - c4gh, err := streaming.NewCrypt4GHReader(fileStream, suite.UserPrivateKey, nil) - assert.NoError(suite.T(), err) + c4gh, err := streaming.NewCrypt4GHReader(fileStream, ts.UserPrivateKey, nil) + assert.NoError(ts.T(), err) data, err := io.ReadAll(c4gh) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "content", string(data)) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "content", string(data)) } diff --git a/sda/cmd/s3inbox/healthchecks.go b/sda/cmd/s3inbox/healthchecks.go index cf1881a70c..9cba134a4d 100644 --- a/sda/cmd/s3inbox/healthchecks.go +++ b/sda/cmd/s3inbox/healthchecks.go @@ -14,7 +14,6 @@ import ( // CheckHealth checks and tries to repair the connections to MQ, DB and S3 func (p *Proxy) CheckHealth(w http.ResponseWriter, _ *http.Request) { - // try to connect to mq, check connection and channel var err error if p.messenger == nil { @@ -74,8 +73,8 @@ func (p *Proxy) CheckHealth(w http.ResponseWriter, _ *http.Request) { } // httpsGetCheck sends a request to the S3 backend and makes sure it is healthy -func (p *Proxy) httpsGetCheck(url string) error { - resp, e := p.client.Get(url) +func (p *Proxy) httpsGetCheck(uri string) error { + resp, e := p.client.Get(uri) if e != nil { return e } @@ -88,7 +87,6 @@ func (p *Proxy) httpsGetCheck(url string) error { } func (p *Proxy) getS3ReadyPath() (string, error) { - s3URL, err := url.Parse(p.s3.URL) if err != nil { return "", err diff --git a/sda/cmd/s3inbox/healthchecks_test.go b/sda/cmd/s3inbox/healthchecks_test.go index f5e8a933b6..870a3fdf9d 100644 --- a/sda/cmd/s3inbox/healthchecks_test.go +++ b/sda/cmd/s3inbox/healthchecks_test.go @@ -9,77 +9,120 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/neicnordic/sensitive-data-archive/internal/helper" + "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type HealthcheckTestSuite struct { - ProxyTests + suite.Suite + S3Fakeconf storage.S3Conf // fakeserver + S3conf storage.S3Conf // actual s3 container + DBConf database.DBConf + fakeServer *FakeServer + MQConf broker.MQConf + messenger *broker.AMQPBroker + database *database.SDAdb } func TestHealthTestSuite(t *testing.T) { suite.Run(t, new(HealthcheckTestSuite)) } -func (suite *HealthcheckTestSuite) SetupTest() { - // Reuse the setup from Proxy - suite.ProxyTests.SetupTest() +func (ts *HealthcheckTestSuite) SetupTest() { + ts.fakeServer = startFakeServer("9024") + + // Create an s3config for the fake server + ts.S3Fakeconf = storage.S3Conf{ + URL: "http://127.0.0.1", + Port: 9024, + AccessKey: "someAccess", + SecretKey: "someSecret", + Bucket: "buckbuck", + Region: "us-east-1", + } + + // Create a configuration for the fake MQ + ts.MQConf = broker.MQConf{ + Host: "127.0.0.1", + Port: MQport, + User: "guest", + Password: "guest", + Vhost: "/", + Exchange: "", + } + + ts.messenger = &broker.AMQPBroker{} + + // Create a database configuration for the fake database + ts.DBConf = database.DBConf{ + Host: "127.0.0.1", + Port: DBport, + User: "postgres", + Password: "rootpasswd", + Database: "sda", + CACert: "", + SslMode: "disable", + ClientCert: "", + ClientKey: "", + } + + ts.database = &database.SDAdb{} } -func (suite *HealthcheckTestSuite) TearDownTest() { - // Reuse the teardown from Proxy - suite.ProxyTests.TearDownTest() +func (ts *HealthcheckTestSuite) TearDownTest() { + ts.fakeServer.Close() + ts.database.Close() } -func (suite *HealthcheckTestSuite) TestHttpsGetCheck() { - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) +func (ts *HealthcheckTestSuite) TestHttpsGetCheck() { + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) url, _ := p.getS3ReadyPath() - assert.NoError(suite.T(), p.httpsGetCheck(url)) - assert.Error(suite.T(), p.httpsGetCheck("http://127.0.0.1:8888/nonexistent"), "404 should fail") + assert.NoError(ts.T(), p.httpsGetCheck(url)) + assert.Error(ts.T(), p.httpsGetCheck("http://127.0.0.1:8888/nonexistent"), "404 should fail") } -func (suite *HealthcheckTestSuite) TestS3URL() { - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) +func (ts *HealthcheckTestSuite) TestS3URL() { + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, ts.messenger, ts.database, new(tls.Config)) _, err := p.getS3ReadyPath() - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) p.s3.URL = "://badurl" url, err := p.getS3ReadyPath() - assert.Empty(suite.T(), url) - assert.Error(suite.T(), err) + assert.Empty(ts.T(), url) + assert.Error(ts.T(), err) } -func (suite *HealthcheckTestSuite) TestHealthchecks() { +func (ts *HealthcheckTestSuite) TestHealthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 200, resp.StatusCode) - + assert.Equal(ts.T(), 200, resp.StatusCode) } -func (suite *HealthcheckTestSuite) TestClosedDBHealthchecks() { +func (ts *HealthcheckTestSuite) TestClosedDBHealthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Check that 200 is reported w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 200, resp.StatusCode) + assert.Equal(ts.T(), 200, resp.StatusCode) // Close connection to DB, check that connection is restored and 200 returned w = httptest.NewRecorder() @@ -87,15 +130,15 @@ func (suite *HealthcheckTestSuite) TestClosedDBHealthchecks() { p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp = w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 200, resp.StatusCode) + assert.Equal(ts.T(), 200, resp.StatusCode) } -func (suite *HealthcheckTestSuite) TestNoS3Healthchecks() { +func (ts *HealthcheckTestSuite) TestNoS3Healthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // S3 unavailable, check that 503 is reported w := httptest.NewRecorder() @@ -103,23 +146,23 @@ func (suite *HealthcheckTestSuite) TestNoS3Healthchecks() { p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 503, resp.StatusCode) + assert.Equal(ts.T(), 503, resp.StatusCode) } -func (suite *HealthcheckTestSuite) TestNoMQHealthchecks() { +func (ts *HealthcheckTestSuite) TestNoMQHealthchecks() { // Setup - database, _ := database.NewSDAdb(suite.DBConf) - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - p := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, messenger, database, new(tls.Config)) + db, _ := database.NewSDAdb(ts.DBConf) + messenger, err := broker.NewMQ(ts.MQConf) + assert.NoError(ts.T(), err) + p := NewProxy(ts.S3Fakeconf, &helper.AlwaysAllow{}, messenger, db, new(tls.Config)) // Messenger unavailable, check that 503 is reported p.messenger.Conf.Port = 123456 p.messenger.Connection.Close() - assert.True(suite.T(), p.messenger.Connection.IsClosed()) + assert.True(ts.T(), p.messenger.Connection.IsClosed()) w := httptest.NewRecorder() p.CheckHealth(w, httptest.NewRequest(http.MethodGet, "https://dummy/health", nil)) resp := w.Result() defer resp.Body.Close() - assert.Equal(suite.T(), 503, resp.StatusCode) + assert.Equal(ts.T(), 503, resp.StatusCode) } diff --git a/sda/cmd/s3inbox/proxy.go b/sda/cmd/s3inbox/proxy.go index 7bc63cf11b..ebf18b8e4b 100644 --- a/sda/cmd/s3inbox/proxy.go +++ b/sda/cmd/s3inbox/proxy.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "encoding/json" "encoding/xml" + "errors" "fmt" "io" "net/http" @@ -21,6 +22,7 @@ import ( "github.com/minio/minio-go/v6/pkg/signer" "github.com/neicnordic/sensitive-data-archive/internal/broker" "github.com/neicnordic/sensitive-data-archive/internal/database" + "github.com/neicnordic/sensitive-data-archive/internal/schema" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" log "github.com/sirupsen/logrus" @@ -33,16 +35,16 @@ type Proxy struct { messenger *broker.AMQPBroker database *database.SDAdb client *http.Client - fileIds map[string]string + fileIDs map[string]string } // The Event struct type Event struct { - Operation string `json:"operation"` - Username string `json:"user"` - Filepath string `json:"filepath"` - Filesize int64 `json:"filesize"` - Checksum []interface{} `json:"encrypted_checksums"` + Operation string `json:"operation"` + Username string `json:"user"` + Filepath string `json:"filepath"` + Filesize int64 `json:"filesize"` + Checksum []any `json:"encrypted_checksums"` } // Checksum used in the message @@ -75,11 +77,11 @@ const ( ) // NewProxy creates a new S3Proxy. This implements the ServerHTTP interface. -func NewProxy(s3conf storage.S3Conf, auth userauth.Authenticator, messenger *broker.AMQPBroker, database *database.SDAdb, tls *tls.Config) *Proxy { - tr := &http.Transport{TLSClientConfig: tls} +func NewProxy(s3conf storage.S3Conf, auth userauth.Authenticator, messenger *broker.AMQPBroker, db *database.SDAdb, tlsConf *tls.Config) *Proxy { + tr := &http.Transport{TLSClientConfig: tlsConf} client := &http.Client{Transport: tr, Timeout: 30 * time.Second} - return &Proxy{s3conf, auth, messenger, database, client, make(map[string]string)} + return &Proxy{s3conf, auth, messenger, db, client, make(map[string]string)} } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { @@ -156,16 +158,26 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } - // register file in database if it's the start of an upload - if p.detectRequestType(r) == Put && p.fileIds[r.URL.Path] == "" { + // if this is an upload request + if p.detectRequestType(r) == Put && p.fileIDs[r.URL.Path] == "" { + // register file in database log.Debugf("registering file %v in the database", r.URL.Path) - p.fileIds[r.URL.Path], err = p.database.RegisterFile(filepath, username) - log.Debugf("fileId: %v", p.fileIds[r.URL.Path]) + p.fileIDs[r.URL.Path], err = p.database.RegisterFile(filepath, username) + log.Debugf("fileId: %v", p.fileIDs[r.URL.Path]) if err != nil { p.internalServerError(w, r, fmt.Sprintf("failed to register file in database: %v", err)) return } + + // check if the file already exists, in that case send an overwrite message, + // so that the FEGA portal is informed that a new version of the file exists. + err = p.sendMessageOnOverwrite(r, rawFilepath, token) + if err != nil { + p.internalServerError(w, r, err.Error()) + + return + } } log.Debug("Forwarding to backend") @@ -177,6 +189,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw } // Send message to upstream and set file as uploaded in the database + // nolint: nestif // We need a nested if statement for checking whether fileId is persisted during possible reconnections if p.uploadFinishedSuccessfully(r, s3response) { log.Debug("create message") message, err := p.CreateMessageFromRequest(r, token) @@ -199,15 +212,35 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw return } - log.Debugf("marking file %v as 'uploaded' in database", p.fileIds[r.URL.Path]) - err = p.database.UpdateFileEventLog(p.fileIds[r.URL.Path], "uploaded", p.fileIds[r.URL.Path], "inbox", "{}", string(jsonMessage)) + // The following block is for treating the case when the client loses connection to the server and then it reconnects to a + // different instance of s3inbox. For more details see #1358. + if p.fileIDs[r.URL.Path] == "" { + p.fileIDs[r.URL.Path], err = p.database.GetFileIDByUserPathAndStatus(username, filepath, "registered") + if err != nil { + p.internalServerError(w, r, fmt.Sprintf("failed to retrieve fileID from database: %v", err)) + + return + } + + log.Debugf("resuming work on file with fileId: %v", p.fileIDs[r.URL.Path]) + } + + if err := p.storeObjectSizeInDB(rawFilepath, p.fileIDs[r.URL.Path]); err != nil { + log.Errorf("storeObjectSizeInDB failed because: %s", err.Error()) + p.internalServerError(w, r, "storeObjectSizeInDB failed") + + return + } + + log.Debugf("marking file %v as 'uploaded' in database", p.fileIDs[r.URL.Path]) + err = p.database.UpdateFileEventLog(p.fileIDs[r.URL.Path], "uploaded", p.fileIDs[r.URL.Path], "inbox", "{}", string(jsonMessage)) if err != nil { p.internalServerError(w, r, fmt.Sprintf("could not connect to db: %v", err)) return } - delete(p.fileIds, r.URL.Path) + delete(p.fileIDs, r.URL.Path) } // Writing non-200 to the response before the headers propagate the error @@ -244,7 +277,7 @@ func (p *Proxy) allowedResponse(w http.ResponseWriter, r *http.Request, token jw func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { var err error if p.messenger == nil { - return fmt.Errorf("messenger is down") + return errors.New("messenger is down") } if p.messenger.IsConnClosed() { log.Warning("connection is closed, reconnecting...") @@ -262,7 +295,7 @@ func (p *Proxy) checkAndSendMessage(jsonMessage []byte, r *http.Request) error { } } - if err := p.messenger.SendMessage(p.fileIds[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { + if err := p.messenger.SendMessage(p.fileIDs[r.URL.Path], p.messenger.Conf.Exchange, p.messenger.Conf.RoutingKey, jsonMessage); err != nil { return fmt.Errorf("error when sending message to broker: %v", err) } @@ -464,7 +497,7 @@ func (p *Proxy) CreateMessageFromRequest(r *http.Request, claims jwt.Token) (Eve event.Filepath = strings.Replace(r.URL.Path, "/"+p.s3.Bucket+"/", "", 1) event.Username = claims.Subject() checksum.Type = "sha256" - event.Checksum = []interface{}{checksum} + event.Checksum = []any{checksum} privateClaims := claims.PrivateClaims() log.Info("user ", event.Username, " with pilot ", privateClaims["pilot"], " uploaded file ", event.Filepath, " with checksum ", checksum.Value, " at ", time.Now()) @@ -494,7 +527,59 @@ func (p *Proxy) requestInfo(fullPath string) (string, int64, error) { } return fmt.Sprintf("%x", sha256.Sum256([]byte(strings.ReplaceAll(*result.Contents[0].ETag, "\"", "")))), *result.Contents[0].Size, nil +} + +// checkFileExists makes a request to the S3 to check whether the file already +// is uploaded. Returns a bool indicating whether the file was found. +func (p *Proxy) checkFileExists(fullPath string) (bool, error) { + filePath := strings.Replace(fullPath, "/"+p.s3.Bucket+"/", "", 1) + client, err := storage.NewS3Client(p.s3) + if err != nil { + return false, fmt.Errorf("could not connect to s3: %v", err) + } + + result, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: &p.s3.Bucket, + Key: &filePath, + }) + + if err != nil && strings.Contains(err.Error(), "StatusCode: 404") { + log.Debugf("s3 could not find file %s: %s", fullPath, err.Error()) + + return false, nil + } + + return result != nil, err +} +func (p *Proxy) sendMessageOnOverwrite(r *http.Request, rawFilepath string, token jwt.Token) error { + exist, err := p.checkFileExists(r.URL.Path) + if err != nil { + return err + } + if exist { + username := token.Subject() + msg := schema.InboxRemove{ + User: username, + FilePath: rawFilepath, + Operation: "remove", + } + + privateClaims := token.PrivateClaims() + log.Info("user ", msg.User, " with pilot ", privateClaims["pilot"], " will overwrite file ", msg.FilePath, " at ", time.Now()) + + jsonMessage, err := json.Marshal(msg) + if err != nil { + return err + } + + err = p.checkAndSendMessage(jsonMessage, r) + if err != nil { + return err + } + } + + return nil } // FormatUploadFilePath ensures that path separators are "/", and returns error if the @@ -503,7 +588,7 @@ func formatUploadFilePath(filePath string) (string, error) { // Check for mixed "\" and "/" in filepath. Stop and throw an error if true so that // we do not end up with unintended folder structure when applying ReplaceAll below if strings.Contains(filePath, "\\") && strings.Contains(filePath, "/") { - return filePath, fmt.Errorf("filepath contains mixed '\\' and '/' characters") + return filePath, errors.New("filepath contains mixed '\\' and '/' characters") } // make any windows path separators linux compatible @@ -542,3 +627,28 @@ func reportError(errorCode int, message string, w http.ResponseWriter) { log.Error(err) } } + +func (p *Proxy) storeObjectSizeInDB(path, fileID string) error { + client, err := storage.NewS3Client(p.s3) + if err != nil { + return err + } + + o, err := client.HeadObject(context.TODO(), &s3.HeadObjectInput{ + Bucket: &p.s3.Bucket, + Key: &path, + }) + if err != nil { + return err + } + + if err := p.database.DB.Ping(); err != nil { + p.database.Close() + _ = p.database.Connect() + } + + const setObjectSize = "UPDATE sda.files set submission_file_size = $1 where id = $2;" + _, err = p.database.DB.Exec(setObjectSize, *o.ContentLength, fileID) + + return err +} diff --git a/sda/cmd/s3inbox/proxy_test.go b/sda/cmd/s3inbox/proxy_test.go index 08d44de61f..313e389661 100644 --- a/sda/cmd/s3inbox/proxy_test.go +++ b/sda/cmd/s3inbox/proxy_test.go @@ -1,17 +1,24 @@ package main import ( + "context" "crypto/tls" "database/sql" "encoding/json" + "errors" "fmt" "net" "net/http" "net/http/httptest" "net/url" "os" + "strings" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/lestrrat-go/jwx/v2/jwk" "github.com/lestrrat-go/jwx/v2/jws" "github.com/lestrrat-go/jwx/v2/jwt" @@ -20,13 +27,15 @@ import ( "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/neicnordic/sensitive-data-archive/internal/storage" "github.com/neicnordic/sensitive-data-archive/internal/userauth" + log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type ProxyTests struct { suite.Suite - S3conf storage.S3Conf + S3Fakeconf storage.S3Conf // fakeserver + S3conf storage.S3Conf // actual s3 container DBConf database.DBConf fakeServer *FakeServer MQConf broker.MQConf @@ -40,13 +49,12 @@ func TestProxyTestSuite(t *testing.T) { suite.Run(t, new(ProxyTests)) } -func (suite *ProxyTests) SetupTest() { - +func (s *ProxyTests) SetupTest() { // Create fake server - suite.fakeServer = startFakeServer("9024") + s.fakeServer = startFakeServer("9024") // Create an s3config for the fake server - suite.S3conf = storage.S3Conf{ + s.S3Fakeconf = storage.S3Conf{ URL: "http://127.0.0.1", Port: 9024, AccessKey: "someAccess", @@ -55,8 +63,17 @@ func (suite *ProxyTests) SetupTest() { Region: "us-east-1", } + s.S3conf = storage.S3Conf{ + URL: "http://127.0.0.1", + Port: s3Port, + AccessKey: "access", + SecretKey: "secretKey", + Bucket: "buckbuck", + Region: "us-east-1", + } + // Create a configuration for the fake MQ - suite.MQConf = broker.MQConf{ + s.MQConf = broker.MQConf{ Host: "127.0.0.1", Port: MQport, User: "guest", @@ -65,10 +82,10 @@ func (suite *ProxyTests) SetupTest() { Exchange: "", } - suite.messenger = &broker.AMQPBroker{} + s.messenger = &broker.AMQPBroker{} // Create a database configuration for the fake database - suite.DBConf = database.DBConf{ + s.DBConf = database.DBConf{ Host: "127.0.0.1", Port: DBport, User: "postgres", @@ -80,38 +97,65 @@ func (suite *ProxyTests) SetupTest() { ClientKey: "", } - suite.database = &database.SDAdb{} + s.database = &database.SDAdb{} // Create temp demo rsa key pair demoKeysPath := "demo-rsa-keys" defer os.RemoveAll(demoKeysPath) prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // Parse demo private key prKeyParsed, err := helper.ParsePrivateRSAKey(prKeyPath, "/rsa") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) // Create token and set up request defaults defaultToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" - suite.auth = userauth.NewValidateFromToken(jwk.NewSet()) - _ = suite.auth.ReadJwtPubKeyPath(jwtpubkeypath) + s.auth = userauth.NewValidateFromToken(jwk.NewSet()) + _ = s.auth.ReadJwtPubKeyPath(jwtpubkeypath) + + s.token, err = jwt.Parse([]byte(defaultToken), jwt.WithKeySet(s.auth.Keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) + if err != nil { + s.T().FailNow() + } - suite.token, err = jwt.Parse([]byte(defaultToken), jwt.WithKeySet(suite.auth.Keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) + s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) + if err != nil { + s.FailNow("bad") + } + s3Client := s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", s.S3conf.URL, s.S3conf.Port)) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.URL, "http:") + o.Region = s.S3conf.Region + o.UsePathStyle = true + }, + ) + _, _ = s3Client.CreateBucket(context.TODO(), &s3.CreateBucketInput{Bucket: aws.String(s.S3conf.Bucket)}) if err != nil { - suite.T().FailNow() + fmt.Println(err.Error()) } + + output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader("This is a test"), + Bucket: aws.String(s.S3conf.Bucket), + Key: aws.String("/dummy/file"), + ContentEncoding: aws.String("application/octet-stream"), + }) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), output, output) } -func (suite *ProxyTests) TearDownTest() { - suite.fakeServer.Close() - suite.database.Close() +func (s *ProxyTests) TearDownTest() { + s.fakeServer.Close() + s.database.Close() } type FakeServer struct { @@ -125,10 +169,15 @@ func startFakeServer(port string) *FakeServer { if err != nil { panic(fmt.Errorf("can't create mock server for testing: %s", err)) } + log.Warnf("fake server running") f := FakeServer{} foo := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { f.pinged = true - fmt.Fprint(w, f.resp) + log.Warnf("hello fake will return %s", f.resp) + if f.resp != "" { + log.Warnf("fake writes %s", f.resp) + fmt.Fprint(w, f.resp) + } }) ts := httptest.NewUnstartedServer(foo) ts.Listener.Close() @@ -165,76 +214,78 @@ func NewMockMessenger() *MockMessenger { func (m *MockMessenger) SendMessage(uuid string, body []byte) error { if uuid == "" || body == nil { - return fmt.Errorf("bad message") + return errors.New("bad message") } return nil } // nolint:bodyclose -func (suite *ProxyTests) TestServeHTTP_disallowed() { - proxy := NewProxy(suite.S3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) +func (s *ProxyTests) TestServeHTTP_disallowed() { + proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() + log.Warnf("using proxy on port %d", s.S3Fakeconf.Port) // Remove bucket disallowed r.Method = "DELETE" r.URL, _ = url.Parse("/asdf/") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Deletion of files are disallowed r.Method = "DELETE" r.URL, _ = url.Parse("/asdf/asdf") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) + log.Warnf("getting to not allowed stuff") // Policy methods are not allowed w = httptest.NewRecorder() r.Method = "GET" r.URL, _ = url.Parse("/asdf?acl=rw") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Normal get is dissallowed w = httptest.NewRecorder() r.Method = "GET" r.URL, _ = url.Parse("/asdf/") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Put policy is disallowed w = httptest.NewRecorder() r.Method = "PUT" r.URL, _ = url.Parse("/asdf?policy=rw") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Create bucket disallowed w = httptest.NewRecorder() r.Method = "PUT" r.URL, _ = url.Parse("/asdf/") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 403, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 403, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Not authorized user get 401 response - proxy = NewProxy(suite.S3conf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) + proxy = NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) w = httptest.NewRecorder() r.Method = "GET" r.URL, _ = url.Parse("/username/file") proxy.ServeHTTP(w, r) - assert.Equal(suite.T(), 401, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 401, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) } -func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { +func (s *ProxyTests) TestServeHTTPS3Unresponsive() { s3conf := storage.S3Conf{ URL: "http://localhost:40211", AccessKey: "someAccess", @@ -242,7 +293,7 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { Bucket: "buckbuck", Region: "us-east-1", } - proxy := NewProxy(s3conf, &helper.AlwaysAllow{}, suite.messenger, suite.database, new(tls.Config)) + proxy := NewProxy(s3conf, &helper.AlwaysAllow{}, s.messenger, s.database, new(tls.Config)) r, _ := http.NewRequest("", "", nil) w := httptest.NewRecorder() @@ -250,119 +301,119 @@ func (suite *ProxyTests) TestServeHTTPS3Unresponsive() { // Just try to list the files r.Method = "GET" r.URL, _ = url.Parse("/dummy/asdf") - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 500, w.Result().StatusCode) // nolint:bodyclose + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 500, w.Result().StatusCode) // nolint:bodyclose } -func (suite *ProxyTests) TestServeHTTP_MQConnectionClosed() { +func (s *ProxyTests) TestServeHTTP_MQConnectionClosed() { // Set up - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the mq connection will be restored when needed proxy.messenger.Connection.Close() - assert.True(suite.T(), proxy.messenger.Connection.IsClosed()) + assert.True(s.T(), proxy.messenger.Connection.IsClosed()) r, _ := http.NewRequest("PUT", "/dummy/connectionclosed-file", nil) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) // nolint:bodyclose - assert.False(suite.T(), proxy.messenger.Connection.IsClosed()) + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose + assert.False(s.T(), proxy.messenger.Connection.IsClosed()) } -func (suite *ProxyTests) TestServeHTTP_MQChannelClosed() { +func (s *ProxyTests) TestServeHTTP_MQChannelClosed() { // Set up - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the mq channel will be restored when needed proxy.messenger.Channel.Close() - assert.True(suite.T(), proxy.messenger.Channel.IsClosed()) + assert.True(s.T(), proxy.messenger.Channel.IsClosed()) r, _ := http.NewRequest("PUT", "/dummy/channelclosed-file", nil) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) // nolint:bodyclose - assert.False(suite.T(), proxy.messenger.Channel.IsClosed()) + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) // nolint:bodyclose + assert.False(s.T(), proxy.messenger.Channel.IsClosed()) } -func (suite *ProxyTests) TestServeHTTP_MQ_Unavailable() { +func (s *ProxyTests) TestServeHTTP_MQ_Unavailable() { // Set up - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // Test that the correct status code is returned when mq connection can't be created proxy.messenger.Conf.Port = 123456 proxy.messenger.Connection.Close() - assert.True(suite.T(), proxy.messenger.Connection.IsClosed()) + assert.True(s.T(), proxy.messenger.Connection.IsClosed()) r, _ := http.NewRequest("PUT", "/dummy/mqunavailable-file", nil) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 500, w.Result().StatusCode) // nolint:bodyclose + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 500, w.Result().StatusCode) // nolint:bodyclose } // nolint:bodyclose -func (suite *ProxyTests) TestServeHTTP_allowed() { - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) - database, _ := database.NewSDAdb(suite.DBConf) - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) +func (s *ProxyTests) TestServeHTTP_allowed() { + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + db, _ := database.NewSDAdb(s.DBConf) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // List files works r, err := http.NewRequest("GET", "/dummy/file", nil) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) w := httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) // Testing the pinged interface + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) // Testing the pinged interface // Put file works w = httptest.NewRecorder() r.Method = "PUT" - suite.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + s.fakeServer.resp = "test/elixirid/file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Put with partnumber sends no message w = httptest.NewRecorder() r.Method = "PUT" r.URL, _ = url.Parse("/dummy/file?partNumber=5") - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Post with uploadId sends message r.Method = "POST" r.URL, _ = url.Parse("/dummy/file?uploadId=5") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Post without uploadId sends no message r.Method = "POST" r.URL, _ = url.Parse("/dummy/file") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Abort multipart works r.Method = "DELETE" r.URL, _ = url.Parse("/dummy/asdf?uploadId=123") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Going through the different extra stuff that can be in the get request // that trigger different code paths in the code. @@ -370,45 +421,45 @@ func (suite *ProxyTests) TestServeHTTP_allowed() { r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?delimiter=puppe") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Show multiparts uploads r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?uploads") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Delimiter alone together with prefix r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?delimiter=puppe&prefix=asdf") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Location parameter r.Method = "GET" r.URL, _ = url.Parse("/dummy/file?location=fnuffe") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 200, w.Result().StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 200, w.Result().StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Filenames with platform incompatible characters are disallowed // not checked in TestServeHTTP_allowed() because we look for a non 403 response r.Method = "PUT" r.URL, _ = url.Parse("/dummy/fi|le") w = httptest.NewRecorder() - proxy.allowedResponse(w, r, suite.token) - assert.Equal(suite.T(), 406, w.Result().StatusCode) - assert.Equal(suite.T(), false, suite.fakeServer.PingedAndRestore()) + proxy.allowedResponse(w, r, s.token) + assert.Equal(s.T(), 406, w.Result().StatusCode) + assert.Equal(s.T(), false, s.fakeServer.PingedAndRestore()) } -func (suite *ProxyTests) TestMessageFormatting() { +func (s *ProxyTests) TestMessageFormatting() { // Set up basic request for multipart upload r, _ := http.NewRequest("POST", "/buckbuck/user/new_file.txt", nil) r.Host = "localhost" @@ -416,90 +467,267 @@ func (suite *ProxyTests) TestMessageFormatting() { r.Header.Set("x-amz-content-sha256", "checksum") claims := jwt.New() - assert.NoError(suite.T(), claims.Set("sub", "user@host.domain")) + assert.NoError(s.T(), claims.Set("sub", "user@host.domain")) // start proxy that denies everything - proxy := NewProxy(suite.S3conf, &helper.AlwaysDeny{}, suite.messenger, suite.database, new(tls.Config)) - suite.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" + proxy := NewProxy(s.S3Fakeconf, &helper.AlwaysDeny{}, s.messenger, s.database, new(tls.Config)) + s.fakeServer.resp = "test/user/new_file.txt12false/user/new_file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"1234STANDARD" msg, err := proxy.CreateMessageFromRequest(r, claims) - assert.Nil(suite.T(), err) - assert.IsType(suite.T(), Event{}, msg) + assert.Nil(s.T(), err) + assert.IsType(s.T(), Event{}, msg) - assert.Equal(suite.T(), int64(1234), msg.Filesize) - assert.Equal(suite.T(), "user/new_file.txt", msg.Filepath) - assert.Equal(suite.T(), "user@host.domain", msg.Username) + assert.Equal(s.T(), int64(1234), msg.Filesize) + assert.Equal(s.T(), "user/new_file.txt", msg.Filepath) + assert.Equal(s.T(), "user@host.domain", msg.Username) c, _ := json.Marshal(msg.Checksum[0]) checksum := Checksum{} _ = json.Unmarshal(c, &checksum) - assert.Equal(suite.T(), "sha256", checksum.Type) - assert.Equal(suite.T(), "5b233b981dc12e7ccf4c242b99c042b7842b73b956ad662e4fe0f8354151538b", checksum.Value) + assert.Equal(s.T(), "sha256", checksum.Type) + assert.Equal(s.T(), "5b233b981dc12e7ccf4c242b99c042b7842b73b956ad662e4fe0f8354151538b", checksum.Value) // Test single shot upload r.Method = "PUT" msg, err = proxy.CreateMessageFromRequest(r, jwt.New()) - assert.Nil(suite.T(), err) - assert.IsType(suite.T(), Event{}, msg) - assert.Equal(suite.T(), "upload", msg.Operation) + assert.Nil(s.T(), err) + assert.IsType(s.T(), Event{}, msg) + assert.Equal(s.T(), "upload", msg.Operation) } -func (suite *ProxyTests) TestDatabaseConnection() { - database, err := database.NewSDAdb(suite.DBConf) - assert.NoError(suite.T(), err) - defer database.Close() - messenger, err := broker.NewMQ(suite.MQConf) - assert.NoError(suite.T(), err) +func (s *ProxyTests) TestDatabaseConnection() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) defer messenger.Connection.Close() // Start proxy that allows everything - proxy := NewProxy(suite.S3conf, helper.NewAlwaysAllow(), messenger, database, new(tls.Config)) + proxy := NewProxy(s.S3Fakeconf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) // PUT a file into the system filename := "/dummy/db-test-file" - r, _ := http.NewRequest("PUT", filename, nil) + + stringReader := strings.NewReader("a brand new string") + r, _ := http.NewRequest("PUT", filename, stringReader) w := httptest.NewRecorder() - suite.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" - proxy.allowedResponse(w, r, suite.token) + s.fakeServer.resp = "test/elixirid/db-test-file.txt12false/elixirid/file.txt2020-03-10T13:20:15.000Z"0a44282bd39178db9680f24813c41aec-1"5STANDARD" + proxy.allowedResponse(w, r, s.token) res := w.Result() defer res.Body.Close() - assert.Equal(suite.T(), 200, res.StatusCode) - assert.Equal(suite.T(), true, suite.fakeServer.PingedAndRestore()) + assert.Equal(s.T(), 200, res.StatusCode) + assert.Equal(s.T(), true, s.fakeServer.PingedAndRestore()) // Check that the file is registered and uploaded in the database // connect to the database - db, err := sql.Open(suite.DBConf.PgDataSource()) - assert.Nil(suite.T(), err, "Failed to connect to database") + db.DB, err = sql.Open(s.DBConf.PgDataSource()) + assert.Nil(s.T(), err, "Failed to connect to database") // Check that the file is in the database var fileID string query := "SELECT id FROM sda.files WHERE submission_file_path = $1;" - err = db.QueryRow(query, filename[1:]).Scan(&fileID) - assert.Nil(suite.T(), err, "Failed to query database") - assert.NotNil(suite.T(), fileID, "File not found in database") + err = db.DB.QueryRow(query, filename[1:]).Scan(&fileID) + assert.Nil(s.T(), err, "Failed to query database") + assert.NotNil(s.T(), fileID, "File not found in database") // Check that the "registered" status is in the database for this file for _, status := range []string{"registered", "uploaded"} { var exists int query = "SELECT 1 FROM sda.file_event_log WHERE event = $1 AND file_id = $2;" - err = db.QueryRow(query, status, fileID).Scan(&exists) - assert.Nil(suite.T(), err, "Failed to find '%v' event in database", status) - assert.Equal(suite.T(), exists, 1, "File '%v' event does not exist", status) + err = db.DB.QueryRow(query, status, fileID).Scan(&exists) + assert.Nil(s.T(), err, "Failed to find '%v' event in database", status) + assert.Equal(s.T(), exists, 1, "File '%v' event does not exist", status) } } -func (suite *ProxyTests) TestFormatUploadFilePath() { +func (s *ProxyTests) TestFormatUploadFilePath() { unixPath := "a/b/c.c4gh" testPath := "a\\b\\c.c4gh" uploadPath, err := formatUploadFilePath(testPath) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), unixPath, uploadPath) + assert.NoError(s.T(), err) + assert.Equal(s.T(), unixPath, uploadPath) // mixed "\" and "/" weirdPath := `dq\sw:*?"<>|\t\s/df.c4gh` _, err = formatUploadFilePath(weirdPath) - assert.EqualError(suite.T(), err, "filepath contains mixed '\\' and '/' characters") + assert.EqualError(s.T(), err, "filepath contains mixed '\\' and '/' characters") // no mixed "\" and "/" but not allowed weirdPath = `dq\sw:*?"<>|\t\sdf!s'(a);w@4&f=+e$,g#[]d%.c4gh` _, err = formatUploadFilePath(weirdPath) - assert.EqualError(suite.T(), err, "filepath contains disallowed characters: :, *, ?, \", <, >, |, !, ', (, ), ;, @, &, =, +, $, ,, #, [, ], %") + assert.EqualError(s.T(), err, "filepath contains disallowed characters: :, *, ?, \", <, >, |, !, ', (, ), ;, @, &, =, +, $, ,, #, [, ], %") +} + +func (s *ProxyTests) TestCheckFileExists() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + defer messenger.Connection.Close() + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), messenger, db, new(tls.Config)) + + res, err := proxy.checkFileExists("/dummy/file") + assert.True(s.T(), res) + assert.Nil(s.T(), err) +} + +func (s *ProxyTests) TestCheckFileExists_nonExistingFile() { + // Check that looking for a non-existing file gives (false, nil) + // from checkFileExists + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + defer messenger.Connection.Close() + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + + res, err := proxy.checkFileExists("nonexistingfilepath") + assert.False(s.T(), res) + assert.Nil(s.T(), err) +} + +func (s *ProxyTests) TestCheckFileExists_unresponsive() { + // Check that errors when connecting to S3 are forwarded + // and that checkFileExists return (false, someError) + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + messenger, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + defer messenger.Connection.Close() + + // Unaccessible S3 (wrong port) + proxy := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + proxy.s3.Port = 1111 + + res, err := proxy.checkFileExists("nonexistingfilepath") + assert.False(s.T(), res) + assert.NotNil(s.T(), err) + assert.Contains(s.T(), err.Error(), "S3: HeadObject") + + // Bad access key gives 403 + proxy.s3.Port = s.S3conf.Port + proxy.s3.AccessKey = "invaild" + res, err = proxy.checkFileExists("nonexistingfilepath") + assert.False(s.T(), res) + assert.NotNil(s.T(), err) + assert.Contains(s.T(), err.Error(), "StatusCode: 403") +} + +func (s *ProxyTests) TestStoreObjectSizeInDB() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + defer mq.Connection.Close() + + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/dummy/file", "test-user") + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) + + assert.NoError(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + + const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" + var objectSize int64 + assert.NoError(s.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) + assert.Equal(s.T(), int64(14), objectSize) +} + +func (s *ProxyTests) TestStoreObjectSizeInDB_dbFailure() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + defer mq.Connection.Close() + + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/dummy/file", "test-user") + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) + + db.Close() + assert.NoError(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) +} + +func (s *ProxyTests) TestStoreObjectSizeInDB_s3Failure() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + defer mq.Connection.Close() + + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/dummy/file", "test-user") + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) + + // Detect autentication failure + p.s3.AccessKey = "badKey" + assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) + + // Detect unresponsive backend service + p.s3.Port = 1234 + assert.Error(s.T(), p.storeObjectSizeInDB("/dummy/file", fileID)) +} + +// This test is intended to try to catch some issues we sometimes see when a query to the S3 backend +// happens to fast so that it is not ready and returns a false 404. +func (s *ProxyTests) TestStoreObjectSizeInDB_fastCheck() { + db, err := database.NewSDAdb(s.DBConf) + assert.NoError(s.T(), err) + defer db.Close() + + mq, err := broker.NewMQ(s.MQConf) + assert.NoError(s.T(), err) + defer mq.Connection.Close() + + p := NewProxy(s.S3conf, helper.NewAlwaysAllow(), s.messenger, s.database, new(tls.Config)) + p.database = db + + fileID, err := db.RegisterFile("/test/new_file", "test-user") + assert.NoError(s.T(), err) + assert.NotNil(s.T(), fileID) + + s3cfg, err := config.LoadDefaultConfig(context.TODO(), config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(s.S3conf.AccessKey, s.S3conf.SecretKey, ""))) + if err != nil { + s.FailNow("bad") + } + s3Client := s3.NewFromConfig( + s3cfg, + func(o *s3.Options) { + o.BaseEndpoint = aws.String(fmt.Sprintf("%s:%d", s.S3conf.URL, s.S3conf.Port)) + o.EndpointOptions.DisableHTTPS = strings.HasPrefix(s.S3conf.URL, "http:") + o.Region = s.S3conf.Region + o.UsePathStyle = true + }, + ) + + output, err := s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader(strings.Repeat("A", 10*1024*1024)), + Bucket: aws.String(s.S3conf.Bucket), + Key: aws.String("/test/new_file"), + ContentEncoding: aws.String("application/octet-stream"), + }) + assert.NoError(s.T(), err) + assert.NotNil(s.T(), output, output) + + assert.NoError(s.T(), p.storeObjectSizeInDB("/test/new_file", fileID)) + + const getObjectSize = "SELECT submission_file_size FROM sda.files WHERE id = $1;" + var objectSize int64 + // If the S3 backend haven't had the time to process the request above correctly this will generate an error. + assert.NoError(s.T(), p.database.DB.QueryRow(getObjectSize, fileID).Scan(&objectSize)) + assert.Equal(s.T(), int64(10*1024*1024), objectSize) } diff --git a/sda/cmd/s3inbox/s3inbox.go b/sda/cmd/s3inbox/s3inbox.go index ade30d9078..ae7898bd96 100644 --- a/sda/cmd/s3inbox/s3inbox.go +++ b/sda/cmd/s3inbox/s3inbox.go @@ -19,9 +19,6 @@ import ( log "github.com/sirupsen/logrus" ) -// Export Conf so we can access it in the other modules -var Conf *config.Config - func main() { sigc := make(chan os.Signal, 5) signal.Notify(sigc, os.Interrupt, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT) @@ -34,21 +31,21 @@ func main() { } }() - Conf, err := config.NewConfig("s3inbox") + conf, err := config.NewConfig("s3inbox") if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - tlsProxy, err := config.TLSConfigProxy(Conf) + tlsProxy, err := config.TLSConfigProxy(conf) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - sdaDB, err := database.NewSDAdb(Conf.Database) + sdaDB, err := database.NewSDAdb(conf.Database) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -62,21 +59,21 @@ func main() { log.Debugf("Connected to sda-db (v%v)", sdaDB.Version) - s3, err := storage.NewS3Client(Conf.Inbox.S3) + s3, err := storage.NewS3Client(conf.Inbox.S3) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - err = storage.CheckS3Bucket(Conf.Inbox.S3.Bucket, s3) + err = storage.CheckS3Bucket(conf.Inbox.S3.Bucket, s3) if err != nil { log.Error(err) sigc <- syscall.SIGINT panic(err) } - messenger, err := broker.NewMQ(Conf.Broker) + messenger, err := broker.NewMQ(conf.Broker) if err != nil { log.Error(err) sigc <- syscall.SIGINT @@ -92,21 +89,21 @@ func main() { }() auth := userauth.NewValidateFromToken(jwk.NewSet()) // Load keys for JWT verification - if Conf.Server.Jwtpubkeyurl != "" { - if err := auth.FetchJwtPubKeyURL(Conf.Server.Jwtpubkeyurl); err != nil { - log.Panicf("Error while getting key %s: %v", Conf.Server.Jwtpubkeyurl, err) + if conf.Server.Jwtpubkeyurl != "" { + if err := auth.FetchJwtPubKeyURL(conf.Server.Jwtpubkeyurl); err != nil { + log.Panicf("Error while getting key %s: %v", conf.Server.Jwtpubkeyurl, err) } } - if Conf.Server.Jwtpubkeypath != "" { - if err := auth.ReadJwtPubKeyPath(Conf.Server.Jwtpubkeypath); err != nil { - log.Panicf("Error while getting key %s: %v", Conf.Server.Jwtpubkeypath, err) + if conf.Server.Jwtpubkeypath != "" { + if err := auth.ReadJwtPubKeyPath(conf.Server.Jwtpubkeypath); err != nil { + log.Panicf("Error while getting key %s: %v", conf.Server.Jwtpubkeypath, err) } } - mux := mux.NewRouter() - proxy := NewProxy(Conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy) - mux.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") - mux.HandleFunc("/health", proxy.CheckHealth) - mux.PathPrefix("/").Handler(proxy) + router := mux.NewRouter() + proxy := NewProxy(conf.Inbox.S3, auth, messenger, sdaDB, tlsProxy) + router.HandleFunc("/", proxy.CheckHealth).Methods("HEAD") + router.HandleFunc("/health", proxy.CheckHealth) + router.PathPrefix("/").Handler(proxy) server := &http.Server{ Addr: ":8000", @@ -114,11 +111,11 @@ func main() { WriteTimeout: 30 * time.Second, IdleTimeout: 30 * time.Second, ReadHeaderTimeout: 30 * time.Second, - Handler: mux, + Handler: router, } - if Conf.Server.Cert != "" && Conf.Server.Key != "" { - if err := server.ListenAndServeTLS(Conf.Server.Cert, Conf.Server.Key); err != nil { + if conf.Server.Cert != "" && conf.Server.Key != "" { + if err := server.ListenAndServeTLS(conf.Server.Cert, conf.Server.Key); err != nil { panic(err) } } else { diff --git a/sda/cmd/s3inbox/s3inbox_test.go b/sda/cmd/s3inbox/s3inbox_test.go index 224cc80b7d..5f2c7eae2e 100644 --- a/sda/cmd/s3inbox/s3inbox_test.go +++ b/sda/cmd/s3inbox/s3inbox_test.go @@ -11,14 +11,14 @@ import ( "testing" "time" + "github.com/neicnordic/sensitive-data-archive/internal/helper" log "github.com/sirupsen/logrus" - "github.com/neicnordic/sensitive-data-archive/internal/helper" "github.com/ory/dockertest/v3" "github.com/ory/dockertest/v3/docker" ) -var DBport, MQport, OIDCport int +var DBport, MQport, OIDCport, s3Port int func TestMain(m *testing.M) { if _, err := os.Stat("/.dockerenv"); err == nil { @@ -39,6 +39,54 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to Docker: %s", err) } + // pulls an image, creates a container based on it and runs it + minio, err := pool.RunWithOptions(&dockertest.RunOptions{ + Name: "s3test", + Repository: "minio/minio", + Tag: "RELEASE.2023-05-18T00-05-36Z", + Cmd: []string{"server", "/data", "--console-address", ":9001"}, + Env: []string{ + "MINIO_ROOT_USER=access", + "MINIO_ROOT_PASSWORD=secretKey", + "MINIO_SERVER_URL=http://127.0.0.1:9000", + }, + }, func(config *docker.HostConfig) { + // set AutoRemove to true so that stopped container goes away by itself + config.AutoRemove = true + config.RestartPolicy = docker.RestartPolicy{ + Name: "no", + } + }) + if err != nil { + log.Panicf("Could not start resource: %s", err) + } + + s3HostAndPort := minio.GetHostPort("9000/tcp") + s3Port, _ = strconv.Atoi(minio.GetPort("9000/tcp")) + log.Warnf("found host+port %s and port %d", s3HostAndPort, s3Port) + + client := http.Client{Timeout: 5 * time.Second} + req, err := http.NewRequest(http.MethodGet, "http://"+s3HostAndPort+"/minio/health/live", http.NoBody) + log.Warnf("request to %v", req) + if err != nil { + log.Panic(err) + } + + // exponential backoff-retry, because the application in the container might not be ready to accept connections yet + if err := pool.Retry(func() error { + res, err := client.Do(req) + if err != nil { + return err + } + res.Body.Close() + + return nil + }); err != nil { + if err := pool.Purge(minio); err != nil { + log.Panicf("Could not purge resource: %s", err) + } + log.Panicf("Could not connect to minio: %s", err) + } // pulls an image, creates a container based on it and runs it postgres, err := pool.RunWithOptions(&dockertest.RunOptions{ Repository: "postgres", @@ -100,8 +148,8 @@ func TestMain(m *testing.M) { MQport, _ = strconv.Atoi(rabbitmq.GetPort("5672/tcp")) mqHostAndPort := rabbitmq.GetHostPort("15672/tcp") - client := http.Client{Timeout: 5 * time.Second} - req, err := http.NewRequest(http.MethodPut, "http://"+mqHostAndPort+"/api/queues/%2F/inbox", http.NoBody) + client = http.Client{Timeout: 5 * time.Second} + req, err = http.NewRequest(http.MethodPut, "http://"+mqHostAndPort+"/api/queues/%2F/inbox", http.NoBody) if err != nil { log.Fatal(err) } @@ -123,12 +171,12 @@ func TestMain(m *testing.M) { log.Fatalf("Could not connect to rabbitmq: %s", err) } - RSAPath, _ := os.MkdirTemp("", "RSA") - if err := helper.CreateRSAkeys(RSAPath, RSAPath); err != nil { + rsaPath, _ := os.MkdirTemp("", "RSA") + if err := helper.CreateRSAkeys(rsaPath, rsaPath); err != nil { log.Panic("Failed to create RSA keys") } - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { log.Panic("Failed to create EC keys") } // pulls an image, creates a container based on it and runs it @@ -156,10 +204,10 @@ func TestMain(m *testing.M) { } OIDCport, _ = strconv.Atoi(oidc.GetPort("8080/tcp")) - OIDCHostAndPort := oidc.GetHostPort("8080/tcp") + oidcHostAndPort := oidc.GetHostPort("8080/tcp") client = http.Client{Timeout: 5 * time.Second} - req, err = http.NewRequest(http.MethodGet, "http://"+OIDCHostAndPort+"/jwk", http.NoBody) + req, err = http.NewRequest(http.MethodGet, "http://"+oidcHostAndPort+"/jwk", http.NoBody) if err != nil { log.Panic(err) } @@ -193,6 +241,9 @@ func TestMain(m *testing.M) { if err := pool.Purge(oidc); err != nil { log.Fatalf("Could not purge resource: %s", err) } + if err := pool.Purge(minio); err != nil { + log.Fatalf("Could not purge resource: %s", err) + } os.Exit(code) } diff --git a/sda/cmd/sync/sync.go b/sda/cmd/sync/sync.go index 4f120a0184..8dcf42e3e8 100644 --- a/sda/cmd/sync/sync.go +++ b/sda/cmd/sync/sync.go @@ -5,6 +5,7 @@ package main import ( "bytes" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -172,7 +173,7 @@ func syncFiles(stableID string) error { return fmt.Errorf("failed to get archive path for file with stable ID: %s", stableID) } - fileSize, err := archive.GetFileSize(archivePath) + fileSize, err := archive.GetFileSize(archivePath, false) if err != nil { return err } @@ -211,7 +212,7 @@ func syncFiles(stableID string) error { if err != nil || copiedSize != int64(fileSize) { switch { case copiedSize != int64(fileSize): - return fmt.Errorf("copied size does not match file size") + return errors.New("copied size does not match file size") default: return err } @@ -242,12 +243,12 @@ func buildSyncDatasetJSON(b []byte) ([]byte, error) { dataset.User = data.User } - json, err := json.Marshal(dataset) + datasetJSON, err := json.Marshal(dataset) if err != nil { return nil, err } - return json, nil + return datasetJSON, nil } func sendPOST(payload []byte) error { @@ -255,12 +256,12 @@ func sendPOST(payload []byte) error { Timeout: 30 * time.Second, } - URL, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) + uri, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) if err != nil { return err } - req, err := http.NewRequest(http.MethodPost, URL, bytes.NewBuffer(payload)) + req, err := http.NewRequest(http.MethodPost, uri, bytes.NewBuffer(payload)) if err != nil { return err } @@ -279,14 +280,14 @@ func sendPOST(payload []byte) error { } func createHostURL(host string, port int) (string, error) { - url, err := url.ParseRequestURI(host) + uri, err := url.ParseRequestURI(host) if err != nil { return "", err } - if url.Port() == "" && port != 0 { - url.Host += fmt.Sprintf(":%d", port) + if uri.Port() == "" && port != 0 { + uri.Host += fmt.Sprintf(":%d", port) } - url.Path = "/dataset" + uri.Path = "/dataset" - return url.String(), nil + return uri.String(), nil } diff --git a/sda/cmd/sync/sync_test.go b/sda/cmd/sync/sync_test.go index 3a86ca9222..95d35a9810 100644 --- a/sda/cmd/sync/sync_test.go +++ b/sda/cmd/sync/sync_test.go @@ -14,7 +14,6 @@ import ( "testing" "time" - "github.com/google/uuid" "github.com/neicnordic/sensitive-data-archive/internal/config" "github.com/neicnordic/sensitive-data-archive/internal/database" "github.com/ory/dockertest/v3" @@ -112,7 +111,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *SyncTest) SetupTest() { +func (s *SyncTest) SetupTest() { viper.Set("log.level", "debug") viper.Set("archive.type", "posix") viper.Set("archive.location", "../../dev_utils") @@ -138,69 +137,70 @@ func (suite *SyncTest) SetupTest() { key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" keyPath, _ := os.MkdirTemp("", "key") err := os.WriteFile(keyPath+"/c4gh.key", []byte(key), 0600) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) viper.Set("c4gh.filepath", keyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" err = os.WriteFile(keyPath+"/c4gh.pub", []byte(pubKey), 0600) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) viper.Set("c4gh.syncPubKeyPath", keyPath+"/c4gh.pub") defer os.RemoveAll(keyPath) } -func (suite *SyncTest) TestBuildSyncDatasetJSON() { - suite.SetupTest() - Conf, err := config.NewConfig("sync") - assert.NoError(suite.T(), err) +func (s *SyncTest) TestBuildSyncDatasetJSON() { + s.SetupTest() + conf, err := config.NewConfig("sync") + assert.NoError(s.T(), err) - db, err = database.NewSDAdb(Conf.Database) - assert.NoError(suite.T(), err) + db, err = database.NewSDAdb(conf.Database) + assert.NoError(s.T(), err) fileID, err := db.RegisterFile("dummy.user/test/file1.c4gh", "dummy.user") - assert.NoError(suite.T(), err, "failed to register file in database") + assert.NoError(s.T(), err, "failed to register file in database") err = db.SetAccessionID("ed6af454-d910-49e3-8cda-488a6f246e67", fileID) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := database.FileInfo{Checksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} - corrID := uuid.New().String() + fileInfo := database.FileInfo{ArchiveChecksum: fmt.Sprintf("%x", sha256.New().Sum(nil)), Size: 1234, Path: "dummy.user/test/file1.c4gh", DecryptedChecksum: checksum, DecryptedSize: 999} - err = db.SetArchived(fileInfo, fileID, corrID) - assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) - assert.NoError(suite.T(), err, "failed to mark file as Verified") + err = db.SetArchived(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Archived") + err = db.SetVerified(fileInfo, fileID) + assert.NoError(s.T(), err, "failed to mark file as Verified") accessions := []string{"ed6af454-d910-49e3-8cda-488a6f246e67"} - assert.NoError(suite.T(), db.MapFilesToDataset("cd532362-e06e-4461-8490-b9ce64b8d9e7", accessions), "failed to map file to dataset") + assert.NoError(s.T(), db.MapFilesToDataset("cd532362-e06e-4461-8490-b9ce64b8d9e7", accessions), "failed to map file to dataset") m := []byte(`{"type":"mapping", "dataset_id": "cd532362-e06e-4461-8490-b9ce64b8d9e7", "accession_ids": ["ed6af454-d910-49e3-8cda-488a6f246e67"]}`) jsonData, err := buildSyncDatasetJSON(m) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) dataset := []byte(`{"dataset_id":"cd532362-e06e-4461-8490-b9ce64b8d9e7","dataset_files":[{"filepath":"dummy.user/test/file1.c4gh","file_id":"ed6af454-d910-49e3-8cda-488a6f246e67","sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}],"user":"dummy.user"}`) - assert.Equal(suite.T(), string(dataset), string(jsonData)) + assert.Equal(s.T(), string(dataset), string(jsonData)) } -func (suite *SyncTest) TestCreateHostURL() { +func (s *SyncTest) TestCreateHostURL() { conf = &config.Config{} conf.Sync = config.Sync{ RemoteHost: "http://localhost", RemotePort: 443, } - s, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "http://localhost:443/dataset", s) + h, err := createHostURL(conf.Sync.RemoteHost, conf.Sync.RemotePort) + assert.NoError(s.T(), err) + assert.Equal(s.T(), "http://localhost:443/dataset", h) } -func (suite *SyncTest) TestSendPOST() { +func (s *SyncTest) TestSendPOST() { r := http.NewServeMux() r.HandleFunc("/dataset", func(w http.ResponseWriter, r *http.Request) { username, _, ok := r.BasicAuth() if ok && username == "foo" { w.WriteHeader(http.StatusUnauthorized) + + return } w.WriteHeader(http.StatusOK) @@ -216,12 +216,12 @@ func (suite *SyncTest) TestSendPOST() { } syncJSON := []byte(`{"user":"test.user@example.com", "dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "dataset_files": [{"filepath": "inbox/user/file1.c4gh","file_id": "5fe7b660-afea-4c3a-88a9-3daabf055ebb", "sha256": "82E4e60e7beb3db2e06A00a079788F7d71f75b61a4b75f28c4c942703dabb6d6"}, {"filepath": "inbox/user/file2.c4gh","file_id": "ed6af454-d910-49e3-8cda-488a6f246e76", "sha256": "c967d96e56dec0f0cfee8f661846238b7f15771796ee1c345cae73cd812acc2b"}]}`) err := sendPOST(syncJSON) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) conf.Sync = config.Sync{ RemoteHost: ts.URL, RemoteUser: "foo", RemotePassword: "bar", } - assert.EqualError(suite.T(), sendPOST(syncJSON), "401 Unauthorized") + assert.EqualError(s.T(), sendPOST(syncJSON), "401 Unauthorized") } diff --git a/sda/cmd/syncapi/syncapi.go b/sda/cmd/syncapi/syncapi.go index 04a1bafd09..672df53f37 100644 --- a/sda/cmd/syncapi/syncapi.go +++ b/sda/cmd/syncapi/syncapi.go @@ -72,7 +72,7 @@ func main() { } } -func setup(config *config.Config) *http.Server { +func setup(conf *config.Config) *http.Server { r := mux.NewRouter().SkipClean(true) r.HandleFunc("/ready", readinessResponse).Methods("GET") @@ -82,7 +82,7 @@ func setup(config *config.Config) *http.Server { cfg := &tls.Config{MinVersion: tls.VersionTLS12} srv := &http.Server{ - Addr: config.API.Host + ":" + fmt.Sprint(config.API.Port), + Addr: conf.API.Host + ":" + fmt.Sprint(conf.API.Port), Handler: r, TLSConfig: cfg, TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), @@ -137,13 +137,16 @@ func dataset(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() if err := schema.ValidateJSON(fmt.Sprintf("%s/../bigpicture/file-sync.json", Conf.Broker.SchemasPath), b); err != nil { - respondWithError(w, http.StatusBadRequest, fmt.Sprintf("eror on JSON validation: %s", err.Error())) + respondWithError(w, http.StatusBadRequest, fmt.Sprintf("error on JSON validation: %s", err.Error())) return } if err := parseDatasetMessage(b); err != nil { - w.WriteHeader(http.StatusInternalServerError) + log.Errorf("error on parsing dataset message: %v", err) + respondWithError(w, http.StatusInternalServerError, "error while processing message") + + return } w.WriteHeader(http.StatusOK) @@ -210,7 +213,7 @@ func respondWithError(w http.ResponseWriter, code int, message string) { respondWithJSON(w, code, map[string]string{"error": message}) } -func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) { +func respondWithJSON(w http.ResponseWriter, code int, payload any) { log.Infoln(payload) response, _ := json.Marshal(payload) diff --git a/sda/cmd/syncapi/syncapi_test.go b/sda/cmd/syncapi/syncapi_test.go index 0a0012969a..da21dbab48 100644 --- a/sda/cmd/syncapi/syncapi_test.go +++ b/sda/cmd/syncapi/syncapi_test.go @@ -105,7 +105,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *SyncAPITest) SetupTest() { +func (s *SyncAPITest) SetupTest() { viper.Set("log.level", "debug") viper.Set("log.format", "json") @@ -125,43 +125,43 @@ func (suite *SyncAPITest) SetupTest() { viper.Set("sync.api.password", "admin") } -func (suite *SyncAPITest) TestSetup() { - suite.SetupTest() +func (s *SyncAPITest) TestSetup() { + s.SetupTest() conf, err := config.NewConfig("sync-api") - assert.NoError(suite.T(), err, "Failed to setup config") - assert.Equal(suite.T(), mqPort, conf.Broker.Port) - assert.Equal(suite.T(), mqPort, viper.GetInt("broker.port")) + assert.NoError(s.T(), err, "Failed to setup config") + assert.Equal(s.T(), mqPort, conf.Broker.Port) + assert.Equal(s.T(), mqPort, viper.GetInt("broker.port")) server := setup(conf) - assert.Equal(suite.T(), "0.0.0.0:8080", server.Addr) + assert.Equal(s.T(), "0.0.0.0:8080", server.Addr) } -func (suite *SyncAPITest) TestShutdown() { - suite.SetupTest() +func (s *SyncAPITest) TestShutdown() { + s.SetupTest() Conf, err = config.NewConfig("sync-api") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "127.0.0.1", Conf.API.MQ.Conf.Host) + assert.NoError(s.T(), err) + assert.Equal(s.T(), "127.0.0.1", Conf.API.MQ.Conf.Host) // make sure all conections are alive - assert.Equal(suite.T(), false, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), false, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), false, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), false, Conf.API.MQ.Connection.IsClosed()) shutdown() - assert.Equal(suite.T(), true, Conf.API.MQ.Channel.IsClosed()) - assert.Equal(suite.T(), true, Conf.API.MQ.Connection.IsClosed()) + assert.Equal(s.T(), true, Conf.API.MQ.Channel.IsClosed()) + assert.Equal(s.T(), true, Conf.API.MQ.Connection.IsClosed()) } -func (suite *SyncAPITest) TestReadinessResponse() { - suite.SetupTest() +func (s *SyncAPITest) TestReadinessResponse() { + s.SetupTest() Conf, err = config.NewConfig("sync-api") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) r := mux.NewRouter() r.HandleFunc("/ready", readinessResponse) @@ -169,44 +169,44 @@ func (suite *SyncAPITest) TestReadinessResponse() { defer ts.Close() res, err := http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the connection to force a reconneciton Conf.API.MQ.Connection.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() // close the channel to force a reconneciton Conf.API.MQ.Channel.Close() res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusServiceUnavailable, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusServiceUnavailable, res.StatusCode) defer res.Body.Close() // reconnect should be fast so now this should pass res, err = http.Get(ts.URL + "/ready") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, res.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, res.StatusCode) defer res.Body.Close() } -func (suite *SyncAPITest) TestDatasetRoute() { - suite.SetupTest() +func (s *SyncAPITest) TestDatasetRoute() { + s.SetupTest() Conf, err = config.NewConfig("sync-api") - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.API.MQ, err = broker.NewMQ(Conf.Broker) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) Conf.Broker.SchemasPath = "../../schemas/isolated/" @@ -217,18 +217,18 @@ func (suite *SyncAPITest) TestDatasetRoute() { goodJSON := []byte(`{"user": "test.user@example.com", "dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e6", "dataset_files": [{"filepath": "inbox/user/file-1.c4gh","file_id": "5fe7b660-afea-4c3a-88a9-3daabf055ebb", "sha256": "82E4e60e7beb3db2e06A00a079788F7d71f75b61a4b75f28c4c942703dabb6d6"}, {"filepath": "inbox/user/file2.c4gh","file_id": "ed6af454-d910-49e3-8cda-488a6f246e76", "sha256": "c967d96e56dec0f0cfee8f661846238b7f15771796ee1c345cae73cd812acc2b"}]}`) good, err := http.Post(ts.URL+"/dataset", "application/json", bytes.NewBuffer(goodJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, good.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, good.StatusCode) defer good.Body.Close() badJSON := []byte(`{"dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "dataset_files": []}`) bad, err := http.Post(ts.URL+"/dataset", "application/json", bytes.NewBuffer(badJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, bad.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, bad.StatusCode) defer bad.Body.Close() } -func (suite *SyncAPITest) TestMetadataRoute() { +func (s *SyncAPITest) TestMetadataRoute() { Conf = &config.Config{} Conf.Broker.SchemasPath = "../../schemas" @@ -239,18 +239,18 @@ func (suite *SyncAPITest) TestMetadataRoute() { goodJSON := []byte(`{"dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "metadata": {"dummy":"data"}}`) good, err := http.Post(ts.URL+"/metadata", "application/json", bytes.NewBuffer(goodJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, good.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, good.StatusCode) defer good.Body.Close() badJSON := []byte(`{"dataset_id": "phail", "metadata": {}}`) bad, err := http.Post(ts.URL+"/metadata", "application/json", bytes.NewBuffer(badJSON)) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusBadRequest, bad.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusBadRequest, bad.StatusCode) defer bad.Body.Close() } -func (suite *SyncAPITest) TestBasicAuth() { +func (s *SyncAPITest) TestBasicAuth() { Conf = &config.Config{} Conf.Broker.SchemasPath = "../../schemas" Conf.SyncAPI = config.SyncAPIConf{ @@ -265,16 +265,16 @@ func (suite *SyncAPITest) TestBasicAuth() { goodJSON := []byte(`{"dataset_id": "cd532362-e06e-4460-8490-b9ce64b8d9e7", "metadata": {"dummy":"data"}}`) req, err := http.NewRequest("POST", ts.URL+"/metadata", bytes.NewBuffer(goodJSON)) - assert.NoError(suite.T(), err) + assert.NoError(s.T(), err) req.SetBasicAuth(Conf.SyncAPI.APIUser, Conf.SyncAPI.APIPassword) good, err := ts.Client().Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusOK, good.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusOK, good.StatusCode) defer good.Body.Close() req.SetBasicAuth(Conf.SyncAPI.APIUser, "wrongpass") bad, err := ts.Client().Do(req) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), http.StatusUnauthorized, bad.StatusCode) + assert.NoError(s.T(), err) + assert.Equal(s.T(), http.StatusUnauthorized, bad.StatusCode) defer bad.Body.Close() } diff --git a/sda/cmd/verify/verify.go b/sda/cmd/verify/verify.go index e8a723d420..908a2546a6 100644 --- a/sda/cmd/verify/verify.go +++ b/sda/cmd/verify/verify.go @@ -4,7 +4,7 @@ package main import ( "bytes" - "crypto/md5" // #nosec + "crypto/md5" "crypto/sha256" "encoding/json" "fmt" @@ -137,7 +137,6 @@ func main() { log.Errorf("GetHeader failed for file with ID: %v, readon: %v", message.FileID, err.Error()) if err := delivered.Ack(false); err != nil { log.Errorf("Failed to nack following getheader error message") - } // store full message info in case we want to fix the db entry and retry infoErrorMessage := broker.InfoError{ @@ -157,7 +156,7 @@ func main() { } var file database.FileInfo - file.Size, err = archive.GetFileSize(message.ArchivePath) + file.Size, err = archive.GetFileSize(message.ArchivePath, false) if err != nil { //nolint:nestif log.Errorf("Failed to get archived file size, reson: (%s)", err.Error()) if strings.Contains(err.Error(), "no such file or directory") || strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { @@ -229,7 +228,7 @@ func main() { continue } - md5hash := md5.New() // #nosec + md5hash := md5.New() sha256hash := sha256.New() stream := io.TeeReader(c4ghr, md5hash) @@ -257,7 +256,7 @@ func main() { // At this point we should do checksum comparison - file.Checksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) + file.ArchiveChecksum = fmt.Sprintf("%x", archiveFileHash.Sum(nil)) file.DecryptedChecksum = fmt.Sprintf("%x", sha256hash.Sum(nil)) switch { @@ -289,8 +288,8 @@ func main() { continue } - if file.Checksum != message.EncryptedChecksums[0].Value { - log.Errorf("encrypted checksum don't match for file: %s, expected %s, got %s", message.FilePath, message.EncryptedChecksums[0].Value, file.Checksum) + if file.ArchiveChecksum != message.EncryptedChecksums[0].Value { + log.Errorf("encrypted checksum don't match for file: %s, expected %s, got %s", message.FilePath, message.EncryptedChecksums[0].Value, file.ArchiveChecksum) if err := db.UpdateFileEventLog(message.FileID, "error", delivered.CorrelationId, "verify", `{"error":"encrypted checksum don't match"}`, string(delivered.Body)); err != nil { log.Errorf("set status ready failed, reason: (%v)", err) if err := delivered.Nack(false, true); err != nil { @@ -345,46 +344,41 @@ func main() { continue } - switch status { - case "disabled": + + if status == "disabled" { log.Infof("file with correlation ID: %s is disabled, stopping verification", delivered.CorrelationId) if err := delivered.Ack(false); err != nil { log.Errorf("Failed acking canceled work, reason: (%s)", err.Error()) } continue - case "enabled": - fileInfo, err := db.GetFileInfo(message.FileID) - if err != nil { - log.Errorf("failed to get info for file: %s", message.FileID) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } + } - continue + fileInfo, err := db.GetFileInfo(message.FileID) + if err != nil { + log.Errorf("failed to get info for file: %s", message.FileID) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } - if fileInfo.DecryptedChecksum != "" { - log.Debugln("file already verified") - if err := mq.SendMessage(delivered.CorrelationId, conf.Broker.Exchange, conf.Broker.RoutingKey, verifiedMessage); err != nil { - log.Errorf("failed to publish message, reason: (%s)", err.Error()) - if err := delivered.Nack(false, true); err != nil { - log.Errorf("failed to Nack message, reason: (%s)", err.Error()) - } - - continue - } + continue + } - if err := delivered.Ack(false); err != nil { - log.Errorf("failed to Ack message, reason: (%s)", err.Error()) + if fileInfo.DecryptedChecksum != fmt.Sprintf("%x", sha256hash.Sum(nil)) { + if err := db.SetVerified(file, message.FileID); err != nil { + log.Errorf("SetVerified failed, reason: (%s)", err.Error()) + if err := delivered.Nack(false, true); err != nil { + log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } continue } + } else { + log.Infof("file with correlation id: %s is already verified", delivered.CorrelationId) } - if err := db.SetVerified(file, message.FileID, delivered.CorrelationId); err != nil { - log.Errorf("SetVerified failed, reason: (%s)", err.Error()) + if err := db.UpdateFileEventLog(message.FileID, "verified", delivered.CorrelationId, "ingest", "{}", string(verifiedMessage)); err != nil { + log.Errorf("failed to set event log status for file: %s", delivered.CorrelationId) if err := delivered.Nack(false, true); err != nil { log.Errorf("failed to Nack message, reason: (%s)", err.Error()) } diff --git a/sda/cmd/verify/verify_test.go b/sda/cmd/verify/verify_test.go index 0f8a2819d2..aea1d90847 100644 --- a/sda/cmd/verify/verify_test.go +++ b/sda/cmd/verify/verify_test.go @@ -15,6 +15,6 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(TestSuite)) } -func (suite *TestSuite) SetupTest() { +func (ts *TestSuite) SetupTest() { viper.Set("log.level", "debug") } diff --git a/sda/go.mod b/sda/go.mod index 9f3b06f12c..d41bb2cefc 100644 --- a/sda/go.mod +++ b/sda/go.mod @@ -1,40 +1,42 @@ module github.com/neicnordic/sensitive-data-archive -go 1.23.1 +go 1.24.0 + +toolchain go1.24.1 require ( - github.com/aws/aws-sdk-go-v2 v1.32.8 - github.com/aws/aws-sdk-go-v2/config v1.28.10 - github.com/aws/aws-sdk-go-v2/credentials v1.17.51 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 - github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 - github.com/aws/smithy-go v1.22.1 - github.com/casbin/casbin/v2 v2.103.0 - github.com/coreos/go-oidc/v3 v3.12.0 + github.com/aws/aws-sdk-go-v2 v1.36.3 + github.com/aws/aws-sdk-go-v2/config v1.29.14 + github.com/aws/aws-sdk-go-v2/credentials v1.17.67 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 + github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 + github.com/aws/smithy-go v1.22.3 + github.com/casbin/casbin/v2 v2.105.0 + github.com/coreos/go-oidc/v3 v3.14.1 github.com/gin-gonic/gin v1.10.0 github.com/google/uuid v1.6.0 github.com/gorilla/mux v1.8.1 github.com/iris-contrib/middleware/cors v0.0.0-20240502084239-34f27409ce72 github.com/kataras/iris/v12 v12.2.11 - github.com/lestrrat-go/jwx/v2 v2.1.3 + github.com/lestrrat-go/jwx/v2 v2.1.6 github.com/lib/pq v1.10.9 github.com/minio/minio-go/v6 v6.0.57 github.com/mocktools/go-smtp-mock v1.10.0 - github.com/neicnordic/crypt4gh v1.13.0 + github.com/neicnordic/crypt4gh v1.14.0 github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 github.com/ory/dockertest v3.3.5+incompatible - github.com/ory/dockertest/v3 v3.11.0 + github.com/ory/dockertest/v3 v3.12.0 github.com/pkg/errors v0.9.1 - github.com/pkg/sftp v1.13.7 + github.com/pkg/sftp v1.13.9 github.com/rabbitmq/amqp091-go v1.10.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/viper v1.19.0 + github.com/spf13/viper v1.20.1 github.com/stretchr/testify v1.10.0 - golang.org/x/crypto v0.32.0 - golang.org/x/oauth2 v0.25.0 - google.golang.org/grpc v1.69.4 - google.golang.org/protobuf v1.36.2 + golang.org/x/crypto v0.38.0 + golang.org/x/oauth2 v0.30.0 + google.golang.org/grpc v1.72.0 + google.golang.org/protobuf v1.36.6 ) require ( @@ -49,19 +51,19 @@ require ( github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/Shopify/goreferrer v0.0.0-20240724165105-aceaa0259138 // indirect github.com/andybalholm/brotli v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/bmatcuk/doublestar/v4 v4.6.1 // indirect github.com/bytedance/sonic v1.12.2 // indirect @@ -71,34 +73,33 @@ require ( github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect - github.com/containerd/continuity v0.4.3 // indirect + github.com/containerd/continuity v0.4.5 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect - github.com/docker/cli v27.2.1+incompatible // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/docker/cli v27.4.1+incompatible // indirect github.com/docker/docker v27.2.1+incompatible // indirect github.com/docker/go-connections v0.5.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/flosch/pongo2/v4 v4.0.2 // indirect - github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/gabriel-vasile/mimetype v1.4.5 // indirect github.com/gin-contrib/sse v0.1.0 // indirect - github.com/go-jose/go-jose/v3 v3.0.3 // indirect - github.com/go-jose/go-jose/v4 v4.0.4 // indirect + github.com/go-jose/go-jose/v3 v3.0.4 // indirect + github.com/go-jose/go-jose/v4 v4.0.5 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.22.1 // indirect - github.com/go-viper/mapstructure/v2 v2.1.0 // indirect + github.com/go-viper/mapstructure/v2 v2.2.1 // indirect github.com/goccy/go-json v0.10.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang-jwt/jwt/v5 v5.2.1 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/gomarkdown/markdown v0.0.0-20240730141124-034f12af3bf6 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/gorilla/css v1.0.1 // indirect github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect - github.com/hashicorp/hcl v1.0.0 // indirect github.com/iris-contrib/schema v0.0.6 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -111,37 +112,35 @@ require ( github.com/klauspost/cpuid/v2 v2.2.8 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect - github.com/lestrrat-go/blackmagic v1.0.2 // indirect + github.com/lestrrat-go/blackmagic v1.0.3 // indirect github.com/lestrrat-go/httpcc v1.0.1 // indirect github.com/lestrrat-go/httprc v1.0.6 // indirect github.com/lestrrat-go/iter v1.0.2 // indirect github.com/lestrrat-go/option v1.0.1 // indirect - github.com/magiconair/properties v1.8.7 // indirect github.com/mailgun/raymond/v2 v2.0.48 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microcosm-cc/bluemonday v1.0.27 // indirect github.com/minio/sha256-simd v1.0.1 // indirect - github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/sys/user v0.3.0 // indirect github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect - github.com/opencontainers/runc v1.1.14 // indirect + github.com/opencontainers/runc v1.2.3 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sagikazarmark/locafero v0.6.0 // indirect - github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/sagikazarmark/locafero v0.7.0 // indirect github.com/schollz/closestmatch v2.1.0+incompatible // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect - github.com/spf13/cast v1.7.0 // indirect - github.com/spf13/pflag v1.0.5 // indirect + github.com/spf13/afero v1.12.0 // indirect + github.com/spf13/cast v1.7.1 // indirect + github.com/spf13/pflag v1.0.6 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tdewolff/minify/v2 v2.20.37 // indirect github.com/tdewolff/parse/v2 v2.7.15 // indirect @@ -157,11 +156,11 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/arch v0.10.0 // indirect golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect - golang.org/x/net v0.30.0 // indirect - golang.org/x/sys v0.29.0 // indirect - golang.org/x/text v0.21.0 // indirect - golang.org/x/time v0.6.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 // indirect + golang.org/x/net v0.38.0 // indirect + golang.org/x/sys v0.33.0 // indirect + golang.org/x/text v0.25.0 // indirect + golang.org/x/time v0.8.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/sda/go.sum b/sda/go.sum index 3d5f5d7244..6933adbf9a 100644 --- a/sda/go.sum +++ b/sda/go.sum @@ -24,44 +24,44 @@ github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= -github.com/aws/aws-sdk-go-v2 v1.32.8 h1:cZV+NUS/eGxKXMtmyhtYPJ7Z4YLoI/V8bkTdRZfYhGo= -github.com/aws/aws-sdk-go-v2 v1.32.8/go.mod h1:P5WJBrYqqbWVaOxgH0X/FYYD47/nooaPOZPlQdmiN2U= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 h1:lL7IfaFzngfx0ZwUGOZdsFFnQ5uLvR0hWqqhyE7Q9M8= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7/go.mod h1:QraP0UcVlQJsmHfioCrveWOC1nbiWUl3ej08h4mXWoc= -github.com/aws/aws-sdk-go-v2/config v1.28.10 h1:fKODZHfqQu06pCzR69KJ3GuttraRJkhlC8g80RZ0Dfg= -github.com/aws/aws-sdk-go-v2/config v1.28.10/go.mod h1:PvdxRYZ5Um9QMq9PQ0zHHNdtKK+he2NHtFCUFMXWXeg= -github.com/aws/aws-sdk-go-v2/credentials v1.17.51 h1:F/9Sm6Y6k4LqDesZDPJCLxQGXNNHd/ZtJiWd0lCZKRk= -github.com/aws/aws-sdk-go-v2/credentials v1.17.51/go.mod h1:TKbzCHm43AoPyA+iLGGcruXd4AFhF8tOmLex2R9jWNQ= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23 h1:IBAoD/1d8A8/1aA8g4MBVtTRHhXRiNAgwdbo/xRM2DI= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.23/go.mod h1:vfENuCM7dofkgKpYzuzf1VT1UKkA/YL3qanfBn7HCaA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48 h1:XnXVe2zRyPf0+fAW5L05esmngvBpC6DQZK7oZB/z/Co= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.48/go.mod h1:S3wey90OrS4f7kYxH6PT175YyEcHTORY07++HurMaRM= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27 h1:jSJjSBzw8VDIbWv+mmvBSP8ezsztMYJGH+eKqi9AmNs= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.27/go.mod h1:/DAhLbFRgwhmvJdOfSm+WwikZrCuUJiA4WgJG0fTNSw= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27 h1:l+X4K77Dui85pIj5foXDhPlnqcNRG2QUyvca300lXh8= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.27/go.mod h1:KvZXSFEXm6x84yE8qffKvT3x8J5clWnVFXphpohhzJ8= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 h1:VaRN3TlFdd6KxX1x3ILT5ynH6HvKgqdiXoTxAF4HQcQ= -github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1/go.mod h1:FbtygfRFze9usAadmnGJNc8KsP346kEe+y2/oyhGAGc= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27 h1:AmB5QxnD+fBFrg9LcqzkgF/CaYvMyU/BTlejG4t1S7Q= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.27/go.mod h1:Sai7P3xTiyv9ZUYO3IFxMnmiIP759/67iQbU4kdmkyU= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 h1:iXtILhvDxB6kPvEXgsDhGaZCSC6LQET5ZHSdJozeI0Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1/go.mod h1:9nu0fVANtYiAePIBh2/pFUSwtJ402hLnp854CNoDOeE= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8 h1:iwYS40JnrBeA9e9aI5S6KKN4EB2zR4iUVYN0nwVivz4= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.8/go.mod h1:Fm9Mi+ApqmFiknZtGpohVcBGvpTu542VC4XO9YudRi0= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8 h1:cWno7lefSH6Pp+mSznagKCgfDGeZRin66UvYUqAkyeA= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.8/go.mod h1:tPD+VjU3ABTBoEJ3nctu5Nyg4P4yjqSH5bJGGkY4+XE= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8 h1:/Mn7gTedG86nbpjT4QEKsN1D/fThiYe1qvq7WsBGNHg= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.8/go.mod h1:Ae3va9LPmvjj231ukHB6UeT8nS7wTPfC3tMZSZMwNYg= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2 h1:a7aQ3RW+ug4IbhoQp29NZdc7vqrzKZZfWZSaQAXOZvQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.72.2/go.mod h1:xMekrnhmJ5aqmyxtmALs7mlvXw5xRh+eYjOjvrIIFJ4= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.9 h1:YqtxripbjWb2QLyzRK9pByfEDvgg95gpC2AyDq4hFE8= -github.com/aws/aws-sdk-go-v2/service/sso v1.24.9/go.mod h1:lV8iQpg6OLOfBnqbGMBKYjilBlf633qwHnBEiMSPoHY= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8 h1:6dBT1Lz8fK11m22R+AqfRsFn8320K0T5DTGxxOQBSMw= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.8/go.mod h1:/kiBvRQXBc6xeJTYzhSdGvJ5vm1tjaDEjH+MSeRJnlY= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.6 h1:VwhTrsTuVn52an4mXx29PqRzs2Dvu921NpGk7y43tAM= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.6/go.mod h1:+8h7PZb3yY5ftmVLD7ocEoE98hdc8PoKS0H3wfx1dlc= -github.com/aws/smithy-go v1.22.1 h1:/HPHZQ0g7f4eUeK6HKglFz8uwVfZKgoI25rb/J+dnro= -github.com/aws/smithy-go v1.22.1/go.mod h1:irrKGvNn1InZwb2d7fkIRNucdfwR8R+Ts3wxYa/cJHg= +github.com/aws/aws-sdk-go-v2 v1.36.3 h1:mJoei2CxPutQVxaATCzDUjcZEjVRdpsiiXi2o38yqWM= +github.com/aws/aws-sdk-go-v2 v1.36.3/go.mod h1:LLXuLpgzEbD766Z5ECcRmi8AzSwfZItDtmABVkRLGzg= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10 h1:zAybnyUQXIZ5mok5Jqwlf58/TFE7uvd3IAsa1aF9cXs= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.10/go.mod h1:qqvMj6gHLR/EXWZw4ZbqlPbQUyenf4h82UQUlKc+l14= +github.com/aws/aws-sdk-go-v2/config v1.29.14 h1:f+eEi/2cKCg9pqKBoAIwRGzVb70MRKqWX4dg1BDcSJM= +github.com/aws/aws-sdk-go-v2/config v1.29.14/go.mod h1:wVPHWcIFv3WO89w0rE10gzf17ZYy+UVS1Geq8Iei34g= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67 h1:9KxtdcIA/5xPNQyZRgUSpYOE6j9Bc4+D7nZua0KGYOM= +github.com/aws/aws-sdk-go-v2/credentials v1.17.67/go.mod h1:p3C44m+cfnbv763s52gCqrjaqyPikj9Sg47kUVaNZQQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75 h1:S61/E3N01oral6B3y9hZ2E1iFDqCZPPOBoBQretCnBI= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.75/go.mod h1:bDMQbkI1vJbNjnvJYpPTSNYBkI/VIv18ngWb/K84tkk= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34 h1:ZK5jHhnrioRkUNOc+hOgQKlUL5JeC3S6JgLxtQ+Rm0Q= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.34/go.mod h1:p4VfIceZokChbA9FzMbRGz5OV+lekcVtHlPKEO0gSZY= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34 h1:SZwFm17ZUNNg5Np0ioo/gq8Mn6u9w19Mri8DnJ15Jf0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.34/go.mod h1:dFZsC0BLo346mvKQLWmoJxT+Sjp+qcVR1tRVHQGOH9Q= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 h1:bIqFDwgGXXN1Kpp99pDOdKMTTb5d2KyU5X/BZxjOkRo= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3/go.mod h1:H5O/EsxDWyU+LP/V8i5sm8cxoZgc2fdNR9bxlOFrQTo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34 h1:ZNTqv4nIdE/DiBfUUfXcLZ/Spcuz+RjeziUtNJackkM= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.34/go.mod h1:zf7Vcd1ViW7cPqYWEHLHJkS50X0JS2IKz9Cgaj6ugrs= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3 h1:eAh2A4b5IzM/lum78bZ590jy36+d/aFLgKF/4Vd1xPE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.3/go.mod h1:0yKJC/kb8sAnmlYa6Zs3QVYqaC8ug2AbnNChv5Ox3uA= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1 h1:4nm2G6A4pV9rdlWzGMPv4BNtQp22v1hg3yrtkYpeLl8= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.7.1/go.mod h1:iu6FSzgt+M2/x3Dk8zhycdIcHjEFb36IS8HVUVFoMg0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15 h1:dM9/92u2F1JbDaGooxTq18wmmFzbJRfXfVfy96/1CXM= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.15/go.mod h1:SwFBy2vjtA0vZbjjaFtfN045boopadnoVPhu4Fv66vY= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15 h1:moLQUoVq91LiqT1nbvzDukyqAlCv89ZmwaHw/ZFlFZg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.15/go.mod h1:ZH34PJUc8ApjBIfgQCFvkWcUDBtl/WTD+uiYHjd8igA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3 h1:BRXS0U76Z8wfF+bnkilA2QwpIch6URlm++yPUt9QPmQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.79.3/go.mod h1:bNXKFFyaiVvWuR6O16h/I1724+aXe/tAkA9/QS01t5k= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19 h1:1XuUZ8mYJw9B6lzAkXhqHlJd/XvaX32evhproijJEZY= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.19/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/smithy-go v1.22.3 h1:Z//5NuZCSW6R4PhQ93hShNbyBbn8BWCmCVCt+Q8Io5k= +github.com/aws/smithy-go v1.22.3/go.mod h1:t1ufH5HMublsJYulve2RKmHDC15xu1f26kHCp/HgceI= github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk= github.com/aymerick/douceur v0.2.0/go.mod h1:wlT5vV2O3h55X9m7iVYN0TBM0NH/MmbLnd30/FjWUq4= github.com/bmatcuk/doublestar/v4 v4.6.1 h1:FH9SifrbvJhnlQpztAx++wlkk70QBf0iBWDwNy7PA4I= @@ -71,8 +71,8 @@ github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKz github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/casbin/casbin/v2 v2.103.0 h1:dHElatNXNrr8XcseUov0ZSiWjauwmZZE6YMV3eU1yic= -github.com/casbin/casbin/v2 v2.103.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= +github.com/casbin/casbin/v2 v2.105.0 h1:dLj5P6pLApBRat9SADGiLxLZjiDPvA1bsPkyV4PGx6I= +github.com/casbin/casbin/v2 v2.105.0/go.mod h1:Ee33aqGrmES+GNL17L0h9X28wXuo829wnNUnS0edAco= github.com/casbin/govaluate v1.3.0 h1:VA0eSY0M2lA86dYd5kPPuNZMUD9QkWnOCnavGrw9myc= github.com/casbin/govaluate v1.3.0/go.mod h1:G/UnbIjZk/0uMNaLwZZmFQrR72tYRZWQkO70si/iR7A= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -83,10 +83,10 @@ github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/ github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= -github.com/containerd/continuity v0.4.3 h1:6HVkalIp+2u1ZLH1J/pYX2oBVXlJZvh1X1A7bEZ9Su8= -github.com/containerd/continuity v0.4.3/go.mod h1:F6PTNCKepoxEaXLQp3wDAjygEnImnZ/7o4JzpodfroQ= -github.com/coreos/go-oidc/v3 v3.12.0 h1:sJk+8G2qq94rDI6ehZ71Bol3oUHy63qNYmkiSjrc/Jo= -github.com/coreos/go-oidc/v3 v3.12.0/go.mod h1:gE3LgjOgFoHi9a4ce4/tJczr0Ai2/BoDhf0r5lltWI0= +github.com/containerd/continuity v0.4.5 h1:ZRoN1sXq9u7V6QoHMcVWGhOwDFqZ4B9i5H6un1Wh0x4= +github.com/containerd/continuity v0.4.5/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= +github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= +github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -95,10 +95,10 @@ github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1 github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a h1:saTgr5tMLFnmy/yg3qDTft4rE5DY2uJ/cCxCe3q0XTU= github.com/dchest/bcrypt_pbkdf v0.0.0-20150205184540-83f37f9c154a/go.mod h1:Bw9BbhOJVNR+t0jCqx2GC6zv0TGBsShs56Y3gfSCvl0= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 h1:rpfIENRNNilwHwZeG5+P150SMrnNEcHYvcCuK6dPZSg= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= -github.com/docker/cli v27.2.1+incompatible h1:U5BPtiD0viUzjGAjV1p0MGB8eVA3L3cbIrnyWmSJI70= -github.com/docker/cli v27.2.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI= +github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/docker v27.2.1+incompatible h1:fQdiLfW7VLscyoeYEBz7/J8soYFDZV1u6VW6gJEjNMI= github.com/docker/docker v27.2.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= @@ -114,18 +114,18 @@ github.com/flosch/pongo2/v4 v4.0.2 h1:gv+5Pe3vaSVmiJvh/BZa82b7/00YUGm0PIyVVLop0H github.com/flosch/pongo2/v4 v4.0.2/go.mod h1:B5ObFANs/36VwxxlgKpdchIJHMvHB562PW+BWPhwZD8= github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= +github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/gabriel-vasile/mimetype v1.4.5 h1:J7wGKdGu33ocBOhGy0z653k/lFKLFDPJMG8Gql0kxn4= github.com/gabriel-vasile/mimetype v1.4.5/go.mod h1:ibHel+/kbxn9x2407k1izTA1S81ku1z/DlgOW2QE0M4= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k= -github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= -github.com/go-jose/go-jose/v4 v4.0.4 h1:VsjPI33J0SB9vQM6PLmNjoHqMQNGPiZ0rHL7Ni7Q6/E= -github.com/go-jose/go-jose/v4 v4.0.4/go.mod h1:NKb5HO1EZccyMpiZNbdUw/14tiXNyUJh188dfnMCAfc= +github.com/go-jose/go-jose/v3 v3.0.4 h1:Wp5HA7bLQcKnf6YYao/4kpRpVMp/yf6+pJKV8WFSaNY= +github.com/go-jose/go-jose/v3 v3.0.4/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= +github.com/go-jose/go-jose/v4 v4.0.5 h1:M6T8+mKZl/+fNNuFHvGIzDz7BTLQPIounk/b9dw3AaE= +github.com/go-jose/go-jose/v4 v4.0.5/go.mod h1:s3P1lRrkT8igV8D9OjyL4WRyHvjB6a4JSllnOrmmBOA= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -140,16 +140,16 @@ github.com/go-playground/validator/v10 v10.22.1 h1:40JcKH+bBNGFczGuoBYgX4I6m/i27 github.com/go-playground/validator/v10 v10.22.1/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-viper/mapstructure/v2 v2.1.0 h1:gHnMa2Y/pIxElCH2GlZZ1lZSsn6XMtufpGyP1XxdC/w= -github.com/go-viper/mapstructure/v2 v2.1.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= +github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA= github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= @@ -177,8 +177,6 @@ github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/ github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= -github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/imkira/go-interpol v1.1.0 h1:KIiKr0VSG2CUW1hl1jpiyuzuJeKUUpC8iM1AIE7N1Vk= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/iris-contrib/httpexpect/v2 v2.15.2 h1:T9THsdP1woyAqKHwjkEsbCnMefsAFvk8iJJKokcJ3Go= @@ -226,22 +224,20 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= -github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= -github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= +github.com/lestrrat-go/blackmagic v1.0.3 h1:94HXkVLxkZO9vJI/w2u1T0DAoprShFd13xtnSINtDWs= +github.com/lestrrat-go/blackmagic v1.0.3/go.mod h1:6AWFyKNNj0zEXQYfTMPfZrAXUWUfTIZ5ECEUEJaijtw= github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= -github.com/lestrrat-go/jwx/v2 v2.1.3 h1:Ud4lb2QuxRClYAmRleF50KrbKIoM1TddXgBrneT5/Jo= -github.com/lestrrat-go/jwx/v2 v2.1.3/go.mod h1:q6uFgbgZfEmQrfJfrCo90QcQOcXFMfbI/fO0NqRtvZo= +github.com/lestrrat-go/jwx/v2 v2.1.6 h1:hxM1gfDILk/l5ylers6BX/Eq1m/pnxe9NBwW6lVfecA= +github.com/lestrrat-go/jwx/v2 v2.1.6/go.mod h1:Y722kU5r/8mV7fYDifjug0r8FK8mZdw0K0GpJw/l8pU= github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= -github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailgun/raymond/v2 v2.0.48 h1:5dmlB680ZkFG2RN/0lvTAghrSxIESeu9/2aeDqACtjw= github.com/mailgun/raymond/v2 v2.0.48/go.mod h1:lsgvL50kgt1ylcFJYZiULi5fjPBkkhNfj4KA0W54Z18= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= @@ -261,10 +257,10 @@ github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/sys/user v0.3.0 h1:9ni5DlcW5an3SvRSx4MouotOygvzaXbaSrc/wGDFWPo= +github.com/moby/sys/user v0.3.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/mocktools/go-smtp-mock v1.10.0 h1:glrRmjNqASyy+jf1IJ2nCWgEbJScD3Amf2IGcXgdEVg= @@ -275,8 +271,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/neicnordic/crypt4gh v1.13.0 h1:NbSAPx1+zFpG6a8GCVwW80y/TGHfGdXJF/zqQKqlHZ8= -github.com/neicnordic/crypt4gh v1.13.0/go.mod h1:lfNIrhlcQrSf5awgCaW+poCsRBlvKOrNjR3CBvXU5Ek= +github.com/neicnordic/crypt4gh v1.14.0 h1:81gNvxD+mWAaPwFDRItYHoVFnuZ66pLayqNjDPrkA3U= +github.com/neicnordic/crypt4gh v1.14.0/go.mod h1:JxD2EAllRLlD91mNTVKa7RQvcbZZEtobI2gVgoUTRYI= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25 h1:9bCMuD3TcnjeqjPT2gSlha4asp8NvgcFRYExCaikCxk= github.com/oauth2-proxy/mockoidc v0.0.0-20240214162133-caebfff84d25/go.mod h1:eDjgYHYDJbPLBLsyZ6qRaugP0mX8vePOhZ5id1fdzJw= @@ -284,18 +280,18 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/opencontainers/runc v1.1.14 h1:rgSuzbmgz5DUJjeSnw337TxDbRuqjs6iqQck/2weR6w= -github.com/opencontainers/runc v1.1.14/go.mod h1:E4C2z+7BxR7GHXp0hAY53mek+x49X1LjPNeMTfRGvOA= +github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80= +github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= -github.com/ory/dockertest/v3 v3.11.0 h1:OiHcxKAvSDUwsEVh2BjxQQc/5EHz9n0va9awCtNGuyA= -github.com/ory/dockertest/v3 v3.11.0/go.mod h1:VIPxS1gwT9NpPOrfD3rACs8Y9Z7yhzO4SB194iUDnUI= +github.com/ory/dockertest/v3 v3.12.0 h1:3oV9d0sDzlSQfHtIaB5k6ghUCVMVLpAY8hwrqoCyRCw= +github.com/ory/dockertest/v3 v3.12.0/go.mod h1:aKNDTva3cp8dwOWwb9cWuX84aH5akkxXRvO7KCwWVjE= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/sftp v1.13.7 h1:uv+I3nNJvlKZIQGSr8JVQLNHFU9YhhNpvC14Y6KgmSM= -github.com/pkg/sftp v1.13.7/go.mod h1:KMKI0t3T6hfA+lTR/ssZdunHo+uwq7ghoN09/FSu3DY= +github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= +github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -305,10 +301,8 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= -github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= -github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= -github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo= +github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k= github.com/sanity-io/litter v1.5.5 h1:iE+sBxPBzoK6uaEP5Lt3fHNgpKcHXc/A2HGETy0uJQo= github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF7bU2UI5U= github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6NgVqpn3+iol9aGu4= @@ -327,14 +321,14 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= -github.com/spf13/cast v1.7.0 h1:ntdiHjuueXFgm5nzDRdOS4yfT43P5Fnud6DH50rz/7w= -github.com/spf13/cast v1.7.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= -github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= -github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= +github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y= +github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= +github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.20.1 h1:ZMi+z/lvLyPSCoNtFCpqjy0S4kPbirhpTMwl8BkW9X4= +github.com/spf13/viper v1.20.1/go.mod h1:P9Mdzt1zoHIG8m2eZQinpiBjo6kCmZSKBClNNqjJvu4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -387,16 +381,18 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= -go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= -go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= -go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= -go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= -go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= -go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= -go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= -go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= -go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= +go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= +go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= +go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= +go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= +go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= +go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= +go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= +go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= +go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -408,10 +404,12 @@ golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= -golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= -golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= +golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 h1:e66Fs6Z+fZTbFBAxKfP3PALWBtpfqks2bwGcexMxgtk= golang.org/x/exp v0.0.0-20240909161429-701f63a606c0/go.mod h1:2TbTHSBQa924w8M6Xs1QcRcFwyucIwBGpK1p2f1YFFY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -419,6 +417,9 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -431,16 +432,23 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= -golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= -golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= -golang.org/x/oauth2 v0.25.0 h1:CY4y7XT9v0cRI9oupztF8AgiIu99L/ksR/Xp/6jrZ70= -golang.org/x/oauth2 v0.25.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= +golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/oauth2 v0.30.0 h1:dnDm7JmhM45NNpd8FDDeLhK6FwqbOf4MLCM9zb1BOHI= +golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKlU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -457,29 +465,37 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= -golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw= +golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= -golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= -golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.32.0 h1:DR4lr0TjUs3epypdhTOkMmuF5CDFJ/8pOnbzMZPQ7bg= +golang.org/x/term v0.32.0/go.mod h1:uZG1FhGx848Sqfsq4/DlJr3xGGsYMu/L5GW4abiaEPQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4= +golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA= +golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg= +golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= @@ -489,16 +505,18 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53 h1:X58yt85/IXCx0Y3ZwN6sEIKZzQtDEYaBWrDvErdXrRE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20241015192408-796eee8c2d53/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI= -google.golang.org/grpc v1.69.4 h1:MF5TftSMkd8GLw/m0KM6V8CMOCY6NZ1NQDPGFgbTt4A= -google.golang.org/grpc v1.69.4/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= -google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= -google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a h1:51aaUVRocpvUOSQKM6Q7VuoaktNIaMCLuhZB6DKksq4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250218202821-56aae31c358a/go.mod h1:uRxBH1mhmO8PGhU89cMcHaXKZqO+OfakD8QQO0oYwlQ= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/sda/internal/broker/broker.go b/sda/internal/broker/broker.go index ff7a9b87f0..48a9c094e4 100644 --- a/sda/internal/broker/broker.go +++ b/sda/internal/broker/broker.go @@ -44,9 +44,9 @@ type MQConf struct { // The empty interface allows for appending various json msgs but also broken json msgs as strings. // It is ok as long as we do not need to access fields in the msg, which we don't. type InfoError struct { - Error string `json:"error"` - Reason string `json:"reason"` - OriginalMessage interface{} `json:"original-message"` + Error string `json:"error"` + Reason string `json:"reason"` + OriginalMessage any `json:"original-message"` } // buildMQURI builds the MQ connection URI diff --git a/sda/internal/broker/broker_test.go b/sda/internal/broker/broker_test.go index 37ffe08411..99c61420d9 100644 --- a/sda/internal/broker/broker_test.go +++ b/sda/internal/broker/broker_test.go @@ -107,7 +107,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *BrokerTestSuite) SetupTest() { +func (ts *BrokerTestSuite) SetupTest() { tMqconf = MQConf{ "127.0.0.1", mqPort, @@ -132,14 +132,14 @@ func TestBrokerTestSuite(t *testing.T) { suite.Run(t, new(BrokerTestSuite)) } -func (suite *BrokerTestSuite) TestBuildMqURI() { - amqps := buildMQURI("localhost", "user", "pass", "/vhost", 5555, true) - assert.Equal(suite.T(), "amqps://user:pass@localhost:5555/vhost", amqps) - amqp := buildMQURI("localhost", "user", "pass", "/vhost", 5555, false) - assert.Equal(suite.T(), "amqp://user:pass@localhost:5555/vhost", amqp) +func (ts *BrokerTestSuite) TestBuildMqURI() { + amqpsURL := buildMQURI("localhost", "user", "pass", "/vhost", 5555, true) + assert.Equal(ts.T(), "amqps://user:pass@localhost:5555/vhost", amqpsURL) + amqpURL := buildMQURI("localhost", "user", "pass", "/vhost", 5555, false) + assert.Equal(ts.T(), "amqp://user:pass@localhost:5555/vhost", amqpURL) } -func (suite *BrokerTestSuite) TestTLSConfigBroker() { +func (ts *BrokerTestSuite) TestTLSConfigBroker() { confOK := tMqconf confOK.Ssl = true confOK.VerifyPeer = true @@ -148,15 +148,15 @@ func (suite *BrokerTestSuite) TestTLSConfigBroker() { confOK.ClientKey = certPath + "/tls.key" tlsConfig, err := TLSConfigBroker(confOK) - assert.NoError(suite.T(), err, "Unexpected error") - assert.NotZero(suite.T(), tlsConfig.Certificates, "Expected warnings were missing") - assert.NotZero(suite.T(), tlsConfig.RootCAs, "Expected warnings were missing") - assert.EqualValues(suite.T(), tlsConfig.ServerName, "mq") + assert.NoError(ts.T(), err, "Unexpected error") + assert.NotZero(ts.T(), tlsConfig.Certificates, "Expected warnings were missing") + assert.NotZero(ts.T(), tlsConfig.RootCAs, "Expected warnings were missing") + assert.EqualValues(ts.T(), tlsConfig.ServerName, "mq") noCa := confOK noCa.CACert = "" _, err = TLSConfigBroker(noCa) - assert.NoError(suite.T(), err, "Unexpected error") + assert.NoError(ts.T(), err, "Unexpected error") var buf bytes.Buffer log.SetOutput(&buf) @@ -165,23 +165,23 @@ func (suite *BrokerTestSuite) TestTLSConfigBroker() { }() noCa.CACert = certPath + "/tls.key" _, err = TLSConfigBroker(noCa) - assert.NoError(suite.T(), err, "Unexpected error") - assert.Contains(suite.T(), buf.String(), "No certs appended, using system certs only") + assert.NoError(ts.T(), err, "Unexpected error") + assert.Contains(ts.T(), buf.String(), "No certs appended, using system certs only") badCertConf := confOK badCertConf.ClientCert = certPath + "/bar" _, err = CatchTLSConfigBrokerPanic(badCertConf) - assert.EqualError(suite.T(), err, "open "+certPath+"/bar: no such file or directory") + assert.EqualError(ts.T(), err, "open "+certPath+"/bar: no such file or directory") badKeyConf := confOK badKeyConf.ClientKey = certPath + "/foo" _, err = CatchTLSConfigBrokerPanic(badKeyConf) - assert.EqualError(suite.T(), err, "open "+certPath+"/foo: no such file or directory") + assert.EqualError(ts.T(), err, "open "+certPath+"/foo: no such file or directory") noPemFile := confOK noPemFile.ClientKey = "broker.go" _, err = CatchTLSConfigBrokerPanic(noPemFile) - assert.EqualError(suite.T(), err, "tls: failed to find any PEM data in key input") + assert.EqualError(ts.T(), err, "tls: failed to find any PEM data in key input") } func CatchTLSConfigBrokerPanic(c MQConf) (cfg *tls.Config, err error) { @@ -196,49 +196,49 @@ func CatchTLSConfigBrokerPanic(c MQConf) (cfg *tls.Config, err error) { return cfg, err } -func (suite *BrokerTestSuite) TestNewMQNoTLS() { +func (ts *BrokerTestSuite) TestNewMQNoTLS() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) b.Channel.Close() b.Connection.Close() } -func (suite *BrokerTestSuite) TestNewMQTLS() { - SslConf := tMqconf - SslConf.Port = tlsPort - SslConf.Ssl = true - SslConf.VerifyPeer = true +func (ts *BrokerTestSuite) TestNewMQTLS() { + sslConf := tMqconf + sslConf.Port = tlsPort + sslConf.Ssl = true + sslConf.VerifyPeer = true - b, err := NewMQ(SslConf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + b, err := NewMQ(sslConf) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) b.Channel.Close() b.Connection.Close() } -func (suite *BrokerTestSuite) TestSendMessage() { +func (ts *BrokerTestSuite) TestSendMessage() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) err = b.SendMessage("1", "", "ingest", []byte("test message")) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) b.Channel.Close() b.Connection.Close() } -func (suite *BrokerTestSuite) TestGetMessages() { +func (ts *BrokerTestSuite) TestGetMessages() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), b, "NewMQ without ssl did not return a broker") - assert.False(suite.T(), b.Connection.IsClosed()) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), b, "NewMQ without ssl did not return a broker") + assert.False(ts.T(), b.Connection.IsClosed()) ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -259,15 +259,15 @@ func (suite *BrokerTestSuite) TestGetMessages() { Body: []byte("test message"), }, ) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) d, err := b.GetMessages("ingest") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) for message := range d { if string(message.Body) == "test message" { err := message.Ack(false) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) break } @@ -277,17 +277,16 @@ func (suite *BrokerTestSuite) TestGetMessages() { b.Connection.Close() } -func (suite *BrokerTestSuite) TestCreateNewChannel() { +func (ts *BrokerTestSuite) TestCreateNewChannel() { b, err := NewMQ(tMqconf) - assert.NoError(suite.T(), err) - assert.False(suite.T(), b.Channel.IsClosed()) + assert.NoError(ts.T(), err) + assert.False(ts.T(), b.Channel.IsClosed()) b.Channel.Close() - assert.True(suite.T(), b.Channel.IsClosed()) - - assert.NoError(suite.T(), b.CreateNewChannel()) - assert.False(suite.T(), b.Channel.IsClosed()) + assert.True(ts.T(), b.Channel.IsClosed()) + assert.NoError(ts.T(), b.CreateNewChannel()) + assert.False(ts.T(), b.Channel.IsClosed()) } // Helper functions below this line diff --git a/sda/internal/config/config.go b/sda/internal/config/config.go index 0320778f81..27de641f69 100644 --- a/sda/internal/config/config.go +++ b/sda/internal/config/config.go @@ -180,13 +180,12 @@ func NewConfig(app string) (*Config, error) { log.Infoln("reading config") if err := viper.ReadInConfig(); err != nil { log.Infoln(err.Error()) - if _, ok := err.(viper.ConfigFileNotFoundError); ok { - log.Infoln("No config file found, using ENVs only") - } else { + if _, ok := err.(viper.ConfigFileNotFoundError); !ok { log.Infoln("ReadInConfig Error") return nil, err } + log.Infoln("No config file found, using ENVs only") } if viper.IsSet("log.format") { @@ -227,7 +226,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) default: - return nil, fmt.Errorf("inbox.type not set") + return nil, errors.New("inbox.type not set") } case "auth": requiredConfVars = []string{ @@ -273,7 +272,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) default: - return nil, fmt.Errorf("archive.type not set") + return nil, errors.New("archive.type not set") } switch viper.GetString("inbox.type") { @@ -282,7 +281,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"inbox.location"}...) default: - return nil, fmt.Errorf("inbox.type not set") + return nil, errors.New("inbox.type not set") } case "finalize": requiredConfVars = []string{ @@ -408,7 +407,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) default: - return nil, fmt.Errorf("archive.type not set") + return nil, errors.New("archive.type not set") } switch viper.GetString("sync.destination.type") { @@ -419,7 +418,7 @@ func NewConfig(app string) (*Config, error) { case SFTP: requiredConfVars = append(requiredConfVars, []string{"sync.destination.sftp.host", "sync.destination.sftp.port", "sync.destination.sftp.userName", "sync.destination.sftp.pemKeyPath", "sync.destination.sftp.pemKeyPass"}...) default: - return nil, fmt.Errorf("sync.destination.type not set") + return nil, errors.New("sync.destination.type not set") } case "sync-api": requiredConfVars = []string{ @@ -452,7 +451,7 @@ func NewConfig(app string) (*Config, error) { case POSIX: requiredConfVars = append(requiredConfVars, []string{"archive.location"}...) default: - return nil, fmt.Errorf("archive.type not set") + return nil, errors.New("archive.type not set") } default: @@ -508,7 +507,7 @@ func NewConfig(app string) (*Config, error) { } if (c.Auth.OIDC.ID == "" || c.Auth.OIDC.Secret == "") && (c.Auth.Cega.ID == "" || c.Auth.Cega.Secret == "") { - return nil, fmt.Errorf("neither cega or oidc login configured") + return nil, errors.New("neither cega or oidc login configured") } c.Auth.InfoURL = viper.GetString("auth.infoUrl") @@ -744,62 +743,62 @@ func (c *Config) configBackup() { // configBroker provides configuration for the message broker func (c *Config) configBroker() error { // Setup broker - broker := broker.MQConf{} + mq := broker.MQConf{} - broker.Host = viper.GetString("broker.host") - broker.Port = viper.GetInt("broker.port") - broker.User = viper.GetString("broker.user") - broker.Password = viper.GetString("broker.password") + mq.Host = viper.GetString("broker.host") + mq.Port = viper.GetInt("broker.port") + mq.User = viper.GetString("broker.user") + mq.Password = viper.GetString("broker.password") - broker.Queue = viper.GetString("broker.queue") + mq.Queue = viper.GetString("broker.queue") if viper.IsSet("broker.serverName") { - broker.ServerName = viper.GetString("broker.serverName") + mq.ServerName = viper.GetString("broker.serverName") } if viper.IsSet("broker.routingkey") { - broker.RoutingKey = viper.GetString("broker.routingkey") + mq.RoutingKey = viper.GetString("broker.routingkey") } if viper.IsSet("broker.exchange") { - broker.Exchange = viper.GetString("broker.exchange") + mq.Exchange = viper.GetString("broker.exchange") } if viper.IsSet("broker.vhost") { if strings.HasPrefix(viper.GetString("broker.vhost"), "/") { - broker.Vhost = viper.GetString("broker.vhost") + mq.Vhost = viper.GetString("broker.vhost") } else { - broker.Vhost = "/" + viper.GetString("broker.vhost") + mq.Vhost = "/" + viper.GetString("broker.vhost") } } else { - broker.Vhost = "/" + mq.Vhost = "/" } if viper.IsSet("broker.ssl") { - broker.Ssl = viper.GetBool("broker.ssl") + mq.Ssl = viper.GetBool("broker.ssl") } if viper.IsSet("broker.verifyPeer") { - broker.VerifyPeer = viper.GetBool("broker.verifyPeer") - if broker.VerifyPeer { + mq.VerifyPeer = viper.GetBool("broker.verifyPeer") + if mq.VerifyPeer { // Since verifyPeer is specified, these are required. - if !(viper.IsSet("broker.clientCert") && viper.IsSet("broker.clientKey")) { + if !viper.IsSet("broker.clientCert") && !viper.IsSet("broker.clientKey") { return errors.New("when broker.verifyPeer is set both broker.clientCert and broker.clientKey is needed") } - broker.ClientCert = viper.GetString("broker.clientCert") - broker.ClientKey = viper.GetString("broker.clientKey") + mq.ClientCert = viper.GetString("broker.clientCert") + mq.ClientKey = viper.GetString("broker.clientKey") } } if viper.IsSet("broker.cacert") { - broker.CACert = viper.GetString("broker.cacert") + mq.CACert = viper.GetString("broker.cacert") } - broker.PrefetchCount = 2 + mq.PrefetchCount = 2 if viper.IsSet("broker.prefetchCount") { - broker.PrefetchCount = viper.GetInt("broker.prefetchCount") + mq.PrefetchCount = viper.GetInt("broker.prefetchCount") } - c.Broker = broker + c.Broker = mq return nil } @@ -819,7 +818,7 @@ func (c *Config) configDatabase() error { // Optional settings if db.SslMode == "verify-full" { // Since verify-full is specified, these are required. - if !(viper.IsSet("db.clientCert") && viper.IsSet("db.clientKey")) { + if !viper.IsSet("db.clientCert") && !viper.IsSet("db.clientKey") { return errors.New("when db.sslMode is set to verify-full both db.clientCert and db.clientKey are needed") } } @@ -1047,7 +1046,6 @@ func (c *Config) configSyncAPI() { if viper.IsSet("sync.api.MappingRouting") { c.SyncAPI.MappingRouting = viper.GetString("sync.api.MappingRouting") } - } // GetC4GHKey reads and decrypts and returns the c4gh key @@ -1121,7 +1119,7 @@ func GetC4GHPublicKey() (*[32]byte, error) { func (c *Config) configServer() error { s := ServerConfig{} - if !(viper.IsSet("server.jwtpubkeypath") || viper.IsSet("server.jwtpubkeyurl")) { + if !viper.IsSet("server.jwtpubkeypath") && !viper.IsSet("server.jwtpubkeyurl") { return errors.New("either server.pubkeypath or server.jwtpubkeyurl should be present to start the service") } diff --git a/sda/internal/config/config_test.go b/sda/internal/config/config_test.go index ea3d7a32e6..4c33844ff2 100644 --- a/sda/internal/config/config_test.go +++ b/sda/internal/config/config_test.go @@ -24,7 +24,7 @@ type ConfigTestSuite struct { var certPath, rootDir string -func (suite *ConfigTestSuite) SetupTest() { +func (ts *ConfigTestSuite) SetupTest() { _, b, _, _ := runtime.Caller(0) rootDir = path.Join(path.Dir(b), "../../../") // pwd, _ = os.Getwd() @@ -32,7 +32,7 @@ func (suite *ConfigTestSuite) SetupTest() { helper.MakeCerts(certPath) rbacFile, err := os.CreateTemp(certPath, "admins") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) rbac := []byte(`{"policy":[ {"role":"admin","path":"/c4gh-keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, @@ -43,7 +43,7 @@ func (suite *ConfigTestSuite) SetupTest() { {"role":"dummy@example.org","rolebinding":"admin"}, {"role":"foo@example.org","rolebinding":"submission"}]}`) _, err = rbacFile.Write(rbac) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("api.rbacFile", rbacFile.Name()) viper.Set("broker.host", "testhost") @@ -68,7 +68,7 @@ func (suite *ConfigTestSuite) SetupTest() { viper.Set("log.level", "debug") } -func (suite *ConfigTestSuite) TearDownTest() { +func (ts *ConfigTestSuite) TearDownTest() { viper.Reset() defer os.RemoveAll(certPath) } @@ -77,172 +77,172 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(ConfigTestSuite)) } -func (suite *ConfigTestSuite) TestNonExistingApplication() { +func (ts *ConfigTestSuite) TestNonExistingApplication() { expectedError := errors.New("application 'test' doesn't exist") config, err := NewConfig("test") - assert.Nil(suite.T(), config) - if assert.Error(suite.T(), err) { - assert.Equal(suite.T(), expectedError, err) + assert.Nil(ts.T(), config) + if assert.Error(ts.T(), err) { + assert.Equal(ts.T(), expectedError, err) } } -func (suite *ConfigTestSuite) TestConfigFile() { +func (ts *ConfigTestSuite) TestConfigFile() { viper.Set("configFile", rootDir+"/.github/integration/sda/config.yaml") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/sda/config.yaml") - assert.Equal(suite.T(), absPath, viper.ConfigFileUsed()) + assert.Equal(ts.T(), absPath, viper.ConfigFileUsed()) } -func (suite *ConfigTestSuite) TestWrongConfigFile() { +func (ts *ConfigTestSuite) TestWrongConfigFile() { viper.Set("configFile", rootDir+"/.github/integration/rabbitmq/cega.conf") config, err := NewConfig("s3inbox") - assert.Nil(suite.T(), config) - assert.Error(suite.T(), err) + assert.Nil(ts.T(), config) + assert.Error(ts.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/rabbitmq/cega.conf") - assert.Equal(suite.T(), absPath, viper.ConfigFileUsed()) + assert.Equal(ts.T(), absPath, viper.ConfigFileUsed()) } -func (suite *ConfigTestSuite) TestConfigPath() { +func (ts *ConfigTestSuite) TestConfigPath() { viper.Reset() viper.Set("configPath", rootDir+"/.github/integration/sda/") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) absPath, _ := filepath.Abs(rootDir + "/.github/integration/sda/config.yaml") - assert.Equal(suite.T(), absPath, viper.ConfigFileUsed()) + assert.Equal(ts.T(), absPath, viper.ConfigFileUsed()) } -func (suite *ConfigTestSuite) TestNoConfig() { +func (ts *ConfigTestSuite) TestNoConfig() { viper.Reset() config, err := NewConfig("s3inbox") - assert.Nil(suite.T(), config) - assert.Error(suite.T(), err) + assert.Nil(ts.T(), config) + assert.Error(ts.T(), err) } -func (suite *ConfigTestSuite) TestMissingRequiredConfVar() { +func (ts *ConfigTestSuite) TestMissingRequiredConfVar() { for _, requiredConfVar := range requiredConfVars { requiredConfVarValue := viper.Get(requiredConfVar) viper.Set(requiredConfVar, nil) expectedError := fmt.Errorf("%s not set", requiredConfVar) config, err := NewConfig("s3inbox") - assert.Nil(suite.T(), config) - if assert.Error(suite.T(), err) { - assert.Equal(suite.T(), expectedError, err) + assert.Nil(ts.T(), config) + if assert.Error(ts.T(), err) { + assert.Equal(ts.T(), expectedError, err) } viper.Set(requiredConfVar, requiredConfVarValue) } } -func (suite *ConfigTestSuite) TestConfigS3Storage() { +func (ts *ConfigTestSuite) TestConfigS3Storage() { config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.Inbox.S3) - assert.Equal(suite.T(), "testurl", config.Inbox.S3.URL) - assert.Equal(suite.T(), "testaccess", config.Inbox.S3.AccessKey) - assert.Equal(suite.T(), "testsecret", config.Inbox.S3.SecretKey) - assert.Equal(suite.T(), "testbucket", config.Inbox.S3.Bucket) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.Inbox.S3) + assert.Equal(ts.T(), "testurl", config.Inbox.S3.URL) + assert.Equal(ts.T(), "testaccess", config.Inbox.S3.AccessKey) + assert.Equal(ts.T(), "testsecret", config.Inbox.S3.SecretKey) + assert.Equal(ts.T(), "testbucket", config.Inbox.S3.Bucket) } -func (suite *ConfigTestSuite) TestConfigBroker() { +func (ts *ConfigTestSuite) TestConfigBroker() { config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.Inbox.S3) - assert.Equal(suite.T(), "/testvhost", config.Broker.Vhost) - assert.Equal(suite.T(), false, config.Broker.Ssl) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.Inbox.S3) + assert.Equal(ts.T(), "/testvhost", config.Broker.Vhost) + assert.Equal(ts.T(), false, config.Broker.Ssl) viper.Set("broker.ssl", true) viper.Set("broker.verifyPeer", true) _, err = NewConfig("s3inbox") - assert.Error(suite.T(), err, "Error expected") + assert.Error(ts.T(), err, "Error expected") viper.Set("broker.clientCert", "dummy-value") viper.Set("broker.clientKey", "dummy-value") _, err = NewConfig("s3inbox") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("broker.vhost", nil) config, err = NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "/", config.Broker.Vhost) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "/", config.Broker.Vhost) } -func (suite *ConfigTestSuite) TestTLSConfigBroker() { +func (ts *ConfigTestSuite) TestTLSConfigBroker() { viper.Set("broker.serverName", "broker") viper.Set("broker.ssl", true) viper.Set("broker.cacert", certPath+"/ca.crt") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsBroker, err := TLSConfigBroker(config) - assert.NotNil(suite.T(), tlsBroker) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), tlsBroker) + assert.NoError(ts.T(), err) viper.Set("broker.verifyPeer", true) viper.Set("broker.clientCert", certPath+"/tls.crt") viper.Set("broker.clientKey", certPath+"/tls.key") config, err = NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsBroker, err = TLSConfigBroker(config) - assert.NotNil(suite.T(), tlsBroker) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), tlsBroker) + assert.NoError(ts.T(), err) viper.Set("broker.clientCert", certPath+"tls.crt") viper.Set("broker.clientKey", certPath+"/tls.key") config, err = NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsBroker, err = TLSConfigBroker(config) - assert.Nil(suite.T(), tlsBroker) - assert.Error(suite.T(), err) + assert.Nil(ts.T(), tlsBroker) + assert.Error(ts.T(), err) } -func (suite *ConfigTestSuite) TestTLSConfigProxy() { +func (ts *ConfigTestSuite) TestTLSConfigProxy() { viper.Set("inbox.cacert", certPath+"/ca.crt") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) tlsProxy, err := TLSConfigProxy(config) - assert.NotNil(suite.T(), tlsProxy) - assert.NoError(suite.T(), err) + assert.NotNil(ts.T(), tlsProxy) + assert.NoError(ts.T(), err) } -func (suite *ConfigTestSuite) TestDefaultLogLevel() { +func (ts *ConfigTestSuite) TestDefaultLogLevel() { viper.Set("log.level", "test") config, err := NewConfig("s3inbox") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), log.TraceLevel, log.GetLevel()) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), log.TraceLevel, log.GetLevel()) } -func (suite *ConfigTestSuite) TestAPIConfiguration() { +func (ts *ConfigTestSuite) TestAPIConfiguration() { // At this point we should fail because we lack configuration viper.Reset() config, err := NewConfig("api") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), config) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), config) // testing deafult values - suite.SetupTest() + ts.SetupTest() config, err = NewConfig("api") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.API) - assert.Equal(suite.T(), "0.0.0.0", config.API.Host) - assert.Equal(suite.T(), 8080, config.API.Port) - assert.Equal(suite.T(), true, config.API.Session.Secure) - assert.Equal(suite.T(), true, config.API.Session.HTTPOnly) - assert.Equal(suite.T(), "api_session_key", config.API.Session.Name) - assert.Equal(suite.T(), -1*time.Second, config.API.Session.Expiration) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.API) + assert.Equal(ts.T(), "0.0.0.0", config.API.Host) + assert.Equal(ts.T(), 8080, config.API.Port) + assert.Equal(ts.T(), true, config.API.Session.Secure) + assert.Equal(ts.T(), true, config.API.Session.HTTPOnly) + assert.Equal(ts.T(), "api_session_key", config.API.Session.Name) + assert.Equal(ts.T(), -1*time.Second, config.API.Session.Expiration) rbac, _ := os.ReadFile(viper.GetString("api.rbacFile")) - assert.Equal(suite.T(), rbac, config.API.RBACpolicy) + assert.Equal(ts.T(), rbac, config.API.RBACpolicy) viper.Reset() - suite.SetupTest() + ts.SetupTest() // over write defaults viper.Set("api.port", 8443) viper.Set("api.session.secure", false) @@ -250,21 +250,21 @@ func (suite *ConfigTestSuite) TestAPIConfiguration() { viper.Set("api.session.expiration", 60) config, err = NewConfig("api") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.API) - assert.Equal(suite.T(), "0.0.0.0", config.API.Host) - assert.Equal(suite.T(), 8443, config.API.Port) - assert.Equal(suite.T(), false, config.API.Session.Secure) - assert.Equal(suite.T(), "test", config.API.Session.Domain) - assert.Equal(suite.T(), 60*time.Second, config.API.Session.Expiration) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.API) + assert.Equal(ts.T(), "0.0.0.0", config.API.Host) + assert.Equal(ts.T(), 8443, config.API.Port) + assert.Equal(ts.T(), false, config.API.Session.Secure) + assert.Equal(ts.T(), "test", config.API.Session.Domain) + assert.Equal(ts.T(), 60*time.Second, config.API.Session.Expiration) } -func (suite *ConfigTestSuite) TestNotifyConfiguration() { +func (ts *ConfigTestSuite) TestNotifyConfiguration() { // At this point we should fail because we lack configuration config, err := NewConfig("notify") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), config) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), config) viper.Set("broker.host", "test") viper.Set("broker.port", 123) @@ -280,16 +280,16 @@ func (suite *ConfigTestSuite) TestNotifyConfiguration() { viper.Set("smtp.from", "noreply") config, err = NewConfig("notify") - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config) } -func (suite *ConfigTestSuite) TestSyncConfig() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestSyncConfig() { + ts.SetupTest() // At this point we should fail because we lack configuration config, err := NewConfig("backup") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), config) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), config) viper.Set("archive.type", "posix") viper.Set("archive.location", "test") @@ -303,32 +303,32 @@ func (suite *ConfigTestSuite) TestSyncConfig() { viper.Set("c4gh.passphrase", "pass") viper.Set("c4gh.syncPubKeyPath", "/keys/recipient") config, err = NewConfig("sync") - assert.NotNil(suite.T(), config) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), config.Broker) - assert.Equal(suite.T(), "testhost", config.Broker.Host) - assert.Equal(suite.T(), 123, config.Broker.Port) - assert.Equal(suite.T(), "testuser", config.Broker.User) - assert.Equal(suite.T(), "testpassword", config.Broker.Password) - assert.Equal(suite.T(), "testqueue", config.Broker.Queue) - assert.NotNil(suite.T(), config.Database) - assert.Equal(suite.T(), "test", config.Database.Host) - assert.Equal(suite.T(), 123, config.Database.Port) - assert.Equal(suite.T(), "test", config.Database.User) - assert.Equal(suite.T(), "test", config.Database.Password) - assert.Equal(suite.T(), "test", config.Database.Database) - assert.NotNil(suite.T(), config.Archive) - assert.NotNil(suite.T(), config.Archive.Posix) - assert.Equal(suite.T(), "test", config.Archive.Posix.Location) - assert.NotNil(suite.T(), config.Sync) - assert.NotNil(suite.T(), config.Sync.Destination.Posix) - assert.Equal(suite.T(), "test", config.Sync.Destination.Posix.Location) + assert.NotNil(ts.T(), config) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), config.Broker) + assert.Equal(ts.T(), "testhost", config.Broker.Host) + assert.Equal(ts.T(), 123, config.Broker.Port) + assert.Equal(ts.T(), "testuser", config.Broker.User) + assert.Equal(ts.T(), "testpassword", config.Broker.Password) + assert.Equal(ts.T(), "testqueue", config.Broker.Queue) + assert.NotNil(ts.T(), config.Database) + assert.Equal(ts.T(), "test", config.Database.Host) + assert.Equal(ts.T(), 123, config.Database.Port) + assert.Equal(ts.T(), "test", config.Database.User) + assert.Equal(ts.T(), "test", config.Database.Password) + assert.Equal(ts.T(), "test", config.Database.Database) + assert.NotNil(ts.T(), config.Archive) + assert.NotNil(ts.T(), config.Archive.Posix) + assert.Equal(ts.T(), "test", config.Archive.Posix.Location) + assert.NotNil(ts.T(), config.Sync) + assert.NotNil(ts.T(), config.Sync.Destination.Posix) + assert.Equal(ts.T(), "test", config.Sync.Destination.Posix.Location) } -func (suite *ConfigTestSuite) TestGetC4GHPublicKey() { +func (ts *ConfigTestSuite) TestGetC4GHPublicKey() { pubKey := "-----BEGIN CRYPT4GH PUBLIC KEY-----\nuQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=\n-----END CRYPT4GH PUBLIC KEY-----" pubKeyPath, _ := os.MkdirTemp("", "pubkey") err := os.WriteFile(pubKeyPath+"/c4gh.pub", []byte(pubKey), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) var kb [32]byte k, _ := base64.StdEncoding.DecodeString("uQO46R56f/Jx0YJjBAkZa2J6n72r6HW/JPMS4tfepBs=") @@ -336,41 +336,41 @@ func (suite *ConfigTestSuite) TestGetC4GHPublicKey() { viper.Set("c4gh.syncPubKeyPath", pubKeyPath+"/c4gh.pub") pkBytes, err := GetC4GHPublicKey() - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), pkBytes) - assert.Equal(suite.T(), pkBytes, &kb, "GetC4GHPublicKey didn't return correct pubKey") + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), pkBytes) + assert.Equal(ts.T(), pkBytes, &kb, "GetC4GHPublicKey didn't return correct pubKey") defer os.RemoveAll(pubKeyPath) } -func (suite *ConfigTestSuite) TestGetC4GHKey() { +func (ts *ConfigTestSuite) TestGetC4GHKey() { key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" keyPath, _ := os.MkdirTemp("", "key") err := os.WriteFile(keyPath+"/c4gh.key", []byte(key), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.filepath", keyPath+"/c4gh.key") pkBytes, err := GetC4GHKey() - assert.EqualError(suite.T(), err, "chacha20poly1305: message authentication failed") - assert.Nil(suite.T(), pkBytes) + assert.EqualError(ts.T(), err, "chacha20poly1305: message authentication failed") + assert.Nil(ts.T(), pkBytes) viper.Set("c4gh.filepath", keyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") pkBytes, err = GetC4GHKey() - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), pkBytes) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), pkBytes) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_AllOK() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_AllOK() { keyPath, _ := os.MkdirTemp("", "key") keyFile1 := keyPath + "/c4gh1.key" keyFile2 := keyPath + "/c4gh2.key" _, err := helper.CreatePrivateKeyFile(keyFile1, "test") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) _, err = helper.CreatePrivateKeyFile(keyFile2, "test") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: keyFile1, Passphrase: "test"}, @@ -378,121 +378,121 @@ func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_AllOK() { }) privateKeys, err := GetC4GHprivateKeys() - assert.NoError(suite.T(), err) - assert.Len(suite.T(), privateKeys, 2) + assert.NoError(ts.T(), err) + assert.Len(ts.T(), privateKeys, 2) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_MissingKeyPath() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_MissingKeyPath() { viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: "/non/existent/path1", Passphrase: "test"}, }) privateKeys, err := GetC4GHprivateKeys() - assert.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "failed to open key file") - assert.Nil(suite.T(), privateKeys) + assert.Error(ts.T(), err) + assert.Contains(ts.T(), err.Error(), "failed to open key file") + assert.Nil(ts.T(), privateKeys) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_WrongPassphrase() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_WrongPassphrase() { keyPath, _ := os.MkdirTemp("", "key") keyFile := keyPath + "/c4gh1.key" _, err := helper.CreatePrivateKeyFile(keyFile, "test") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: keyFile, Passphrase: "wrong"}, }) privateKeys, err := GetC4GHprivateKeys() - assert.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "chacha20poly1305: message authentication faile") - assert.Nil(suite.T(), privateKeys) + assert.Error(ts.T(), err) + assert.Contains(ts.T(), err.Error(), "chacha20poly1305: message authentication faile") + assert.Nil(ts.T(), privateKeys) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestGetC4GHprivateKeys_InvalidKey() { +func (ts *ConfigTestSuite) TestGetC4GHprivateKeys_InvalidKey() { key := "not a valid key" keyPath, _ := os.MkdirTemp("", "key") keyFile := keyPath + "/c4gh1.key" err := os.WriteFile(keyFile, []byte(key), 0600) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) viper.Set("c4gh.privateKeys", []C4GHprivateKeyConf{ {FilePath: keyFile, Passphrase: "wrong"}, }) privateKeys, err := GetC4GHprivateKeys() - assert.Error(suite.T(), err) - assert.Contains(suite.T(), err.Error(), "read of unrecognized private key format") - assert.Nil(suite.T(), privateKeys) + assert.Error(ts.T(), err) + assert.Contains(ts.T(), err.Error(), "read of unrecognized private key format") + assert.Nil(ts.T(), privateKeys) defer os.RemoveAll(keyPath) } -func (suite *ConfigTestSuite) TestConfigSyncAPI() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigSyncAPI() { + ts.SetupTest() noConfig, err := NewConfig("sync-api") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) viper.Set("sync.api.user", "user") viper.Set("sync.api.password", "password") config, err := NewConfig("sync-api") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "user", config.SyncAPI.APIUser) - assert.Equal(suite.T(), "password", config.SyncAPI.APIPassword) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "user", config.SyncAPI.APIUser) + assert.Equal(ts.T(), "password", config.SyncAPI.APIPassword) viper.Set("sync.api.AccessionRouting", "wrong") config, err = NewConfig("sync-api") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), "wrong", config.SyncAPI.AccessionRouting) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), "wrong", config.SyncAPI.AccessionRouting) } -func (suite *ConfigTestSuite) TestConfigReEncryptServer() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigReEncryptServer() { + ts.SetupTest() noConfig, err := NewConfig("reencrypt") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) key := "-----BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY-----\nYzRnaC12MQAGc2NyeXB0ABQAAAAAEna8op+BzhTVrqtO5Rx7OgARY2hhY2hhMjBfcG9seTEzMDUAPMx2Gbtxdva0M2B0tb205DJT9RzZmvy/9ZQGDx9zjlObj11JCqg57z60F0KhJW+j/fzWL57leTEcIffRTA==\n-----END CRYPT4GH ENCRYPTED PRIVATE KEY-----" keyPath, _ := os.MkdirTemp("", "key") defer os.RemoveAll(keyPath) if err := os.WriteFile(keyPath+"/c4gh.key", []byte(key), 0600); err != nil { - suite.T().FailNow() + ts.T().FailNow() } viper.Set("c4gh.filepath", keyPath+"/c4gh.key") viper.Set("c4gh.passphrase", "test") config, err := NewConfig("reencrypt") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), 50051, config.ReEncrypt.Port) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 50051, config.ReEncrypt.Port) viper.Set("grpc.CACert", certPath+"/ca.crt") viper.Set("grpc.serverCert", certPath+"/tls.crt") viper.Set("grpc.serverKey", certPath+"/tls.key") config, err = NewConfig("reencrypt") - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), certPath+"/ca.crt", config.ReEncrypt.CACert) - assert.Equal(suite.T(), certPath+"/tls.crt", config.ReEncrypt.ServerCert) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), certPath+"/ca.crt", config.ReEncrypt.CACert) + assert.Equal(ts.T(), certPath+"/tls.crt", config.ReEncrypt.ServerCert) } -func (suite *ConfigTestSuite) TestConfigAuth_CEGA() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigAuth_CEGA() { + ts.SetupTest() - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { - suite.T().FailNow() + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { + ts.T().FailNow() } - defer os.RemoveAll(ECPath) + defer os.RemoveAll(ecPath) noConfig, err := NewConfig("auth") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) viper.Set("auth.s3Inbox", "http://inbox:8000") viper.Set("auth.publicFile", "no-file") @@ -504,38 +504,38 @@ func (suite *ConfigTestSuite) TestConfigAuth_CEGA() { viper.Set("auth.Jwt.signatureAlg", "ES256") viper.Set("auth.Jwt.tokenTTL", 168) _, err = NewConfig("auth") - assert.ErrorContains(suite.T(), err, "no such file or directory") + assert.ErrorContains(ts.T(), err, "no such file or directory") - viper.Set("auth.publicFile", ECPath+"/ec.pub") - viper.Set("auth.Jwt.privateKey", ECPath+"/ec") + viper.Set("auth.publicFile", ecPath+"/ec.pub") + viper.Set("auth.Jwt.privateKey", ecPath+"/ec") c, err := NewConfig("auth") - assert.Equal(suite.T(), c.Auth.JwtPrivateKey, fmt.Sprintf("%s/ec", ECPath)) - assert.Equal(suite.T(), c.Auth.JwtTTL, 168) - assert.NoError(suite.T(), err, "unexpected failure") + assert.Equal(ts.T(), c.Auth.JwtPrivateKey, fmt.Sprintf("%s/ec", ecPath)) + assert.Equal(ts.T(), c.Auth.JwtTTL, 168) + assert.NoError(ts.T(), err, "unexpected failure") } -func (suite *ConfigTestSuite) TestConfigAuth_OIDC() { - suite.SetupTest() +func (ts *ConfigTestSuite) TestConfigAuth_OIDC() { + ts.SetupTest() - ECPath, _ := os.MkdirTemp("", "EC") - if err := helper.CreateECkeys(ECPath, ECPath); err != nil { - suite.T().FailNow() + ecPath, _ := os.MkdirTemp("", "EC") + if err := helper.CreateECkeys(ecPath, ecPath); err != nil { + ts.T().FailNow() } - defer os.RemoveAll(ECPath) + defer os.RemoveAll(ecPath) noConfig, err := NewConfig("auth") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), noConfig) + assert.Error(ts.T(), err) + assert.Nil(ts.T(), noConfig) viper.Set("auth.s3Inbox", "http://inbox:8000") - viper.Set("auth.publicFile", ECPath+"/ec.pub") + viper.Set("auth.publicFile", ecPath+"/ec.pub") viper.Set("oidc.id", "oidcTestID") viper.Set("oidc.secret", "oidcTestIssuer") _, err = NewConfig("auth") - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) viper.Set("oidc.provider", "http://provider:9000") viper.Set("oidc.redirectUrl", "http://auth/oidc/login") _, err = NewConfig("auth") - assert.NoError(suite.T(), err, "unexpected failure") + assert.NoError(ts.T(), err, "unexpected failure") } diff --git a/sda/internal/database/database.go b/sda/internal/database/database.go index eed386a8c7..bb00a755f9 100644 --- a/sda/internal/database/database.go +++ b/sda/internal/database/database.go @@ -4,6 +4,7 @@ package database import ( "database/sql" + "errors" "fmt" "time" @@ -32,11 +33,12 @@ type SDAdb struct { // FileInfo is used by ingest for file metadata (path, size, checksum) type FileInfo struct { - Checksum string + ArchiveChecksum string Size int64 Path string DecryptedChecksum string DecryptedSize int64 + UploadedChecksum string } type SyncData struct { @@ -46,10 +48,11 @@ type SyncData struct { } type SubmissionFileInfo struct { - FileID string `json:"fileID"` - InboxPath string `json:"inboxPath"` - Status string `json:"fileStatus"` - CreateAt string `json:"createAt"` + AccessionID string `json:"accessionID,omitempty"` + FileID string `json:"fileID"` + InboxPath string `json:"inboxPath"` + Status string `json:"fileStatus"` + CreateAt string `json:"createAt"` } type DatasetInfo struct { @@ -83,7 +86,6 @@ var RetryTimes = 5 // NewSDAdb creates a new DB connection from the given DBConf variables. // Currently, only postgresql connections are supported. func NewSDAdb(config DBConf) (*SDAdb, error) { - dbs := SDAdb{DB: nil, Version: -1, Config: config} err := dbs.Connect() @@ -116,7 +118,7 @@ func (dbs *SDAdb) Connect() error { } // default error - err := fmt.Errorf("failed to connect within reconnect time") + err := errors.New("failed to connect within reconnect time") log.Infoln("Connecting to database") log.Debugf("host: %s:%d, database: %s, user: %s", dbs.Config.Host, dbs.Config.Port, dbs.Config.Database, dbs.Config.User) @@ -171,7 +173,6 @@ func (config *DBConf) PgDataSource() (string, string) { // getVersion fetches the database schema version. This function return -1 when // the version could not be fetched. func (dbs *SDAdb) getVersion() (int, error) { - dbs.checkAndReconnectIfNeeded() log.Debug("Fetching database schema version") diff --git a/sda/internal/database/db_functions.go b/sda/internal/database/db_functions.go index adb2a505f9..66be75d5c6 100644 --- a/sda/internal/database/db_functions.go +++ b/sda/internal/database/db_functions.go @@ -20,7 +20,6 @@ import ( // event. If the file already exists in the database, the entry is updated, but // a new file event is always inserted. func (dbs *SDAdb) RegisterFile(uploadPath, uploadUser string) (string, error) { - dbs.checkAndReconnectIfNeeded() if dbs.Version < 4 { @@ -100,6 +99,45 @@ func (dbs *SDAdb) getInboxFilePathFromID(submissionUser, fileID string) (string, return filePath, nil } +// GetFileIDByUserPathAndStatus checks if a file exists in the database for a given user and submission filepath +// and returns its fileID for the latest specified status +func (dbs *SDAdb) GetFileIDByUserPathAndStatus(submissionUser, filePath, status string) (string, error) { + var ( + err error + fileID string + ) + // 2, 4, 8, 16, 32 seconds between each retry event. + for count := 1; count <= RetryTimes; count++ { + fileID, err = dbs.getFileIDByUserPathAndStatus(submissionUser, filePath, status) + if err == nil || strings.Contains(err.Error(), "sql: no rows in result set") { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) + } + + return fileID, err +} + +func (dbs *SDAdb) getFileIDByUserPathAndStatus(submissionUser, filePath, status string) (string, error) { + dbs.checkAndReconnectIfNeeded() + db := dbs.DB + + const getFileID = "SELECT id from sda.files " + + "WHERE submission_user=$1 and submission_file_path =$2 and stable_id IS null " + + "AND EXISTS (SELECT 1 FROM " + + "(SELECT event from sda.file_event_log JOIN sda.files ON sda.files.id=sda.file_event_log.file_id " + + "WHERE submission_user=$1 and submission_file_path =$2 order by started_at desc limit 1) " + + "AS subquery WHERE event = $3)" + + var fileID string + err := db.QueryRow(getFileID, submissionUser, filePath, status).Scan(&fileID) + if err != nil { + return "", err + } + + return fileID, nil +} + // UpdateFileEventLog updates the status in of the file in the database. // The message parameter is the rabbitmq message sent on file upload. func (dbs *SDAdb) UpdateFileEventLog(fileUUID, event, corrID, user, details, message string) error { @@ -163,34 +201,37 @@ func (dbs *SDAdb) storeHeader(header []byte, id string) error { } // SetArchived marks the file as 'ARCHIVED' -func (dbs *SDAdb) SetArchived(file FileInfo, fileID, corrID string) error { - var ( - err error - count int - ) +func (dbs *SDAdb) SetArchived(file FileInfo, fileID string) error { + var err error - for count == 0 || (err != nil && count < RetryTimes) { - err = dbs.setArchived(file, fileID, corrID) - count++ + for count := 1; count <= RetryTimes; count++ { + err = dbs.setArchived(file, fileID) + if err == nil { + break + } + time.Sleep(time.Duration(math.Pow(2, float64(count))) * time.Second) } return err } -func (dbs *SDAdb) setArchived(file FileInfo, fileID, corrID string) error { +func (dbs *SDAdb) setArchived(file FileInfo, fileID string) error { dbs.checkAndReconnectIfNeeded() db := dbs.DB - const query = "SELECT sda.set_archived($1, $2, $3, $4, $5, $6);" - _, err := db.Exec(query, - fileID, - corrID, - file.Path, - file.Size, - file.Checksum, - "SHA256", - ) + const setArchived = "UPDATE sda.files SET archive_file_path = $1, archive_file_size = $2 WHERE id = $3;" + if _, err := db.Exec(setArchived, file.Path, file.Size, fileID); err != nil { + return fmt.Errorf("setArchived error: %s", err.Error()) + } - return err + log.Debugf("checksum: %s", file.UploadedChecksum) + const addChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + + "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UPLOADED')::sda.checksum_source)" + + "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + if _, err := db.Exec(addChecksum, fileID, file.UploadedChecksum, "SHA256"); err != nil { + return fmt.Errorf("addChecksum error: %s", err.Error()) + } + + return nil } func (dbs *SDAdb) GetFileStatus(corrID string) (string, error) { @@ -256,35 +297,42 @@ func (dbs *SDAdb) getHeader(fileID string) ([]byte, error) { } // MarkCompleted marks the file as "COMPLETED" -func (dbs *SDAdb) SetVerified(file FileInfo, fileID, corrID string) error { +func (dbs *SDAdb) SetVerified(file FileInfo, fileID string) error { var ( err error count int ) for count == 0 || (err != nil && count < RetryTimes) { - err = dbs.setVerified(file, fileID, corrID) + err = dbs.setVerified(file, fileID) count++ } return err } -func (dbs *SDAdb) setVerified(file FileInfo, fileID, corrID string) error { +func (dbs *SDAdb) setVerified(file FileInfo, fileID string) error { dbs.checkAndReconnectIfNeeded() - db := dbs.DB - const completed = "SELECT sda.set_verified($1, $2, $3, $4, $5, $6, $7);" - _, err := db.Exec(completed, - fileID, - corrID, - file.Checksum, - "SHA256", - file.DecryptedSize, - file.DecryptedChecksum, - "SHA256", - ) + const verified = "UPDATE sda.files SET decrypted_file_size = $1 WHERE id = $2;" + if _, err := dbs.DB.Exec(verified, file.DecryptedSize, fileID); err != nil { + return fmt.Errorf("setVerified error: %s", err.Error()) + } - return err + const addArchiveChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + + "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('ARCHIVED')::sda.checksum_source)" + + "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + if _, err := dbs.DB.Exec(addArchiveChecksum, fileID, file.ArchiveChecksum, "SHA256"); err != nil { + return fmt.Errorf("addArchiveChecksum error: %s", err.Error()) + } + + const addUnencryptedChecksum = "INSERT INTO sda.checksums(file_id, checksum, type, source)" + + "VALUES($1, $2, upper($3)::sda.checksum_algorithm, upper('UNENCRYPTED')::sda.checksum_source)" + + "ON CONFLICT ON CONSTRAINT unique_checksum DO UPDATE SET checksum = EXCLUDED.checksum;" + if _, err := dbs.DB.Exec(addUnencryptedChecksum, fileID, file.DecryptedChecksum, "SHA256"); err != nil { + return fmt.Errorf("addUnencryptedChecksum error: %s", err.Error()) + } + + return nil } // GetArchived retrieves the location and size of archive @@ -502,7 +550,6 @@ func (dbs *SDAdb) updateDatasetEvent(datasetID, status, message string) error { } return nil - } // GetFileInfo returns info on a ingested file @@ -524,15 +571,21 @@ func (dbs *SDAdb) getFileInfo(id string) (FileInfo, error) { dbs.checkAndReconnectIfNeeded() db := dbs.DB const getFileID = "SELECT archive_file_path, archive_file_size from sda.files where id = $1;" - const checkSum = "SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted from sda.checksums where file_id = $1;" + const checkSum = "SELECT MAX(checksum) FILTER(where source = 'ARCHIVED') as Archived, " + + "MAX(checksum) FILTER(where source = 'UNENCRYPTED') as Unencrypted, " + + "MAX(checksum) FILTER(where source = 'UPLOADED') as Uploaded from sda.checksums where file_id = $1;" var info FileInfo if err := db.QueryRow(getFileID, id).Scan(&info.Path, &info.Size); err != nil { return FileInfo{}, err } - if err := db.QueryRow(checkSum, id).Scan(&info.Checksum, &info.DecryptedChecksum); err != nil { + var archivedChecksum, decryptedChecksum, uploadedChecksum sql.NullString + if err := db.QueryRow(checkSum, id).Scan(&archivedChecksum, &decryptedChecksum, &uploadedChecksum); err != nil { return FileInfo{}, err } + info.ArchiveChecksum = archivedChecksum.String + info.DecryptedChecksum = decryptedChecksum.String + info.UploadedChecksum = uploadedChecksum.String return info, nil } @@ -651,14 +704,14 @@ func (dbs *SDAdb) getArchivePath(stableID string) (string, error) { } // GetUserFiles retrieves all the files a user submitted -func (dbs *SDAdb) GetUserFiles(userID string) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) GetUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { var err error files := []*SubmissionFileInfo{} // 2, 4, 8, 16, 32 seconds between each retry event. for count := 1; count <= RetryTimes; count++ { - files, err = dbs.getUserFiles(userID) + files, err = dbs.getUserFiles(userID, allData) if err == nil { break } @@ -669,14 +722,14 @@ func (dbs *SDAdb) GetUserFiles(userID string) ([]*SubmissionFileInfo, error) { } // getUserFiles is the actual function performing work for GetUserFiles -func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { +func (dbs *SDAdb) getUserFiles(userID string, allData bool) ([]*SubmissionFileInfo, error) { dbs.checkAndReconnectIfNeeded() files := []*SubmissionFileInfo{} db := dbs.DB // select all files (that are not part of a dataset) of the user, each one annotated with its latest event - const query = "SELECT f.id, f.submission_file_path, e.event, f.created_at FROM sda.files f " + + const query = "SELECT f.id, f.submission_file_path, f.stable_id, e.event, f.created_at FROM sda.files f " + "LEFT JOIN (SELECT DISTINCT ON (file_id) file_id, started_at, event FROM sda.file_event_log ORDER BY file_id, started_at DESC) e ON f.id = e.file_id WHERE f.submission_user = $1 " + "AND f.id NOT IN (SELECT f.id FROM sda.files f RIGHT JOIN sda.file_dataset d ON f.id = d.file_id); " @@ -689,13 +742,18 @@ func (dbs *SDAdb) getUserFiles(userID string) ([]*SubmissionFileInfo, error) { // Iterate rows for rows.Next() { + var accessionID sql.NullString // Read rows into struct fi := &SubmissionFileInfo{} - err := rows.Scan(&fi.FileID, &fi.InboxPath, &fi.Status, &fi.CreateAt) + err := rows.Scan(&fi.FileID, &fi.InboxPath, &accessionID, &fi.Status, &fi.CreateAt) if err != nil { return nil, err } + if allData { + fi.AccessionID = accessionID.String + } + // Add instance of struct (file) to array if the status is not disabled if fi.Status != "disabled" { files = append(files, fi) @@ -748,7 +806,7 @@ func (dbs *SDAdb) getCorrID(user, path, accession string) (string, error) { return "", rows.Err() } - return "", fmt.Errorf("sql: no rows in result set") + return "", errors.New("sql: no rows in result set") } // list all users with files not yet assigned to a dataset @@ -847,7 +905,6 @@ func (dbs *SDAdb) SetKeyHash(keyHash, fileID string) error { query := "UPDATE sda.files SET key_hash = $1 WHERE id = $2;" result, err := db.Exec(query, keyHash, fileID) if err != nil { - return err } if rowsAffected, _ := result.RowsAffected(); rowsAffected == 0 { diff --git a/sda/internal/database/db_functions_test.go b/sda/internal/database/db_functions_test.go index b169831cd9..369220aea3 100644 --- a/sda/internal/database/db_functions_test.go +++ b/sda/internal/database/db_functions_test.go @@ -14,7 +14,6 @@ import ( // TestRegisterFile tests that RegisterFile() behaves as intended func (suite *DatabaseTests) TestRegisterFile() { - // create database connection db, err := NewSDAdb(suite.dbConf) assert.NoError(suite.T(), err, "got %v when creating new connection", err) @@ -38,7 +37,6 @@ func (suite *DatabaseTests) TestRegisterFile() { err = db.DB.QueryRow("SELECT EXISTS(SELECT 1 FROM sda.file_event_log WHERE file_id=$1 AND event='registered')", fileID).Scan(&exists) assert.NoError(suite.T(), err, "Failed to check if registered file event exists") assert.True(suite.T(), exists, "RegisterFile() did not insert a row into sda.file_event_log with id: "+fileID) - } func (suite *DatabaseTests) TestGetFileID() { @@ -105,16 +103,15 @@ func (suite *DatabaseTests) TestSetArchived() { fileID, err := db.RegisterFile("/testuser/TestSetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), -1} - corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), -1, fmt.Sprintf("%x", sha256.New())} + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = db.SetArchived(fileInfo, "00000000-0000-0000-0000-000000000000", corrID) + err = db.SetArchived(fileInfo, "00000000-0000-0000-0000-000000000000") assert.ErrorContains(suite.T(), err, "violates foreign key constraint") - err = db.SetArchived(fileInfo, fileID, "00000000-0000-0000-0000-000000000000") - assert.ErrorContains(suite.T(), err, "duplicate key value violates unique constraint") + err = db.SetArchived(fileInfo, fileID) + assert.NoError(suite.T(), err, "failed to mark file as Archived") } func (suite *DatabaseTests) TestGetFileStatus() { @@ -158,9 +155,11 @@ func (suite *DatabaseTests) TestSetVerified() { fileID, err := db.RegisterFile("/testuser/TestSetVerified.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - corrID := uuid.New().String() - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/testuser/TestSetVerified.c4gh", fmt.Sprintf("%x", sha256.New()), 948} - err = db.SetVerified(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/testuser/TestSetVerified.c4gh", fmt.Sprintf("%x", sha256.New()), 948, fmt.Sprintf("%x", sha256.New())} + err = db.SetVerified(fileInfo, fileID) + assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) + + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) } @@ -172,11 +171,11 @@ func (suite *DatabaseTests) TestGetArchived() { fileID, err := db.RegisterFile("/testuser/TestGetArchived.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987} - corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestGetArchived.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} + + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) filePath, fileSize, err := db.GetArchived(fileID) @@ -192,11 +191,11 @@ func (suite *DatabaseTests) TestSetAccessionID() { // register a file in the database fileID, err := db.RegisterFile("/testuser/TestSetAccessionID.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987} - corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestSetAccessionID.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} + + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) stableID := "TEST:000-1234-4567" err = db.SetAccessionID(stableID, fileID) @@ -210,11 +209,11 @@ func (suite *DatabaseTests) TestCheckAccessionIDExists() { // register a file in the database fileID, err := db.RegisterFile("/testuser/TestCheckAccessionIDExists.c4gh", "testuser") assert.NoError(suite.T(), err, "failed to register file in database") - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987} - corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1000, "/tmp/TestCheckAccessionIDExists.c4gh", fmt.Sprintf("%x", sha256.New()), 987, fmt.Sprintf("%x", sha256.New())} + + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) stableID := "TEST:111-1234-4567" err = db.SetAccessionID(stableID, fileID) @@ -245,18 +244,18 @@ func (suite *DatabaseTests) TestGetFileInfo() { _, err = decSha.Write([]byte("DecryptedChecksum")) assert.NoError(suite.T(), err) - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/tmp/TestGetFileInfo.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} - corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/tmp/TestGetFileInfo.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} + + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "got (%v) when marking file as verified", err) info, err := db.GetFileInfo(fileID) assert.NoError(suite.T(), err, "got (%v) when getting file archive information", err) assert.Equal(suite.T(), int64(2000), info.Size) assert.Equal(suite.T(), "/tmp/TestGetFileInfo.c4gh", info.Path) - assert.Equal(suite.T(), "11c94bc7fb13afeb2b3fb16c1dbe9206dc09560f1b31420f2d46210ca4ded0a8", info.Checksum) + assert.Equal(suite.T(), "11c94bc7fb13afeb2b3fb16c1dbe9206dc09560f1b31420f2d46210ca4ded0a8", info.ArchiveChecksum) assert.Equal(suite.T(), "a671218c2418aa51adf97e33c5c91a720289ba3c9fd0d36f6f4bf9610730749f", info.DecryptedChecksum) } @@ -286,6 +285,15 @@ func (suite *DatabaseTests) TestMapFilesToDataset() { err := db.MapFilesToDataset(di, acs) assert.NoError(suite.T(), err, "failed to map file to dataset") } + + // Append files to an existing dataset + assert.NoError(suite.T(), db.MapFilesToDataset("dataset1", accessions[9:11]), "failed to append files to a dataset") + var dsMembers int + const q1 = "SELECT count(file_id) from sda.file_dataset WHERE dataset_id = (SELECT id from sda.datasets WHERE stable_id = $1);" + if err := db.DB.QueryRow(q1, "dataset1").Scan(&dsMembers); err != nil { + suite.FailNow("failed to get dataset members from database") + } + assert.Equal(suite.T(), 5, dsMembers) } func (suite *DatabaseTests) TestGetInboxPath() { @@ -372,12 +380,12 @@ func (suite *DatabaseTests) TestGetSyncData() { assert.NoError(suite.T(), err, "failed to register file in database") checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/tmp/TestGetGetSyncData.c4gh", checksum, 999} - corrID := uuid.New().String() - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, "/tmp/TestGetGetSyncData.c4gh", checksum, 999, fmt.Sprintf("%x", sha256.New())} + + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Verified") stableID := "TEST:000-1111-2222" @@ -433,8 +441,8 @@ func (suite *DatabaseTests) TestGetArchivePath() { checksum := fmt.Sprintf("%x", sha256.New()) corrID := uuid.New().String() - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1234, corrID, checksum, 999} - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New()), 1234, corrID, checksum, 999, fmt.Sprintf("%x", sha256.New())} + err = db.SetArchived(fileInfo, fileID) assert.NoError(suite.T(), err, "failed to mark file as Archived") err = db.SetAccessionID("acession-0001", fileID) @@ -456,19 +464,22 @@ func (suite *DatabaseTests) TestGetUserFiles() { assert.NoError(suite.T(), err, "failed to register file in database") err = db.UpdateFileEventLog(fileID, "uploaded", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") + err = db.SetAccessionID(fmt.Sprintf("stableID-00%d", i), fileID) + assert.NoError(suite.T(), err, "failed to update satus of file in database") err = db.UpdateFileEventLog(fileID, "ready", fileID, testUser, "{}", "{}") assert.NoError(suite.T(), err, "failed to update satus of file in database") } - filelist, err := db.GetUserFiles("unknownuser") + filelist, err := db.GetUserFiles("unknownuser", true) assert.NoError(suite.T(), err, "failed to get (empty) file list of unknown user") assert.Empty(suite.T(), filelist, "file list of unknown user is not empty") - filelist, err = db.GetUserFiles(testUser) + filelist, err = db.GetUserFiles(testUser, true) assert.NoError(suite.T(), err, "failed to get file list") assert.Equal(suite.T(), testCases, len(filelist), "file list is of incorrect length") for _, fileInfo := range filelist { assert.Equal(suite.T(), "ready", fileInfo.Status, "incorrect file status") + assert.Contains(suite.T(), fileInfo.AccessionID, "stableID-00", "incorrect file accession ID") } } @@ -505,8 +516,8 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { } checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, fileID, checksum, 999} - if err := db.SetArchived(fileInfo, fileID, fileID); err != nil { + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, fileID, checksum, 999, fmt.Sprintf("%x", sha256.New())} + if err := db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as archived") } if err := db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}"); err != nil { @@ -526,7 +537,6 @@ func (suite *DatabaseTests) TestGetCorrID_sameFilePath() { corrID, err := db.GetCorrID(user, filePath, "") assert.NoError(suite.T(), err, "failed to get correlation ID of file in database") assert.Equal(suite.T(), fileID2, corrID) - } func (suite *DatabaseTests) TestGetCorrID_wrongFilePath() { @@ -592,13 +602,13 @@ func (suite *DatabaseTests) TestListActiveUsers() { assert.Equal(suite.T(), fileID, corrID) checksum := fmt.Sprintf("%x", sha256.New().Sum(nil)) - fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999} - err = db.SetArchived(fileInfo, fileID, corrID) + fileInfo := FileInfo{fmt.Sprintf("%x", sha256.New().Sum(nil)), 1234, filePath, checksum, 999, fmt.Sprintf("%x", sha256.New())} + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -655,13 +665,14 @@ func (suite *DatabaseTests) TestGetDatasetStatus() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -839,13 +850,14 @@ func (suite *DatabaseTests) TestListDatasets() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -922,13 +934,14 @@ func (suite *DatabaseTests) TestListUserDatasets() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -938,7 +951,6 @@ func (suite *DatabaseTests) TestListUserDatasets() { if err != nil { suite.FailNowf("got (%s) when setting stable ID: %s, %s", err.Error(), stableID, fileID) } - } if err := db.MapFilesToDataset("test-user-dataset-01", []string{"accession_User-Q_00", "accession_User-Q_01", "accession_User-Q_02"}); err != nil { suite.FailNow("failed to map files to dataset") @@ -1056,12 +1068,11 @@ func (suite *DatabaseTests) TestGetReVerificationData() { suite.FailNow("failed to generate checksum") } - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} - corrID := uuid.New().String() - if err = db.SetArchived(fileInfo, fileID, corrID); err != nil { + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } - if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { + if err = db.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as verified") } accession := "acession-001" @@ -1095,12 +1106,12 @@ func (suite *DatabaseTests) TestGetReVerificationData_wrongAccessionID() { suite.FailNow("failed to generate checksum") } - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} - corrID := uuid.New().String() - if err = db.SetArchived(fileInfo, fileID, corrID); err != nil { + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetReVerificationData.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} + + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } - if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { + if err = db.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as verified") } accession := "acession-001" @@ -1134,12 +1145,12 @@ func (suite *DatabaseTests) TestGetDecryptedChecksum() { suite.FailNow("failed to generate checksum") } - fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetDecryptedChecksum.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987} - corrID := uuid.New().String() - if err = db.SetArchived(fileInfo, fileID, corrID); err != nil { + fileInfo := FileInfo{fmt.Sprintf("%x", encSha.Sum(nil)), 2000, "/archive/TestGetDecryptedChecksum.c4gh", fmt.Sprintf("%x", decSha.Sum(nil)), 1987, fmt.Sprintf("%x", sha256.New())} + + if err = db.SetArchived(fileInfo, fileID); err != nil { suite.FailNow("failed to archive file") } - if err = db.SetVerified(fileInfo, fileID, corrID); err != nil { + if err = db.SetVerified(fileInfo, fileID); err != nil { suite.FailNow("failed to mark file as verified") } @@ -1177,13 +1188,14 @@ func (suite *DatabaseTests) TestGetDsatasetFiles() { filePath, checksum, 999, + fmt.Sprintf("%x", sha256.New()), } - err = db.SetArchived(fileInfo, fileID, corrID) + err = db.SetArchived(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Archived") } - err = db.SetVerified(fileInfo, fileID, corrID) + err = db.SetVerified(fileInfo, fileID) if err != nil { suite.FailNow("failed to mark file as Verified") } @@ -1228,3 +1240,39 @@ func (suite *DatabaseTests) TestGetInboxFilePathFromID() { _, err = db.getInboxFilePathFromID(user, fileID) assert.Error(suite.T(), err) } + +func (suite *DatabaseTests) TestGetFileIDByUserPathAndStatus() { + db, err := NewSDAdb(suite.dbConf) + assert.NoError(suite.T(), err, "got (%v) when creating new connection", err) + + user := "UserX" + filePath := fmt.Sprintf("/%v/Deletefile1.c4gh", user) + fileID, err := db.RegisterFile(filePath, user) + if err != nil { + suite.FailNow("Failed to register file") + } + // sanity check - should fail + _, err = db.GetFileIDByUserPathAndStatus("wrong-user", filePath, "registered") + assert.EqualError(suite.T(), err, "sql: no rows in result set") + + // check happy path + fileID2, err := db.getFileIDByUserPathAndStatus(user, filePath, "registered") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), fileID, fileID2) + + // update the status of the file + err = db.UpdateFileEventLog(fileID, "archived", fileID, user, "{}", "{}") + if err != nil { + suite.FailNow("Failed to update file event log") + } + + // check that an error and an empty fileID are returned when the most recent status is not registered + fileID2, err = db.getFileIDByUserPathAndStatus(user, filePath, "registered") + assert.Error(suite.T(), err) + assert.Equal(suite.T(), "", fileID2) + + // check that the function works for other statuses as well + fileID2, err = db.getFileIDByUserPathAndStatus(user, filePath, "archived") + assert.NoError(suite.T(), err) + assert.Equal(suite.T(), fileID, fileID2) +} diff --git a/sda/internal/database/db_test.go b/sda/internal/database/db_test.go index 554a2385b2..6027aaa6d5 100644 --- a/sda/internal/database/db_test.go +++ b/sda/internal/database/db_test.go @@ -104,9 +104,9 @@ func TestMain(m *testing.M) { os.Exit(code) } -func (suite *DatabaseTests) SetupTest() { +func (ts *DatabaseTests) SetupTest() { // Database connection variables - suite.dbConf = DBConf{ + ts.dbConf = DBConf{ Host: "127.0.0.1", Port: dbPort, User: "postgres", @@ -118,21 +118,20 @@ func (suite *DatabaseTests) SetupTest() { ClientKey: "", } - db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + db, err := NewSDAdb(ts.dbConf) + assert.Nil(ts.T(), err, "got %v when creating new connection", err) _, err = db.DB.Exec("TRUNCATE sda.files, sda.encryption_keys CASCADE") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) } -func (suite *DatabaseTests) TearDownTest() {} +func (ts *DatabaseTests) TearDownTest() {} // TestNewSDAdb tests creation of new database connections, as well as fetching // of the database schema version. -func (suite *DatabaseTests) TestNewSDAdb() { - +func (ts *DatabaseTests) TestNewSDAdb() { // test working database connection - db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + db, err := NewSDAdb(ts.dbConf) + assert.Nil(ts.T(), err, "got %v when creating new connection", err) db.Close() @@ -150,22 +149,20 @@ func (suite *DatabaseTests) TestNewSDAdb() { } _, err = NewSDAdb(wrongConf) - assert.NotNil(suite.T(), err, "connection allowed with wrong credentials") - + assert.NotNil(ts.T(), err, "connection allowed with wrong credentials") } // TestConnect tests creation of new database connections -func (suite *DatabaseTests) TestConnect() { - +func (ts *DatabaseTests) TestConnect() { // test connecting to a database - db := SDAdb{DB: nil, Version: -1, Config: suite.dbConf} + db := SDAdb{DB: nil, Version: -1, Config: ts.dbConf} err := db.Connect() - assert.Nil(suite.T(), err, "failed connecting: %s", err) + assert.Nil(ts.T(), err, "failed connecting: %s", err) // test that nothing happens if you connect when already connected err = db.Connect() - assert.Nil(suite.T(), err, "Connect() should return nil when called on an"+ + assert.Nil(ts.T(), err, "Connect() should return nil when called on an"+ " already open connection: %s", err) // test querying a closed connection @@ -173,11 +170,11 @@ func (suite *DatabaseTests) TestConnect() { query := "SELECT MAX(version) FROM sda.dbschema_version" var dbVersion = -1 err = db.DB.QueryRow(query).Scan(&dbVersion) - assert.NotNil(suite.T(), err, "query possible on closed connection") + assert.NotNil(ts.T(), err, "query possible on closed connection") // test reconnection by using getVersion() _, err = db.getVersion() - assert.Nil(suite.T(), err, "failed reconnecting: %s", err) + assert.Nil(ts.T(), err, "failed reconnecting: %s", err) db.Close() @@ -196,16 +193,14 @@ func (suite *DatabaseTests) TestConnect() { db.Config = wrongConf err = db.Connect() - assert.NotNil(suite.T(), err, "connection allowed with wrong credentials") - + assert.NotNil(ts.T(), err, "connection allowed with wrong credentials") } // TestClose tests that the connection is properly closed -func (suite *DatabaseTests) TestClose() { - +func (ts *DatabaseTests) TestClose() { // test working database connection - db, err := NewSDAdb(suite.dbConf) - assert.Nil(suite.T(), err, "got %v when creating new connection", err) + db, err := NewSDAdb(ts.dbConf) + assert.Nil(ts.T(), err, "got %v when creating new connection", err) db.Close() @@ -213,10 +208,10 @@ func (suite *DatabaseTests) TestClose() { query := "SELECT MAX(version) FROM sda.dbschema_version" var dbVersion = -1 err = db.DB.QueryRow(query).Scan(&dbVersion) - assert.NotNil(suite.T(), err, "query possible on closed connection") + assert.NotNil(ts.T(), err, "query possible on closed connection") // check that nothing happens if Close is called on a closed connection db.Close() - assert.NotPanics(suite.T(), db.Close, + assert.NotPanics(ts.T(), db.Close, "Close paniced when called on closed connection") } diff --git a/sda/internal/helper/helper.go b/sda/internal/helper/helper.go index bc54713733..e337768d83 100644 --- a/sda/internal/helper/helper.go +++ b/sda/internal/helper/helper.go @@ -8,6 +8,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/pem" + "errors" "fmt" "log" "math/big" @@ -26,14 +27,14 @@ import ( // Global variables for test token creation var ( - DefaultTokenClaims = map[string]interface{}{ + DefaultTokenClaims = map[string]any{ "iss": "https://dummy.ega.nbis.se", "sub": "dummy", "exp": time.Now().Add(time.Hour * 2).Unix(), "pilot": "dummy-pilot", } - WrongUserClaims = map[string]interface{}{ + WrongUserClaims = map[string]any{ "sub": "c5773f41d17d27bd53b1e6794aedc32d7906e779@elixir-europe.org", "aud": "15137645-3153-4d49-9ddb-594027cd4ca7", "azp": "15137645-3153-4d49-9ddb-594027cd4ca7", @@ -44,7 +45,7 @@ var ( "jti": "5e6c6d24-42eb-408e-ba20-203904e388a1", } - ExpiredClaims = map[string]interface{}{ + ExpiredClaims = map[string]any{ "sub": "dummy", "aud": "15137645-3153-4d49-9ddb-594027cd4ca7", "azp": "15137645-3153-4d49-9ddb-594027cd4ca7", @@ -55,7 +56,7 @@ var ( "jti": "5e6c6d24-42eb-408e-ba20-203904e388a1", } - NonValidClaims = map[string]interface{}{ + NonValidClaims = map[string]any{ "sub": "c5773f41d17d27bd53b1e6794aedc32d7906e779@elixir-europe.org", "aud": "15137645-3153-4d49-9ddb-594027cd4ca7", "azp": "15137645-3153-4d49-9ddb-594027cd4ca7", @@ -66,7 +67,7 @@ var ( "jti": "5e6c6d24-42eb-408e-ba20-203904e388a1", } - WrongTokenAlgClaims = map[string]interface{}{ + WrongTokenAlgClaims = map[string]any{ "iss": "Online JWT Builder", "iat": time.Now().Unix(), "exp": time.Now().Add(time.Hour * 2).Unix(), @@ -96,7 +97,7 @@ type AlwaysDeny struct{} // Authenticate does not authenticate anyone. func (u *AlwaysDeny) Authenticate(_ *http.Request) (jwt.Token, error) { - return nil, fmt.Errorf("denied") + return nil, errors.New("denied") } // MakeFolder creates a folder and subfolders for the keys pair @@ -208,7 +209,7 @@ func CreateSSHKey(path string) error { } // CreateRSAToken creates an RSA token -func CreateRSAToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]interface{}) (string, error) { +func CreateRSAToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]any) (string, error) { if err := jwk.AssignKeyID(jwtKey); err != nil { return "AssignKeyID failed", err } @@ -232,7 +233,7 @@ func CreateRSAToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]int } // CreateECToken creates an EC token -func CreateECToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]interface{}) (string, error) { +func CreateECToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]any) (string, error) { if err := jwk.AssignKeyID(jwtKey); err != nil { return "AssignKeyID failed", err } @@ -256,13 +257,12 @@ func CreateECToken(jwtKey jwk.Key, headerAlg string, tokenClaims map[string]inte } // CreateHSToken creates an HS token -func CreateHSToken(key []byte, tokenClaims map[string]interface{}) (string, error) { +func CreateHSToken(key []byte, tokenClaims map[string]any) (string, error) { token := jwt.New() for key, value := range tokenClaims { if err := token.Set(key, value); err != nil { return "failed to set claim", err } - } jwtKey, err := jwk.FromRaw(key) @@ -343,28 +343,28 @@ func MakeCerts(outDir string) { // create our private and public key caPrivKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } // create the CA certificate caBytes, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caPrivKey.PublicKey, caPrivKey) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } err = TLScertToFile(outDir+"/ca.crt", caBytes) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } tlsKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } err = TLSkeyToFile(outDir+"/tls.key", tlsKey) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } // set up our server certificate @@ -386,12 +386,12 @@ func MakeCerts(outDir string) { // create the TLS certificate certBytes, err := x509.CreateCertificate(rand.Reader, certTemplate, caTemplate, &tlsKey.PublicKey, caPrivKey) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } err = TLScertToFile(outDir+"/tls.crt", certBytes) if err != nil { - log.Fatalln(err) + log.Fatalln(err) // nolint # FIXME Fatal should only be called from main } log.Printf("certificartes written to: %s", outDir) } diff --git a/sda/internal/helper/helper_test.go b/sda/internal/helper/helper_test.go index 384cc0df2a..42f538817c 100644 --- a/sda/internal/helper/helper_test.go +++ b/sda/internal/helper/helper_test.go @@ -21,114 +21,114 @@ func TestUserAuthTestSuite(t *testing.T) { suite.Run(t, new(HelperTest)) } -func (suite *HelperTest) SetupTest() { +func (ts *HelperTest) SetupTest() { } -func (suite *HelperTest) TestMakeFolder() { +func (ts *HelperTest) TestMakeFolder() { privateK, publicK, _ := MakeFolder("dummy-folder") - assert.Equal(suite.T(), "dummy-folder/private-key", privateK) - assert.Equal(suite.T(), "dummy-folder/public-key", publicK) + assert.Equal(ts.T(), "dummy-folder/private-key", privateK) + assert.Equal(ts.T(), "dummy-folder/public-key", publicK) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateRSAkeys() { +func (ts *HelperTest) TestCreateRSAkeys() { privateK, publicK, _ := MakeFolder("dummy-folder") - assert.Nil(suite.T(), CreateRSAkeys(privateK, publicK)) + assert.Nil(ts.T(), CreateRSAkeys(privateK, publicK)) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestParsePrivateRSAKey() { +func (ts *HelperTest) TestParsePrivateRSAKey() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateRSAkeys(privateK, publicK) - assert.Nil(suite.T(), e) + assert.Nil(ts.T(), e) _, err := ParsePrivateRSAKey(privateK, "/rsa") - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateRSAToken() { +func (ts *HelperTest) TestCreateRSAToken() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateRSAkeys(privateK, publicK) - assert.Nil(suite.T(), e) - ParsedPrKey, _ := ParsePrivateRSAKey(privateK, "/rsa") - tok, err := CreateRSAToken(ParsedPrKey, "RS256", DefaultTokenClaims) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), e) + parsedPrKey, _ := ParsePrivateRSAKey(privateK, "/rsa") + tok, err := CreateRSAToken(parsedPrKey, "RS256", DefaultTokenClaims) + assert.Nil(ts.T(), err) set := jwk.NewSet() keyData, err := os.ReadFile(filepath.Join(filepath.Clean(publicK), "/rsa.pub")) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) key, err := jwk.ParseKey(keyData, jwk.WithPEM(true)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = jwk.AssignKeyID(key) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), set.AddKey(key)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), set.AddKey(key)) fmt.Println(tok) _, err = jwt.Parse([]byte(tok), jwt.WithKeySet(set, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateECkeys() { +func (ts *HelperTest) TestCreateECkeys() { privateK, publicK, _ := MakeFolder("dummy-folder") - assert.Nil(suite.T(), CreateECkeys(privateK, publicK)) + assert.Nil(ts.T(), CreateECkeys(privateK, publicK)) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestParsePrivateECKey() { +func (ts *HelperTest) TestParsePrivateECKey() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateECkeys(privateK, publicK) - assert.Nil(suite.T(), e) + assert.Nil(ts.T(), e) k, err := ParsePrivateECKey(privateK, "/ec") - assert.Nil(suite.T(), err) - assert.Equal(suite.T(), "EC", fmt.Sprintf("%v", k.KeyType())) + assert.Nil(ts.T(), err) + assert.Equal(ts.T(), "EC", fmt.Sprintf("%v", k.KeyType())) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateECToken() { +func (ts *HelperTest) TestCreateECToken() { privateK, publicK, _ := MakeFolder("dummy-folder") e := CreateECkeys(privateK, publicK) - assert.Nil(suite.T(), e) - ParsedPrKey, err := ParsePrivateECKey(privateK, "/ec") - assert.NoError(suite.T(), err) - _, err = CreateECToken(ParsedPrKey, "ES256", DefaultTokenClaims) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), e) + parsedPrKey, err := ParsePrivateECKey(privateK, "/ec") + assert.NoError(ts.T(), err) + _, err = CreateECToken(parsedPrKey, "ES256", DefaultTokenClaims) + assert.Nil(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreateHSToken() { +func (ts *HelperTest) TestCreateHSToken() { key := make([]byte, 256) tok, err := CreateHSToken(key, DefaultTokenClaims) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) set := jwk.NewSet() jwtKey, err := jwk.FromRaw(key) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = jwk.AssignKeyID(jwtKey) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) - assert.NoError(suite.T(), set.AddKey(jwtKey)) + assert.NoError(ts.T(), set.AddKey(jwtKey)) fmt.Println(tok) _, err = jwt.Parse([]byte(tok), jwt.WithKeySet(set, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer os.RemoveAll("dummy-folder") } -func (suite *HelperTest) TestCreatePrivateKeyFile() { +func (ts *HelperTest) TestCreatePrivateKeyFile() { tempDir, err := os.MkdirTemp("", "key") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) defer os.RemoveAll(tempDir) // Define the key file path and passphrase @@ -137,17 +137,17 @@ func (suite *HelperTest) TestCreatePrivateKeyFile() { // Call the function under test publicKey, err := CreatePrivateKeyFile(keyFile, passphrase) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Verify the file was created _, err = os.Stat(keyFile) - assert.NoError(suite.T(), err, "Private key file should exist") + assert.NoError(ts.T(), err, "Private key file should exist") // Read and verify the file contents fileContent, err := os.ReadFile(keyFile) - assert.NoError(suite.T(), err, "Should be able to read the private key file") - assert.Contains(suite.T(), string(fileContent), "BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY", "File should contain a valid private key") + assert.NoError(ts.T(), err, "Should be able to read the private key file") + assert.Contains(ts.T(), string(fileContent), "BEGIN CRYPT4GH ENCRYPTED PRIVATE KEY", "File should contain a valid private key") // Verify the public key is not empty - assert.NotEqual(suite.T(), [32]byte{}, publicKey, "Public key should not be empty") + assert.NotEqual(ts.T(), [32]byte{}, publicKey, "Public key should not be empty") } diff --git a/sda/internal/jsonadapter/jsonadapter.go b/sda/internal/jsonadapter/jsonadapter.go index 970fbc9e9b..74ee928876 100644 --- a/sda/internal/jsonadapter/jsonadapter.go +++ b/sda/internal/jsonadapter/jsonadapter.go @@ -54,8 +54,8 @@ func NewAdapter(source *[]byte) *Adapter { return &Adapter{policy: []CasbinRule{}, source: source} } -func (a *Adapter) LoadPolicy(model model.Model) error { - err := a.loadFromBuffer(model) +func (a *Adapter) LoadPolicy(m model.Model) error { + err := a.loadFromBuffer(m) if err != nil { return err } @@ -63,7 +63,7 @@ func (a *Adapter) LoadPolicy(model model.Model) error { return nil } -func (a *Adapter) loadFromBuffer(model model.Model) error { +func (a *Adapter) loadFromBuffer(m model.Model) error { if len(*a.source) == 0 { return nil } @@ -75,13 +75,13 @@ func (a *Adapter) loadFromBuffer(model model.Model) error { } for _, p := range input.Policy { - if err := persist.LoadPolicyLine(fmt.Sprintf("p,%s,%s,%s", p.Role, p.Path, p.Action), model); err != nil { + if err := persist.LoadPolicyLine(fmt.Sprintf("p,%s,%s,%s", p.Role, p.Path, p.Action), m); err != nil { return err } } for _, r := range input.Roles { - if err := persist.LoadPolicyLine(fmt.Sprintf("g,%s,%s", r.Role, r.RoleBinding), model); err != nil { + if err := persist.LoadPolicyLine(fmt.Sprintf("g,%s,%s", r.Role, r.RoleBinding), m); err != nil { return err } } @@ -105,16 +105,16 @@ func (a *Adapter) RemoveFilteredPolicy(_ string, _ string, _ int, _ ...string) e } // SavePolicy saves policy -func (a *Adapter) SavePolicy(model model.Model) error { +func (a *Adapter) SavePolicy(m model.Model) error { a.policy = []CasbinRule{} var rules []CasbinRule - for ptype, ast := range model["p"] { + for ptype, ast := range m["p"] { for _, rule := range ast.Policy { rules = append(rules, CasbinRule{PType: ptype, V0: rule[0], V1: rule[1], V2: rule[2]}) } } - for ptype, ast := range model["g"] { + for ptype, ast := range m["g"] { for _, rule := range ast.Policy { rules = append(rules, CasbinRule{PType: ptype, V0: rule[0], V1: rule[1]}) } diff --git a/sda/internal/jsonadapter/jsonadapter_test.go b/sda/internal/jsonadapter/jsonadapter_test.go index 8aa655cbcf..406a02c527 100644 --- a/sda/internal/jsonadapter/jsonadapter_test.go +++ b/sda/internal/jsonadapter/jsonadapter_test.go @@ -26,10 +26,10 @@ func TestConfigTestSuite(t *testing.T) { suite.Run(t, new(AdapterTestSuite)) } -func (suite *AdapterTestSuite) SetupSuite() { - suite.Model, _ = model.NewModelFromString(Model) - suite.EmptyPolicy = []byte(`{"policy":[],"roles":[]}`) - suite.DefaultPolicy = []byte(`{"policy":[{"role":"admin","path":"/keys/*","action":"(GET)|(POST)|(PUT)"}, +func (ts *AdapterTestSuite) SetupSuite() { + ts.Model, _ = model.NewModelFromString(Model) + ts.EmptyPolicy = []byte(`{"policy":[],"roles":[]}`) + ts.DefaultPolicy = []byte(`{"policy":[{"role":"admin","path":"/keys/*","action":"(GET)|(POST)|(PUT)"}, {"role":"submission","path":"/dataset/create","action":"POST"}, {"role":"submission","path":"/dataset/release/*dataset","action":"POST"}, {"role":"submission","path":"/file/ingest","action":"POST"}, @@ -37,81 +37,80 @@ func (suite *AdapterTestSuite) SetupSuite() { "roles":[{"role":"admin","rolebinding":"submission"}, {"role":"dummy@example.org","rolebinding":"admin"}, {"role":"foo@example.org","rolebinding":"submission"}]}`) - suite.ExpectedPolicy = [][]string{{"admin", "/keys/*", "(GET)|(POST)|(PUT)"}, + ts.ExpectedPolicy = [][]string{{"admin", "/keys/*", "(GET)|(POST)|(PUT)"}, {"submission", "/dataset/create", "POST"}, {"submission", "/dataset/release/*dataset", "POST"}, {"submission", "/file/ingest", "POST"}, {"submission", "/file/accession", "POST"}} - suite.ExpectedGroups = [][]string{{"admin", "submission"}, {"dummy@example.org", "admin"}, {"foo@example.org", "submission"}} + ts.ExpectedGroups = [][]string{{"admin", "submission"}, {"dummy@example.org", "admin"}, {"foo@example.org", "submission"}} - suite.File, _ = os.CreateTemp("", "policy") - _, err := suite.File.Write(suite.DefaultPolicy) + ts.File, _ = os.CreateTemp("", "policy") + _, err := ts.File.Write(ts.DefaultPolicy) if err != nil { - suite.FailNow("failed to write policy file to disk") + ts.FailNow("failed to write policy file to disk") } - } -func (suite *AdapterTestSuite) TearDownSuite() { - os.RemoveAll(suite.File.Name()) +func (ts *AdapterTestSuite) TearDownSuite() { + os.RemoveAll(ts.File.Name()) } -func (suite *AdapterTestSuite) TestAdapter_empty() { - a := NewAdapter(&suite.EmptyPolicy) - e, err := casbin.NewEnforcer(suite.Model, a) - assert.NoError(suite.T(), err, "New enforcer failed on empty policy") +func (ts *AdapterTestSuite) TestAdapter_empty() { + a := NewAdapter(&ts.EmptyPolicy) + e, err := casbin.NewEnforcer(ts.Model, a) + assert.NoError(ts.T(), err, "New enforcer failed on empty policy") p, err := e.GetPolicy() - assert.NoError(suite.T(), err, "failed to get policy") - assert.Equal(suite.T(), [][]string(nil), p) + assert.NoError(ts.T(), err, "failed to get policy") + assert.Equal(ts.T(), [][]string(nil), p) } -func (suite *AdapterTestSuite) TestAdapter() { - a := NewAdapter(&suite.DefaultPolicy) - e, err := casbin.NewEnforcer(suite.Model, a) - assert.NoError(suite.T(), err, "New enforcer failed withpolicy") +func (ts *AdapterTestSuite) TestAdapter() { + a := NewAdapter(&ts.DefaultPolicy) + e, err := casbin.NewEnforcer(ts.Model, a) + assert.NoError(ts.T(), err, "New enforcer failed withpolicy") p, err := e.GetPolicy() - assert.Equal(suite.T(), len(suite.ExpectedPolicy), len(p)) - assert.NoError(suite.T(), err, "failed to get policy") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, suite.ExpectedPolicy)) + assert.Equal(ts.T(), len(ts.ExpectedPolicy), len(p)) + assert.NoError(ts.T(), err, "failed to get policy") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, ts.ExpectedPolicy)) g, err := e.GetGroupingPolicy() - assert.NoError(suite.T(), err, "failed to get groups") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, suite.ExpectedGroups)) + assert.NoError(ts.T(), err, "failed to get groups") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, ts.ExpectedGroups)) } -func (suite *AdapterTestSuite) TestAdapter_fromFile() { - b, err := os.ReadFile(suite.File.Name()) - assert.NoError(suite.T(), err, "failed to read json file") - e, err := casbin.NewEnforcer(suite.Model, NewAdapter(&b)) - assert.NoError(suite.T(), err, "New enforcer failed with policy from file: %s", suite.File.Name()) +func (ts *AdapterTestSuite) TestAdapter_fromFile() { + b, err := os.ReadFile(ts.File.Name()) + assert.NoError(ts.T(), err, "failed to read json file") + e, err := casbin.NewEnforcer(ts.Model, NewAdapter(&b)) + assert.NoError(ts.T(), err, "New enforcer failed with policy from file: %s", ts.File.Name()) p, err := e.GetPolicy() - assert.Equal(suite.T(), len(suite.ExpectedPolicy), len(p)) - assert.NoError(suite.T(), err, "failed to get policy") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, suite.ExpectedPolicy)) + assert.Equal(ts.T(), len(ts.ExpectedPolicy), len(p)) + assert.NoError(ts.T(), err, "failed to get policy") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedPolicy, p), fmt.Sprintf("Policy: %v, supposed to be %v", p, ts.ExpectedPolicy)) g, err := e.GetGroupingPolicy() - assert.NoError(suite.T(), err, "failed to get groups") - assert.True(suite.T(), util.Array2DEquals(suite.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, suite.ExpectedGroups)) + assert.NoError(ts.T(), err, "failed to get groups") + assert.True(ts.T(), util.Array2DEquals(ts.ExpectedGroups, g), fmt.Sprintf("Groups: %v, supposed to be %v", g, ts.ExpectedGroups)) } -func (suite *AdapterTestSuite) TestAdapter_save() { - a := NewAdapter(&suite.EmptyPolicy) - e, err := casbin.NewEnforcer(suite.Model, a) - assert.NoError(suite.T(), err, "New enforcer failed without policy") +func (ts *AdapterTestSuite) TestAdapter_save() { + a := NewAdapter(&ts.EmptyPolicy) + e, err := casbin.NewEnforcer(ts.Model, a) + assert.NoError(ts.T(), err, "New enforcer failed without policy") _, err = e.AddPolicy("foo", "/data/*", "GET") - assert.NoError(suite.T(), err, "failed to add policy") + assert.NoError(ts.T(), err, "failed to add policy") _, err = e.AddPolicy("foo", "/data/:value", "POST") - assert.NoError(suite.T(), err, "failed to add policy") + assert.NoError(ts.T(), err, "failed to add policy") - assert.NoError(suite.T(), e.SavePolicy(), "failed to save policy") - assert.Equal(suite.T(), 2, len(a.policy)) + assert.NoError(ts.T(), e.SavePolicy(), "failed to save policy") + assert.Equal(ts.T(), 2, len(a.policy)) } -func (suite *AdapterTestSuite) TestAdapter_notImplemented() { - a := NewAdapter(&suite.EmptyPolicy) - assert.Error(suite.T(), a.AddPolicy("", "", []string{""})) - assert.Error(suite.T(), a.RemovePolicy("", "", []string{""})) - assert.Error(suite.T(), a.RemoveFilteredPolicy("", "", 0, "")) +func (ts *AdapterTestSuite) TestAdapter_notImplemented() { + a := NewAdapter(&ts.EmptyPolicy) + assert.Error(ts.T(), a.AddPolicy("", "", []string{""})) + assert.Error(ts.T(), a.RemovePolicy("", "", []string{""})) + assert.Error(ts.T(), a.RemoveFilteredPolicy("", "", 0, "")) } diff --git a/sda/internal/schema/schema.go b/sda/internal/schema/schema.go index 503e7e3088..f249ed52f7 100644 --- a/sda/internal/schema/schema.go +++ b/sda/internal/schema/schema.go @@ -2,6 +2,7 @@ package schema import ( "encoding/json" + "errors" "fmt" "path/filepath" "strings" @@ -12,7 +13,7 @@ import ( func ValidateJSON(reference string, body []byte) error { dest := getStructName(reference) if dest == "" { - return fmt.Errorf("unknown reference schema") + return errors.New("unknown reference schema") } compiler := jsonschema.NewCompiler() compiler.Draft = jsonschema.Draft7 @@ -22,7 +23,7 @@ func ValidateJSON(reference string, body []byte) error { return err } - var v interface{} + var v any if err := json.Unmarshal(body, &v); err != nil { return err } @@ -34,7 +35,7 @@ func ValidateJSON(reference string, body []byte) error { return nil } -func getStructName(path string) interface{} { +func getStructName(path string) any { switch strings.TrimSuffix(filepath.Base(path), filepath.Ext(path)) { case "dataset-deprecate": return new(DatasetDeprecate) @@ -93,9 +94,9 @@ type DatasetRelease struct { } type InfoError struct { - Error string `json:"error"` - Reason string `json:"reason"` - OriginalMessage interface{} `json:"original-message"` + Error string `json:"error"` + Reason string `json:"reason"` + OriginalMessage any `json:"original-message"` } type InboxRemove struct { @@ -172,12 +173,12 @@ type DatasetFiles struct { } type SyncMetadata struct { - DatasetID string `json:"dataset_id"` - Metadata interface{} `json:"metadata"` + DatasetID string `json:"dataset_id"` + Metadata any `json:"metadata"` } type Metadata struct { - Metadata interface{} + Metadata any } type C4ghPubKey struct { diff --git a/sda/internal/storage/storage.go b/sda/internal/storage/storage.go index d3edcb10fe..3788301770 100644 --- a/sda/internal/storage/storage.go +++ b/sda/internal/storage/storage.go @@ -31,7 +31,7 @@ import ( // Backend defines methods to be implemented by PosixBackend, S3Backend and sftpBackend type Backend interface { - GetFileSize(filePath string) (int64, error) + GetFileSize(filePath string, expectDelay bool) (int64, error) RemoveFile(filePath string) error NewFileReader(filePath string) (io.ReadCloser, error) NewFileWriter(filePath string) (io.WriteCloser, error) @@ -56,35 +56,35 @@ type posixConf struct { } // NewBackend initiates a storage backend -func NewBackend(config Conf) (Backend, error) { - switch config.Type { +func NewBackend(conf Conf) (Backend, error) { + switch conf.Type { case "s3": - return newS3Backend(config.S3) + return newS3Backend(conf.S3) case "sftp": - return newSftpBackend(config.SFTP) + return newSftpBackend(conf.SFTP) default: - return newPosixBackend(config.Posix) + return newPosixBackend(conf.Posix) } } -func newPosixBackend(config posixConf) (*posixBackend, error) { - fileInfo, err := os.Stat(config.Location) +func newPosixBackend(conf posixConf) (*posixBackend, error) { + fileInfo, err := os.Stat(conf.Location) if err != nil { return nil, err } if !fileInfo.IsDir() { - return nil, fmt.Errorf("%s is not a directory", config.Location) + return nil, fmt.Errorf("%s is not a directory", conf.Location) } - return &posixBackend{Location: config.Location}, nil + return &posixBackend{Location: conf.Location}, nil } // NewFileReader returns an io.Reader instance func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if pb == nil { - return nil, fmt.Errorf("invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.Open(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -100,7 +100,7 @@ func (pb *posixBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // NewFileWriter returns an io.Writer instance func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if pb == nil { - return nil, fmt.Errorf("invalid posixBackend") + return nil, errors.New("invalid posixBackend") } file, err := os.OpenFile(filepath.Join(filepath.Clean(pb.Location), filePath), os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0640) @@ -114,9 +114,9 @@ func (pb *posixBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { } // GetFileSize returns the size of the file -func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { +func (pb *posixBackend) GetFileSize(filePath string, _ bool) (int64, error) { if pb == nil { - return 0, fmt.Errorf("invalid posixBackend") + return 0, errors.New("invalid posixBackend") } stat, err := os.Stat(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -132,7 +132,7 @@ func (pb *posixBackend) GetFileSize(filePath string) (int64, error) { // RemoveFile removes a file from a given path func (pb *posixBackend) RemoveFile(filePath string) error { if pb == nil { - return fmt.Errorf("invalid posixBackend") + return errors.New("invalid posixBackend") } err := os.Remove(filepath.Join(filepath.Clean(pb.Location), filePath)) @@ -289,7 +289,7 @@ func (sb *s3Backend) NewFileWriter(filePath string) (io.WriteCloser, error) { } // GetFileSize returns the size of a specific object -func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { +func (sb *s3Backend) GetFileSize(filePath string, expectDelay bool) (int64, error) { r, err := sb.Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ Bucket: &sb.Bucket, Key: &filePath, @@ -305,15 +305,15 @@ func (sb *s3Backend) GetFileSize(filePath string) (int64, error) { // Retry on error up to five minutes to allow for // "slow writes' or s3 eventual consistency for err != nil && time.Since(start) < retryTime { - if strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:") { + if !expectDelay && (strings.Contains(err.Error(), "NoSuchKey:") || strings.Contains(err.Error(), "NotFound:")) { return 0, err } time.Sleep(1 * time.Second) + r, err = sb.Client.HeadObject(context.TODO(), &s3.HeadObjectInput{ Bucket: &sb.Bucket, Key: &filePath, }) - } if err != nil { @@ -352,7 +352,7 @@ func transportConfigS3(conf S3Conf) http.RoundTripper { if conf.CAcert != "" { cacert, e := os.ReadFile(conf.CAcert) // #nosec this file comes from our config if e != nil { - log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) + log.Fatalf("failed to append %q to RootCAs: %v", cacert, e) // nolint # FIXME Fatal should only be called from main } if ok := cfg.RootCAs.AppendCertsFromPEM(cacert); !ok { log.Debug("no certs appended, using system certs only") @@ -378,29 +378,29 @@ type SftpConf struct { HostKey string } -func newSftpBackend(config SftpConf) (*sftpBackend, error) { +func newSftpBackend(conf SftpConf) (*sftpBackend, error) { // read in and parse pem key - key, err := os.ReadFile(config.PemKeyPath) + key, err := os.ReadFile(conf.PemKeyPath) if err != nil { return nil, fmt.Errorf("failed to read from key file, %v", err) } var signer ssh.Signer - if config.PemKeyPass == "" { + if conf.PemKeyPass == "" { signer, err = ssh.ParsePrivateKey(key) } else { - signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(config.PemKeyPass)) + signer, err = ssh.ParsePrivateKeyWithPassphrase(key, []byte(conf.PemKeyPass)) } if err != nil { return nil, fmt.Errorf("failed to parse private key, %v", err) } // connect - conn, err := ssh.Dial("tcp", config.Host+":"+config.Port, + conn, err := ssh.Dial("tcp", conf.Host+":"+conf.Port, &ssh.ClientConfig{ - User: config.UserName, + User: conf.UserName, Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)}, - HostKeyCallback: TrustedHostKeyCallback(config.HostKey), + HostKeyCallback: TrustedHostKeyCallback(conf.HostKey), }, ) if err != nil { @@ -416,7 +416,7 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { sfb := &sftpBackend{ Connection: conn, Client: client, - Conf: &config, + Conf: &conf, } _, err = client.ReadDir("./") @@ -431,7 +431,7 @@ func newSftpBackend(config SftpConf) (*sftpBackend, error) { // NewFileWriter returns an io.Writer instance for the sftp remote func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { if sfb == nil { - return nil, fmt.Errorf("invalid sftpBackend") + return nil, errors.New("invalid sftpBackend") } // Make remote directories parent := filepath.Dir(filePath) @@ -449,9 +449,9 @@ func (sfb *sftpBackend) NewFileWriter(filePath string) (io.WriteCloser, error) { } // GetFileSize returns the size of the file -func (sfb *sftpBackend) GetFileSize(filePath string) (int64, error) { +func (sfb *sftpBackend) GetFileSize(filePath string, _ bool) (int64, error) { if sfb == nil { - return 0, fmt.Errorf("invalid sftpBackend") + return 0, errors.New("invalid sftpBackend") } stat, err := sfb.Client.Lstat(filePath) @@ -465,7 +465,7 @@ func (sfb *sftpBackend) GetFileSize(filePath string) (int64, error) { // NewFileReader returns an io.Reader instance func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { if sfb == nil { - return nil, fmt.Errorf("invalid sftpBackend") + return nil, errors.New("invalid sftpBackend") } file, err := sfb.Client.Open(filePath) @@ -479,7 +479,7 @@ func (sfb *sftpBackend) NewFileReader(filePath string) (io.ReadCloser, error) { // RemoveFile removes a file or an empty directory. func (sfb *sftpBackend) RemoveFile(filePath string) error { if sfb == nil { - return fmt.Errorf("invalid sftpBackend") + return errors.New("invalid sftpBackend") } err := sfb.Client.Remove(filePath) diff --git a/sda/internal/storage/storage_test.go b/sda/internal/storage/storage_test.go index d07d5e905f..a97819b191 100644 --- a/sda/internal/storage/storage_test.go +++ b/sda/internal/storage/storage_test.go @@ -140,7 +140,7 @@ func TestStorageTestSuite(t *testing.T) { suite.Run(t, new(StorageTestSuite)) } -func (suite *StorageTestSuite) SetupTest() { +func (ts *StorageTestSuite) SetupTest() { testS3Conf := S3Conf{ "http://127.0.0.1", s3Port, @@ -171,281 +171,300 @@ func (suite *StorageTestSuite) SetupTest() { testConf = Conf{posixType, testS3Conf, testPosixConf, testSftpConf} } -func (suite *StorageTestSuite) TestNewBackend() { +func (ts *StorageTestSuite) TestNewBackend() { testConf.Type = posixType p, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend posix failed") - assert.IsType(suite.T(), p, &posixBackend{}, "Wrong type from NewBackend with posix") + assert.NoError(ts.T(), err, "Backend posix failed") + assert.IsType(ts.T(), p, &posixBackend{}, "Wrong type from NewBackend with posix") var buf bytes.Buffer log.SetOutput(&buf) testConf.Type = sftpType sf, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend sftp failed") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") - assert.IsType(suite.T(), sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") + assert.NoError(ts.T(), err, "Backend sftp failed") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") + assert.IsType(ts.T(), sf, &sftpBackend{}, "Wrong type from NewBackend with SFTP") buf.Reset() testConf.Type = s3Type s, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend s3 failed") - assert.IsType(suite.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") + assert.NoError(ts.T(), err, "Backend s3 failed") + assert.IsType(ts.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") // test some extra ssl handling testConf.S3.CAcert = "/dev/null" s, err = NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend s3 failed") - assert.IsType(suite.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") + assert.NoError(ts.T(), err, "Backend s3 failed") + assert.IsType(ts.T(), s, &s3Backend{}, "Wrong type from NewBackend with S3") } -func (suite *StorageTestSuite) TestNewS3Client() { +func (ts *StorageTestSuite) TestNewS3Client() { c, err := NewS3Client(testConf.S3) - assert.NoError(suite.T(), err) - assert.NotNil(suite.T(), c) + assert.NoError(ts.T(), err) + assert.NotNil(ts.T(), c) } -func (suite *StorageTestSuite) TestCheckS3Bucket() { +func (ts *StorageTestSuite) TestCheckS3Bucket() { s3, err := newS3Backend(testConf.S3) - assert.NoError(suite.T(), err) - assert.NoError(suite.T(), CheckS3Bucket(testConf.S3.Bucket, s3.Client)) + assert.NoError(ts.T(), err) + assert.NoError(ts.T(), CheckS3Bucket(testConf.S3.Bucket, s3.Client)) testConf.S3.URL = "file://tmp/" bad, err := newS3Backend(testConf.S3) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) err = CheckS3Bucket(testConf.S3.Bucket, bad.Client) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) } -func (suite *StorageTestSuite) TestPosixBackend() { +func (ts *StorageTestSuite) TestPosixBackend() { posixPath, _ := os.MkdirTemp("", "posix") defer os.RemoveAll(posixPath) testConf.Type = posixType testConf.Posix = posixConf{posixPath} backend, err := NewBackend(testConf) - assert.Nil(suite.T(), err, "POSIX backend failed unexpectedly") + assert.Nil(ts.T(), err, "POSIX backend failed unexpectedly") log.SetOutput(os.Stdout) writer, err := backend.NewFileWriter("testFile") - assert.NotNil(suite.T(), writer, "Got a nil reader for writer from posix") - assert.NoError(suite.T(), err, "posix NewFileWriter failed when it shouldn't") + assert.NotNil(ts.T(), writer, "Got a nil reader for writer from posix") + assert.NoError(ts.T(), err, "posix NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - assert.NoError(suite.T(), err, "Failure when writing to posix writer") - assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") + assert.NoError(ts.T(), err, "Failure when writing to posix writer") + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") writer.Close() reader, err := backend.NewFileReader("testFile") - assert.Nil(suite.T(), err, "posix NewFileReader failed when it should work") - assert.NotNil(suite.T(), reader, "Reader that should be usable is nosuite.T(), bailing out") + assert.Nil(ts.T(), err, "posix NewFileReader failed when it should work") + assert.NotNil(ts.T(), reader, "Reader that should be usable is nosuite.T(), bailing out") var buf bytes.Buffer log.SetOutput(&buf) writer, err = backend.NewFileWriter("posix/Not/Creatable") - assert.Nil(suite.T(), writer, "Got a non-nil reader for writer from posix") - assert.Error(suite.T(), err, "posix NewFileWriter worked when it shouldn't") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + assert.Nil(ts.T(), writer, "Got a non-nil reader for writer from posix") + assert.Error(ts.T(), err, "posix NewFileWriter worked when it shouldn't") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") buf.Reset() log.SetOutput(os.Stdout) var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") - assert.Nil(suite.T(), err, "unexpected error when reading back data") + assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Nil(ts.T(), err, "unexpected error when reading back data") - size, err := backend.GetFileSize("testFile") - assert.Nil(suite.T(), err, "posix NewFileReader failed when it should work") - assert.NotNil(suite.T(), size, "Got a nil size for posix") + size, err := backend.GetFileSize("testFile", false) + assert.Nil(ts.T(), err, "posix NewFileReader failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for posix") err = backend.RemoveFile("testFile") - assert.Nil(suite.T(), err, "posix RemoveFile failed when it should work") + assert.Nil(ts.T(), err, "posix RemoveFile failed when it should work") log.SetOutput(&buf) reader, err = backend.NewFileReader("posixDoesNotExist") - assert.Error(suite.T(), err, "posix NewFileReader worked when it should not") - assert.Nil(suite.T(), reader, "Got a non-nil reader for posix") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + assert.Error(ts.T(), err, "posix NewFileReader worked when it should not") + assert.Nil(ts.T(), reader, "Got a non-nil reader for posix") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") buf.Reset() - _, err = backend.GetFileSize("posixDoesNotExist") - assert.Error(suite.T(), err, "posix GetFileSize worked when it should not") - assert.NotZero(suite.T(), buf.Len(), "Expected warning missing") + _, err = backend.GetFileSize("posixDoesNotExist", false) + assert.Error(ts.T(), err, "posix GetFileSize worked when it should not") + assert.NotZero(ts.T(), buf.Len(), "Expected warning missing") log.SetOutput(os.Stdout) testConf.Posix.Location = "/thisdoesnotexist" backEnd, err := NewBackend(testConf) - assert.NotNil(suite.T(), err, "Backend worked when it should not") - assert.Nil(suite.T(), backEnd, "Got a backend when expected not to") + assert.NotNil(ts.T(), err, "Backend worked when it should not") + assert.Nil(ts.T(), backEnd, "Got a backend when expected not to") testConf.Posix.Location = "/etc/passwd" backEnd, err = NewBackend(testConf) - assert.NotNil(suite.T(), err, "Backend worked when it should not") - assert.Nil(suite.T(), backEnd, "Got a backend when expected not to") + assert.NotNil(ts.T(), err, "Backend worked when it should not") + assert.Nil(ts.T(), backEnd, "Got a backend when expected not to") var dummyBackend *posixBackend failReader, err := dummyBackend.NewFileReader("/") - assert.NotNil(suite.T(), err, "NewFileReader worked when it should not") - assert.Nil(suite.T(), failReader, "Got a Reader when expected not to") + assert.NotNil(ts.T(), err, "NewFileReader worked when it should not") + assert.Nil(ts.T(), failReader, "Got a Reader when expected not to") failWriter, err := dummyBackend.NewFileWriter("/") - assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") - assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") + assert.NotNil(ts.T(), err, "NewFileWriter worked when it should not") + assert.Nil(ts.T(), failWriter, "Got a Writer when expected not to") - _, err = dummyBackend.GetFileSize("/") - assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") + _, err = dummyBackend.GetFileSize("/", false) + assert.NotNil(ts.T(), err, "GetFileSize worked when it should not") err = dummyBackend.RemoveFile("/") - assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") + assert.NotNil(ts.T(), err, "RemoveFile worked when it should not") } -func (suite *StorageTestSuite) TestS3Backend() { +func (ts *StorageTestSuite) TestS3Backend() { testConf.Type = s3Type s3back, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend failed") + assert.NoError(ts.T(), err, "Backend failed") writer, err := s3back.NewFileWriter("s3Creatable") - assert.NotNil(suite.T(), writer, "Got a nil reader for writer from s3") - assert.Nil(suite.T(), err, "s3 NewFileWriter failed when it shouldn't") + assert.NotNil(ts.T(), writer, "Got a nil reader for writer from s3") + assert.Nil(ts.T(), err, "s3 NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - assert.Nil(suite.T(), err, "Failure when writing to s3 writer") - assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") + assert.Nil(ts.T(), err, "Failure when writing to s3 writer") + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") writer.Close() // sleep to allow the write to complete, otherwise the next step will fail due to timing issues. time.Sleep(1 * time.Second) reader, err := s3back.NewFileReader("s3Creatable") - assert.NoError(suite.T(), err, "s3 NewFileReader failed when it should work") - assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") + assert.NoError(ts.T(), err, "s3 NewFileReader failed when it should work") + assert.NotNil(ts.T(), reader, "Reader that should be usable is not, bailing out") - size, err := s3back.GetFileSize("s3Creatable") - assert.Nil(suite.T(), err, "s3 GetFileSize failed when it should work") - assert.NotNil(suite.T(), size, "Got a nil size for s3") - assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") + size, err := s3back.GetFileSize("s3Creatable", false) + assert.Nil(ts.T(), err, "s3 GetFileSize failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for s3") + assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") + + // make sure expectDelay=true works + // delete file and make sure the file size can not be retrieved anymore + err = s3back.RemoveFile("s3Creatable") + assert.Nil(ts.T(), err, "s3 RemoveFile failed when it should work") + _, err = s3back.GetFileSize("s3Creatable", true) + assert.NotNil(ts.T(), err, "s3 GetFileSize worked when it should not") + assert.Error(ts.T(), err) + // rewrite file, do not wait before retrieving file size + writer, err = s3back.NewFileWriter("s3Creatable") + assert.Nil(ts.T(), err, "s3 NewFileWriter failed when it shouldn't") + written, err = writer.Write(writeData) + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") + assert.Nil(ts.T(), err, "Failure when writing to s3 writer") + writer.Close() + size, err = s3back.GetFileSize("s3Creatable", true) + assert.Nil(ts.T(), err, "s3 GetFileSize with expected delay failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for s3") + assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") if err != nil && err != io.EOF { - assert.Nil(suite.T(), err, "unexpected error when reading back data") + assert.Nil(ts.T(), err, "unexpected error when reading back data") } err = s3back.RemoveFile("s3Creatable") - assert.Nil(suite.T(), err, "s3 RemoveFile failed when it should work") + assert.Nil(ts.T(), err, "s3 RemoveFile failed when it should work") - _, err = s3back.GetFileSize("s3DoesNotExist") - assert.NotNil(suite.T(), err, "s3 GetFileSize worked when it should not") - assert.Error(suite.T(), err) + _, err = s3back.GetFileSize("s3DoesNotExist", false) + assert.NotNil(ts.T(), err, "s3 GetFileSize worked when it should not") + assert.Error(ts.T(), err) reader, err = s3back.NewFileReader("s3DoesNotExist") - assert.NotNil(suite.T(), err, "s3 NewFileReader worked when it should not") - assert.Error(suite.T(), err) - assert.Nil(suite.T(), reader, "Got a non-nil reader for s3") + assert.NotNil(ts.T(), err, "s3 NewFileReader worked when it should not") + assert.Error(ts.T(), err) + assert.Nil(ts.T(), reader, "Got a non-nil reader for s3") testConf.S3.URL = "file://tmp/" _, err = NewBackend(testConf) - assert.Error(suite.T(), err, "Backend worked when it should not") + assert.Error(ts.T(), err, "Backend worked when it should not") } -func (suite *StorageTestSuite) TestSftpBackend() { +func (ts *StorageTestSuite) TestSftpBackend() { var buf bytes.Buffer log.SetOutput(&buf) testConf.Type = sftpType sftpBack, err := NewBackend(testConf) - assert.NoError(suite.T(), err, "Backend failed") + assert.NoError(ts.T(), err, "Backend failed") buf.Reset() var sftpDoesNotExist = "nonexistent/file" var sftpCreatable = "/share/file/exists" writer, err := sftpBack.NewFileWriter(sftpCreatable) - assert.NotNil(suite.T(), writer, "Got a nil reader for writer from sftp") - assert.Nil(suite.T(), err, "sftp NewFileWriter failed when it shouldn't") + assert.NotNil(ts.T(), writer, "Got a nil reader for writer from sftp") + assert.Nil(ts.T(), err, "sftp NewFileWriter failed when it shouldn't") written, err := writer.Write(writeData) - assert.Nil(suite.T(), err, "Failure when writing to sftp writer") - assert.Equal(suite.T(), len(writeData), written, "Did not write all writeData") + assert.Nil(ts.T(), err, "Failure when writing to sftp writer") + assert.Equal(ts.T(), len(writeData), written, "Did not write all writeData") writer.Close() reader, err := sftpBack.NewFileReader(sftpCreatable) - assert.Nil(suite.T(), err, "sftp NewFileReader failed when it should work") - assert.NotNil(suite.T(), reader, "Reader that should be usable is not, bailing out") + assert.Nil(ts.T(), err, "sftp NewFileReader failed when it should work") + assert.NotNil(ts.T(), reader, "Reader that should be usable is not, bailing out") - size, err := sftpBack.GetFileSize(sftpCreatable) - assert.Nil(suite.T(), err, "sftp GetFileSize failed when it should work") - assert.NotNil(suite.T(), size, "Got a nil size for sftp") - assert.Equal(suite.T(), int64(len(writeData)), size, "Got an incorrect file size") + size, err := sftpBack.GetFileSize(sftpCreatable, false) + assert.Nil(ts.T(), err, "sftp GetFileSize failed when it should work") + assert.NotNil(ts.T(), size, "Got a nil size for sftp") + assert.Equal(ts.T(), int64(len(writeData)), size, "Got an incorrect file size") err = sftpBack.RemoveFile(sftpCreatable) - assert.Nil(suite.T(), err, "sftp RemoveFile failed when it should work") + assert.Nil(ts.T(), err, "sftp RemoveFile failed when it should work") err = sftpBack.RemoveFile(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "failed to remove file with sftp, file does not exist") + assert.ErrorContains(ts.T(), err, "file does not exist") var readBackBuffer [4096]byte readBack, err := reader.Read(readBackBuffer[0:4096]) - assert.Equal(suite.T(), len(writeData), readBack, "did not read back data as expected") - assert.Equal(suite.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") + assert.Equal(ts.T(), len(writeData), readBack, "did not read back data as expected") + assert.Equal(ts.T(), writeData, readBackBuffer[:readBack], "did not read back data as expected") if err != nil && err != io.EOF { - assert.Nil(suite.T(), err, "unexpected error when reading back data") + assert.Nil(ts.T(), err, "unexpected error when reading back data") } - _, err = sftpBack.GetFileSize(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "failed to get file size with sftp, file does not exist") + _, err = sftpBack.GetFileSize(sftpDoesNotExist, false) + assert.EqualError(ts.T(), err, "failed to get file size with sftp, file does not exist") reader, err = sftpBack.NewFileReader(sftpDoesNotExist) - assert.EqualError(suite.T(), err, "failed to open file with sftp, file does not exist") - assert.Nil(suite.T(), reader, "Got a non-nil reader for sftp") + assert.EqualError(ts.T(), err, "failed to open file with sftp, file does not exist") + assert.Nil(ts.T(), reader, "Got a non-nil reader for sftp") // wrong host key testConf.SFTP.HostKey = "wronghostkey" _, err = NewBackend(testConf) - assert.ErrorContains(suite.T(), err, "failed to start ssh connection, ssh: handshake failed: host key verification expected") + assert.ErrorContains(ts.T(), err, "failed to start ssh connection, ssh: handshake failed: host key verification expected") // wrong key password testConf.SFTP.PemKeyPass = "wrongkey" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to parse private key, x509: decryption password incorrect") + assert.EqualError(ts.T(), err, "failed to parse private key, x509: decryption password incorrect") // missing key password testConf.SFTP.PemKeyPass = "" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to parse private key, ssh: this private key is passphrase protected") + assert.EqualError(ts.T(), err, "failed to parse private key, ssh: this private key is passphrase protected") // wrong key testConf.SFTP.PemKeyPath = "nonexistentkey" _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to read from key file, open nonexistentkey: no such file or directory") + assert.EqualError(ts.T(), err, "failed to read from key file, open nonexistentkey: no such file or directory") f, _ := os.CreateTemp(sshPath, "dummy") testConf.SFTP.PemKeyPath = f.Name() _, err = NewBackend(testConf) - assert.EqualError(suite.T(), err, "failed to parse private key, ssh: no key found") + assert.EqualError(ts.T(), err, "failed to parse private key, ssh: no key found") testConf.SFTP.Host = "nonexistenthost" _, err = NewBackend(testConf) - assert.NotNil(suite.T(), err, "Backend worked when it should not") + assert.NotNil(ts.T(), err, "Backend worked when it should not") var dummyBackend *sftpBackend failReader, err := dummyBackend.NewFileReader("/") - assert.NotNil(suite.T(), err, "NewFileReader worked when it should not") - assert.Nil(suite.T(), failReader, "Got a Reader when expected not to") + assert.NotNil(ts.T(), err, "NewFileReader worked when it should not") + assert.Nil(ts.T(), failReader, "Got a Reader when expected not to") failWriter, err := dummyBackend.NewFileWriter("/") - assert.NotNil(suite.T(), err, "NewFileWriter worked when it should not") - assert.Nil(suite.T(), failWriter, "Got a Writer when expected not to") + assert.NotNil(ts.T(), err, "NewFileWriter worked when it should not") + assert.Nil(ts.T(), failWriter, "Got a Writer when expected not to") - _, err = dummyBackend.GetFileSize("/") - assert.NotNil(suite.T(), err, "GetFileSize worked when it should not") + _, err = dummyBackend.GetFileSize("/", false) + assert.NotNil(ts.T(), err, "GetFileSize worked when it should not") err = dummyBackend.RemoveFile("/") - assert.NotNil(suite.T(), err, "RemoveFile worked when it should not") - assert.EqualError(suite.T(), err, "invalid sftpBackend") + assert.NotNil(ts.T(), err, "RemoveFile worked when it should not") + assert.EqualError(ts.T(), err, "invalid sftpBackend") } diff --git a/sda/internal/userauth/userauth.go b/sda/internal/userauth/userauth.go index ba9b8c0dff..13c73005e5 100644 --- a/sda/internal/userauth/userauth.go +++ b/sda/internal/userauth/userauth.go @@ -2,6 +2,7 @@ package userauth import ( "context" + "errors" "fmt" "net/http" "net/url" @@ -39,14 +40,13 @@ func NewValidateFromToken(keyset jwk.Set) *ValidateFromToken { func (u *ValidateFromToken) Authenticate(r *http.Request) (jwt.Token, error) { // Verify signature by parsing the token with the given key if u == nil { - return nil, fmt.Errorf("error validating token keyset") + return nil, errors.New("error validating token keyset") } switch { - case r.Header.Get("X-Amz-Security-Token") != "": tokenStr := r.Header.Get("X-Amz-Security-Token") if tokenStr == "" { - return nil, fmt.Errorf("no access token supplied") + return nil, errors.New("no access token supplied") } token, err := jwt.Parse([]byte(tokenStr), jwt.WithKeySet(u.Keyset, jws.WithInferAlgorithmFromKey(true)), jwt.WithValidate(true)) if err != nil { @@ -79,8 +79,7 @@ func (u *ValidateFromToken) Authenticate(r *http.Request) (jwt.Token, error) { return token, nil default: - return nil, fmt.Errorf("no access token supplied") - + return nil, errors.New("no access token supplied") } } @@ -155,10 +154,10 @@ func (u *ValidateFromToken) FetchJwtPubKeyURL(jwtpubkeyurl string) error { func readTokenFromHeader(authStr string) (string, error) { headerParts := strings.Split(authStr, " ") if headerParts[0] != "Bearer" { - return "", fmt.Errorf("authorization scheme must be bearer") + return "", errors.New("authorization scheme must be bearer") } if len(headerParts) != 2 || headerParts[1] == "" { - return "", fmt.Errorf("token string is missing from authorization header") + return "", errors.New("token string is missing from authorization header") } return headerParts[1], nil diff --git a/sda/internal/userauth/userauth_test.go b/sda/internal/userauth/userauth_test.go index f181f783a9..1a984172c1 100644 --- a/sda/internal/userauth/userauth_test.go +++ b/sda/internal/userauth/userauth_test.go @@ -33,7 +33,7 @@ func TestUserAuthTestSuite(t *testing.T) { suite.Run(t, new(UserAuthTest)) } -func (suite *UserAuthTest) SetupTest() { +func (ts *UserAuthTest) SetupTest() { } func TestMain(m *testing.M) { @@ -80,10 +80,10 @@ func TestMain(m *testing.M) { } OIDCport, _ = strconv.Atoi(oidc.GetPort("8080/tcp")) - OIDCHostAndPort := oidc.GetHostPort("8080/tcp") + oidcHostAndPort := oidc.GetHostPort("8080/tcp") client := http.Client{Timeout: 5 * time.Second} - req, err := http.NewRequest(http.MethodGet, "http://"+OIDCHostAndPort+"/jwk", http.NoBody) + req, err := http.NewRequest(http.MethodGet, "http://"+oidcHostAndPort+"/jwk", http.NoBody) if err != nil { log.Panic(err) } @@ -115,75 +115,75 @@ func TestMain(m *testing.M) { os.Exit(0) } -func (suite *UserAuthTest) TestAlwaysAuthenticator() { +func (ts *UserAuthTest) TestAlwaysAuthenticator() { a := helper.NewAlwaysAllow() r, _ := http.NewRequest("Get", "/", nil) _, err := a.Authenticate(r) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_NoFile() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_NoFile() { a := NewValidateFromToken(jwk.NewSet()) err := a.ReadJwtPubKeyPath("") - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_GetFile() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_GetFile() { demoKeysPath := "temp-keys" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateECkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" a := NewValidateFromToken(jwk.NewSet()) err = a.ReadJwtPubKeyPath(jwtpubkeypath) - assert.NoError(suite.T(), err) - assert.Equal(suite.T(), 2, a.Keyset.Len()) + assert.NoError(ts.T(), err) + assert.Equal(ts.T(), 2, a.Keyset.Len()) defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_WrongURL() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_WrongURL() { a := NewValidateFromToken(jwk.NewSet()) jwtpubkeyurl := "/dummy/" err := a.FetchJwtPubKeyURL(jwtpubkeyurl) - assert.Equal(suite.T(), "jwtpubkeyurl is not a proper URL (/dummy/)", err.Error()) + assert.Equal(ts.T(), "jwtpubkeyurl is not a proper URL (/dummy/)", err.Error()) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_BadURL() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_BadURL() { a := NewValidateFromToken(jwk.NewSet()) jwtpubkeyurl := "dummy.com/jwk" err := a.FetchJwtPubKeyURL(jwtpubkeyurl) - assert.Equal(suite.T(), "parse \"dummy.com/jwk\": invalid URI for request", err.Error()) + assert.Equal(ts.T(), "parse \"dummy.com/jwk\": invalid URI for request", err.Error()) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_GoodURL() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_GoodURL() { a := NewValidateFromToken(jwk.NewSet()) jwtpubkeyurl := fmt.Sprintf("http://localhost:%d/jwk", OIDCport) err := a.FetchJwtPubKeyURL(jwtpubkeyurl) - assert.NoError(suite.T(), err, "failed to fetch remote JWK") - assert.Equal(suite.T(), 3, a.Keyset.Len()) + assert.NoError(ts.T(), err, "failed to fetch remote JWK") + assert.Equal(ts.T(), 3, a.Keyset.Len()) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { // These tests should be possible to reuse with all correct authenticators somehow // Create temp demo rsa key pair demoKeysPath := "demo-rsa-keys" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" a := NewValidateFromToken(jwk.NewSet()) @@ -191,11 +191,11 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { // Parse demo private key prKeyParsed, err := helper.ParsePrivateRSAKey(prKeyPath, "/rsa") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create token and set up request defaults defaultToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ := http.NewRequest("", "/", nil) r.Host = "localhost" @@ -204,7 +204,7 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.Header.Set("X-Amz-Security-Token", "notJWT") r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) r.Header.Set("X-Amz-Security-Token", defaultToken) @@ -212,24 +212,24 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.URL.Path = "/dummy/" signer.SignV4(*r, "username", "testpass", "", "us-east-1") token, err := a.Authenticate(r) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) privateClaims := token.PrivateClaims() - assert.Equal(suite.T(), privateClaims["pilot"], helper.DefaultTokenClaims["pilot"]) + assert.Equal(ts.T(), privateClaims["pilot"], helper.DefaultTokenClaims["pilot"]) // Create and test expired Elixir token expiredToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.ExpiredClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", expiredToken) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) // Elixir token is not valid (e.g. issued in a future time) nonValidToken, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.NonValidClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" @@ -237,7 +237,7 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.URL.Path = "/username/" _, nonvalidToken := a.Authenticate(r) // The error output is huge so a smaller part is compared - assert.Equal(suite.T(), "\"iat\" not satisfied", nonvalidToken.Error()) + assert.Equal(ts.T(), "\"iat\" not satisfied", nonvalidToken.Error()) // Elixir tokens broken r, _ = http.NewRequest("", "/", nil) @@ -245,40 +245,40 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_RSA() { r.Header.Set("X-Amz-Security-Token", defaultToken[3:]) r.URL.Path = "/username/" _, brokenToken := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", "random"+defaultToken) r.URL.Path = "/username/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) _, brokenToken2 := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") // Bad issuer basIss, err := helper.CreateRSAToken(prKeyParsed, "RS256", helper.WrongTokenAlgClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", basIss) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Contains(suite.T(), err.Error(), "failed to get issuer from token") + assert.Contains(ts.T(), err.Error(), "failed to get issuer from token") // Delete the keys when testing is done or failed defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { // Create temp demo ec key pair demoKeysPath := "demo-ec-keys" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateECkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" @@ -287,11 +287,11 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { // Parse demo private key prKeyParsed, err := helper.ParsePrivateECKey(prKeyPath, "/ec") - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create token and set up request defaults defaultToken, err := helper.CreateECToken(prKeyParsed, "ES256", helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ := http.NewRequest("", "/", nil) r.Host = "localhost" @@ -301,22 +301,22 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { r.URL.Path = "/dummy/" signer.SignV4(*r, "username", "testpass", "", "us-east-1") _, err = a.Authenticate(r) - assert.Nil(suite.T(), err) + assert.Nil(ts.T(), err) // Create and test expired Elixir token expiredToken, err := helper.CreateECToken(prKeyParsed, "ES256", helper.ExpiredClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", expiredToken) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Error(suite.T(), err) + assert.Error(ts.T(), err) // Elixir token is not valid nonValidToken, err := helper.CreateECToken(prKeyParsed, "ES256", helper.NonValidClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" @@ -324,7 +324,7 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { r.URL.Path = "/username/" _, nonvalidToken := a.Authenticate(r) // The error output is huge so a smaller part is compared - assert.Equal(suite.T(), "\"iat\" not satisfied", nonvalidToken.Error()) + assert.Equal(ts.T(), "\"iat\" not satisfied", nonvalidToken.Error()) // Elixir tokens broken r, _ = http.NewRequest("", "/", nil) @@ -332,38 +332,38 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_EC() { r.Header.Set("X-Amz-Security-Token", defaultToken[3:]) r.URL.Path = "/username/" _, brokenToken := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken.Error(), "failed to parse jws: failed to parse JOSE headers:") r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", "random"+defaultToken) r.URL.Path = "/username/" _, brokenToken2 := a.Authenticate(r) - assert.Contains(suite.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") + assert.Contains(ts.T(), brokenToken2.Error(), "failed to parse jws: failed to parse JOSE headers:") // Bad issuer basIss, err := helper.CreateECToken(prKeyParsed, "ES256", helper.WrongTokenAlgClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) r, _ = http.NewRequest("", "/", nil) r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", basIss) r.URL.Path = "/dummy/" _, err = a.Authenticate(r) - assert.Contains(suite.T(), err.Error(), "failed to get issuer from token") + assert.Contains(ts.T(), err.Error(), "failed to get issuer from token") defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestWrongKeyType_RSA() { +func (ts *UserAuthTest) TestWrongKeyType_RSA() { // Create temp demo ec key pair demoKeysPath := "demo-ec-keys" demoPrKeyName := "/ec" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateECkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" @@ -372,20 +372,20 @@ func (suite *UserAuthTest) TestWrongKeyType_RSA() { // Parse demo private key _, err = helper.ParsePrivateRSAKey(prKeyPath, demoPrKeyName) - assert.Equal(suite.T(), "bad key format, expected RSA got EC", err.Error()) + assert.Equal(ts.T(), "bad key format, expected RSA got EC", err.Error()) defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestWrongKeyType_EC() { +func (ts *UserAuthTest) TestWrongKeyType_EC() { // Create temp demo ec key pair demoKeysPath := "demo-rsa-keys" demoPrKeyName := "/rsa" prKeyPath, pubKeyPath, err := helper.MakeFolder(demoKeysPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) err = helper.CreateRSAkeys(prKeyPath, pubKeyPath) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) jwtpubkeypath := demoKeysPath + "/public-key/" @@ -394,20 +394,20 @@ func (suite *UserAuthTest) TestWrongKeyType_EC() { // Parse demo private key _, err = helper.ParsePrivateECKey(prKeyPath, demoPrKeyName) - assert.Equal(suite.T(), "bad key format, expected EC got RSA", err.Error()) + assert.Equal(ts.T(), "bad key format, expected EC got RSA", err.Error()) defer os.RemoveAll(demoKeysPath) } -func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_HS() { +func (ts *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_HS() { // Create random secret key := make([]byte, 256) _, err := rand.Read(key) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) // Create HS256 token wrongAlgToken, err := helper.CreateHSToken(key, helper.DefaultTokenClaims) - assert.NoError(suite.T(), err) + assert.NoError(ts.T(), err) a := NewValidateFromToken(jwk.NewSet()) @@ -415,8 +415,8 @@ func (suite *UserAuthTest) TestUserTokenAuthenticator_ValidateSignature_HS() { r.Host = "localhost" r.Header.Set("X-Amz-Security-Token", wrongAlgToken) r.URL.Path = "/username/" - _, WrongAlg := a.Authenticate(r) - assert.Contains(suite.T(), WrongAlg.Error(), "failed to find key with key ID") + _, wrongAlg := a.Authenticate(r) + assert.Contains(ts.T(), wrongAlg.Error(), "failed to find key with key ID") } func TestGetBearerToken(t *testing.T) {