diff --git a/Dockerfile.rocky9 b/Dockerfile.rocky9 index 3104dcf7..4f712902 100755 --- a/Dockerfile.rocky9 +++ b/Dockerfile.rocky9 @@ -127,6 +127,7 @@ ARG ENABLE_TWOFACTOR_STRICT_ADDRESS=False ARG TWOFACTOR_AUTH_APPS="" ARG ENABLE_PEERS=True ARG ENABLE_QUOTA=False +ARG ENABLE_ACCOUNTING=False ARG PEERS_MANDATORY=False ARG PEERS_EXPLICIT_FIELDS="" ARG PEERS_CONTACT_HINT="authorized to invite you as peer" @@ -230,6 +231,7 @@ ARG QUOTA_LUSTRE_VERSION="2.15.4" ARG QUOTA_LUSTRE_BASE="/dev/null" ARG QUOTA_GOCRYPTFS_XRAY="/dev/null" ARG QUOTA_GOCRYPTFS_SOCK="/dev/null" +ARG ACCOUNTING_UPDATE_INTERVAL=3600 # Jupyter Arguments ARG JUPYTER_SERVICES="" @@ -1030,6 +1032,7 @@ ARG ENABLE_TWOFACTOR_STRICT_ADDRESS ARG TWOFACTOR_AUTH_APPS ARG ENABLE_PEERS ARG ENABLE_QUOTA +ARG ENABLE_ACCOUNTING ARG PEERS_MANDATORY ARG PEERS_EXPLICIT_FIELDS ARG PEERS_CONTACT_HINT @@ -1111,6 +1114,7 @@ ARG QUOTA_UPDATE_INTERVAL ARG QUOTA_USER_LIMIT ARG QUOTA_VGRID_LIMIT ARG QUOTA_LUSTRE_VERSION +ARG ACCOUNTING_UPDATE_INTERVAL # TODO: do we still need the ~/.local/ wrapper now that update-alternatives run? ENV PYTHONPATH=${MIG_ROOT} @@ -1244,6 +1248,7 @@ RUN ./generateconfs.py --source=. \ --enable_quota=${ENABLE_QUOTA} --quota_backend="${QUOTA_BACKEND}" \ --quota_update_interval=${QUOTA_UPDATE_INTERVAL} \ --quota_user_limit=${QUOTA_USER_LIMIT} --quota_vgrid_limit=${QUOTA_VGRID_LIMIT} \ + --enable_accounting=${ENABLE_ACCOUNTING} --accounting_update_interval=${ACCOUNTING_UPDATE_INTERVAL} \ --storage_protocols="${STORAGE_PROTOCOLS}" \ --wwwserve_max_bytes=${WWWSERVE_MAX_BYTES} \ --password_policy=${MIG_PASSWORD_POLICY} \ diff --git a/Makefile b/Makefile index 37f09f71..8734b1b3 100755 --- a/Makefile +++ b/Makefile @@ -112,6 +112,7 @@ initdirs: initcomposevars mkdir -p ${PERSISTENT_ROOT}/events_home mkdir -p ${PERSISTENT_ROOT}/sitestats_home mkdir -p ${PERSISTENT_ROOT}/quota_home + mkdir -p ${PERSISTENT_ROOT}/accounting_home mkdir -p ${PERSISTENT_ROOT}/sandbox_home mkdir -p ${PERSISTENT_ROOT}/sss_home mkdir -p ${PERSISTENT_ROOT}/workflows_db_home @@ -143,6 +144,7 @@ initdirs: initcomposevars mkdir -p ${LOG_ROOT}/syslog/migrid-webdavs mkdir -p ${LOG_ROOT}/syslog/migrid-ftps mkdir -p ${LOG_ROOT}/syslog/migrid-quota + mkdir -p ${LOG_ROOT}/syslog/migrid-accounting initcomposevars: @echo "creating env variable map in docker-compose_shared.yml" @@ -180,6 +182,10 @@ initservices: && "${ENABLE_QUOTA}" == "True" ]]; then @ENABLED_SERVICES+=" $$service" @fi + @if [[ "$$service" == "migrid-accounting" \ + && "${ENABLE_ACCOUNTING}" == "True" ]]; then + @ENABLED_SERVICES+=" $$service" + @fi @done; @echo $$ENABLED_SERVICES > ./.migrid_enabled_services diff --git a/development.env b/development.env index 42133383..cfd21a84 100644 --- a/development.env +++ b/development.env @@ -111,6 +111,7 @@ ENABLE_JOBS=True ENABLE_RESOURCES=True ENABLE_EVENTS=True ENABLE_QUOTA=False +ENABLE_ACCOUNTING=False ENABLE_GRAVATARS=True ENABLE_SITESTATUS=True STATUS_SYSTEM_MATCH=ANY diff --git a/development_gdp.env b/development_gdp.env index 87a9bd41..d4dd91be 100644 --- a/development_gdp.env +++ b/development_gdp.env @@ -111,6 +111,7 @@ ENABLE_JOBS=False ENABLE_RESOURCES=False ENABLE_EVENTS=False ENABLE_QUOTA=False +ENABLE_ACCOUNTING=False ENABLE_GRAVATARS=False ENABLE_SITESTATUS=True STATUS_SYSTEM_MATCH=ANY diff --git a/doc/source/sections/configuration/variables.rst b/doc/source/sections/configuration/variables.rst index 96e6dae4..96a28fa1 100644 --- a/doc/source/sections/configuration/variables.rst +++ b/doc/source/sections/configuration/variables.rst @@ -436,7 +436,10 @@ Variables - Enable the built-in janitor service to handle recurring tasks like clean up and cache update * - ENABLE_QUOTA - False - - Enable additional quota integration in the user pages if fundamentally enabled with the QUOTA_X variables. + - Enable additional storage quota daemon if fundamentally enabled with the QUOTA_X variables. + * - ENABLE_ACCOUNTING + - False + - Enable additional storage accounting daemon and integration in the user pages. * - ENABLE_GDP - False - Enable GDP mode for sensitive data with a lot of restrictions on access and logging @@ -656,4 +659,7 @@ Variables * - QUOTA_GOCRYPTFS_SOCK - "/dev/null" - A gocryptfs socket is needed by gocryptfs-xray to resolve encoded data paths from MiG data paths. + * - ACCOUNTING_UPDATE_INTERVAL + - 3600 + - The frequency (seconds) of storage accounting updates. diff --git a/docker-compose_production.yml b/docker-compose_production.yml index 5fd9c3f0..64ef5bc4 100644 --- a/docker-compose_production.yml +++ b/docker-compose_production.yml @@ -33,6 +33,7 @@ services: # Available target service names follow migrid init script and are: # httpd script monitor sshmux events cron transfers janitor # openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota + # accounting RUN_SERVICES: volumes: - type: volume @@ -154,6 +155,9 @@ services: - type: volume source: quota_home target: /home/mig/state/quota_home + - type: volume + source: accounting_home + target: /home/mig/state/accounting_home - type: volume source: sandbox_home target: /home/mig/state/sandbox_home @@ -236,6 +240,7 @@ services: # Available target service names follow migrid init script and are: # httpd script monitor sshmux events cron transfers janitor # openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota + # accounting RUN_SERVICES: openid rsyslogd depends_on: - migrid @@ -306,6 +311,9 @@ services: # source: quota_home # target: /home/mig/state/quota_home #- type: volume + # source: accounting_home + # target: /home/mig/state/accounting_home + #- type: volume # source: sandbox_home # target: /home/mig/state/sandbox_home #- type: volume @@ -381,6 +389,7 @@ services: # Available target service names follow migrid init script and are: # httpd script monitor sshmux events cron transfers janitor # openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota + # accounting RUN_SERVICES: sftp sftpsubsys rsyslogd depends_on: - migrid @@ -452,6 +461,9 @@ services: # source: quota_home # target: /home/mig/state/quota_home #- type: volume + # source: accounting_home + # target: /home/mig/state/accounting_home + #- type: volume # source: sandbox_home # target: /home/mig/state/sandbox_home #- type: volume @@ -527,6 +539,7 @@ services: # Available target service names follow migrid init script and are: # httpd script monitor sshmux events cron transfers janitor # openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota + # accounting RUN_SERVICES: ftps rsyslogd depends_on: - migrid @@ -597,6 +610,9 @@ services: # source: quota_home # target: /home/mig/state/quota_home #- type: volume + # source: accounting_home + # target: /home/mig/state/accounting_home + #- type: volume # source: sandbox_home # target: /home/mig/state/sandbox_home #- type: volume @@ -672,6 +688,7 @@ services: # Available target service names follow migrid init script and are: # httpd script monitor sshmux events cron transfers janitor # openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota + # accounting RUN_SERVICES: webdavs rsyslogd depends_on: - migrid @@ -817,6 +834,7 @@ services: # Available target service names follow migrid init script and are: # httpd script monitor sshmux events cron transfers janitor # openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota + # accounting RUN_SERVICES: quota rsyslogd depends_on: - migrid @@ -870,6 +888,58 @@ services: target: /home/mig/state/vgrid_files_writable command: /app/docker-entry.sh -k + migrid-accounting: + container_name: migrid-accounting + image: ${CONTAINER_REGISTRY}/ucphhpc/migrid${CONTAINER_TAG} + network_mode: host + environment: + TZ: ${TZ} + # RUN_SERVICES specifies which daemons to launch + # Available target service names follow migrid init script and are: + # httpd script monitor sshmux events cron transfers janitor + # openid sftp sftpsubsys webdavs ftps notify imnotify vmproxy quota + # accounting + RUN_SERVICES: accounting rsyslogd + depends_on: + # IMPORTANT: finish volume init before migrid launch as explained above + migrid-volume-init: + condition: service_completed_successfully + volumes: + - type: volume + source: mig + target: /home/mig/mig + - type: volume + source: hotfixes + target: /hotfixes + - type: volume + source: mig-server-extconfs + target: /home/mig/mig/server/MiGserver.d + - type: volume + source: state + target: /home/mig/state + - type: volume + source: vgrid_home + target: /home/mig/state/vgrid_home + - type: volume + source: freeze_home + target: /home/mig/state/freeze_home + - type: volume + source: user_settings + target: /home/mig/state/user_settings + - type: volume + source: migrid-accounting-syslog + target: /var/log + - type: volume + source: log + target: /home/mig/state/log + - type: volume + source: quota_home + target: /home/mig/state/quota_home + - type: volume + source: accounting_home + target: /home/mig/state/accounting_home + command: /app/docker-entry.sh -k + # NOTE: not used in stand-alone production mode # nginx-proxy: # image: ${CONTAINER_REGISTRY}/jwilder/nginx-proxy @@ -1045,6 +1115,14 @@ volumes: device: ${PERSISTENT_ROOT}/quota_home o: bind + accounting_home: + # Volume used to contain the migrid accounting_home + driver: local + driver_opts: + type: none + device: ${PERSISTENT_ROOT}/accounting_home + o: bind + sandbox_home: # Volume used to contain the migrid sandbox_home driver: local @@ -1222,9 +1300,17 @@ volumes: o: bind migrid-quota-syslog: - # Volume used for exposing migrid lustre quota container system log + # Volume used for exposing migrid quota container system log driver: local driver_opts: type: none device: ${LOG_ROOT}/syslog/migrid-quota o: bind + + migrid-accounting-syslog: + # Volume used for exposing migrid accounting container system log + driver: local + driver_opts: + type: none + device: ${LOG_ROOT}/syslog/migrid-accounting + o: bind diff --git a/production.env b/production.env index 8ad28638..0f0c7309 100644 --- a/production.env +++ b/production.env @@ -111,6 +111,7 @@ ENABLE_JOBS=True ENABLE_RESOURCES=True ENABLE_EVENTS=True ENABLE_QUOTA=False +ENABLE_ACCOUNTING=False ENABLE_GRAVATARS=True ENABLE_SITESTATUS=True STATUS_SYSTEM_MATCH=ANY @@ -274,3 +275,6 @@ QUOTA_LUSTRE_VERSION=2.15.4 QUOTA_LUSTRE_BASE="/dev/null" QUOTA_GOCRYPTFS_XRAY="/dev/null" QUOTA_GOCRYPTFS_SOCK="/dev/null" + +# Accounting settings +ACCOUNTING_UPDATE_INTERVAL=3600