diff --git a/news/209.feature b/news/209.feature new file mode 100644 index 0000000..fd14765 --- /dev/null +++ b/news/209.feature @@ -0,0 +1 @@ +Added support for zodb-pgjsonb storage, configured with `ZODB_PGJSONB_DSN`, with optional blob storage in S3 enabled by `ZODB_PGJSONB_S3BLOBS_ENABLED`. Projects built on these images need to add `zodb-pgjsonb[s3]` to their dependencies, which requires Python 3.12 or later. @erral diff --git a/skeleton/docker-entrypoint.sh b/skeleton/docker-entrypoint.sh index 779bb1d..e8c7ea4 100755 --- a/skeleton/docker-entrypoint.sh +++ b/skeleton/docker-entrypoint.sh @@ -64,6 +64,42 @@ elif [[ -v ZEO_ADDRESS ]]; then [ -z ${ZEO_STORAGE+x} ] && export ZEO_STORAGE=1 [ -z ${ZEO_CLIENT_CACHE_SIZE+x} ] && export ZEO_CLIENT_CACHE_SIZE=128MB [ -z ${ZEO_DROP_CACHE_RATHER_VERIFY+x} ] && export ZEO_DROP_CACHE_RATHER_VERIFY=false +elif [[ -v ZODB_PGJSONB_DSN ]]; then + MSG="Using zodb-pgjsonb configuration" + CONF=zodb-pgjsonb.conf + # Check zodb-pgjsonb variables + [ -z ${ZODB_PGJSONB_HISTORY_PRESERVING+x} ] && export ZODB_PGJSONB_HISTORY_PRESERVING=false + + if [[ -v ZODB_PGJSONB_S3BLOBS_ENABLED && $ZODB_PGJSONB_S3BLOBS_ENABLED == "true" ]]; then + + if [[ -n "$ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL" && \ + -n "$ZODB_PGJSONB_S3BLOBS_BUCKET_NAME" && \ + -n "$ZODB_PGJSONB_S3BLOBS_ACCESS_KEY" && \ + -n "$ZODB_PGJSONB_S3BLOBS_SECRET_KEY" ]]; then + + MSG="Using zodb-pgjsonb S3 blobs configuration" + CONF=zodb-pgjsonb-s3blobs.conf + + # An endpoint URL with an explicit scheme takes precedence over USE_SSL + [ -z ${ZODB_PGJSONB_S3BLOBS_USE_SSL+x} ] && export ZODB_PGJSONB_S3BLOBS_USE_SSL=true + # An empty region is not passed on, so boto3 picks its own default + [ -z ${ZODB_PGJSONB_S3BLOBS_REGION+x} ] && export ZODB_PGJSONB_S3BLOBS_REGION= + [ -z ${ZODB_PGJSONB_S3BLOBS_PREFIX+x} ] && export ZODB_PGJSONB_S3BLOBS_PREFIX= + [ -z ${ZODB_PGJSONB_S3BLOBS_THRESHOLD+x} ] && export ZODB_PGJSONB_S3BLOBS_THRESHOLD=100KB + [ -z ${ZODB_PGJSONB_S3BLOBS_CACHE_DIR+x} ] && export ZODB_PGJSONB_S3BLOBS_CACHE_DIR=/data/blobcache + [ -z ${ZODB_PGJSONB_S3BLOBS_CACHE_SIZE+x} ] && export ZODB_PGJSONB_S3BLOBS_CACHE_SIZE=1GB + + else + # If the required environment vars are missing, print an error message and exit. + # Credentials are reported as set or missing, never printed. + echo "ERROR: You have enabled the usage of S3 blobs, but some required data is missing:" + echo "- Endpoint URL: ${ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL:-MISSING}" + echo "- Bucket: ${ZODB_PGJSONB_S3BLOBS_BUCKET_NAME:-MISSING}" + echo "- Access Key: $([[ -n "$ZODB_PGJSONB_S3BLOBS_ACCESS_KEY" ]] && echo set || echo MISSING)" + echo "- Secret Key: $([[ -n "$ZODB_PGJSONB_S3BLOBS_SECRET_KEY" ]] && echo set || echo MISSING)" + exit 1 + fi + fi else MSG="Using default configuration" CONF=zope.conf diff --git a/skeleton/etc/zodb-pgjsonb-s3blobs.conf b/skeleton/etc/zodb-pgjsonb-s3blobs.conf new file mode 100644 index 0000000..cc6a068 --- /dev/null +++ b/skeleton/etc/zodb-pgjsonb-s3blobs.conf @@ -0,0 +1,45 @@ +%define INSTANCE /app +instancehome $INSTANCE + +%define CLIENTHOME $(CLIENT_HOME) +clienthome $CLIENTHOME + +debug-mode $(DEBUG_MODE) +security-policy-implementation $(SECURITY_POLICY_IMPLEMENTATION) +verbose-security $(VERBOSE_SECURITY) +default-zpublisher-encoding $(DEFAULT_ZPUBLISHER_ENCODING) + + + CHAMELEON_CACHE $INSTANCE/var/cache + + + + form-memory-limit $(ZOPE_FORM_MEMORY_LIMIT) + form-disk-limit $(ZOPE_FORM_DISK_LIMIT) + form-memfile-limit $(ZOPE_FORM_MEMFILE_LIMIT) + + + + # zodb-pgjsonb database, with large blobs stored in S3 + %import zodb_pgjsonb + + dsn $(ZODB_PGJSONB_DSN) + history-preserving $(ZODB_PGJSONB_HISTORY_PRESERVING) + + s3-endpoint-url $(ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL) + s3-use-ssl $(ZODB_PGJSONB_S3BLOBS_USE_SSL) + s3-bucket-name $(ZODB_PGJSONB_S3BLOBS_BUCKET_NAME) + s3-access-key $(ZODB_PGJSONB_S3BLOBS_ACCESS_KEY) + s3-secret-key $(ZODB_PGJSONB_S3BLOBS_SECRET_KEY) + s3-region $(ZODB_PGJSONB_S3BLOBS_REGION) + s3-prefix $(ZODB_PGJSONB_S3BLOBS_PREFIX) + + blob-threshold $(ZODB_PGJSONB_S3BLOBS_THRESHOLD) + blob-cache-dir $(ZODB_PGJSONB_S3BLOBS_CACHE_DIR) + blob-cache-size $(ZODB_PGJSONB_S3BLOBS_CACHE_SIZE) + + + + mount-point / + cache-size $(ZODB_CACHE_SIZE) + diff --git a/skeleton/etc/zodb-pgjsonb.conf b/skeleton/etc/zodb-pgjsonb.conf new file mode 100644 index 0000000..1a6dfdf --- /dev/null +++ b/skeleton/etc/zodb-pgjsonb.conf @@ -0,0 +1,32 @@ +%define INSTANCE /app +instancehome $INSTANCE + +%define CLIENTHOME $(CLIENT_HOME) +clienthome $CLIENTHOME + +debug-mode $(DEBUG_MODE) +security-policy-implementation $(SECURITY_POLICY_IMPLEMENTATION) +verbose-security $(VERBOSE_SECURITY) +default-zpublisher-encoding $(DEFAULT_ZPUBLISHER_ENCODING) + + + CHAMELEON_CACHE $INSTANCE/var/cache + + + + form-memory-limit $(ZOPE_FORM_MEMORY_LIMIT) + form-disk-limit $(ZOPE_FORM_DISK_LIMIT) + form-memfile-limit $(ZOPE_FORM_MEMFILE_LIMIT) + + + + # Main database + %import zodb_pgjsonb + + dsn $(ZODB_PGJSONB_DSN) + history-preserving $(ZODB_PGJSONB_HISTORY_PRESERVING) + + + mount-point / + cache-size $(ZODB_CACHE_SIZE) + diff --git a/test/config.sh b/test/config.sh index 5f21f3c..0367325 100644 --- a/test/config.sh +++ b/test/config.sh @@ -16,6 +16,8 @@ plone-listenport plone-zeoclient plone-relstorage plone-shared-blob-dir +plone-pgjsonb +plone-pgjsonb-s3blobs ' imageTests+=( diff --git a/test/tests/plone-pgjsonb-s3blobs/run.sh b/test/tests/plone-pgjsonb-s3blobs/run.sh new file mode 100755 index 0000000..11a3456 --- /dev/null +++ b/test/tests/plone-pgjsonb-s3blobs/run.sh @@ -0,0 +1,90 @@ +#!/bin/bash +set -eo pipefail + +dir="$(dirname "$(readlink -f "$BASH_SOURCE")")" + +image="$1" + +PLONE_TEST_SLEEP=10 +PLONE_TEST_TRIES=10 + +DSN="dbname='plone' user='plone' host='db' password='plone'" +S3_ACCESS_KEY="access-$RANDOM-$RANDOM" +S3_SECRET_KEY="s3cr3t-$RANDOM-$RANDOM" +S3_BUCKET="plone" + +# Enabling S3 blobs with missing settings must fail without printing credentials +if out="$(docker run --rm \ + -e ZODB_PGJSONB_DSN="$DSN" \ + -e ZODB_PGJSONB_S3BLOBS_ENABLED=true \ + -e ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL="http://s3:5000" \ + -e ZODB_PGJSONB_S3BLOBS_ACCESS_KEY="$S3_ACCESS_KEY" \ + -e ZODB_PGJSONB_S3BLOBS_SECRET_KEY="$S3_SECRET_KEY" \ + "$image" true 2>&1)"; then + echo >&2 "container started although the S3 bucket name is missing" + false +fi +[[ "$out" == *"Bucket: MISSING"* ]] +[[ "$out" == *"Secret Key: set"* ]] +[[ "$out" != *"$S3_SECRET_KEY"* ]] +[[ "$out" != *"$S3_ACCESS_KEY"* ]] + +# Start Postgres +zname="pgjsonb-container-$RANDOM-$RANDOM" +zpull="$(docker pull postgres:17)" +zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:17)" + +# Start an S3-compatible server +sname="s3-container-$RANDOM-$RANDOM" +spull="$(docker pull motoserver/moto:5.2.3)" +sid="$(docker run -d --name "$sname" motoserver/moto:5.2.3)" + +pname="plone-container-$RANDOM-$RANDOM" + +# Tear down +trap "docker rm -vf $sid $zid > /dev/null; docker rm -vf $pname > /dev/null 2>&1 || true" EXIT + +s3() { + docker run --rm -i \ + --link "$sname":s3 \ + --entrypoint /app/bin/python \ + "$image" \ + -c "import boto3; s3 = boto3.client('s3', endpoint_url='http://s3:5000', region_name='us-east-1', aws_access_key_id='$S3_ACCESS_KEY', aws_secret_access_key='$S3_SECRET_KEY'); $1" +} + +# Create the bucket once the S3 server accepts connections +. "$dir/../../retry.sh" --tries "$PLONE_TEST_TRIES" --sleep 2 s3 "\"s3.create_bucket(Bucket='$S3_BUCKET')\"" + +plone_env=( + -e ZODB_PGJSONB_DSN="$DSN" + -e ZODB_PGJSONB_S3BLOBS_ENABLED=true + -e ZODB_PGJSONB_S3BLOBS_ENDPOINT_URL="http://s3:5000" + -e ZODB_PGJSONB_S3BLOBS_BUCKET_NAME="$S3_BUCKET" + -e ZODB_PGJSONB_S3BLOBS_ACCESS_KEY="$S3_ACCESS_KEY" + -e ZODB_PGJSONB_S3BLOBS_SECRET_KEY="$S3_SECRET_KEY" +) + +# Start Plone as zodb-pgjsonb client with blobs in S3 +pid="$(docker run -d --name "$pname" --link=$zname:db --link=$sname:s3 "${plone_env[@]}" "$image")" + +get() { + docker run --rm -i \ + --link "$pname":plone \ + --entrypoint /app/bin/python \ + "$image" \ + -c "from urllib.request import urlopen; con = urlopen('$1'); print(con.read())" +} + +. "$dir/../../retry.sh" --tries "$PLONE_TEST_TRIES" --sleep "$PLONE_TEST_SLEEP" get "http://plone:8080" + +# Plone is up and running +[[ "$(get 'http://plone:8080')" == *"Welcome to Plone!"* ]] + +# The zodb-pgjsonb S3 blobs configuration was used +[[ "$(docker logs "$pid" 2>&1)" == *"Using zodb-pgjsonb S3 blobs configuration"* ]] + +# A blob above the size threshold ends up in the bucket +docker run --rm --link=$zname:db --link=$sname:s3 "${plone_env[@]}" \ + -v "$dir/write_blob.py:/app/write_blob.py:ro" \ + "$image" run /app/write_blob.py | grep -q "blob committed" +[[ "$(s3 "print(s3.list_objects_v2(Bucket='$S3_BUCKET').get('KeyCount', 0))")" -ge 1 ]] diff --git a/test/tests/plone-pgjsonb-s3blobs/write_blob.py b/test/tests/plone-pgjsonb-s3blobs/write_blob.py new file mode 100644 index 0000000..f0026b1 --- /dev/null +++ b/test/tests/plone-pgjsonb-s3blobs/write_blob.py @@ -0,0 +1,15 @@ +"""Store a blob larger than the S3 threshold in the ZODB root. + +Run with ``zconsole run``, which provides ``app``. +""" + +import transaction +from ZODB.blob import Blob + +blob = Blob() +with blob.open("w") as fh: + fh.write(b"x" * 200 * 1024) + +app._p_jar.root()["s3blob-test"] = blob # noqa: F821 +transaction.commit() +print("blob committed") diff --git a/test/tests/plone-pgjsonb/run.sh b/test/tests/plone-pgjsonb/run.sh new file mode 100755 index 0000000..c1b04dd --- /dev/null +++ b/test/tests/plone-pgjsonb/run.sh @@ -0,0 +1,37 @@ +#!/bin/bash +set -eo pipefail + +dir="$(dirname "$(readlink -f "$BASH_SOURCE")")" + +image="$1" + +PLONE_TEST_SLEEP=10 +PLONE_TEST_TRIES=10 + +# Start Postgres +zname="pgjsonb-container-$RANDOM-$RANDOM" +zpull="$(docker pull postgres:17)" +zid="$(docker run -d --name "$zname" -e POSTGRES_USER=plone -e POSTGRES_PASSWORD=plone -e POSTGRES_DB=plone postgres:17)" + +# Start Plone as zodb-pgjsonb client +pname="plone-container-$RANDOM-$RANDOM" +pid="$(docker run -d --name "$pname" --link=$zname:db -e ZODB_PGJSONB_DSN="dbname='plone' user='plone' host='db' password='plone'" "$image")" + +# Tear down +trap "docker rm -vf $pid $zid > /dev/null" EXIT + +get() { + docker run --rm -i \ + --link "$pname":plone \ + --entrypoint /app/bin/python \ + "$image" \ + -c "from urllib.request import urlopen; con = urlopen('$1'); print(con.read())" +} + +. "$dir/../../retry.sh" --tries "$PLONE_TEST_TRIES" --sleep "$PLONE_TEST_SLEEP" get "http://plone:8080" + +# Plone is up and running +[[ "$(get 'http://plone:8080')" == *"Welcome to Plone!"* ]] + +# The zodb-pgjsonb configuration was used +[[ "$(docker logs "$pid" 2>&1)" == *"Using zodb-pgjsonb configuration"* ]]