diff --git a/ci/dockerfiles/main-postgres/Dockerfile b/ci/dockerfiles/main-postgres/Dockerfile index d58dc64d9c2..4fc8db712d6 100644 --- a/ci/dockerfiles/main-postgres/Dockerfile +++ b/ci/dockerfiles/main-postgres/Dockerfile @@ -18,4 +18,4 @@ RUN echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg m "postgresql-${DB_VERSION}" \ "postgresql-client-${DB_VERSION}" -RUN sed -i 's/port = 5433/port = 5432/g' "/etc/postgresql/${DB_VERSION}/main/postgresql.conf" +RUN sed -i 's/port = 5433/port = 5432/g' "/etc/postgresql/${DB_VERSION}/main/postgresql.conf" 2>/dev/null || true diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 66cf0e6a982..d9d1c7a7ea1 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -12,8 +12,8 @@ groups: - unit-release - unit-director-sqlite - unit-director-mysql-8 - - unit-director-postgres-13 - unit-director-postgres-15 + - unit-director-postgres-17 - run-bundle-audit - integration-postgres - integration-postgres-hotswap @@ -25,6 +25,10 @@ groups: - bats-fips - bats-fips-cleanup-leftovers - brats-acceptance + - brats-postgres-15 + - brats-postgres-16 + - brats-postgres-17 + - brats-postgres-18 - brats-utils - brats-performance - bosh-disaster-recovery-acceptance-tests @@ -44,8 +48,8 @@ groups: - name: container-images jobs: - - build-main-postgres-13 - build-main-postgres-15 + - build-main-postgres-17 - build-main-mysql-8-0 - build-integration - build-docker-cpi @@ -67,8 +71,8 @@ jobs: - unit-release - unit-director-sqlite - unit-director-mysql-8 - - unit-director-postgres-13 - unit-director-postgres-15 + - unit-director-postgres-17 - run-bundle-audit plan: - get: bosh @@ -202,10 +206,10 @@ jobs: DB: mysql RAKE_TASK: spec:unit:director - - name: unit-director-postgres-13 + - name: unit-director-postgres-17 public: true serial: true - serial_groups: [ unit-director-postgres-13 ] + serial_groups: [ unit-director-postgres-17 ] build_log_retention: builds: 250 plan: @@ -214,15 +218,16 @@ jobs: - get: bosh passed: [ gate ] trigger: true - - get: integration-postgres-13-image + - get: integration-postgres-17-image - task: test-rake-task file: bosh-ci/ci/tasks/test-rake-task.yml - image: integration-postgres-13-image + image: integration-postgres-17-image timeout: 2h privileged: true params: COVERAGE: false DB: postgresql + PG_VERSION: "17" RAKE_TASK: spec:unit:director - name: unit-director-postgres-15 @@ -246,6 +251,7 @@ jobs: params: COVERAGE: false DB: postgresql + PG_VERSION: "15" RAKE_TASK: spec:unit:director - name: run-bundle-audit @@ -387,8 +393,8 @@ jobs: - unit-release - unit-director-sqlite - unit-director-mysql-8 - - unit-director-postgres-13 - unit-director-postgres-15 + - unit-director-postgres-17 - integration-mysql - integration-postgres - integration-postgres-hotswap @@ -676,6 +682,138 @@ jobs: vars: *brats-terraform-vars action: destroy + - name: brats-postgres-15 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-15 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "15" + BRATS_FOCUS_REGEXP: "postgres-release" + + - name: brats-postgres-16 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-16 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "16" + BRATS_FOCUS_REGEXP: "postgres-release" + + - name: brats-postgres-17 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-17 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "17" + BRATS_FOCUS_REGEXP: "postgres-release" + + - name: brats-postgres-18 + serial: true + plan: + - in_parallel: + - get: bosh-ci + - get: integration-image + - get: bosh + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-dns-release + - get: stemcell + resource: warden-stemcell + - get: director-stemcell + resource: warden-stemcell + - get: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-release + resource: bosh-candidate-release-tarballs + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-candidate-release-compiled + trigger: true + passed: [ create-bosh-candidate-release-compiled ] + - get: bosh-deployment + - get: docker-cpi-image + - task: test-brats-postgres-18 + file: bosh-ci/ci/shared/brats/test-acceptance.yml + image: docker-cpi-image + privileged: true + params: + STEMCELL_OS: ubuntu-noble + DIRECTOR_STEMCELL_OS: ubuntu-noble + PG_VERSION: "18" + BRATS_FOCUS_REGEXP: "postgres-release" + - name: brats-performance serial: true plan: @@ -1178,7 +1316,7 @@ jobs: params: image: image/image.tar - - name: build-main-postgres-13 + - name: build-main-postgres-17 public: true serial: true plan: @@ -1202,11 +1340,11 @@ jobs: - name: image params: CONTEXT: bosh-ci-dockerfiles/ci/dockerfiles/main-postgres - BUILD_ARG_DB_VERSION: "13" + BUILD_ARG_DB_VERSION: "17" IMAGE_ARG_BASE_IMAGE: integration-image/image.tar run: path: build - - put: integration-postgres-13-image + - put: integration-postgres-17-image no_get: true params: image: image/image.tar @@ -1437,10 +1575,6 @@ jobs: trigger: true - get: mariadb-connector-c-resource trigger: true - - get: postgres-13-src - trigger: true - - get: postgres-15-src - trigger: true - get: ruby-release trigger: true - get: bosh-blobstore-dav @@ -1486,40 +1620,6 @@ jobs: options: credentials_source: static json_key: '((gcp_json_key))' - - task: bump-postgres-13-package - file: bosh-ci/ci/tasks/bump-postgres-packages.yml - image: integration-image - input_mapping: - bosh: bosh-out - postgres-src: postgres-13-src - output_mapping: - bosh: bosh-out - params: - GIT_USER_NAME: *git_user_name - GIT_USER_EMAIL: *git_user_email - MAJOR_VERSION: 13 - PRIVATE_YML: | - blobstore: - options: - credentials_source: static - json_key: '((gcp_json_key))' - - task: bump-postgres-15-package - file: bosh-ci/ci/tasks/bump-postgres-packages.yml - image: integration-image - input_mapping: - bosh: bosh-out - postgres-src: postgres-15-src - output_mapping: - bosh: bosh-out - params: - GIT_USER_NAME: *git_user_name - GIT_USER_EMAIL: *git_user_email - MAJOR_VERSION: 15 - PRIVATE_YML: | - blobstore: - options: - credentials_source: static - json_key: '((gcp_json_key))' - task: bump-ruby-package file: ruby-release/ci/tasks/shared/bump-ruby-package.yml input_mapping: @@ -1685,21 +1785,6 @@ resources: regex: '>(?P3\.\d+.\d+)\d{4}-\d{2}-\d{2}Stable' uri: "https://ftp.osuosl.org/pub/mariadb/connector-c-{version}/mariadb-connector-c-{version}-src.tar.gz" - - name: postgres-13-src - type: http-resource - source: - index: "https://ftp.postgresql.org/pub/source/" - regex: 'href="v(?P13\.[0-9.]+)/"' - uri: "https://ftp.postgresql.org/pub/source/v{version}/postgresql-{version}.tar.gz" - - - name: postgres-15-src - type: http-resource - source: - index: "https://ftp.postgresql.org/pub/source/" - regex: 'href="v(?P15\.[0-9.]+)/"' - uri: "https://ftp.postgresql.org/pub/source/v{version}/postgresql-{version}.tar.gz" - - - name: bosh-blobstore-dav type: s3 source: @@ -1961,10 +2046,10 @@ resources: username: ((github_read_write_packages.username)) password: ((github_read_write_packages.password)) - - name: integration-postgres-13-image + - name: integration-postgres-17-image type: registry-image source: - repository: ghcr.io/cloudfoundry/bosh/main-postgres-13 + repository: ghcr.io/cloudfoundry/bosh/main-postgres-17 tag: *branch_name username: ((github_read_write_packages.username)) password: ((github_read_write_packages.password)) diff --git a/ci/tasks/bump-postgres-packages.sh b/ci/tasks/bump-postgres-packages.sh deleted file mode 100755 index e5c17fb0761..00000000000 --- a/ci/tasks/bump-postgres-packages.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash -NEED_COMMIT=false - -echo "${PRIVATE_YML}" > bosh/config/private.yml - -set -e - -pushd bosh - CURRENT_BLOBS=$(bosh blobs) - BLOB_PATH=$(ls ../postgres-src/postgresql-*.tar.gz) - FILENAME=$( basename "${BLOB_PATH}" ) - OLD_BLOB_PATH=$(grep "postgresql-${MAJOR_VERSION}" config/blobs.yml | cut -f1 -d:) - if ! echo "${CURRENT_BLOBS}" | grep "${FILENAME}" ; then - NEED_COMMIT=true - echo "adding ${FILENAME}" - bosh add-blob --sha2 "${BLOB_PATH}" "postgres/${FILENAME}" - bosh remove-blob "${OLD_BLOB_PATH}" - bosh upload-blobs - fi - - if ${NEED_COMMIT}; then - echo "-----> $(date): Creating git commit" - git config user.name "${GIT_USER_NAME}" - git config user.email "${GIT_USER_EMAIL}" - git add . - - git --no-pager diff --cached - if [[ "$( git status --porcelain )" != "" ]]; then - git commit -am "Bump packages" - fi - fi -popd diff --git a/ci/tasks/bump-postgres-packages.yml b/ci/tasks/bump-postgres-packages.yml deleted file mode 100644 index e44357d0eff..00000000000 --- a/ci/tasks/bump-postgres-packages.yml +++ /dev/null @@ -1,20 +0,0 @@ ---- -platform: linux - -inputs: -- name: bosh-ci -- name: bosh -- name: postgres-src - -outputs: -- name: bosh - -run: - path: bosh-ci/ci/tasks/bump-postgres-packages.sh - -params: - PRIVATE_YML: - MAJOR_VERSION: - GIT_USER_NAME: - GIT_USER_EMAIL: - diff --git a/config/blobs.yml b/config/blobs.yml index 711d44a3ee5..3fdb558ea8a 100644 --- a/config/blobs.yml +++ b/config/blobs.yml @@ -18,18 +18,6 @@ nats/nats-server-v2.14.3-linux-amd64.tar.gz: size: 6843959 object_id: 9c3d83e2-f9d6-43e2-59d4-9c56e0e9cd3c sha: sha256:f3d0c820c749f81d717310fb00d4903919e70e3e66b268bd352a088b9788eb93 -postgres/postgresql-10.23.tar.gz: - size: 25493839 - object_id: 1549746f-1e8d-4ce5-5585-36eca41ab528 - sha: sha256:39886b7fbf375fdd79e94b676d8346a62c3c190ccf00b3f763002c8fc8fb71f0 -postgres/postgresql-13.23.tar.gz: - size: 28506793 - object_id: c2814297-9505-46e7-7b78-9adbd0d4b8f5 - sha: sha256:f5eb642f659d59a9a65df08a1364fed8e83db5df116eff17a2f063a02854069b -postgres/postgresql-15.18.tar.gz: - size: 30728818 - object_id: d0e42b9a-3281-4e39-77eb-32a0d6bdb131 - sha: sha256:d32a533516edc688ebbb96ae221b605be2712edd94d7428b42a4af5644250943 s3cli/s3cli-0.0.413-linux-amd64: size: 15962472 object_id: 8adb4196-1706-432a-40af-b371d7495881 diff --git a/docs/postgres-migration.md b/docs/postgres-migration.md new file mode 100644 index 00000000000..98aa27f0da5 --- /dev/null +++ b/docs/postgres-migration.md @@ -0,0 +1,85 @@ +# Migrating from bosh postgres jobs to cloudfoundry/postgres-release + +## Background + +The `postgres` and `postgres-13` jobs previously shipped inside the bosh release +have been removed. PostgreSQL is now provided by the +[cloudfoundry/postgres-release](https://github.com/cloudfoundry/postgres-release). + +This release ships PostgreSQL versions 15, 16, 17, and 18. + +## Switching from jobs/postgres (bosh release) + +In your director manifest, replace: + +```yaml +releases: +- name: bosh + version: latest + +instance_groups: +- name: bosh + jobs: + - name: postgres + release: bosh + properties: + postgres: + user: bosh + password: secret + database: bosh + listen_address: 127.0.0.1 + port: 5432 + max_connections: 200 +``` + +With: + +```yaml +releases: +- name: bosh + version: latest +- name: postgres + url: https://bosh.io/d/github.com/cloudfoundry/postgres-release + version: latest + +instance_groups: +- name: bosh + jobs: + - name: postgres + release: postgres + properties: + databases: + version: 15 # match your existing on-disk data version + port: 5432 + max_connections: 500 + databases: + - name: bosh + roles: + - name: bosh + password: secret +``` + +## Switching from jobs/postgres-13 + +Same as above, but set `databases.version: 13`. + +## In-place cutover procedure + +1. Set `databases.version: 15` (or 13 if still on postgres-13). + postgres-release detects the existing `/var/vcap/store/postgres-` + data directory and starts without reinitializing. +2. Deploy. BOSH will restart the postgres process using postgres-release. +3. To upgrade to a newer PostgreSQL major version, change `databases.version` + to 16, 17, or 18 and redeploy. postgres-release handles `pg_upgrade`. + +## Property mapping + +| Old (`bosh` release) | New (`postgres-release`) | +|---------------------------------|----------------------------------------------| +| `postgres.user` | `databases.roles[0].name` | +| `postgres.password` | `databases.roles[0].password` | +| `postgres.database` | `databases.databases[0].name` | +| `postgres.additional_databases` | additional `databases.databases` entries | +| `postgres.listen_address` | `databases.address` (default `127.0.0.1`) | +| `postgres.port` | `databases.port` (default `5432`) | +| `postgres.max_connections` | `databases.max_connections` (default `500`) | diff --git a/jobs/postgres-13/spec b/jobs/postgres-13/spec index efc3d021b84..6f7eebacf2f 100644 --- a/jobs/postgres-13/spec +++ b/jobs/postgres-13/spec @@ -1,5 +1,6 @@ --- name: postgres-13 +description: "DEPRECATED: This job will be removed in the next bosh release. Use the postgres job from cloudfoundry/postgres-release instead. See docs/postgres-migration.md for upgrade instructions." templates: bpm.yml: config/bpm.yml diff --git a/jobs/postgres-13/templates/pre-start.erb b/jobs/postgres-13/templates/pre-start.erb index 64a49f9bac7..4e2286364d3 100755 --- a/jobs/postgres-13/templates/pre-start.erb +++ b/jobs/postgres-13/templates/pre-start.erb @@ -1,5 +1,8 @@ #!/bin/bash +echo "WARNING: The postgres-13 job is deprecated and will be removed in the next bosh release." +echo "WARNING: Migrate to the postgres job from cloudfoundry/postgres-release. See docs/postgres-migration.md" + set -eu set -o pipefail diff --git a/jobs/postgres/monit b/jobs/postgres/monit deleted file mode 100644 index 7c3ef102a37..00000000000 --- a/jobs/postgres/monit +++ /dev/null @@ -1,5 +0,0 @@ -check process postgres - with pidfile /var/vcap/sys/run/bpm/postgres/postgres.pid - start program "/var/vcap/jobs/bpm/bin/bpm start postgres" with timeout 300 seconds - stop program "/var/vcap/jobs/bpm/bin/bpm stop postgres" - group vcap diff --git a/jobs/postgres/spec b/jobs/postgres/spec deleted file mode 100644 index 6938ef83029..00000000000 --- a/jobs/postgres/spec +++ /dev/null @@ -1,35 +0,0 @@ ---- -name: postgres - -templates: - bpm.yml: config/bpm.yml - pre-start.erb: bin/pre-start - create-database.erb: bin/create-database - postgres.erb: bin/postgres - postgresql.conf.erb: config/postgresql.conf - -packages: - - postgres-15 - - postgres-13 - -properties: - postgres.user: - description: Username clients must use to access Postgres - default: bosh - postgres.password: - description: Password clients must use to access Postgres - postgres.listen_address: - description: IP address Postgres listens on; use 0.0.0.0 to listen on all IP addresses - default: 127.0.0.1 - postgres.port: - description: TCP port Postgres listens on - default: 5432 - postgres.database: - description: Default Postgres database (it will be created during the startup process) - default: bosh - postgres.additional_databases: - description: Additional Postgres databases to be created (during the startup process) - default: [] - postgres.max_connections: - description: Set max concurrent connections for database - default: 200 diff --git a/jobs/postgres/templates/bpm.yml b/jobs/postgres/templates/bpm.yml deleted file mode 100644 index 65f3c3c4b03..00000000000 --- a/jobs/postgres/templates/bpm.yml +++ /dev/null @@ -1,24 +0,0 @@ -<%= - -postgres_config = { - "name" => "postgres", - "executable" => "/var/vcap/jobs/postgres/bin/postgres", - "shutdown_signal" => "INT", - "additional_volumes" => [ - { - "path" => "/var/vcap/store/postgres-15", - "writable" => true, - }, - ], - "limits" => { - "open_files" => 65536, - }, -} - -config = { - "processes" => [postgres_config], -} - -YAML.dump(config) - -%> diff --git a/jobs/postgres/templates/create-database.erb b/jobs/postgres/templates/create-database.erb deleted file mode 100644 index 14444ca4896..00000000000 --- a/jobs/postgres/templates/create-database.erb +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -e - -PACKAGE_DIR=/var/vcap/packages/postgres-15 - -HOST=<%= p("postgres.listen_address") %> -PORT=<%= p("postgres.port") %> -USER='<%= p("postgres.user") %>' -PASSWORD='<%= p("postgres.password") %>' -DBNAMES=(<%= p("postgres.additional_databases").join(" ") %>) -DBNAMES+=('<%= p("postgres.database") %>') - -export LD_LIBRARY_PATH="$PACKAGE_DIR/lib:$LD_LIBRARY_PATH" - -# wait for database to be running -for i in $( seq 0 30 ); do - if /var/vcap/packages/postgres-15/bin/oid2name -H ${HOST} -U ${USER} -d postgres -q ; then - break - fi - - sleep 1 -done - -set +e - -for DBNAME in "${DBNAMES[@]}"; do - echo "Trying to create database $DBNAME..." - $PACKAGE_DIR/bin/createdb $DBNAME -h $HOST -p $PORT -U vcap - if [ $? != 0 ]; then - echo "Warning: failed to create $DBNAME; ignoring" - fi - - echo "Trying to create user..." - $PACKAGE_DIR/bin/psql -d $DBNAME -h $HOST -p $PORT -U vcap -c "create role \"$USER\" NOSUPERUSER LOGIN INHERIT CREATEDB" - - echo "Trying to alter user (to change password)..." - $PACKAGE_DIR/bin/psql -d $DBNAME -h $HOST -p $PORT -U vcap -c "alter role \"$USER\" with password '$PASSWORD'" - - echo "Grant permission to user" - $PACKAGE_DIR/bin/psql -d $DBNAME -h $HOST -p $PORT -U vcap -c "grant all on schema public to \"$USER\";" -done diff --git a/jobs/postgres/templates/postgres.erb b/jobs/postgres/templates/postgres.erb deleted file mode 100644 index 3e7da7971d4..00000000000 --- a/jobs/postgres/templates/postgres.erb +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -e - -PACKAGE_DIR=/var/vcap/packages/postgres-15 - -STORE_DIR=/var/vcap/store - -# then we switched to including version so it is easier to detect version changes -DATA_DIR=$STORE_DIR/postgres-15 - -JOB_DIR=/var/vcap/jobs/postgres -RUN_DIR=/var/vcap/sys/run/bpm/postgres - -HOST=<%= p("postgres.listen_address") %> -PORT=<%= p("postgres.port") %> -USER='<%= p("postgres.user") %>' - -export LD_LIBRARY_PATH="$PACKAGE_DIR/lib:$LD_LIBRARY_PATH" - - -cp $JOB_DIR/config/postgresql.conf $DATA_DIR - -/var/vcap/jobs/postgres/bin/create-database & - -echo "Starting PostgreSQL:" -exec $PACKAGE_DIR/bin/postgres -h $HOST -p $PORT -D $DATA_DIR diff --git a/jobs/postgres/templates/postgresql.conf.erb b/jobs/postgres/templates/postgresql.conf.erb deleted file mode 100644 index ebd2e702d9b..00000000000 --- a/jobs/postgres/templates/postgresql.conf.erb +++ /dev/null @@ -1,18 +0,0 @@ -data_directory = '/var/vcap/store/postgres-15' - -listen_addresses = '<%= p('postgres.listen_address') %>' -port = <%= p('postgres.port') %> -max_connections = <%= p('postgres.max_connections') %> - -ssl = false - -log_line_prefix = '%t ' - -datestyle = 'iso, mdy' - -lc_messages = 'en_US.UTF-8' -lc_monetary = 'en_US.UTF-8' -lc_numeric = 'en_US.UTF-8' -lc_time = 'en_US.UTF-8' - -default_text_search_config = 'pg_catalog.english' diff --git a/jobs/postgres/templates/pre-start.erb b/jobs/postgres/templates/pre-start.erb deleted file mode 100755 index d320b5c0feb..00000000000 --- a/jobs/postgres/templates/pre-start.erb +++ /dev/null @@ -1,165 +0,0 @@ -#!/bin/bash - -set -eu -set -o pipefail - -CURRENT_POSTGRES_VERSION=15 -OBSOLETE_VERSIONS=(9.4 10) -## These MUST be in "newest supported to oldest supported" order! -UPGRADEABLE_VERSIONS=(13) - -PACKAGE_DIR=/var/vcap/packages/postgres-$CURRENT_POSTGRES_VERSION -PERSISTENT_DISK_DIR=/var/vcap/store -STORE_DIR=${PERSISTENT_DISK_DIR}/postgres-$CURRENT_POSTGRES_VERSION - -USER='<%= p("postgres.user") %>' - -sysctl -w "kernel.shmmax=67108864" - -function print_director_upgrade_recommendation_for { - case $1 in - "9.4") - echo "Please use a previous bosh release version (270.12) to migrate data from postgres-9.4 to postgres-10." - ;; - "10") - echo "Please use a previous bosh release version (280.1.10) to migrate data from postgres-10 to postgres-13." - ;; - # Uncomment once 13 is in ${OBSOLETE_VERSIONS} - # "13") - # echo "Please use a previous bosh release version (280.1.10) to migrate data from postgres-13 to postgres-15." - # ;; - *) - echo "Unknown obsolete postgres version: '$1'!" - echo "Try using a previous bosh release version to migrate data from postgres version '$1' to a supported one." - ;; - esac -} - -UPGRADEABLE_VERSION_INSTALLED=0 -for version in ${UPGRADEABLE_VERSIONS[@]}; do - if [ -d ${PERSISTENT_DISK_DIR}/postgres-${version} ]; then - UPGRADEABLE_VERSION_INSTALLED=1 - break - fi -done - -for version in ${OBSOLETE_VERSIONS[@]}; do - if [ -d ${PERSISTENT_DISK_DIR}/postgres-${version} ]; then - - # Do we have no supported versions installed and also do not have the latest Postgres installed? - if [ $UPGRADEABLE_VERSION_INSTALLED -eq 0 ] && [ ! -d $STORE_DIR ]; then - # We never upgraded this BOSH Director from this obsolete version to a supported one. - # Give a recommendation and exit. - print_director_upgrade_recommendation_for $version - exit 1 - fi - - # delete the obsolete directory to free up space - echo "Deleting obsolete old postgres directory." - rm -rf "${PERSISTENT_DISK_DIR}/postgres-${version}" - fi -done - -# If the latest Postgres has not yet been configured, configure it: -if [[ ! -d ${STORE_DIR} || ! -f ${STORE_DIR}/postgresql.conf ]]; then - mkdir -p "${STORE_DIR}" - chown vcap:vcap "${STORE_DIR}" - - # initdb creates data directories - su - vcap -c "${PACKAGE_DIR}/bin/initdb -E utf8 -D ${STORE_DIR}" - - touch "${STORE_DIR}/fresh" - - if [ $? != 0 ]; then - echo "ERROR: Unable to Initialize Postgres DB" - exit 1 - fi - - echo "host all ${USER} 0.0.0.0/0 md5" >> "${STORE_DIR}/pg_hba.conf" - - mkdir -p "${STORE_DIR}/pg_log" - chown vcap:vcap "${STORE_DIR}/pg_log" -fi - -# If we just configured the latest Postgres, import from the old version -if [[ -f ${STORE_DIR}/fresh ]] ; then - for version in ${UPGRADEABLE_VERSIONS[@]}; do - PACKAGE_DIR_OLD=/var/vcap/packages/postgres-$version - STORE_DIR_OLD=${PERSISTENT_DISK_DIR}/postgres-$version - - if [[ -d ${STORE_DIR_OLD} ]] ; then - echo "copying contents of postgres-$version to postgres-$CURRENT_POSTGRES_VERSION for postgres upgrade..." - su - vcap -c "${PACKAGE_DIR}/bin/pg_upgrade \ - --old-bindir=${PACKAGE_DIR_OLD}/bin \ - --new-bindir=${PACKAGE_DIR}/bin \ - --old-datadir=${STORE_DIR_OLD} \ - --new-datadir=${STORE_DIR}" - - echo "successfully upgraded from postgres-$version" - rm -rf "${STORE_DIR_OLD}" - break - fi - done - - rm "${STORE_DIR}/fresh" -fi - -# "bpm enforces its own locking around process operations to avoid race conditions" -# from docs: https://bosh.io/docs/bpm/runtime/ -# so postmaster.pid is stale if it still exists -# remove it to prevent running into: -# FATAL: lock file "postmaster.pid" already exists -if [[ -f ${STORE_DIR}/postmaster.pid ]] ; then - rm "${STORE_DIR}/postmaster.pid" -fi - -# The below code reindexes Postgres databases if the glibc version has changed since we last performed a reindex, -# or if we don't know when we reindexed and are running a version of glibc that made major changes to the system collation. -# See: -# -# for more information about why we do this, and why we consider glibc versions older than 2.28 to be unaffected. -# -# NOTE: If you edit this code, make sure to update the copies in all of the other Postgres versions. Extracting this into a common -# file and sharing that file amongst the Postgres releases was just as bad as having multiple copies, so this is what we did. - -rebuild_postgres_indexes() -{ - POSTGRES_DATABASE_USER="vcap" - for database in $(echo "select datname from pg_database" | su - vcap -c "${PACKAGE_DIR}/bin/postgres --single -D ${STORE_DIR} postgres | egrep -o '\".*\"'") - do - echo "Going to reindex database '$database' in '${STORE_DIR}'" - echo "REINDEX DATABASE $database" | su - vcap -c "${PACKAGE_DIR}/bin/postgres --single -D ${STORE_DIR} $database" - # This doesn't work on Postgres versions earlier than 15, because REFRESH COLLATION VERSION isn't a thing until 15. - echo "ALTER DATABASE $database REFRESH COLLATION VERSION" | su - vcap -c "${PACKAGE_DIR}/bin/postgres --single -D ${STORE_DIR} $database" || /bin/true - done -} - -PREVIOUS_GLIBC_VERSION_DIR=${PERSISTENT_DISK_DIR}/postgres-previous-glibc-tracking/ -PREVIOUS_GLIBC_VERSION_FILE=${PREVIOUS_GLIBC_VERSION_DIR}/previous-glibc-version.txt -BREAKING_GLIBC_MAJOR=2 -BREAKING_GLIBC_MINOR=28 - -previous_glibc_version=$(cat $PREVIOUS_GLIBC_VERSION_FILE 2>/dev/null || echo "") -previous_glibc_major=$(echo $previous_glibc_version | cut -d "." -f 1) -previous_glibc_minor=$(echo $previous_glibc_version | cut -d "." -f 2) - -current_glibc_version="$(dpkg-query --show --showformat '${Version}\n' libc6 | egrep -o '^[0-9.]+')" -current_glibc_major=$(echo $current_glibc_version | cut -d "." -f 1) -current_glibc_minor=$(echo $current_glibc_version | cut -d "." -f 2) - -if [[ -f $PREVIOUS_GLIBC_VERSION_FILE ]] -then - if [[ $current_glibc_version != $previous_glibc_version ]] - then - rebuild_postgres_indexes - fi -else - # If we're newer or equal to 2.28, we'll rebuild indexes to handle the issue we know about and any future ones. - if [[ $current_glibc_major -gt $BREAKING_GLIBC_MAJOR || $current_glibc_minor -ge $BREAKING_GLIBC_MINOR ]] - then - rebuild_postgres_indexes - fi -fi - -mkdir -p $PREVIOUS_GLIBC_VERSION_DIR 2>/dev/null -echo $current_glibc_version > $PREVIOUS_GLIBC_VERSION_FILE diff --git a/packages/postgres-13/README.md b/packages/postgres-13/README.md deleted file mode 100644 index dcc85f34e34..00000000000 --- a/packages/postgres-13/README.md +++ /dev/null @@ -1,5 +0,0 @@ -postgres-13 package -============ -Postgres! - -The blob contains source code, we probably got it [here?](https://www.postgresql.org/ftp/source/v13.2/) diff --git a/packages/postgres-13/packaging b/packages/postgres-13/packaging deleted file mode 100644 index f3710189c8c..00000000000 --- a/packages/postgres-13/packaging +++ /dev/null @@ -1,29 +0,0 @@ -set -e -x - -tar xzf postgres/postgresql-13.*.tar.gz - -pushd postgresql-* - ./configure --prefix=${BOSH_INSTALL_TARGET} - - pushd src/bin/pg_config - make - make install - popd - - cp -LR src/include ${BOSH_INSTALL_TARGET} - - pushd src/interfaces/libpq - make - make install - popd - - pushd src - make - make install - popd - - pushd contrib - make - make install - popd -popd diff --git a/packages/postgres-13/spec b/packages/postgres-13/spec deleted file mode 100644 index 4127c2ba768..00000000000 --- a/packages/postgres-13/spec +++ /dev/null @@ -1,5 +0,0 @@ ---- -name: postgres-13 - -files: -- postgres/postgresql-13.*.tar.gz diff --git a/packages/postgres-13/spec.yml b/packages/postgres-13/spec.yml deleted file mode 100644 index 4127c2ba768..00000000000 --- a/packages/postgres-13/spec.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -name: postgres-13 - -files: -- postgres/postgresql-13.*.tar.gz diff --git a/packages/postgres-15/README.md b/packages/postgres-15/README.md deleted file mode 100644 index d223d30096b..00000000000 --- a/packages/postgres-15/README.md +++ /dev/null @@ -1,5 +0,0 @@ -postgres-15 package -============ -Postgres! - -The blob contains source code, we probably got it [here?](https://www.postgresql.org/ftp/source/v15.1/) diff --git a/packages/postgres-15/packaging b/packages/postgres-15/packaging deleted file mode 100644 index 44b878bbe11..00000000000 --- a/packages/postgres-15/packaging +++ /dev/null @@ -1,29 +0,0 @@ -set -e -x - -tar xzf postgres/postgresql-15.*.tar.gz - -pushd postgresql-* - ./configure --prefix=${BOSH_INSTALL_TARGET} - - pushd src/bin/pg_config - make - make install - popd - - cp -LR src/include ${BOSH_INSTALL_TARGET} - - pushd src/interfaces/libpq - make - make install - popd - - pushd src - make - make install - popd - - pushd contrib - make - make install - popd -popd diff --git a/packages/postgres-15/spec b/packages/postgres-15/spec deleted file mode 100644 index 7faa07c0a7d..00000000000 --- a/packages/postgres-15/spec +++ /dev/null @@ -1,5 +0,0 @@ ---- -name: postgres-15 - -files: -- postgres/postgresql-15.*.tar.gz diff --git a/spec/postgres_bpm_spec.rb b/spec/postgres_bpm_spec.rb deleted file mode 100644 index 589ca82a07f..00000000000 --- a/spec/postgres_bpm_spec.rb +++ /dev/null @@ -1,43 +0,0 @@ -require 'spec_helper' - -RSpec.shared_examples 'rendered postgres* bpm.yml' do - let(:release) { Bosh::Common::Template::Test::ReleaseDir.new(RELEASE_ROOT) } - let(:template) { job.template('config/bpm.yml') } - - subject(:rendered_template) do - YAML.load(template.render(properties)) - end - - context 'with standard config' do - let(:properties) do - { - postgres: { - adapter: "postgres", - database: "bosh", - host: "127.0.0.1", - listen_address: "127.0.0.1", - password: "secret", - user: "postgres", - }, - } - end - - it 'should use SIGINT as shutdown signal' do - expect(rendered_template['processes'][0]).to include('shutdown_signal' => 'INT') - end - end -end - -RSpec.describe 'postgres job rendering' do - describe 'postgres job' do - it_should_behave_like 'rendered postgres* bpm.yml' do - let(:job) { release.job('postgres') } - end - end - - describe 'postgres-13 job' do - it_should_behave_like 'rendered postgres* bpm.yml' do - let(:job) { release.job('postgres-13') } - end - end -end diff --git a/src/brats/acceptance/postgres_release_test.go b/src/brats/acceptance/postgres_release_test.go new file mode 100644 index 00000000000..c50a0e01cf0 --- /dev/null +++ b/src/brats/acceptance/postgres_release_test.go @@ -0,0 +1,65 @@ +package acceptance_test + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "brats/utils" +) + +var _ = Describe("postgres-release version matrix", func() { + const deployTimeout = 20 * time.Minute + + manifest := utils.AssetPath("postgres-release-manifest.yml") + + type pgEntry struct { + version int + previous int + } + + DescribeTable("PostgreSQL version", + func(e pgEntry) { + deploymentName := fmt.Sprintf("postgres-release-%d-%x", e.version, GinkgoT().RandomSeed()) + + By(fmt.Sprintf("deploying postgres-release at version %d", e.version)) + session := utils.OuterBosh("deploy", "-n", manifest, + "-d", deploymentName, + "-v", fmt.Sprintf("stemcell-os=%s", utils.StemcellOS()), + "-v", fmt.Sprintf("deployment-name=%s", deploymentName), + "-v", fmt.Sprintf("pg-version=%d", e.version), + ) + Eventually(session, deployTimeout).Should(gexec.Exit(0)) + + if e.previous > 0 { + By(fmt.Sprintf("upgrading from postgres-release version %d to %d", e.previous, e.version)) + previousDeploymentName := fmt.Sprintf("postgres-release-%d-to-%d-%x", e.previous, e.version, GinkgoT().RandomSeed()) + + By(fmt.Sprintf("deploying at version %d first", e.previous)) + session = utils.OuterBosh("deploy", "-n", manifest, + "-d", previousDeploymentName, + "-v", fmt.Sprintf("stemcell-os=%s", utils.StemcellOS()), + "-v", fmt.Sprintf("deployment-name=%s", previousDeploymentName), + "-v", fmt.Sprintf("pg-version=%d", e.previous), + ) + Eventually(session, deployTimeout).Should(gexec.Exit(0)) + + By(fmt.Sprintf("upgrading to version %d", e.version)) + session = utils.OuterBosh("deploy", "-n", manifest, + "-d", previousDeploymentName, + "-v", fmt.Sprintf("stemcell-os=%s", utils.StemcellOS()), + "-v", fmt.Sprintf("deployment-name=%s", previousDeploymentName), + "-v", fmt.Sprintf("pg-version=%d", e.version), + ) + Eventually(session, deployTimeout).Should(gexec.Exit(0)) + } + }, + Entry("deploys version 15", pgEntry{version: 15}), + Entry("upgrades 15 -> 16", pgEntry{version: 16, previous: 15}), + Entry("upgrades 16 -> 17", pgEntry{version: 17, previous: 16}), + Entry("upgrades 17 -> 18", pgEntry{version: 18, previous: 17}), + ) +}) diff --git a/src/brats/assets/postgres-13-manifest.yml b/src/brats/assets/postgres-13-manifest.yml index fd132f9dc61..54d3615c2a8 100644 --- a/src/brats/assets/postgres-13-manifest.yml +++ b/src/brats/assets/postgres-13-manifest.yml @@ -6,7 +6,7 @@ instance_groups: instances: 1 jobs: - name: postgres - release: bosh + release: postgres - name: bpm release: bpm vm_type: default @@ -15,10 +15,14 @@ instance_groups: networks: - name: default properties: - postgres: - user: postgres - password: c1oudc0w - database: bosh + databases: + version: 13 + port: 5432 + databases: + - name: bosh + roles: + - name: postgres + password: c1oudc0w stemcells: - alias: default @@ -26,10 +30,12 @@ stemcells: version: latest releases: -- name: "bosh" +- name: bosh version: "276.1.1" url: "https://bosh.io/d/github.com/cloudfoundry/bosh?v=276.1.1" sha1: "f9a625dd8a8fc6e01f1641390ced3ac0fee31523" +- name: postgres + version: latest - name: bpm version: latest diff --git a/src/brats/assets/postgres-manifest.yml b/src/brats/assets/postgres-manifest.yml index 6181a8a6f00..90c0ca73120 100644 --- a/src/brats/assets/postgres-manifest.yml +++ b/src/brats/assets/postgres-manifest.yml @@ -6,7 +6,7 @@ instance_groups: instances: 1 jobs: - name: postgres - release: bosh + release: postgres - name: bpm release: bpm vm_type: default @@ -15,13 +15,14 @@ instance_groups: networks: - name: default properties: - postgres: - listen_address: 127.0.0.1 - host: 127.0.0.1 - user: postgres - password: c1oudc0w - database: bosh - adapter: postgres + databases: + version: 15 + port: 5432 + databases: + - name: bosh + roles: + - name: postgres + password: c1oudc0w stemcells: - alias: default @@ -31,6 +32,8 @@ stemcells: releases: - name: bosh version: latest +- name: postgres + version: latest - name: bpm version: latest diff --git a/src/brats/assets/postgres-release-manifest.yml b/src/brats/assets/postgres-release-manifest.yml new file mode 100644 index 00000000000..885ed2b54d1 --- /dev/null +++ b/src/brats/assets/postgres-release-manifest.yml @@ -0,0 +1,44 @@ +--- +name: ((deployment-name)) +instance_groups: +- name: bosh + azs: [z1] + instances: 1 + jobs: + - name: postgres + release: postgres + - name: bpm + release: bpm + vm_type: default + stemcell: default + persistent_disk_type: default + networks: + - name: default + properties: + databases: + version: ((pg-version)) + port: 5432 + databases: + - name: bosh + roles: + - name: postgres + password: c1oudc0w + +stemcells: +- alias: default + os: ((stemcell-os)) + version: latest + +releases: +- name: bosh + version: latest +- name: postgres + version: latest +- name: bpm + version: latest + +update: + canaries: 1 + max_in_flight: 10 + canary_watch_time: 1000-30000 + update_watch_time: 1000-30000 diff --git a/src/spec/integration_support/postgres_version_helper.rb b/src/spec/integration_support/postgres_version_helper.rb index 4603305548a..cbb4842c1ee 100644 --- a/src/spec/integration_support/postgres_version_helper.rb +++ b/src/spec/integration_support/postgres_version_helper.rb @@ -1,4 +1,3 @@ -require 'yaml' require 'integration_support/constants' module IntegrationSupport @@ -7,8 +6,8 @@ class << self def ensure_version_match!(env_db) return unless env_db == 'postgresql' - unless local_major_version == release_major_version - raise "Postgres major version mismatch: jobs/postgres/spec: #{local_version}; local: #{release_version}." + unless local_major_version == configured_major_version + raise "Postgres major version mismatch: PG_VERSION=#{configured_version}; local: #{local_version}." end end @@ -20,29 +19,20 @@ def local_version `postgres --version`.chomp.split(' ').last end - def release_major_version - release_version.split('.')[0] + def configured_major_version + configured_version.split('.')[0] end - def release_version - @release_version ||= begin - postgres_release_config = - YAML.load_file( - File.join(IntegrationSupport::Constants::BOSH_REPO_ROOT, 'jobs', 'postgres', 'spec'), - permitted_classes: [Symbol], - aliases: true, - ) - - # sort alphanumerics correctly, e.g. 10 > 9 - postgres_version = postgres_release_config['packages'].max_by { |s| s.scan(/\d+/).first.to_i } - - postgres_version.split('-').last + def configured_version + ENV.fetch('PG_VERSION') do + raise 'PG_VERSION environment variable must be set when DB=postgresql' end end end end end + RSpec.configure do |c| c.before(:suite) do IntegrationSupport::PostgresVersionHelper.ensure_version_match!(ENV['DB'])