diff --git a/.github/workflows/build-php-linux.yml b/.github/workflows/build-php-linux.yml new file mode 100644 index 0000000..396bd0a --- /dev/null +++ b/.github/workflows/build-php-linux.yml @@ -0,0 +1,149 @@ +# Builds one PHP release for Linux (x86-64 or arm64) with static-php-cli in its glibc container, +# tests it, and publishes it on the php- release as php--build-.tar.gz: +# bin/php and bin/php-fpm with the compiled-in extensions and the shared ones in ext/ from +# php.linux in versions.json (build number: php.linux.build). +# Called by release.yml for every version without that build yet; run it by hand to test a build +# without publishing. Nothing is published unless every check passed, and a published asset is +# never replaced. +name: Build PHP Linux + +on: + workflow_call: + inputs: + version: + type: string + required: true + platform: + type: string + required: true + publish: + type: boolean + default: false + workflow_dispatch: + inputs: + version: + description: PHP version (major.minor.patch) + required: true + platform: + description: Platform + type: choice + options: [linux-x64, linux-arm64] + publish: + description: Publish the package as a release asset + type: boolean + default: false + +permissions: + contents: read + +env: + VERSION: ${{ inputs.version }} + PLATFORM: ${{ inputs.platform }} + +jobs: + build: + name: build php ${{ inputs.version }} ${{ inputs.platform }} + runs-on: ${{ inputs.platform == 'linux-arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + timeout-minutes: 150 + outputs: + asset: ${{ steps.config.outputs.asset }} + ok: ${{ steps.done.outputs.ok }} + env: + GITHUB_TOKEN: ${{ github.token }} + steps: + - uses: actions/checkout@v7 + + - name: Read versions.json + id: config + run: | + jq -r '.php.linux | "SPC_VERSION=\(.spc)\nMIN_GLIBC=\(.min_glibc)\nSTATIC_EXTS=\(.static_extensions)\nSHARED_EXTS=\(.shared_extensions)"' versions.json >> "$GITHUB_ENV" + echo "MINOR=$(cut -d. -f1-2 <<<"$VERSION")" >> "$GITHUB_ENV" + build="$(jq -r .php.linux.build versions.json)" + echo "BUILD=$build" >> "$GITHUB_ENV" + asset="php-$VERSION-build$build-$PLATFORM.tar.gz" + echo "ASSET=$asset" >> "$GITHUB_ENV" + echo "asset=$asset" >> "$GITHUB_OUTPUT" + + - name: Install patchelf + run: sudo apt-get update -q && sudo apt-get install -yq patchelf + + - name: Cache static-php-cli downloads + uses: actions/cache@v6 + with: + path: spc-src/downloads + key: spc-linux-downloads-${{ env.SPC_VERSION }}-${{ inputs.platform }}-${{ inputs.version }}-${{ hashFiles('versions.json') }} + restore-keys: spc-linux-downloads-${{ env.SPC_VERSION }}-${{ inputs.platform }}-${{ env.MINOR }} + + - name: Build + run: bash packages/php/linux/build.sh "$VERSION" out + + - name: Portability (readelf) + run: bash scripts/verify-linux.sh out "$MIN_GLIBC" | tee -a "$GITHUB_STEP_SUMMARY" + + # Every shared extension in versions.json has to build and load for a release. + - name: Smoke test + run: bash scripts/smoke-php-package.sh php out + + - name: Package + run: | + jq -n --arg version "$VERSION" --argjson build "$BUILD" --arg platform "$PLATFORM" --arg commit "$GITHUB_SHA" \ + --arg spc "$SPC_VERSION" --arg min_glibc "$MIN_GLIBC" --arg static "$STATIC_EXTS" --arg shared "$SHARED_EXTS" \ + --arg built "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + '{package: "php", version: $version, build: $build, platform: $platform, min_glibc: $min_glibc, + built: $built, commit: $commit, static_php_cli: $spc, + extensions: {compiled_in: ($static | split(",")), shared: ($shared | split(","))}}' \ + > out/flock-package.json + tar -C out -czf "$ASSET" . + sha256sum "$ASSET" | tee -a "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v7 + with: + name: ${{ env.ASSET }} + path: ${{ env.ASSET }} + retention-days: 7 + - id: done + run: echo ok=true >> "$GITHUB_OUTPUT" + + # The oldest glibc the build targets, without anything the runner has installed. + portable: + name: run php ${{ inputs.version }} ${{ inputs.platform }} on CentOS 7 (glibc 2.17) + needs: build + if: needs.build.outputs.ok == 'true' + runs-on: ${{ inputs.platform == 'linux-arm64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }} + outputs: + ok: ${{ steps.done.outputs.ok }} + env: + ASSET: ${{ needs.build.outputs.asset }} + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: ${{ env.ASSET }} + - name: Read versions.json + run: jq -r '.php.linux | "STATIC_EXTS=\(.static_extensions)\nSHARED_EXTS=\(.shared_extensions)"' versions.json >> "$GITHUB_ENV" + # In a container: the runner's own glibc and libraries would hide what the package lacks. As + # the runner's user, like PHP-FPM on a user's machine (as root it wants a pool user). + - name: Smoke test + run: | + mkdir pkg && tar -C pkg -xzf "$ASSET" + docker run --rm --user "$(id -u):$(id -g)" -v "$PWD:/w" -w /w -e STATIC_EXTS -e SHARED_EXTS centos:7 \ + bash scripts/smoke-php-package.sh php pkg + - id: done + run: echo ok=true >> "$GITHUB_OUTPUT" + + publish: + needs: [build, portable] + if: inputs.publish && needs.portable.outputs.ok == 'true' + runs-on: ubuntu-latest + permissions: + contents: write + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + ASSET: ${{ needs.build.outputs.asset }} + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: ${{ env.ASSET }} + - run: bash scripts/publish.sh php "$VERSION" "$ASSET" diff --git a/.github/workflows/build-service.yml b/.github/workflows/build-service.yml index 5aaeea5..071a2b0 100644 --- a/.github/workflows/build-service.yml +++ b/.github/workflows/build-service.yml @@ -1,6 +1,7 @@ # Builds one release of a service (PostgreSQL, MariaDB, Valkey, Redis) from source for one -# platform with packages///build.*, checks it (portability on macOS), smoke-tests it -# on a clean runner, and publishes it as --build-. on the +# platform with packages///build.* (Linux: in the manylinux_2_28 container, glibc +# 2.28), checks it (portability on macOS and Linux), smoke-tests it on a clean runner (Linux: an +# almalinux:8 container), and publishes it as --build-. on the # - release, next to the upstream source when the license asks for it. # Called by release.yml for every version without that build yet; run it by hand to test. # Nothing is published unless every check passed, and a published asset is never replaced. @@ -32,7 +33,7 @@ on: platform: description: Platform type: choice - options: [macos-arm64, windows-x64] + options: [macos-arm64, windows-x64, linux-x64, linux-arm64] publish: description: Publish the package as a release asset type: boolean @@ -49,7 +50,7 @@ env: jobs: build: name: build ${{ inputs.package }} ${{ inputs.version }} ${{ inputs.platform }} - runs-on: ${{ inputs.platform == 'windows-x64' && 'windows-2022' || 'macos-15' }} + runs-on: ${{ fromJSON('{"windows-x64":"windows-2022","linux-x64":"ubuntu-24.04","linux-arm64":"ubuntu-24.04-arm"}')[inputs.platform] || 'macos-15' }} timeout-minutes: 180 outputs: asset: ${{ steps.config.outputs.asset }} @@ -85,6 +86,27 @@ jobs: if: runner.os == 'macOS' run: bash scripts/verify-macos.sh out "$MIN_OS" | tee -a "$GITHUB_STEP_SUMMARY" + # Linux: the static OpenSSL only changes with its version. + - name: Cache OpenSSL + if: runner.os == 'Linux' && env.OPENSSL_VERSION != '' + uses: actions/cache@v6 + with: + path: ${{ inputs.package }}-work/openssl + key: openssl-${{ env.OPENSSL_VERSION }}-${{ inputs.platform }}-manylinux_2_28-no-dso + + # Linux: in the manylinux_2_28 container (AlmaLinux 8), so the binaries run on glibc 2.28+. + - name: Build + if: runner.os == 'Linux' + run: | + arch=x86_64; [[ "$PLATFORM" == linux-arm64 ]] && arch=aarch64 + docker run --rm -v "$PWD:/w" -w /w -e OPENSSL_VERSION -e GITHUB_TOKEN -e SOURCES_DIR=/w/sources \ + "quay.io/pypa/manylinux_2_28_$arch" bash "packages/$PKG/linux/build.sh" "$VERSION" out + sudo chown -R "$(id -u):$(id -g)" . + + - name: Portability (readelf) + if: runner.os == 'Linux' + run: bash scripts/verify-linux.sh out "$MIN_GLIBC" | tee -a "$GITHUB_STEP_SUMMARY" + # Windows Valkey: MSYS2 provides fork() (packages/valkey/windows/build.sh). - uses: msys2/setup-msys2@v2 if: runner.os == 'Windows' && inputs.package == 'valkey' @@ -156,11 +178,12 @@ jobs: retention-days: 7 # A clean runner: on macOS the oldest Apple Silicon image with Homebrew moved away (closest to - # a user's Mac); on Windows without MSYS2 runtimes on PATH, so only the bundled DLLs are used. + # a user's Mac); on Windows without MSYS2 runtimes on PATH, so only the bundled DLLs are used; + # on Linux an almalinux:8 container (the oldest glibc the build targets, nothing installed). test: name: run ${{ inputs.package }} ${{ inputs.version }} ${{ inputs.platform }} needs: build - runs-on: ${{ inputs.platform == 'windows-x64' && 'windows-2022' || 'macos-14' }} + runs-on: ${{ fromJSON('{"windows-x64":"windows-2022","linux-x64":"ubuntu-24.04","linux-arm64":"ubuntu-24.04-arm"}')[inputs.platform] || 'macos-14' }} timeout-minutes: 20 env: ASSET: ${{ needs.build.outputs.asset }} @@ -175,6 +198,13 @@ jobs: - name: Smoke test if: runner.os == 'macOS' run: bash scripts/smoke-service.sh "$PKG" "$ASSET" + # As the runner's user, known to the container (initdb looks it up): PostgreSQL refuses root. + - name: Smoke test + if: runner.os == 'Linux' + run: | + docker run --rm --user "$(id -u):$(id -g)" -v /etc/passwd:/etc/passwd:ro -v /etc/group:/etc/group:ro \ + -v "$PWD:/w" -w /w almalinux:8 \ + bash scripts/smoke-service.sh "$PKG" "$ASSET" - name: Remove other MSYS runtimes from PATH if: runner.os == 'Windows' shell: pwsh diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a69532a..c871014 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -32,6 +32,6 @@ jobs: - name: Check run: | go -C tools/flockrel run . check -versions ../../versions.json -php-build ../../packages/php/macos/build.sh \ - -out "$RUNNER_TEMP/findings.json" 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" + -php-build-linux ../../packages/php/linux/build.sh -out "$RUNNER_TEMP/findings.json" 2>&1 | tee -a "$GITHUB_STEP_SUMMARY" - name: Issues run: bash scripts/upstream-issues.sh "$RUNNER_TEMP/findings.json" | tee -a "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/mirror.yml b/.github/workflows/mirror.yml index bbbc3b1..02d9939 100644 --- a/.github/workflows/mirror.yml +++ b/.github/workflows/mirror.yml @@ -35,7 +35,7 @@ on: platform: description: Platform type: choice - options: [macos-arm64, windows-x64] + options: [macos-arm64, windows-x64, linux-x64, linux-arm64] publish: description: Publish the package as a release asset type: boolean @@ -86,7 +86,7 @@ jobs: test: name: run ${{ inputs.package }} ${{ inputs.version }} ${{ inputs.platform }} needs: fetch - runs-on: ${{ inputs.platform == 'windows-x64' && 'windows-2022' || 'macos-15' }} + runs-on: ${{ fromJSON('{"windows-x64":"windows-2022","linux-x64":"ubuntu-24.04","linux-arm64":"ubuntu-24.04-arm"}')[inputs.platform] || 'macos-15' }} timeout-minutes: 20 env: ASSET: ${{ needs.fetch.outputs.asset }} @@ -103,7 +103,7 @@ jobs: if: runner.os == 'macOS' run: sudo mv /opt/homebrew /opt/homebrew.hidden - name: Smoke test - if: runner.os == 'macOS' + if: runner.os != 'Windows' run: bash scripts/smoke-service.sh "$PKG" "$ASSET" - name: Smoke test if: runner.os == 'Windows' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 97ccda7..360a081 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,7 @@ # Flock downloads https://github.com//releases/download/manifest/manifest.json (+ .sig), or # manifest-beta.json / manifest-experimental.json for those channels. # php-macos PHP and FrankenPHP for macOS (build-php-macos.yml) +# php-linux PHP for Linux, x86-64 and arm64 (build-php-linux.yml) # php-exts-windows extension packs for Windows PHP, one per minor (build-php-exts-windows.yml) # frankenphp-windows FrankenPHP for Windows (build-frankenphp-windows.yml) # builds services built from source: PostgreSQL, MariaDB (macOS), Valkey, Redis (build-service.yml) @@ -34,6 +35,7 @@ jobs: outputs: any: ${{ steps.plan.outputs.any }} php_macos: ${{ steps.plan.outputs.php_macos }} + php_linux: ${{ steps.plan.outputs.php_linux }} php_exts_windows: ${{ steps.plan.outputs.php_exts_windows }} frankenphp_windows: ${{ steps.plan.outputs.frankenphp_windows }} builds: ${{ steps.plan.outputs.builds }} @@ -67,6 +69,21 @@ jobs: publish: true optional: ${{ matrix.optional == true }} + php-linux: + needs: plan + if: needs.plan.outputs.php_linux != '[]' + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.php_linux) }} + permissions: + contents: write + uses: ./.github/workflows/build-php-linux.yml + with: + version: ${{ matrix.version }} + platform: ${{ matrix.platform }} + publish: true + php-exts-windows: needs: plan if: needs.plan.outputs.php_exts_windows != '[]' @@ -132,7 +149,7 @@ jobs: publish: true manifest: - needs: [plan, php-macos, php-exts-windows, frankenphp-windows, builds, mirrors] + needs: [plan, php-macos, php-linux, php-exts-windows, frankenphp-windows, builds, mirrors] # Also after failed builds: whatever did get published belongs in the manifests. if: always() && needs.plan.result == 'success' permissions: diff --git a/README.md b/README.md index cd697d6..0ec41b7 100644 --- a/README.md +++ b/README.md @@ -7,31 +7,55 @@ signed manifest, so nothing has to be compiled on your machine. ## What we publish -Platforms are macOS on Apple Silicon (`macos-arm64`, macOS 13+) and Windows x64 (`windows-x64`). +Platforms are macOS on Apple Silicon (`macos-arm64`, macOS 13+), Windows x64 (`windows-x64`) and, +for some packages so far, Linux on x86-64 and arm64 (`linux-x64`, `linux-arm64`). Every package is either **built** here from verified upstream source, or **mirrored**: the vendor's own binaries, checked against the vendor's checksum or signature. -| Package | macOS arm64 | Windows x64 | Versions (`versions.json`) | -|---|---|---|---| -| `php` | built: cli + fpm, extensions compiled in, optional ones in `ext/` | mirrored: the official thread-safe build from windows.php.net | macOS 8.1–8.5, Windows 7.4–8.5 | -| `php-exts` | – | built: extension DLLs for the Windows build (below) | one pack per Windows minor | -| `frankenphp` | built: FrankenPHP with the same PHP and extensions as `php` | built: FrankenPHP next to the official PHP build it links against | PHP 8.2–8.5 | -| `postgres` | built: server, client tools (psql, pg_dump, …), contrib | built | 14–18 | -| `mariadb` | built | mirrored | 10.11, 11.4, 11.8, 12.3 | -| `mysql` | mirrored | mirrored | 8.0, 8.4, 9.7 | -| `mongodb` | mirrored | mirrored | 7.0, 8.0, 9.0 | -| `valkey` | built | built with MSYS2 (`fork()` for persistence) | 9.1 | -| `redis` | built | – (Valkey only) | 7.2, 7.4, 8.4, 8.10 | -| `meilisearch` | mirrored (Community Edition) | mirrored (Community Edition) | 1 | -| `typesense` | mirrored | – (no Windows build upstream) | 29, 30 | -| `rustfs` | mirrored | mirrored | 1 | -| `node` | mirrored | mirrored | 16, 18, 20, 22, 24, 26 | +| Package | macOS arm64 | Windows x64 | Linux x64, arm64 | Versions (`versions.json`) | +|---|---|---|---|---| +| `php` | built: cli + fpm, extensions compiled in, optional ones in `ext/` | mirrored: the official thread-safe build from windows.php.net | built, like macOS (glibc 2.17+, see below) | macOS 8.1–8.5, Linux 8.2–8.5, Windows 7.4–8.5 | +| `php-exts` | – | built: extension DLLs for the Windows build (below) | – | one pack per Windows minor | +| `frankenphp` | built: FrankenPHP with the same PHP and extensions as `php` | built: FrankenPHP next to the official PHP build it links against | – | PHP 8.2–8.5 | +| `postgres` | built: server, client tools (psql, pg_dump, …), contrib | built | built (glibc 2.28+, see below) | 14–18 | +| `mariadb` | built | mirrored | built (glibc 2.28+) | 10.11, 11.4, 11.8, 12.3 | +| `mysql` | mirrored | mirrored | – | 8.0, 8.4, 9.7 | +| `mongodb` | mirrored | mirrored | – | 7.0, 8.0, 9.0 | +| `valkey` | built | built with MSYS2 (`fork()` for persistence) | built (glibc 2.28+) | 9.1 | +| `redis` | built | – (Valkey only) | built (glibc 2.28+) | 7.2, 7.4, 8.4, 8.10 | +| `meilisearch` | mirrored (Community Edition) | mirrored (Community Edition) | mirrored (Community Edition) | 1 | +| `typesense` | mirrored | – (no Windows build upstream) | mirrored | 29, 30 | +| `rustfs` | mirrored | mirrored | mirrored (static musl build) | 1 | +| `node` | mirrored | mirrored | mirrored | 16, 18, 20, 22, 24, 26 | The versions column lists release lines: every day the newest upstream release of each line is looked up (`tools/flockrel/upstream.go`, php.net and windows.php.net for PHP) and built or mirrored when it isn't published yet. Patch releases come in on their own; a new line (PostgreSQL 19, Node.js 28, PHP 8.6) is one entry in `versions.json`. +### PHP for Linux + +`php` for `linux-x64` and `linux-arm64` is built like the macOS one, with static-php-cli and the +same extensions (`php.linux`), but in static-php-cli's glibc container (`bin/spc-gnu-docker`: CentOS +7, glibc 2.17). Every library but glibc's is compiled in, and unlike a fully static musl build, PHP +loads the shared extensions in `ext/`. Before publishing, `scripts/verify-linux.sh` checks that no +binary needs a newer glibc or another shared library, and the smoke test runs again in a `centos:7` +container. Linux starts at PHP 8.2: 8.1 had reached its end of life, and its headers need +`_GNU_SOURCE`, which the mongodb extension's `-std=c17` build doesn't get from 8.1's `php_config.h`. +FrankenPHP isn't built for Linux yet. + +### Services for Linux + +PostgreSQL, MariaDB, Valkey and Redis for Linux are built in the `manylinux_2_28` container (AlmaLinux 8, +the image Python wheels are built in), so they need glibc 2.28 (2018: Debian 10, Ubuntu 20.04, RHEL +8 and later). Valkey and Redis need nothing but glibc. PostgreSQL links OpenSSL statically, as on +macOS, and `scripts/linux/bundle.sh` copies the other libraries it needs (zlib, libuuid, libedit for +psql) to `lib/`, with their licenses, and points every binary there with an `$ORIGIN` RUNPATH, the +Linux counterpart of `scripts/macos/relocate.sh`. MariaDB is configured as on macOS and bundles +libedit and ncurses for its client; it is built without libaio (InnoDB then does its I/O with its +own threads), whose library Ubuntu 24.04 renamed. `scripts/verify-linux.sh` checks that every +library comes from glibc or the package, and the smoke test runs in an `almalinux:8` container. + ### PHP release candidates A PHP minor in `versions.json` that has no release yet gets its release candidates instead @@ -75,19 +99,19 @@ read its configure option from, so `--enable-opentelemetry` is passed explicitly ### Package layouts -Archives are `.tar.gz` on macOS and `.zip` on Windows. Mirrored packages keep the vendor's layout, +Archives are `.tar.gz` on macOS and Linux and `.zip` on Windows. Mirrored packages keep the vendor's layout, usually one top-level folder (`mysql-8.4.11-macos15-arm64/bin/mysqld`); unpack and drop it. What we build has `flock-package.json` at the top (what it is, how it was built) and `LICENSES/`. | Package | Layout | |---|---| -| `php` (macOS) | `bin/php`, `bin/php-fpm`, `ext/*.so` | +| `php` (macOS, Linux) | `bin/php`, `bin/php-fpm`, `ext/*.so` | | `php` (Windows) | the official zip: `php.exe`, `php-cgi.exe`, `ext/` | | `php-exts` | `ext/php_.dll` into PHP's `ext\`; `*.dll` next to `php.exe` (ImageMagick) | | `frankenphp` (macOS) | `bin/frankenphp`, `ext/*.so` | | `frankenphp` (Windows) | `frankenphp.exe` next to the official PHP build of the same version | -| `postgres` | `bin/`, `lib/`, `share/` (relocatable) | -| `mariadb` (macOS) | `bin/`, `scripts/mariadb-install-db`, `share/`, `lib/plugin/`; run with `--basedir` | +| `postgres` | `bin/`, `lib/`, `share/` (relocatable; on Linux `lib/` also has the bundled libraries) | +| `mariadb` (macOS, Linux) | `bin/`, `scripts/mariadb-install-db`, `share/`, `lib/plugin/` (Linux: also the bundled libraries in `lib/`); run with `--basedir` | | `valkey`, `redis` | `-server`, `-cli`, `-benchmark` (Windows: with the MSYS2 DLLs) | | `meilisearch`, `rustfs`, `typesense` | the executable and its license | @@ -126,9 +150,10 @@ the same version. | `promote.yml` | records a channel decision (promote, hold, yank) in `channels.json` | | `check.yml` | weekly: new release lines, end of life, newer pinned versions as issues | | `build-php-macos.yml` | `php` and `frankenphp` for macOS (static-php-cli) | +| `build-php-linux.yml` | `php` for Linux (static-php-cli, glibc container) | | `build-php-exts-windows.yml` | `php-exts` for one PHP minor | | `build-frankenphp-windows.yml` | `frankenphp` for Windows | -| `build-service.yml` | PostgreSQL, MariaDB (macOS), Valkey, Redis from source | +| `build-service.yml` | PostgreSQL, MariaDB (macOS, Linux), Valkey, Redis from source (Linux: in `manylinux_2_28`) | | `mirror.yml` | everything mirrored, including PHP for Windows | ### Licenses and sources @@ -162,7 +187,7 @@ Every published build is in one of three channels, and each channel has its own listing the builds at its level or above: | Channel | Manifest | Who | Contains | -|---|---|---|---| +|---|---|---|---|---| | stable | `manifest.json` | every Flock installation | what has been promoted | | beta | `manifest-beta.json` | opt-in in Flock | stable, patches in their waiting time, approved new lines, pre-releases | | experimental | `manifest-experimental.json` | testing before promotion | everything published | @@ -263,10 +288,12 @@ packages//mirror.sh download + verify a vendor release (mirrored pa packages//source.sh download + verify the upstream source (builds, and published for copyleft) scripts/lib.sh download and checksum helpers for the package scripts scripts/macos/ static OpenSSL build, relocation of macOS installs (@loader_path) +scripts/linux/ static OpenSSL build, bundling of libraries into Linux installs ($ORIGIN) scripts/windows/ php-windows-builder setup, extension packs, Windows smoke tests scripts/publish.sh upload a package (and its sources) to its release scripts/ready-issue.sh keep the Ready for testing issue up to date scripts/upstream-issues.sh turn the weekly check's findings into issues scripts/verify-macos.sh fails when a Mach-O file links outside macOS or targets macOS > 13 +scripts/verify-linux.sh fails when an ELF file needs a library from outside glibc and the package, or a newer glibc scripts/smoke-*.sh smoke tests run against every package before it is published ``` diff --git a/packages/mariadb/linux/build.sh b/packages/mariadb/linux/build.sh new file mode 100755 index 0000000..55a4a73 --- /dev/null +++ b/packages/mariadb/linux/build.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Builds MariaDB for Linux with the configuration of the macOS build (packages/mariadb/macos): +# a static OpenSSL, bundled zlib and PCRE, the STANDALONE layout of MariaDB's own tarballs. The +# client's line editing (libedit, ncurses) comes from the build container and is bundled in lib/ +# (scripts/linux/bundle.sh). No libaio: InnoDB then does its I/O with its own threads, and the +# package doesn't depend on libaio.so.1, which Ubuntu 24.04 renamed. No systemd either. +# Runs in the manylinux_2_28 container (build-service.yml, AlmaLinux 8), so it needs glibc 2.28. +# Usage: packages/mariadb/linux/build.sh +# OPENSSL_VERSION OpenSSL to link statically (required) +set -euo pipefail +. "$(dirname "$0")/../../../scripts/lib.sh" +V="${1:?usage: build.sh }" +OUT="$(mkdir -p "${2:?usage: build.sh }" && cd "$2" && pwd)" +: "${OPENSSL_VERSION:?OPENSSL_VERSION is required}" +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +WORK="${WORK_DIR:-$PWD/mariadb-work}" +JOBS="$(nproc)" + +echo "::group::Build dependencies" +# libedit-devel is in AlmaLinux 8's PowerTools repository; OpenSSL's Configure needs Perl's core +# modules (IPC::Cmd), which the image's minimal Perl lacks; source.sh reads the checksum with jq. +dnf install -y -q --enablerepo=powertools libedit-devel ncurses-devel bison perl jq +echo "::endgroup::" + +mkdir -p "$WORK" && cd "$WORK" +echo "::group::OpenSSL $OPENSSL_VERSION" +bash "$ROOT/scripts/linux/openssl.sh" "$OPENSSL_VERSION" "$WORK/openssl" +echo "::endgroup::" + +echo "::group::Configure MariaDB $V" +bash "$ROOT/packages/mariadb/source.sh" "$V" mariadb.tar.gz +rm -rf src build && mkdir src build +tar -C src --strip-components 1 -xzf mariadb.tar.gz +cmake -S src -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DINSTALL_LAYOUT=STANDALONE \ + -DCMAKE_INSTALL_PREFIX="$OUT" \ + -DWITH_SSL="$WORK/openssl" \ + -DWITH_PCRE=bundled \ + -DWITH_ZLIB=bundled \ + -DWITH_LIBEDIT=OFF -DWITH_READLINE=OFF `# not the bundled copies: the container's libedit` \ + -DWITH_URING=NO -DWITH_LIBAIO=NO -DWITH_SYSTEMD=no \ + -DPLUGIN_TOKUDB=NO -DPLUGIN_ROCKSDB=NO -DPLUGIN_MROONGA=NO -DPLUGIN_SPIDER=NO -DPLUGIN_CONNECT=NO \ + -DPLUGIN_OQGRAPH=NO -DPLUGIN_SPHINX=NO -DPLUGIN_COLUMNSTORE=NO \ + -DWITH_UNIT_TESTS=OFF -DWITH_EMBEDDED_SERVER=OFF -DWITH_WSREP=OFF +echo "::endgroup::" + +echo "::group::Build" +cmake --build build -j "$JOBS" +rm -rf "$OUT" +cmake --install build +echo "::endgroup::" + +rm -rf "$OUT/mysql-test" "$OUT/sql-bench" "$OUT/man" "$OUT/include" "$OUT/share/doc" +# Test, benchmark, backup and storage-engine repair tools, as on macOS. +rm -f "$OUT"/bin/mariadb-test* "$OUT"/bin/mysqltest* "$OUT"/bin/mariadb-client-test* "$OUT"/bin/mysql_client_test* \ + "$OUT"/bin/mariadb-backup "$OUT"/bin/mariabackup "$OUT"/bin/mbstream "$OUT"/bin/mariadb-slap "$OUT"/bin/mysqlslap \ + "$OUT"/bin/aria_* "$OUT"/bin/myisam* "$OUT"/bin/mariadb-ldb "$OUT"/bin/sst_dump +find "$OUT/lib" -name '*.a' -delete +bash "$ROOT/scripts/linux/bundle.sh" "$OUT" + +mkdir -p "$OUT/LICENSES" +cp src/COPYING "$OUT/LICENSES/mariadb.txt" +cp "$WORK/openssl/LICENSE.txt" "$OUT/LICENSES/openssl.txt" +"$OUT/bin/mariadbd" --version diff --git a/packages/meilisearch/mirror.sh b/packages/meilisearch/mirror.sh index 52bf9f4..d1b8a1b 100755 --- a/packages/meilisearch/mirror.sh +++ b/packages/meilisearch/mirror.sh @@ -10,6 +10,8 @@ V="$1"; PLATFORM="$2"; OUT="$3" case "$PLATFORM" in macos-arm64) asset=meilisearch-macos-apple-silicon; bin=meilisearch ;; windows-x64) asset=meilisearch-windows-amd64.exe; bin=meilisearch.exe ;; + linux-x64) asset=meilisearch-linux-amd64; bin=meilisearch ;; + linux-arm64) asset=meilisearch-linux-aarch64; bin=meilisearch ;; esac dir="$(mktemp -d)" fetch "https://github.com/meilisearch/meilisearch/releases/download/v$V/$asset" "$dir/$bin" diff --git a/packages/node/mirror.sh b/packages/node/mirror.sh index 5662cdf..4c00da6 100755 --- a/packages/node/mirror.sh +++ b/packages/node/mirror.sh @@ -10,6 +10,8 @@ V="$1"; PLATFORM="$2"; OUT="$3" case "$PLATFORM" in macos-arm64) name="node-v$V-darwin-arm64.tar.gz" ;; windows-x64) name="node-v$V-win-x64.zip" ;; + linux-x64) name="node-v$V-linux-x64.tar.gz" ;; + linux-arm64) name="node-v$V-linux-arm64.tar.gz" ;; esac tmp="$(mktemp -d)" base="https://nodejs.org/dist/v$V" diff --git a/packages/php/linux/build.sh b/packages/php/linux/build.sh new file mode 100755 index 0000000..53c3935 --- /dev/null +++ b/packages/php/linux/build.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# Builds PHP for Linux with static-php-cli in its glibc container (bin/spc-gnu-docker: CentOS 7, +# glibc 2.17): php + php-fpm (NTS) with the compiled-in extensions, plus shared extensions as .so. +# The binaries link only against glibc (everything else is compiled in), so they run on glibc +# distributions from 2014 on, but not on musl ones (Alpine). Unlike a fully static musl build, +# PHP can load the shared extensions. +# +# Usage: packages/php/linux/build.sh +# major.minor builds the newest patch release; major.minor.patch exactly that release. +# SPC_VERSION static-php-cli release to use (required) +# STATIC_EXTS comma-separated extensions compiled in (required) +# SHARED_EXTS comma-separated extensions built as .so (optional) +# DOWNLOAD_ARGS extra arguments for `spc download`, e.g. --custom-url=xdebug:https://… (optional) +# WORK_DIR static-php-cli source checkout, the container's workspace (default: ./spc-src) +# GITHUB_TOKEN lets static-php-cli query GitHub without the anonymous rate limit +# +# Needs Docker and patchelf. Produces /bin/php, /bin/php-fpm, /ext/*.so and +# /LICENSES/. A shared extension that fails to build is reported, not fatal: PHP is still +# usable without it. +set -euo pipefail + +PHP_VERSION="${1:?usage: build.sh }" +PHP_MINOR="$(cut -d. -f1-2 <<<"$PHP_VERSION")" +OUT="$(mkdir -p "${2:?usage: build.sh }" && cd "$2" && pwd)" +: "${SPC_VERSION:?SPC_VERSION is required}" +: "${STATIC_EXTS:?STATIC_EXTS is required}" +SHARED_EXTS="${SHARED_EXTS:-}" + +# The same Xdebug and opentelemetry releases as packages/php/macos/build.sh (see the reasons there). +xdebug=3.5.3 +[[ "$PHP_MINOR" == 8.6 ]] && xdebug=3.6.0alpha1 +DOWNLOAD_ARGS="${DOWNLOAD_ARGS:-} --custom-url=xdebug:https://xdebug.org/files/xdebug-$xdebug.tgz" +DOWNLOAD_ARGS="$DOWNLOAD_ARGS --custom-url=opentelemetry:https://pecl.php.net/get/opentelemetry-1.4.2.tgz" +# OpenLDAP 2.7 puts declarations right after labels, which the container's GCC 10 rejects (C23 +# allows it, GCC 11 accepts it); static-php-cli takes the newest release, so pin the 2.6 line. +DOWNLOAD_ARGS="$DOWNLOAD_ARGS --custom-url=ldap:https://www.openldap.org/software/download/OpenLDAP/openldap-release/openldap-2.6.15.tgz" + +echo "::group::static-php-cli $SPC_VERSION (glibc container)" +# The container script and the image it builds come with the source, not the spc binary. +mkdir -p "${WORK_DIR:=$PWD/spc-src}" +cd "$WORK_DIR" +if [[ ! -x bin/spc-gnu-docker ]]; then + curl -fsSL "https://github.com/crazywhalecc/static-php-cli/archive/refs/tags/$SPC_VERSION.tar.gz" | tar xz --strip-components 1 +fi +# Downloads run on the host, with the spc binary: CentOS 7's curl can't reach servers that only +# speak TLS 1.3 (xdebug.org). SPC_TARGET has it pick the pre-built libraries for the container's +# glibc (--linux-glibc-2.17.txz). +if [[ ! -x ./spc ]]; then + curl -fsSL "https://github.com/crazywhalecc/static-php-cli/releases/download/$SPC_VERSION/spc-linux-$(uname -m).tar.gz" | tar xz + chmod +x spc +fi +download() { SPC_TARGET=native-native-gnu.2.17 ./spc download "$@"; } +# The container mounts these; created here, they belong to the runner's user rather than root. +mkdir -p buildroot source dist downloads pkgroot log +spc() { bin/spc-gnu-docker "$@"; } +spc --version +echo "::endgroup::" + +# The container runs as root: what it leaves in the mounted folders can only be removed with sudo. +remove() { rm -rf "$@" 2>/dev/null || sudo rm -rf "$@"; } + +all="$STATIC_EXTS${SHARED_EXTS:+,$SHARED_EXTS}" + +echo "::group::Download sources for PHP $PHP_VERSION" +# A release candidate (8.6.0RC3) isn't on php.net/distributions: php.net's list of release +# candidates has its tarball (on the release manager's downloads.php.net page) and SHA-256. +rc_sum="" +with_php="$PHP_VERSION" +if [[ "$PHP_VERSION" =~ RC[0-9]+$ ]]; then + read -r rc_url rc_sum < <(curl -fsSL --retry 3 'https://www.php.net/pre-release-builds.php?format=json' \ + | jq -r --arg v "$PHP_VERSION" '.[] | objects | .release | objects | select(.version == $v) | "\(.files.xz.path) \(.files.xz.sha256)"') + [[ -n "${rc_url:-}" ]] || { echo "::error::php.net doesn't list PHP $PHP_VERSION as a release candidate"; exit 1; } + DOWNLOAD_ARGS="$DOWNLOAD_ARGS --custom-url=php-src:$rc_url" + with_php="$PHP_MINOR" # static-php-cli only takes x.y or x.y.z; the source is the RC tarball +fi +# shellcheck disable=SC2086 +download --with-php="$with_php" --for-extensions="$all" --prefer-pre-built --retry=3 ${DOWNLOAD_ARGS:-} +if [[ -n "$rc_sum" ]]; then + rc_file="downloads/$(basename "$rc_url")" + [[ -f "$rc_file" ]] || { echo "::error::static-php-cli didn't save $rc_file"; ls downloads; exit 1; } + got="$(sha256sum "$rc_file" | cut -d' ' -f1)" + [[ "$got" == "$rc_sum" ]] || { echo "::error::$rc_file: SHA-256 $got, expected $rc_sum"; exit 1; } + echo "✓ $(basename "$rc_file") SHA-256 $got" +fi +echo "::endgroup::" + +# spc hides compiler output; it lands in log/ (wiped by the next spc command), so print the errors. +print_spc_errors() { + echo "::group::static-php-cli log: errors" + grep -h -E ' error:|fatal error|undefined reference|\*\*\* \[' log/*.log 2>/dev/null | sort -u | head -80 || true + echo "::endgroup::" + echo "::group::static-php-cli log: last 80 lines" + tail -n 80 log/spc.shell.log 2>/dev/null || true + echo "::endgroup::" +} + +rm -rf "$OUT" && mkdir -p "$OUT/bin" "$OUT/ext" "$OUT/LICENSES" + +# static-php-cli builds the shared extensions one after another and stops at the first that +# fails. Rebuild without that one until every remaining extension built (at most 4 attempts). +pending="$SHARED_EXTS" +for attempt in 1 2 3 4; do + echo "::group::Build PHP $PHP_VERSION (attempt $attempt${pending:+, shared: $pending})" + # Start clean: libraries built for an extension we dropped would otherwise be linked + # without their dependencies. Pre-built libraries unpack quickly. + remove buildroot source/php-src + mkdir -p buildroot + ok=1 + # static-php-cli's post-build sanity checks are stricter than we need; the smoke test verifies. + spc build "$STATIC_EXTS" ${pending:+--build-shared="$pending"} --build-cli --build-fpm 2>&1 | tee spc-build.out || ok=0 + echo "::endgroup::" + [[ $ok -eq 1 ]] || print_spc_errors + + for bin in php php-fpm; do + [[ -x "buildroot/bin/$bin" ]] || { echo "::error::$bin missing"; exit 1; } + cp "buildroot/bin/$bin" "$OUT/bin/" + done + + remaining="" + for ext in ${pending//,/ }; do + so="$(find buildroot/modules buildroot/lib/php -name "$ext.so" 2>/dev/null | head -1 || true)" + if [[ -n "$so" ]]; then cp "$so" "$OUT/ext/"; else remaining="${remaining:+$remaining,}$ext"; fi + done + [[ -z "$remaining" ]] && break + + failed="$(sed -n 's/.*Failed module: .*shared extension \([a-z0-9_]*\) builder.*/\1/p' spc-build.out | tail -1)" + if [[ -z "$failed" ]]; then + echo "::warning::not built and no failing extension reported: $remaining" + break + fi + echo "::warning::$failed.so failed to build, retrying without it" + pending="$(tr ',' '\n' <<<"$remaining" | { grep -vx "$failed" || true; } | paste -sd, -)" + [[ -n "$pending" ]] || break +done +for ext in ${SHARED_EXTS//,/ }; do + [[ -f "$OUT/ext/$ext.so" ]] || echo "::warning::$ext.so was not built" +done + +if [[ "$PHP_VERSION" == *.*.* ]]; then + built="$("$OUT/bin/php" -n -r 'echo PHP_VERSION;')" + [[ "$built" == "$PHP_VERSION" ]] || { echo "::error::asked for PHP $PHP_VERSION, built $built"; exit 1; } +fi + +# The binaries keep the container's library path (/app/buildroot/lib) as RPATH: nothing is there +# on a user's machine, and whoever could create it would have PHP load their libraries. +find "$OUT/bin" "$OUT/ext" -type f -exec patchelf --remove-rpath {} \; + +# Inside the container, so into a mounted folder. +remove dist/licenses +if spc dump-license --for-extensions="$all" --dump-dir=dist/licenses; then + cp -R dist/licenses/. "$OUT/LICENSES/" +else + echo "::warning::license dump failed" +fi + +ls -la "$OUT/bin" +echo "built PHP $PHP_VERSION into $OUT" diff --git a/packages/postgres/linux/build.sh b/packages/postgres/linux/build.sh new file mode 100644 index 0000000..92547f7 --- /dev/null +++ b/packages/postgres/linux/build.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Builds the PostgreSQL server, client tools and contrib extensions for Linux, with a static OpenSSL +# (pgcrypto, sslinfo). zlib, libuuid (uuid-ossp) and libedit (psql's line editing) come from the +# build container and are bundled in lib/ (scripts/linux/bundle.sh). Relocatable: PostgreSQL finds +# share/ and lib/ relative to bin/, and every binary finds lib/ through $ORIGIN. +# Runs in the manylinux_2_28 container (build-service.yml, AlmaLinux 8), so it needs glibc 2.28. +# Usage: packages/postgres/linux/build.sh +# OPENSSL_VERSION OpenSSL to link statically (required) +set -euo pipefail +. "$(dirname "$0")/../../../scripts/lib.sh" +V="${1:?usage: build.sh }" +OUT="$(mkdir -p "${2:?usage: build.sh }" && cd "$2" && pwd)" +: "${OPENSSL_VERSION:?OPENSSL_VERSION is required}" +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +WORK="${WORK_DIR:-$PWD/postgres-work}" +JOBS="$(nproc)" + +echo "::group::Build dependencies" +# libedit-devel is in AlmaLinux 8's PowerTools repository; OpenSSL's Configure needs Perl's core +# modules (IPC::Cmd), which the image's minimal Perl lacks; PostgreSQL's configure wants flex. +dnf install -y -q --enablerepo=powertools libedit-devel libuuid-devel zlib-devel perl flex +echo "::endgroup::" + +mkdir -p "$WORK" && cd "$WORK" +echo "::group::OpenSSL $OPENSSL_VERSION" +bash "$ROOT/scripts/linux/openssl.sh" "$OPENSSL_VERSION" "$WORK/openssl" +echo "::endgroup::" + +echo "::group::Configure PostgreSQL $V" +bash "$ROOT/packages/postgres/source.sh" "$V" postgresql.tar.bz2 +rm -rf src && mkdir src && tar -C src --strip-components 1 -xjf postgresql.tar.bz2 +cd src +# Installed straight into OUT without an rpath; bundle.sh then gives every binary an $ORIGIN one. +# ICU stays off (Flock uses --locale=C). The static libcrypto needs libdl and libpthread. +./configure --prefix="$OUT" --disable-rpath --with-openssl --with-uuid=e2fs --without-icu \ + --with-libedit-preferred --with-zlib \ + CPPFLAGS="-I$WORK/openssl/include" LDFLAGS="-L$WORK/openssl/lib" LIBS="-ldl -lpthread" +echo "::endgroup::" + +# PostgreSQL 15+ refuses a libpq that references exit functions (libpq-refs-stamp, using nm). +# The static OpenSSL brings two: atexit (its cleanup handler) and pthread_exit, neither of which +# libpq calls on its own. Hide just those two from the check; anything else still fails it. +mkdir -p "$WORK/nm" +cat >"$WORK/nm/nm" <<'NM' +#!/bin/sh +/usr/bin/nm "$@" | grep -v -E ' (atexit|pthread_exit)(@[A-Z_0-9.@]*)?$' +NM +chmod +x "$WORK/nm/nm" + +echo "::group::Build" +rm -rf "$OUT" && mkdir -p "$OUT" +PATH="$WORK/nm:$PATH" make -j"$JOBS" +make -C contrib -j"$JOBS" +make install +make -C contrib install +echo "::endgroup::" + +# Headers, static libraries and pkg-config files are only for building against PostgreSQL. +rm -rf "$OUT/include" "$OUT/lib/pkgconfig" "$OUT/share/doc" "$OUT/share/man" +find "$OUT/lib" -name '*.a' -delete +bash "$ROOT/scripts/linux/bundle.sh" "$OUT" + +mkdir -p "$OUT/LICENSES" +cp COPYRIGHT "$OUT/LICENSES/postgresql.txt" +cp "$WORK/openssl/LICENSE.txt" "$OUT/LICENSES/openssl.txt" +"$OUT/bin/postgres" --version diff --git a/packages/redis/linux/build.sh b/packages/redis/linux/build.sh new file mode 100755 index 0000000..13a5498 --- /dev/null +++ b/packages/redis/linux/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Builds Redis for Linux: redis-server, redis-cli and redis-benchmark, linked only against glibc +# (TLS off: Flock only listens on 127.0.0.1; libc's malloc, as for Valkey). Builds src/ only: +# since 8.x the top-level make also builds modules that need Rust. +# Runs in the manylinux_2_28 container (build-service.yml), so the binaries need glibc 2.28. +# Usage: packages/redis/linux/build.sh +set -euo pipefail +V="${1:?usage: build.sh }" +OUT="$(mkdir -p "${2:?usage: build.sh }" && cd "$2" && pwd)" +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +WORK="${WORK_DIR:-$PWD/redis-work}" + +mkdir -p "$WORK" && cd "$WORK" +# The work directory sits inside this repository: without this, the version string would +# record our commit (mkreleasehdr.sh asks git) instead of 00000000 for a release tarball. +export GIT_DIR="$WORK/no-git" +bash "$ROOT/packages/redis/source.sh" "$V" redis.tar.gz +rm -rf src && mkdir src && tar -C src --strip-components 1 -xzf redis.tar.gz +make -C src/src -j"$(nproc)" BUILD_TLS=no USE_SYSTEMD=no MALLOC=libc + +rm -rf "$OUT" && mkdir -p "$OUT/LICENSES" +cp src/src/redis-server src/src/redis-cli src/src/redis-benchmark "$OUT/" +# 7.2: BSD (COPYING); 7.4: RSALv2/SSPL; 8.x: RSALv2/SSPL/AGPLv3 (LICENSE.txt plus one file each). +for f in src/COPYING src/LICENSE*.txt src/licenses/*; do + [[ -f "$f" ]] && cp "$f" "$OUT/LICENSES/" +done +"$OUT/redis-server" --version diff --git a/packages/rustfs/mirror.sh b/packages/rustfs/mirror.sh index 62e446c..ca001e5 100755 --- a/packages/rustfs/mirror.sh +++ b/packages/rustfs/mirror.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # Mirrors the RustFS binary, checked against GitHub's SHA-256 of the release asset. Upstream's zip -# holds only the executable, so it's repacked with the Apache-2.0 license. +# holds only the executable, so it's repacked with the Apache-2.0 license. On Linux, the static musl +# build, which runs on any distribution. # Usage: packages/rustfs/mirror.sh set -euo pipefail . "$(dirname "$0")/../../scripts/lib.sh" @@ -9,6 +10,8 @@ V="$1"; PLATFORM="$2"; OUT="$3" case "$PLATFORM" in macos-arm64) asset="rustfs-macos-aarch64-v$V.zip" ;; windows-x64) asset="rustfs-windows-x86_64-v$V.zip" ;; + linux-x64) asset="rustfs-linux-x86_64-musl-v$V.zip" ;; + linux-arm64) asset="rustfs-linux-aarch64-musl-v$V.zip" ;; esac tmp="$(mktemp -d)"; dir="$tmp/pkg"; mkdir "$dir" fetch "https://github.com/rustfs/rustfs/releases/download/$V/$asset" "$tmp/$asset" diff --git a/packages/typesense/mirror.sh b/packages/typesense/mirror.sh index 1dc08a0..6603dc7 100755 --- a/packages/typesense/mirror.sh +++ b/packages/typesense/mirror.sh @@ -1,15 +1,20 @@ #!/usr/bin/env bash -# Mirrors the Typesense server binary for macOS (there is no native Windows build). Typesense -# publishes no checksum file; the binary is checked against the MD5 shipped inside the tarball. -# Repacked with the GPLv3 license, which the upstream tarball doesn't include. -# Usage: packages/typesense/mirror.sh macos-arm64 +# Mirrors the Typesense server binary for macOS and Linux (there is no native Windows build). +# Typesense publishes no checksum file; the binary is checked against the MD5 shipped inside the +# tarball. Repacked with the GPLv3 license, which the upstream tarball doesn't include. +# Usage: packages/typesense/mirror.sh set -euo pipefail . "$(dirname "$0")/../../scripts/lib.sh" V="$1"; PLATFORM="$2"; OUT="$3" -[[ "$PLATFORM" == macos-arm64 ]] || { echo "Typesense has no Windows build" >&2; exit 1; } +case "$PLATFORM" in + macos-arm64) name=darwin-arm64 ;; + linux-x64) name=linux-amd64 ;; + linux-arm64) name=linux-arm64 ;; + *) echo "Typesense has no Windows build" >&2; exit 1 ;; +esac tmp="$(mktemp -d)"; dir="$tmp/pkg"; mkdir "$dir" -fetch "https://dl.typesense.org/releases/$V/typesense-server-$V-darwin-arm64.tar.gz" "$tmp/t.tar.gz" +fetch "https://dl.typesense.org/releases/$V/typesense-server-$V-$name.tar.gz" "$tmp/t.tar.gz" tar -C "$dir" -xzf "$tmp/t.tar.gz" check_md5 "$dir/typesense-server" "$(cut -d' ' -f1 "$dir/typesense-server.md5.txt")" fetch "https://raw.githubusercontent.com/typesense/typesense/v$V/LICENSE.txt" "$dir/LICENSE.txt" diff --git a/packages/valkey/linux/build.sh b/packages/valkey/linux/build.sh new file mode 100755 index 0000000..1a95f45 --- /dev/null +++ b/packages/valkey/linux/build.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Builds Valkey for Linux: valkey-server, valkey-cli and valkey-benchmark, linked only against glibc +# (TLS off: Flock only listens on 127.0.0.1; libc's malloc, as on macOS: the bundled jemalloc is +# built for the build machine's page size, and arm64 kernels with 16K or 64K pages exist). +# Runs in the manylinux_2_28 container (build-service.yml), so the binaries need glibc 2.28. +# Usage: packages/valkey/linux/build.sh +set -euo pipefail +V="${1:?usage: build.sh }" +OUT="$(mkdir -p "${2:?usage: build.sh }" && cd "$2" && pwd)" +ROOT="$(cd "$(dirname "$0")/../../.." && pwd)" +WORK="${WORK_DIR:-$PWD/valkey-work}" + +mkdir -p "$WORK" && cd "$WORK" +# The work directory sits inside this repository: without this, the version string would +# record our commit (mkreleasehdr.sh asks git) instead of 00000000 for a release tarball. +export GIT_DIR="$WORK/no-git" +bash "$ROOT/packages/valkey/source.sh" "$V" valkey.tar.gz +rm -rf src && mkdir src && tar -C src --strip-components 1 -xzf valkey.tar.gz +make -C src/src -j"$(nproc)" BUILD_TLS=no USE_SYSTEMD=no MALLOC=libc + +rm -rf "$OUT" && mkdir -p "$OUT/LICENSES" +cp src/src/valkey-server src/src/valkey-cli src/src/valkey-benchmark "$OUT/" +cp src/COPYING "$OUT/LICENSES/valkey.txt" +"$OUT/valkey-server" --version diff --git a/scripts/lib.sh b/scripts/lib.sh index d02e210..3ac42d3 100644 --- a/scripts/lib.sh +++ b/scripts/lib.sh @@ -1,14 +1,20 @@ # Shared helpers for the package scripts (source.sh, mirror.sh, build.sh). Source it: # . "$(dirname "$0")/../../scripts/lib.sh" +# --retry-all-errors needs curl 7.71; the Linux build container (AlmaLinux 8) has 7.61. +retry_all=--retry-all-errors +curl --help all 2>/dev/null | grep -q -- --retry-all-errors || retry_all= + # fetch : downloads with retries, failing on HTTP errors. fetch() { - curl -fsSL --retry 3 --retry-all-errors -o "$2" "$1" + # shellcheck disable=SC2086 + curl -fsSL --retry 3 $retry_all -o "$2" "$1" } # fetch_text : prints a small file (checksum lists, API answers). fetch_text() { - curl -fsSL --retry 3 --retry-all-errors "$1" + # shellcheck disable=SC2086 + curl -fsSL --retry 3 $retry_all "$1" } # Read from stdin: with a file name, Git Bash's sha256sum escapes Windows paths (""). diff --git a/scripts/linux/bundle.sh b/scripts/linux/bundle.sh new file mode 100755 index 0000000..2cb2adc --- /dev/null +++ b/scripts/linux/bundle.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env bash +# Makes a Linux install relocatable, like scripts/macos/relocate.sh: every shared library an ELF +# file under needs from outside it, beyond glibc's own, libgcc_s and libstdc++ (which every +# glibc distribution has), is copied to /lib, with the license files its RPM package lists +# (into /LICENSES). Every ELF file that needs a library in /lib then finds it through a +# RUNPATH relative to itself ($ORIGIN); other rpaths, which point into the build, are dropped. +# A library under the LGPL (libxcrypt) has its source package downloaded to SOURCES_DIR, when set, +# to be published next to the archive; GCC's runtime libraries are exempt (GCC Runtime Library +# Exception). Needs patchelf. scripts/verify-linux.sh checks the result. +# Usage: scripts/linux/bundle.sh +set -euo pipefail +. "$(dirname "$0")/../lib.sh" +DIR="$(cd "${1:?usage: bundle.sh }" && pwd)" +LIB="$DIR/lib" + +# source_package : the source package of a bundled LGPL library into SOURCES_DIR, from +# AlmaLinux's vault (the build container is AlmaLinux 8). +source_package() { + [[ -n "${SOURCES_DIR:-}" ]] || return 0 + case "$1" in libatomic | libgcc | libstdc++ | libgomp) return 0 ;; esac + [[ "$(rpm -q --qf '%{LICENSE}' "$1")" == *LGPL* ]] || return 0 + local srpm repo; srpm="$(rpm -q --qf '%{SOURCERPM}' "$1")" + mkdir -p "$SOURCES_DIR" + for repo in BaseOS AppStream PowerTools; do + fetch "https://vault.almalinux.org/8.10/$repo/Source/Packages/$srpm" "$SOURCES_DIR/$srpm" 2>/dev/null \ + && { echo "source: $srpm"; return 0; } + done + echo "✗ no source package $srpm for $1 (LGPL)" >&2 + return 1 +} + +system() { + case "$1" in + libc.so.* | libm.so.* | libdl.so.* | libpthread.so.* | librt.so.* | libresolv.so.* | libutil.so.* | \ + ld-linux*.so.* | libgcc_s.so.* | libstdc++.so.*) return 0 ;; + esac + return 1 +} +elf() { [[ ! -L "$1" && "$(head -c 4 "$1" | tr -d '\0')" == $'\x7fELF' ]]; } + +# Copied libraries need others in turn: repeat until nothing new is needed. +added=1 +while [[ $added -eq 1 ]]; do + added=0 + while IFS= read -r -d '' f; do + elf "$f" || continue + while read -r name path; do + system "$name" || [[ -e "$LIB/$name" ]] && continue + [[ "$path" == /* && -f "$path" ]] || { echo "✗ $f needs $name, which isn't installed" >&2; exit 1; } + mkdir -p "$LIB" "$DIR/LICENSES" + cp -L "$path" "$LIB/$name" && chmod 755 "$LIB/$name" + echo "bundled $name from $path" + if pkg="$(rpm -qf --qf '%{NAME}' "$path" 2>/dev/null)"; then + licenses="$(rpm -qL "$pkg" | grep '^/' || true)" + # A library package can leave the license to another one built from the same source + # (ncurses-libs to ncurses-base). + if [[ -z "$licenses" ]]; then + src="$(rpm -q --qf '%{SOURCERPM}' "$pkg")" + licenses="$(rpm -qa --qf '%{NAME} %{SOURCERPM}\n' | awk -v s="$src" '$2 == s {print $1}' \ + | xargs -r rpm -qL | grep '^/' || true)" + fi + [[ -n "$licenses" ]] || echo "::warning::no license file for $name ($pkg)" + while read -r license; do + [[ -f "$license" ]] && cp "$license" "$DIR/LICENSES/$pkg-$(basename "$license")" + done <<<"$licenses" + source_package "$pkg" + fi + added=1 + done < <(LD_LIBRARY_PATH="$LIB" ldd "$f" 2>/dev/null | awk '$2 == "=>" {print $1, $3}') + done < <(find "$DIR" -type f -print0) +done + +while IFS= read -r -d '' f; do + elf "$f" || continue + chmod u+w "$f" + uses_lib=0 + for name in $(readelf -d "$f" | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p'); do + [[ -e "$LIB/$name" ]] && uses_lib=1 + done + if [[ $uses_lib -eq 1 ]]; then + rel="$(realpath --relative-to="$(dirname "$f")" "$LIB")" + if [[ "$rel" == . ]]; then rpath='$ORIGIN'; else rpath="\$ORIGIN/$rel"; fi + patchelf --set-rpath "$rpath" "$f" + else + patchelf --remove-rpath "$f" + fi +done < <(find "$DIR" -type f -print0) +echo "✓ bundled the libraries of $DIR in lib/ (\$ORIGIN)" diff --git a/scripts/linux/openssl.sh b/scripts/linux/openssl.sh new file mode 100755 index 0000000..c031c3c --- /dev/null +++ b/scripts/linux/openssl.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Builds a static OpenSSL into (Linux, like scripts/macos/openssl.sh), checked against the +# SHA-256 published with the OpenSSL release. Reuses when it already holds this build. +# Usage: scripts/linux/openssl.sh +set -euo pipefail +. "$(dirname "$0")/../lib.sh" +V="${1:?usage: openssl.sh }"; PREFIX="${2:?usage: openssl.sh }" + +[[ -f "$PREFIX/lib/libssl.a" && -f "$PREFIX/.version-$V-no-dso" ]] && exit 0 +case "$(uname -m)" in + x86_64) target=linux-x86_64 ;; + aarch64) target=linux-aarch64 ;; + *) echo "unsupported machine $(uname -m)" >&2; exit 1 ;; +esac +tmp="$(mktemp -d)" +url="https://github.com/openssl/openssl/releases/download/openssl-$V/openssl-$V.tar.gz" +fetch "$url" "$tmp/openssl.tar.gz" +check_sha256 "$tmp/openssl.tar.gz" "$(fetch_text "$url.sha256" | awk '{print $1}')" +mkdir "$tmp/src" && tar -C "$tmp/src" --strip-components 1 -xzf "$tmp/openssl.tar.gz" +rm -rf "$PREFIX" +# -fPIC: it ends up in shared libraries (libpq.so, MariaDB's plugins). no-dso: nothing loads +# OpenSSL modules here, and without it every library linking libcrypto.a would also need -ldl. +(cd "$tmp/src" \ + && ./Configure "$target" no-shared no-dso no-tests no-docs no-apps -fPIC --prefix="$PREFIX" --libdir=lib \ + && make -j"$(nproc)" && make install_sw) +cp "$tmp/src/LICENSE.txt" "$PREFIX/LICENSE.txt" +touch "$PREFIX/.version-$V-no-dso" +rm -rf "$tmp" diff --git a/scripts/smoke-php-package.sh b/scripts/smoke-php-package.sh index df540b5..acfad64 100755 --- a/scripts/smoke-php-package.sh +++ b/scripts/smoke-php-package.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -# Smoke-tests a macOS PHP or FrankenPHP package directory against php.macos in versions.json: -# every compiled-in extension loaded, every shared extension in ext/ built and loading. +# Smoke-tests a macOS or Linux PHP package directory, or a macOS FrankenPHP one, against php.macos in +# versions.json (or the lists in STATIC_EXTS and SHARED_EXTS, which the Linux workflow takes from +# php.linux): every compiled-in extension loaded, every shared extension in ext/ built and loading. # Usage: scripts/smoke-php-package.sh set -euo pipefail PKG="${1:?usage: smoke-php-package.sh }" diff --git a/scripts/smoke-service.sh b/scripts/smoke-service.sh index a7f4add..3e23934 100644 --- a/scripts/smoke-service.sh +++ b/scripts/smoke-service.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash -# Smoke-tests a service package on macOS the way Flock runs it: unpack (dropping a single +# Smoke-tests a service package on macOS or Linux the way Flock runs it: unpack (dropping a single # top-level folder, as Flock does), initialise a data directory, start on 127.0.0.1, talk to it # with its own client, shut down. With SIGNER_MACOS set, the main binary must carry a valid code -# signature whose authority matches that regex (SIGNER_MACOS in versions.json) (the vendor's, for mirrored packages). +# signature whose authority matches that regex (SIGNER_MACOS in versions.json) (the vendor's, for mirrored packages); +# Linux binaries carry none. # Usage: scripts/smoke-service.sh set -uo pipefail @@ -34,7 +35,7 @@ entries=("$D"/*) if [[ ${#entries[@]} -eq 1 && -d "${entries[0]}" ]]; then D="${entries[0]}"; fi echo "### $PKG: $(basename "$ARCHIVE")" | tee -a "$summary" -if [[ -n "${SIGNER_MACOS:-}" ]]; then +if [[ -n "${SIGNER_MACOS:-}" && "$(uname -s)" == Darwin ]]; then case "$PKG" in mysql) main=bin/mysqld ;; mongodb) main=bin/mongod ;; node) main=bin/node ;; *) main="" ;; esac if codesign --verify --strict "$D/$main" 2>/dev/null \ && authority="$(codesign -dvv "$D/$main" 2>&1 | sed -n 's/^Authority=//p' | head -1)" \ diff --git a/scripts/verify-linux.sh b/scripts/verify-linux.sh new file mode 100755 index 0000000..01b661a --- /dev/null +++ b/scripts/verify-linux.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Checks that every ELF file under a directory runs on other Linux machines: +# - it needs only glibc's libraries, the dynamic loader, libgcc_s and libstdc++ (on every glibc +# distribution), and libraries shipped in the directory itself (scripts/linux/bundle.sh), +# which it finds there, not on this machine +# - its RPATH or RUNPATH, if any, is relative to itself ($ORIGIN): never into the build tree +# - it needs no glibc symbol newer than MAX_GLIBC (default 2.17) +# Usage: scripts/verify-linux.sh [max-glibc] +set -euo pipefail + +DIR="$(cd "${1:?usage: verify-linux.sh [max-glibc]}" && pwd)" +MAX_GLIBC="${2:-2.17}" + +ver_gt() { [[ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -1)" == "$1" && "$1" != "$2" ]]; } +system() { + case "$1" in + libc.so.6 | libm.so.6 | libdl.so.2 | libpthread.so.0 | librt.so.1 | libresolv.so.2 | libutil.so.1 | \ + ld-linux-*.so.* | libgcc_s.so.1 | libstdc++.so.6) return 0 ;; + esac + return 1 +} + +failed=0 +checked=0 +while IFS= read -r -d '' f; do + [[ "$(head -c 4 "$f" | tr -d '\0')" == $'\x7fELF' ]] || continue + checked=$((checked + 1)) + + resolved="$(ldd "$f" 2>/dev/null || true)" + while read -r lib; do + system "$lib" && continue + path="$(awk -v n="$lib" '$1 == n && $2 == "=>" {print $3}' <<<"$resolved")" + [[ "$path" == "$DIR"/* ]] || { echo "✗ $f needs $lib (${path:-not found}), not from the package"; failed=1; } + done < <(readelf -d "$f" | sed -n 's/.*(NEEDED).*\[\(.*\)\]/\1/p') + + while read -r entry; do + [[ "$entry" == '$ORIGIN'* ]] || { echo "✗ $f has the rpath $entry"; failed=1; } + done < <(readelf -d "$f" | sed -n 's/.*(R\(UN\)\{0,1\}PATH).*\[\(.*\)\]/\2/p' | tr ':' '\n') + + newest="$(readelf -V --wide "$f" | grep -o 'GLIBC_[0-9][0-9.]*' | sed 's/GLIBC_//' | sort -V | tail -1 || true)" + if [[ -n "$newest" ]] && ver_gt "$newest" "$MAX_GLIBC"; then + echo "✗ $f needs glibc $newest (max $MAX_GLIBC)" + failed=1 + fi +done < <(find "$DIR" -type f -print0) + +if [[ $checked -eq 0 ]]; then + echo "✗ no ELF files found in $DIR" + exit 1 +fi +if [[ $failed -ne 0 ]]; then + exit 1 +fi +echo "✓ $checked ELF files are portable (glibc ≤ $MAX_GLIBC, other libraries only from the package)" diff --git a/tools/flockrel/check.go b/tools/flockrel/check.go index 6e7c1fd..bb00af3 100644 --- a/tools/flockrel/check.go +++ b/tools/flockrel/check.go @@ -22,6 +22,7 @@ func runCheck(args []string) error { fs := flag.NewFlagSet("check", flag.ExitOnError) versionsPath := fs.String("versions", "versions.json", "versions file") buildSh := fs.String("php-build", "packages/php/macos/build.sh", "PHP macOS build script (pins xdebug and opentelemetry)") + linuxBuildSh := fs.String("php-build-linux", "packages/php/linux/build.sh", "PHP Linux build script (pins xdebug and opentelemetry)") out := fs.String("out", "findings.json", "where to write the findings") fs.Parse(args) @@ -33,7 +34,11 @@ func runCheck(args []string) error { if err != nil { return err } - findings, problems := check(v, string(script), liveSources{}, time.Now().UTC()) + linuxScript, err := os.ReadFile(*linuxBuildSh) + if err != nil { + return err + } + findings, problems := check(v, string(script), string(linuxScript), liveSources{}, time.Now().UTC()) for _, p := range problems { fmt.Fprintf(os.Stderr, "::warning::%s\n", p) } @@ -176,7 +181,7 @@ func newestOf(vs []string) string { } // check returns the findings, and problems (sources that couldn't be read) to warn about. -func check(v *versions, phpBuildScript string, src sources, now time.Time) (findings []finding, problems []string) { +func check(v *versions, phpBuildScript, linuxPHPBuildScript string, src sources, now time.Time) (findings []finding, problems []string) { today := now.Format(time.DateOnly) // lineFindings reports, for one product, lines newer than the newest a config builds and lines @@ -281,6 +286,7 @@ func check(v *versions, phpBuildScript string, src sources, now time.Time) (find } lineFindings("PHP", []lineConfig{ {"`php.macos.minors`", v.PHP.MacOS.Minors, ""}, + {"`php.linux.minors`", v.PHP.Linux.Minors, ""}, {"`php.windows.minors`", v.PHP.Windows.Minors, " (check that the extensions in `php.windows.extensions` support it)"}, {"`frankenphp.minors`", v.FrankenPHP.Minors, ""}, }, cycles, upstream, rcs, false, false) @@ -316,12 +322,20 @@ func check(v *versions, phpBuildScript string, src sources, now time.Time) (find {"FrankenPHP", v.FrankenPHP.Version, "php/frankenphp", "", "`frankenphp.version` (then raise `frankenphp.*.build`)", ""}, {"static-php-cli", v.PHP.MacOS.SPC, "crazywhalecc/static-php-cli", "", "`php.macos.spc` (then raise `php.macos.build` and `frankenphp.macos.build`)", ""}, } + if v.PHP.Linux.SPC != "" { + pins = append(pins, pin{"static-php-cli (Linux)", v.PHP.Linux.SPC, "crazywhalecc/static-php-cli", "", "`php.linux.spc` (then raise `php.linux.build`)", ""}) + } if ossl := opensslPin(v); ossl != "" { - pins = append(pins, pin{"OpenSSL", ossl, "openssl/openssl", "openssl-", "`OPENSSL_VERSION` of postgres and mariadb (then raise their macOS `build`)", truncate(ossl, 2)}) + pins = append(pins, pin{"OpenSSL", ossl, "openssl/openssl", "openssl-", "`OPENSSL_VERSION` of postgres and mariadb (then raise their macOS and, for postgres, Linux `build`)", truncate(ossl, 2)}) } - for _, ext := range []struct{ name, repo string }{{"xdebug", "xdebug/xdebug"}, {"opentelemetry", "open-telemetry/opentelemetry-php-instrumentation"}} { - if m := regexp.MustCompile(ext.name + `-([0-9.]+)\.tgz`).FindStringSubmatch(phpBuildScript); m != nil { - pins = append(pins, pin{ext.name + " (macOS)", m[1], ext.repo, "", "`packages/php/macos/build.sh` (then raise `php.macos.build` and `frankenphp.macos.build`)", ""}) + for _, script := range []struct{ platform, dir, text, raise string }{ + {"macOS", "macos", phpBuildScript, "`php.macos.build` and `frankenphp.macos.build`"}, + {"Linux", "linux", linuxPHPBuildScript, "`php.linux.build`"}, + } { + for _, ext := range []struct{ name, repo string }{{"xdebug", "xdebug/xdebug"}, {"opentelemetry", "open-telemetry/opentelemetry-php-instrumentation"}} { + if m := regexp.MustCompile(ext.name + `-([0-9.]+)\.tgz`).FindStringSubmatch(script.text); m != nil { + pins = append(pins, pin{ext.name + " (" + script.platform + ")", m[1], ext.repo, "", "`packages/php/" + script.dir + "/build.sh` (then raise " + script.raise + ")", ""}) + } } } exts := v.PHP.Windows.Extensions diff --git a/tools/flockrel/check_test.go b/tools/flockrel/check_test.go index ebe3da1..36fd92b 100644 --- a/tools/flockrel/check_test.go +++ b/tools/flockrel/check_test.go @@ -28,6 +28,7 @@ func TestCheck(t *testing.T) { var v versions if err := json.Unmarshal([]byte(`{ "php": {"macos": {"minors": ["8.4", "8.5"], "spc": "2.8.5"}, + "linux": {"minors": ["8.4", "8.5"], "spc": "2.8.5"}, "windows": {"minors": ["8.1", "8.4", "8.5"], "extensions": { "from_source": {"igbinary": {"repo": "igbinary/igbinary", "ref": "3.2.16", "refs": {"8.5": "3.2.17RC1"}}, "apcu": {"repo": "krakjoe/apcu", "ref": "v5.1.28"}}, @@ -65,7 +66,8 @@ func TestCheck(t *testing.T) { }, rcs: map[string]string{"8.5.13": "8.5.13RC1", "8.7.0": "8.7.0RC1", "8.6.1": "8.6.1RC1"}, } - findings, problems := check(&v, "--custom-url=xdebug:https://xdebug.org/files/xdebug-3.5.3.tgz\n--custom-url=opentelemetry:https://pecl.php.net/get/opentelemetry-1.4.2.tgz", src, now) + script := "--custom-url=xdebug:https://xdebug.org/files/xdebug-3.5.3.tgz\n--custom-url=opentelemetry:https://pecl.php.net/get/opentelemetry-1.4.2.tgz" + findings, problems := check(&v, script, script, src, now) if len(problems) > 0 { t.Errorf("problems: %v", problems) } @@ -95,7 +97,7 @@ func TestCheck(t *testing.T) { } t.Errorf("%d findings, want %d", len(findings), len(want)) } - if b := got["New release line: PHP 8.6"].Body; !strings.Contains(b, "`php.macos.minors`, `php.windows.minors` (check that the extensions") || !strings.Contains(b, "`frankenphp.minors`") { + if b := got["New release line: PHP 8.6"].Body; !strings.Contains(b, "`php.macos.minors`, `php.linux.minors`, `php.windows.minors` (check that the extensions") || !strings.Contains(b, "`frankenphp.minors`") { t.Errorf("PHP 8.6 body: %s", b) } if b := got["New release line: PHP 8.7"].Body; !strings.Contains(b, "release candidate phase (8.7.0RC1)") || !strings.Contains(b, "beta channel only") { @@ -110,6 +112,7 @@ func TestCheck(t *testing.T) { "| OpenSSL | 3.5.9 | 3.5.10 |", // the same line, not 4.0 "| xdebug (macOS) | 3.5.3 | 3.6.0 |", "| opentelemetry (macOS) | 1.4.2 | 1.5.0 |", + "| xdebug (Linux) | 3.5.3 | 3.6.0 |", "| igbinary for PHP 8.5 (Windows) | 3.2.17RC1 | 3.2.17 |", "| igbinary (Windows) | 3.2.16 | 3.2.17 |", } { diff --git a/tools/flockrel/main_test.go b/tools/flockrel/main_test.go index bf16869..88a17a1 100644 --- a/tools/flockrel/main_test.go +++ b/tools/flockrel/main_test.go @@ -37,6 +37,7 @@ func TestTagAndAssetNames(t *testing.T) { assets := map[string]parsed{ "php-8.4.26-build1-macos-arm64.tar.gz": {1, "macos-arm64", true}, "php-8.4.26-build12-windows-x64.zip": {12, "windows-x64", true}, + "php-8.4.26-build3-linux-arm64.tar.gz": {3, "linux-arm64", true}, "php-8.4.26-macos-arm64.tar.gz": {}, "php-8.4.26-build0-macos-arm64.tar.gz": {}, "php-8.4.26-build1-macos-arm64.tar.xz": {}, diff --git a/tools/flockrel/plan.go b/tools/flockrel/plan.go index f82d437..70cb3c8 100644 --- a/tools/flockrel/plan.go +++ b/tools/flockrel/plan.go @@ -22,6 +22,7 @@ type build struct { // runPlan prints, and writes to $GITHUB_OUTPUT, the builds whose release asset doesn't exist yet: // // php_macos=[{"package":"php","version":"8.4.26","platform":"macos-arm64","build":1}, ...] (php and frankenphp) +// php_linux=[{"package":"php","version":"8.4.26","platform":"linux-arm64","build":1}, ...] // php_exts_windows=[{"package":"php-exts","version":"8.4","platform":"windows-x64","build":1}, ...] // frankenphp_windows=[{"package":"frankenphp","version":"8.4.26","platform":"windows-x64","build":1}, ...] // builds=[{"package":"postgres","version":"18.6","platform":"windows-x64","build":1}, ...] @@ -69,18 +70,18 @@ func runPlan(args []string) error { return true } - phpMacOS, phpExtsWindows, frankenWindows := []build{}, []build{}, []build{} + phpMacOS, phpLinux, phpExtsWindows, frankenWindows := []build{}, []build{}, []build{}, []build{} builds, mirrors := []build{}, []build{} - // PHP versions per minor: php.net for macOS (built from source), windows.php.net for Windows - // (mirrored, and the base of the extension packs and FrankenPHP), which can trail by a day. - macOSPHP := map[string]string{} - macOSVersion := func(minor string) (string, error) { - if v, ok := macOSPHP[minor]; ok { + // PHP versions per minor: php.net for macOS and Linux (built from source), windows.php.net for + // Windows (mirrored, and the base of the extension packs and FrankenPHP), which can trail by a day. + phpNetPHP := map[string]string{} + phpNetVersion := func(minor string) (string, error) { + if v, ok := phpNetPHP[minor]; ok { return v, nil } latest, err := latestPHP(minor) - macOSPHP[minor] = latest + phpNetPHP[minor] = latest return latest, err } var windowsPHP map[string]string @@ -100,7 +101,7 @@ func runPlan(args []string) error { // The release is out, but windows.php.net doesn't have its build yet: wait for it rather // than building the release candidate it still lists. if ok && newMinorRC.MatchString(ver) { - if released, err := macOSVersion(minor); err == nil && !newMinorRC.MatchString(released) { + if released, err := phpNetVersion(minor); err == nil && !newMinorRC.MatchString(released) { fmt.Fprintf(os.Stderr, "waiting PHP %s for Windows (windows.php.net still lists %s)\n", released, ver) return "", false } @@ -114,7 +115,7 @@ func runPlan(args []string) error { phpVersions = strings.Split(*only, ",") } else { for _, minor := range v.PHP.MacOS.Minors { - latest, err := macOSVersion(minor) + latest, err := phpNetVersion(minor) if err != nil { return err } @@ -127,6 +128,25 @@ func runPlan(args []string) error { phpMacOS = append(phpMacOS, b) } } + var linuxVersions []string + if *only != "" { + linuxVersions = strings.Split(*only, ",") + } else { + for _, minor := range v.PHP.Linux.Minors { + latest, err := phpNetVersion(minor) + if err != nil { + return err + } + linuxVersions = append(linuxVersions, latest) + } + } + for _, ver := range linuxVersions { + for _, platform := range []string{"linux-x64", "linux-arm64"} { + if b := (build{Package: "php", Version: strings.TrimSpace(ver), Platform: platform, Build: v.PHP.Linux.Build}); needed(b) { + phpLinux = append(phpLinux, b) + } + } + } if *only == "" { for _, minor := range v.PHP.Windows.Minors { if ver, ok := windowsVersion(minor); ok { @@ -153,7 +173,7 @@ func runPlan(args []string) error { // FrankenPHP embeds PHP, so it is built per PHP version on both platforms. if selected("frankenphp") && *only == "" { for _, minor := range v.FrankenPHP.Minors { - latest, err := macOSVersion(minor) + latest, err := phpNetVersion(minor) if err != nil { return err } @@ -178,7 +198,7 @@ func runPlan(args []string) error { fmt.Fprintf(os.Stderr, "::warning::%s\n", msg) } for _, ver := range vers { - for _, platform := range []string{"macos-arm64", "windows-x64"} { + for _, platform := range platforms { c, ok := p.Platforms[platform] if !ok { continue @@ -200,11 +220,11 @@ func runPlan(args []string) error { for _, o := range []struct { name string list []build - }{{"php_macos", phpMacOS}, {"php_exts_windows", phpExtsWindows}, {"frankenphp_windows", frankenWindows}, {"builds", builds}, {"mirrors", mirrors}} { + }{{"php_macos", phpMacOS}, {"php_linux", phpLinux}, {"php_exts_windows", phpExtsWindows}, {"frankenphp_windows", frankenWindows}, {"builds", builds}, {"mirrors", mirrors}} { j, _ := json.Marshal(o.list) fmt.Fprintf(&out, "%s=%s\n", o.name, j) } - fmt.Fprintf(&out, "any=%t\n", len(phpMacOS)+len(phpExtsWindows)+len(frankenWindows)+len(builds)+len(mirrors) > 0) + fmt.Fprintf(&out, "any=%t\n", len(phpMacOS)+len(phpLinux)+len(phpExtsWindows)+len(frankenWindows)+len(builds)+len(mirrors) > 0) fmt.Print(out.String()) if path := os.Getenv("GITHUB_OUTPUT"); path != "" { f, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644) diff --git a/tools/flockrel/versions.go b/tools/flockrel/versions.go index de106d9..3006e45 100644 --- a/tools/flockrel/versions.go +++ b/tools/flockrel/versions.go @@ -21,6 +21,15 @@ type versions struct { StaticExtensions string `json:"static_extensions"` SharedExtensions string `json:"shared_extensions"` } `json:"macos"` + // Linux is built like macOS, in static-php-cli's glibc container, for linux-x64 and linux-arm64. + Linux struct { + Minors []string `json:"minors"` + Build int `json:"build"` + SPC string `json:"spc"` + MinGlibc string `json:"min_glibc"` + StaticExtensions string `json:"static_extensions"` + SharedExtensions string `json:"shared_extensions"` + } `json:"linux"` // Windows mirrors the official thread-safe x64 builds from windows.php.net. Windows struct { Build int `json:"build"` @@ -76,7 +85,10 @@ type platformConfig struct { Build int `json:"build"` } -var platformExt = map[string]string{"macos-arm64": "tar.gz", "windows-x64": "zip"} +// platforms are the platforms we publish for, in the order the plan lists them. +var platforms = []string{"macos-arm64", "windows-x64", "linux-x64", "linux-arm64"} + +var platformExt = map[string]string{"macos-arm64": "tar.gz", "windows-x64": "zip", "linux-x64": "tar.gz", "linux-arm64": "tar.gz"} func loadVersions(path string) (*versions, error) { data, err := os.ReadFile(path) diff --git a/versions.json b/versions.json index b8c2fb3..165cebd 100644 --- a/versions.json +++ b/versions.json @@ -8,6 +8,14 @@ "static_extensions": "bcmath,bz2,calendar,ctype,curl,dom,exif,fileinfo,filter,ftp,gd,gmp,iconv,igbinary,intl,ldap,mbregex,mbstring,mysqli,mysqlnd,opcache,openssl,pcntl,pdo,pdo_mysql,pdo_pgsql,pdo_sqlite,pgsql,phar,posix,readline,redis,session,shmop,simplexml,soap,sockets,sodium,sqlite3,sysvmsg,sysvsem,sysvshm,tokenizer,xml,xmlreader,xmlwriter,xsl,zip,zlib,apcu", "shared_extensions": "xdebug,imagick,pcov,mongodb,memcached,opentelemetry" }, + "linux": { + "minors": ["8.2", "8.3", "8.4", "8.5"], + "build": 1, + "spc": "2.8.5", + "min_glibc": "2.17", + "static_extensions": "bcmath,bz2,calendar,ctype,curl,dom,exif,fileinfo,filter,ftp,gd,gmp,iconv,igbinary,intl,ldap,mbregex,mbstring,mysqli,mysqlnd,opcache,openssl,pcntl,pdo,pdo_mysql,pdo_pgsql,pdo_sqlite,pgsql,phar,posix,readline,redis,session,shmop,simplexml,soap,sockets,sodium,sqlite3,sysvmsg,sysvsem,sysvshm,tokenizer,xml,xmlreader,xmlwriter,xsl,zip,zlib,apcu", + "shared_extensions": "xdebug,imagick,pcov,mongodb,memcached,opentelemetry" + }, "windows": { "minors": ["7.4", "8.0", "8.1", "8.2", "8.3", "8.4", "8.5"], "build": 1, @@ -37,33 +45,41 @@ "lines": ["14", "15", "16", "17", "18"], "platforms": { "macos-arm64": { "how": "build", "build": 1 }, - "windows-x64": { "how": "build", "build": 1 } + "windows-x64": { "how": "build", "build": 1 }, + "linux-x64": { "how": "build", "build": 1 }, + "linux-arm64": { "how": "build", "build": 1 } }, - "env": { "OPENSSL_VERSION": "3.5.9", "MIN_OS": "13.0" } + "env": { "MIN_GLIBC": "2.28", "OPENSSL_VERSION": "3.5.9", "MIN_OS": "13.0" } }, "mariadb": { "lines": ["10.11", "11.4", "11.8", "12.3"], "platforms": { "macos-arm64": { "how": "build", "build": 1 }, - "windows-x64": { "how": "mirror", "build": 1 } + "windows-x64": { "how": "mirror", "build": 1 }, + "linux-x64": { "how": "build", "build": 1 }, + "linux-arm64": { "how": "build", "build": 1 } }, - "env": { "OPENSSL_VERSION": "3.5.9", "MIN_OS": "13.0" }, + "env": { "MIN_GLIBC": "2.28", "OPENSSL_VERSION": "3.5.9", "MIN_OS": "13.0" }, "publish_source": true }, "valkey": { "lines": ["9.1"], "platforms": { "macos-arm64": { "how": "build", "build": 1 }, - "windows-x64": { "how": "build", "build": 1 } + "windows-x64": { "how": "build", "build": 1 }, + "linux-x64": { "how": "build", "build": 1 }, + "linux-arm64": { "how": "build", "build": 1 } }, - "env": { "MIN_OS": "13.0" } + "env": { "MIN_GLIBC": "2.28", "MIN_OS": "13.0" } }, "redis": { "lines": ["7.2", "7.4", "8.4", "8.10"], "platforms": { - "macos-arm64": { "how": "build", "build": 1 } + "macos-arm64": { "how": "build", "build": 1 }, + "linux-x64": { "how": "build", "build": 1 }, + "linux-arm64": { "how": "build", "build": 1 } }, - "env": { "MIN_OS": "13.0" }, + "env": { "MIN_GLIBC": "2.28", "MIN_OS": "13.0" }, "publish_source": true }, "mysql": { @@ -88,13 +104,17 @@ "lines": ["1"], "platforms": { "macos-arm64": { "how": "mirror", "build": 1 }, - "windows-x64": { "how": "mirror", "build": 1 } + "windows-x64": { "how": "mirror", "build": 1 }, + "linux-x64": { "how": "mirror", "build": 1 }, + "linux-arm64": { "how": "mirror", "build": 1 } } }, "typesense": { "lines": ["29", "30"], "platforms": { - "macos-arm64": { "how": "mirror", "build": 1 } + "macos-arm64": { "how": "mirror", "build": 1 }, + "linux-x64": { "how": "mirror", "build": 1 }, + "linux-arm64": { "how": "mirror", "build": 1 } }, "publish_source": true }, @@ -102,14 +122,18 @@ "lines": ["1"], "platforms": { "macos-arm64": { "how": "mirror", "build": 1 }, - "windows-x64": { "how": "mirror", "build": 1 } + "windows-x64": { "how": "mirror", "build": 1 }, + "linux-x64": { "how": "mirror", "build": 1 }, + "linux-arm64": { "how": "mirror", "build": 1 } } }, "node": { "lines": ["16", "18", "20", "22", "24", "26"], "platforms": { "macos-arm64": { "how": "mirror", "build": 1 }, - "windows-x64": { "how": "mirror", "build": 1 } + "windows-x64": { "how": "mirror", "build": 1 }, + "linux-x64": { "how": "mirror", "build": 1 }, + "linux-arm64": { "how": "mirror", "build": 1 } }, "env": { "SIGNER_MACOS": "Node.js|OpenJS", "SIGNER_WINDOWS": "Node.js|OpenJS" } }