diff --git a/.dockerignore b/.dockerignore index 88bc099..d295451 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,4 @@ docs/ Dockerfile .gitignore data +launch_sysimage.sh diff --git a/.env.dist b/.env.dist index add792a..f2618b6 100644 --- a/.env.dist +++ b/.env.dist @@ -14,7 +14,7 @@ WORKER_USERNAME=worker@email.com WORKER_PASSWORD=password # Debug -JULIA_DEBUG=ReefGuideWorker +JULIA_DEBUG=ReefGuideWorker,ReefGuide # Configuration for the worker POLL_INTERVAL_MS=5000 diff --git a/.github/workflows/PublishDockerImage.yml b/.github/workflows/PublishDockerImage.yml index 0a2dfe4..56c7759 100644 --- a/.github/workflows/PublishDockerImage.yml +++ b/.github/workflows/PublishDockerImage.yml @@ -60,7 +60,6 @@ jobs: uses: docker/build-push-action@v6 with: context: . - target: app-src # Specifies which stage of the Dockerfile to build push: true # Pushes the image to the registry tags: ${{ steps.meta.outputs.tags }} # Uses the tags generated in the metadata step labels: ${{ steps.meta.outputs.labels }} # Uses the labels generated in the metadata step diff --git a/.gitignore b/.gitignore index 08346a9..3483db6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ sandbox/* data .env +.env.* +!.env.dist +sysimages diff --git a/Dockerfile b/Dockerfile index 5750927..211c2a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,31 @@ +#============================================================================== +# ReefGuide Worker Base Image +#============================================================================== +# This Dockerfile creates a Julia-based container image for the ReefGuide Worker +# application. It provides a complete Julia environment with the ReefGuideWorker +# package installed from source as a development dependency. +# +# Usage: +# Build> docker build -t reefguide-worker . +# Run> docker build -t reefguide-worker . +#============================================================================== + # See https://hub.docker.com/_/julia for valid versions. ARG JULIA_VERSION="1.11.6" -#------------------------------------------------------------------------------ -# internal-base build target: julia with OS updates and an empty @app -# Julia environment prepared for use. NOT intended for standalone use. -#------------------------------------------------------------------------------ -FROM julia:${JULIA_VERSION}-bookworm AS internal-base +FROM ${BASE_IMAGE} + +# Redeclare +ARG BASE_IMAGE -# Record the actual base image used from the FROM command as label in the compiled image -ARG BASE_IMAGE="julia:${JULIA_VERSION}-bookworm" +# Record the actual base image used from the FROM command as label in the +# compiled image LABEL org.opencontainers.image.base.name=${BASE_IMAGE} -# Update all pre-installed OS packages (to get security updates) -# and add a few extra utilities -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - apt-get update \ + +# Update all pre-installed OS packages (to get security updates) and add a few +# extra utilities +RUN apt-get update \ && apt-get -y upgrade \ && apt-get install --no-install-recommends -y \ git \ @@ -32,32 +42,35 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ && rm -rf /var/lib/apt/lists/* # Tweak the JULIA_DEPOT_PATH setting so that our shared environments will end up -# in a user-agnostic location, not in ~/.julia => /root/.julia which is the default. -# See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH -# This allows apps derived from this image to drop privileges and run as non-root -# user accounts, but still activate environments configured by this dockerfile. -ENV JULIA_DEPOT_PATH="/usr/local/share/julia" -ENV PRJ_PATH="/usr/local/share/julia/environments/app" +# in a user-agnostic location, not in ~/.julia => /root/.julia which is the +# default. See +# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH +# This allows apps derived from this image to drop privileges and run as +# non-root user accounts, but still activate environments configured by this +# dockerfile. +ENV JULIA_DEPOT_PATH=/usr/local/share/julia + +# Ensure the @app environment is in the load path for Julia, so that apps +# derived from this image can access any packages installed to there. (See +# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_LOAD_PATH) +ENV JULIA_LOAD_PATH="@:@app:@v#.#:@stdlib" + +# This tells Julia's package manager to use the CLI installation of Git rather +# than an internal lib version - this works better with auth for example in +# CI/CD environments ENV JULIA_PKG_USE_CLI_GIT=true -# Coerce Julia to build across multiple targets -# Generic targets taken from: cpu_targets taken from: +# Coerce Julia to build across multiple targets. See: # https://docs.julialang.org/en/v1/devdocs/sysimg/#Specifying-multiple-system-image-targets -ENV JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1) - -# Alternate set that was found to initially alleviate issues on AWS at expense of very -# long build times. -# ENV JULIA_CPU_TARGET=x86_64;haswell;skylake;skylake-avx512;tigerlake +# Alternate set that was found to initially alleviate excessive +# (re)precompilation issues on AWS at expense of very long build times... +# x86_64;haswell;skylake;skylake-avx512;tigerlake +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" -# Prepare an empty @app Julia environment for derived images to use - this is created in the shared depot path +# Prepare an empty @app Julia environment for derived images to use - this is +# created in the shared depot path RUN mkdir -p "${JULIA_DEPOT_PATH}" && \ - chmod 0755 "${JULIA_DEPOT_PATH}" && \ - julia -e 'using Pkg; Pkg.activate("app", shared=true)' - -# Ensure the @app environment is in the load path for Julia, so that apps derived -# from this image can access any packages installed to there. -# (See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_LOAD_PATH) -ENV JULIA_LOAD_PATH="@:@app:@v#.#:@stdlib" + chmod 0755 "${JULIA_DEPOT_PATH}" # Copy project and manifest - includes Manifest-v1.11 etc COPY Project.toml Manifest*.toml ./ @@ -68,35 +81,15 @@ RUN julia --project=@app \ # Install ReefGuideWorker from source and configure it as a development # package in the @app shared environment. -# Should be v speedy if the .toml file is unchanged, because all the -# dependencies *should* already be installed. COPY ./src src -RUN julia --project=@app \ - -e 'using Pkg; \ - Pkg.add("MKL"); \ +RUN julia --project=@app -e \ + 'using Pkg; \ Pkg.develop(PackageSpec(path=pwd())); \ - Pkg.precompile(); \ - using ReefGuideWorker;' - -# Run Julia commands by default as the container launches. -# Derived applications should override the command. -ENTRYPOINT ["julia", "--project=@app"] - -#------------------------------------------------------------------------------ -# app-src build target: installs directly from source files in this repo. -#------------------------------------------------------------------------------ -FROM internal-base AS app-src - -ENV APP_ENV_DIR="${JULIA_DEPOT_PATH}/environments/app" \ - APP_SRC_DIR="/usr/local/src/app" \ - JULIA_PKG_USE_CLI_GIT=true - -# Expect to include the prepped data at /data/app and the config at -# /data/.config.toml -VOLUME ["/data/app"] + Pkg.instantiate(); \ + Pkg.precompile();' # By default, drops the user into a julia shell with ReefGuideWorker activated -ENTRYPOINT ["julia", "--project=@app", "-t", "auto,1", "-e"] +ENTRYPOINT ["julia", "--project=@app", "-e"] # Derived applications should override the command e.g. to start CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage new file mode 100644 index 0000000..82438b4 --- /dev/null +++ b/Dockerfile.build_sysimage @@ -0,0 +1,84 @@ +#============================================================================== +# ReefGuide Worker: Sysimage Builder +#============================================================================== +# Builds a Julia system image for faster ReefGuide Worker startup times. +# The system image is compiled using PackageCompiler.jl and can be extracted +# from the resulting container. +# +# Following advice found in this Discourse thread: +# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 +# +# Also found this relevant issue: +# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 +# +# Build: +# docker build --target export-sysimage -f Dockerfile.build_sysimage -t reefguide-sysimage . +# +# Extract sysimage: +# docker create --name temp-sysimage reefguide-sysimage +# docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so +# docker rm temp-sysimage +#============================================================================== + +ARG JULIA_VERSION="1.11.5" +FROM julia:${JULIA_VERSION}-bookworm AS internal-base + +# Since 1.9.0 Julia, the CPU target is set to "native" by default. This settings +# avoids the need to compile the Julia packages for the specific CPU +# architecture of the host machine Make sure the image can be used on any x86_64 +# machine by setting JULIA_CPU_TARGET to the same value used by the generic +# julia binaries, see +# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" + +ENV JULIA_VERSION=1.11.5 +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=/usr/local/share/julia +ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app +ENV APP_SRC_DIR=/usr/local/src/app +ENV JULIA_PKG_USE_CLI_GIT=true + +# Update all pre-installed OS packages (to get security updates) +# and add a few extra utilities +RUN apt-get update \ + && apt-get -y upgrade \ + && apt-get install --no-install-recommends -y \ + git \ + openssl \ + libssl-dev \ + g++ \ + curl \ + ca-certificates \ + gdal-bin \ + libgdal-dev \ + libfftw3-dev \ + && apt-get clean \ + && apt-get autoremove --purge \ + && rm -rf /var/lib/apt/lists/* + +# Setup shared environment and add packages +RUN mkdir -p "${JULIA_DEPOT_PATH}" \ + && chmod 0755 "${JULIA_DEPOT_PATH}" + +WORKDIR "${APP_SRC_DIR}" + +COPY Project.toml Manifest*.toml ./ +COPY src/ src/ + +# Build sysimage +RUN julia -t auto --project=@app -e \ + 'using Pkg; \ + Pkg.add("PackageCompiler"); \ + Pkg.develop(PackageSpec(path=pwd())); \ + Pkg.instantiate(); ' + +# Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation +# https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 +RUN julia --project=@app -t auto -e 'include("src/sysimage.jl")' + +# Export Julia sysimage to host filesystem +# From project root +# docker build --target export-sysimage -t reefguide-sysimage -f sandbox/smaller_sysimage/Dockerfile . +FROM scratch AS export-sysimage +COPY --from=internal-base /usr/local/src/app/reefguide_img.so /reefguide_img.so diff --git a/Project.toml b/Project.toml index 6c7bb0f..10b5702 100644 --- a/Project.toml +++ b/Project.toml @@ -33,6 +33,6 @@ Logging = "1.11.0" Minio = "0.2.2" PrecompileSignatures = "3.0.3" Random = "1.11.0" -ReefGuide = "0.1.2" +ReefGuide = "0.1.6" SentryIntegration = "0.7.1" Serialization = "1.11.0" diff --git a/README.md b/README.md index de0f99a..d5050d9 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,47 @@ This worker template integrates with the ReefGuide ecosystem, connecting to the docker run --env-file .env worker ``` +#### Building the sysimage + +To build, using docker, a current sysimage + +``` +docker build --target export-sysimage -f Dockerfile.build_sysimage -t sysimage . +``` + +Then copy the file out: + +``` +docker create --name temp-sysimage sysimage bash +docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so +docker rm temp-sysimage +``` + +#### Running the worker with the sysimage + +The main Dockerfile, once built, can be run in a sysimage mode. + +You need to + +1. mount a volume containing the sysimage file (as above) +2. override the entrypoint and cmd to run using the sysimage + +The script `./launch_sysimage.sh` does this for you + +``` +./launch_sysimage.sh +``` + +So for example, if you have a file `sysimage.so` in the folder `sysimages`, you would run + +``` +./run-worker.sh ./sysimages sysimage.so worker +``` + +**NOTE**: the `./` relative path is important in the first argument. + +Where worker was the `-t` argument when building the main dockerfile. + ## How It Works ### Core Architecture diff --git a/launch_sysimage.sh b/launch_sysimage.sh new file mode 100755 index 0000000..90c5f13 --- /dev/null +++ b/launch_sysimage.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Script to run the ReefGuide worker container +# Usage: ./run-worker.sh + +set -e + +# Check if required arguments are provided +if [ $# -lt 3 ]; then + echo "Usage: $0 " + echo "Example: $0 ./sysimages reefguide_img.so worker" + exit 1 +fi + +SYSIMAGE_DIR="$1" +SYSIMAGE_NAME="$2" +CONTAINER_NAME="$3" + +# Check if sysimage directory exists +if [ ! -d "$SYSIMAGE_DIR" ]; then + echo "Error: System image directory not found at $SYSIMAGE_DIR" + exit 1 +fi + +# Check if sysimage file exists in the directory +if [ ! -f "$SYSIMAGE_DIR/$SYSIMAGE_NAME" ]; then + echo "Error: System image file $SYSIMAGE_NAME not found in $SYSIMAGE_DIR" + exit 1 +fi + +echo "Starting ReefGuide worker container..." +echo "System image directory: $SYSIMAGE_DIR" +echo "System image file: $SYSIMAGE_NAME" +echo "Worker container name: $CONTAINER_NAME" + +# Run the container +docker run \ + -v "$SYSIMAGE_DIR:/sysimage" \ + -v "./data:/data" \ + --network host \ + --env-file=.env \ + --entrypoint julia \ + ${CONTAINER_NAME} \ + --project=@app \ + -t auto \ + -J "/sysimage/$SYSIMAGE_NAME" \ + --sysimage-native-code=yes \ + -e "using ReefGuideWorker; ReefGuideWorker.start_worker()" + +echo "Container $CONTAINER_NAME started successfully!" +echo "To view logs: docker logs $CONTAINER_NAME" +echo "To stop: docker stop $CONTAINER_NAME" +echo "To remove: docker rm $CONTAINER_NAME"