From aaa53a37076a22f650a7347b1c8282230a70fd04 Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:42:45 +1000 Subject: [PATCH 01/13] Clean up dockerfile for container without sysimage --- Dockerfile | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5750927..b32870a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +# Worker container without sysimage + # See https://hub.docker.com/_/julia for valid versions. ARG JULIA_VERSION="1.11.6" @@ -5,10 +7,9 @@ ARG JULIA_VERSION="1.11.6" # internal-base build target: julia with OS updates and an empty @app # Julia environment prepared for use. NOT intended for standalone use. #------------------------------------------------------------------------------ -FROM julia:${JULIA_VERSION}-bookworm AS internal-base +FROM ${BASE_IMAGE} AS internal-base # Record the actual base image used from the FROM command as label in the compiled image -ARG BASE_IMAGE="julia:${JULIA_VERSION}-bookworm" LABEL org.opencontainers.image.base.name=${BASE_IMAGE} # Update all pre-installed OS packages (to get security updates) @@ -36,8 +37,8 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ # See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH # This allows apps derived from this image to drop privileges and run as non-root # user accounts, but still activate environments configured by this dockerfile. -ENV JULIA_DEPOT_PATH="/usr/local/share/julia" -ENV PRJ_PATH="/usr/local/share/julia/environments/app" +ENV JULIA_DEPOT_PATH=/usr/local/share/julia +ENV PRJ_PATH=/usr/local/share/julia/environments/app ENV JULIA_PKG_USE_CLI_GIT=true # Coerce Julia to build across multiple targets @@ -51,8 +52,7 @@ ENV JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base # Prepare an empty @app Julia environment for derived images to use - this is created in the shared depot path RUN mkdir -p "${JULIA_DEPOT_PATH}" && \ - chmod 0755 "${JULIA_DEPOT_PATH}" && \ - julia -e 'using Pkg; Pkg.activate("app", shared=true)' + chmod 0755 "${JULIA_DEPOT_PATH}" # Ensure the @app environment is in the load path for Julia, so that apps derived # from this image can access any packages installed to there. @@ -71,12 +71,14 @@ RUN julia --project=@app \ # Should be v speedy if the .toml file is unchanged, because all the # dependencies *should* already be installed. COPY ./src src -RUN julia --project=@app \ - -e 'using Pkg; \ - Pkg.add("MKL"); \ +RUN julia --project=@app -e \ + 'using Pkg; \ + Pkg.add("PackageCompiler"); \ Pkg.develop(PackageSpec(path=pwd())); \ - Pkg.precompile(); \ - using ReefGuideWorker;' + Pkg.instantiate(); \ + Pkg.precompile();' + +# Pkg.add(name="MKL_jll", version="2024.2.0"); \ # Run Julia commands by default as the container launches. # Derived applications should override the command. @@ -86,9 +88,10 @@ ENTRYPOINT ["julia", "--project=@app"] # app-src build target: installs directly from source files in this repo. #------------------------------------------------------------------------------ FROM internal-base AS app-src +ENV JULIA_DEPOT_PATH=/usr/local/share/julia -ENV APP_ENV_DIR="${JULIA_DEPOT_PATH}/environments/app" \ - APP_SRC_DIR="/usr/local/src/app" \ +ENV APP_ENV_DIR=${JULIA_DEPOT_PATH}/environments/app \ + APP_SRC_DIR=/usr/local/src/app \ JULIA_PKG_USE_CLI_GIT=true # Expect to include the prepped data at /data/app and the config at @@ -96,7 +99,7 @@ ENV APP_ENV_DIR="${JULIA_DEPOT_PATH}/environments/app" \ VOLUME ["/data/app"] # By default, drops the user into a julia shell with ReefGuideWorker activated -ENTRYPOINT ["julia", "--project=@app", "-t", "auto,1", "-e"] +ENTRYPOINT ["julia", "--project=@app", "-e"] # Derived applications should override the command e.g. to start CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] From 2298ad5aa2dada16334b3fa6c1f5840dde24e769 Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:42:54 +1000 Subject: [PATCH 02/13] Add dockerfile with sysimage --- Dockerfile_sysimage | 74 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 Dockerfile_sysimage diff --git a/Dockerfile_sysimage b/Dockerfile_sysimage new file mode 100644 index 0000000..8e76d1b --- /dev/null +++ b/Dockerfile_sysimage @@ -0,0 +1,74 @@ +# Worker container with sysimage + +# Following advice found in this Discourse thread: +# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 + +ARG JULIA_VERSION="1.11.5" +FROM julia:${JULIA_VERSION}-bookworm AS internal-base + +# Since 1.9.0 Julia, the CPU target is set to "native" by default. +# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine +# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET +# to the same value used by the generic julia binaries, see +# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" + +ENV JULIA_VERSION=1.11.5 +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=usr/local/share/julia +ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app +ENV APP_SRC_DIR=/usr/local/src/app + +# Update all pre-installed OS packages (to get security updates) +# and add a few extra utilities +RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ + --mount=target=/var/cache/apt,type=cache,sharing=locked \ + apt-get update \ + && apt-get -y upgrade \ + && apt-get install --no-install-recommends -y \ + git \ + openssl \ + libssl-dev \ + g++ \ + curl \ + ca-certificates \ + && apt-get clean \ + && apt-get autoremove --purge \ + && rm -rf /var/lib/apt/lists/* + +# Setup shared environment and add packages +RUN mkdir -p "${JULIA_DEPOT_PATH}" \ + && chmod 0755 "${JULIA_DEPOT_PATH}" + +WORKDIR "${APP_SRC_DIR}" + +COPY Project.toml Manifest*.toml ./ +COPY src/ src/ + +# Expect to include the prepped data at /data/app and the config at +# /data/.config.toml +VOLUME ["/data", "/data/app"] + +# # By default, drops the user into a julia shell with ReefGuideWorker activated +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=usr/local/share/julia + +# If dependencies draw in specific Artifacts (additional and lazily downloaded +# dependencies), then the environment using sysimages must also contain those Artifacts. +# So we recreate the conditions here. +# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 +RUN julia --project=@app -e \ + 'using Pkg; \ + Pkg.develop(PackageSpec(path=pwd())); \ + Pkg.instantiate(); \ + Pkg.precompile();' + +ENTRYPOINT [ \ + "julia", "--project=@app", \ + "-J", "/data/reefguide_img.so", \ + "--sysimage-native-code=yes", \ + "-e"] + +CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] From 5c21a3359007c46747033b18594489c796423bcf Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:44:38 +1000 Subject: [PATCH 03/13] Indicate usage --- Dockerfile_sysimage | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile_sysimage b/Dockerfile_sysimage index 8e76d1b..329ec6d 100644 --- a/Dockerfile_sysimage +++ b/Dockerfile_sysimage @@ -1,4 +1,5 @@ # Worker container with sysimage +# docker build -t reefguide-img-worker -f Dockerfile_sysimage . # Following advice found in this Discourse thread: # https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 From 425f25acca95516c08dd10add90dfa79000a8139 Mon Sep 17 00:00:00 2001 From: Takuya Iwanaga Date: Mon, 14 Jul 2025 12:53:25 +1000 Subject: [PATCH 04/13] Add sysimage build file and rename to match docker conventions --- Dockerfile.build_sysimage | 77 ++++++++++++++++++++++ Dockerfile_sysimage => Dockerfile.sysimage | 2 +- 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 Dockerfile.build_sysimage rename Dockerfile_sysimage => Dockerfile.sysimage (97%) diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage new file mode 100644 index 0000000..cda3954 --- /dev/null +++ b/Dockerfile.build_sysimage @@ -0,0 +1,77 @@ +# Build process to generate sysimage for ReefGuideWorker. +# docker build -t reefguide-img-worker -f Dockerfile.build_sysimage . + +# Following advice found in this Discourse thread: +# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 + +# Also found this relevant issue: +# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 + +# docker build --target export-sysimage -t my-sysimage . +# docker run --target app-src --env-file .env -it -v [src dir]:/data/app reefguide-worker + +ARG JULIA_VERSION="1.11.5" +FROM julia:${JULIA_VERSION}-bookworm AS internal-base + +# Since 1.9.0 Julia, the CPU target is set to "native" by default. +# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine +# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET +# to the same value used by the generic julia binaries, see +# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" + +ENV JULIA_VERSION=1.11.5 +ENV JULIA_DIR=/usr/local/julia +ENV JULIA_PATH=${JULIA_DIR} +ENV JULIA_DEPOT_PATH=usr/local/share/julia +ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app +ENV APP_SRC_DIR=/usr/local/src/app +ENV SYSIMAGE_SRC_DIR=/usr/local/src/app_sysimage +ENV JULIA_PKG_USE_CLI_GIT=true + +# Update all pre-installed OS packages (to get security updates) +# and add a few extra utilities +RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ + --mount=target=/var/cache/apt,type=cache,sharing=locked \ + apt-get update \ + && apt-get -y upgrade \ + && apt-get install --no-install-recommends -y \ + git \ + openssl \ + libssl-dev \ + g++ \ + curl \ + ca-certificates \ + && apt-get clean \ + && apt-get autoremove --purge \ + && rm -rf /var/lib/apt/lists/* + +# Setup shared environment and add packages +RUN mkdir -p "${JULIA_DEPOT_PATH}" \ + && chmod 0755 "${JULIA_DEPOT_PATH}" + +WORKDIR "${APP_SRC_DIR}" + +COPY Project.toml Manifest*.toml ./ +COPY src/ src/ + +# Build sysimage +RUN julia --project=@app -e \ + 'using Pkg; \ + Pkg.add("PackageCompiler"); \ + Pkg.develop(PackageSpec(path=pwd())); \ + Pkg.instantiate(); ' + +# Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation +# https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 +ENV JULIA_NUM_PRECOMPILE_TAKS=2 +RUN julia --project=@app -t 1,1 -e 'include("src/sysimage.jl")' + +# Export Julia sysimage to host filesystem +# From project root +# docker build --target export-sysimage -t reefguide-sysimage -f sandbox/smaller_sysimage/Dockerfile . +FROM scratch AS export-sysimage +ENV APP_SRC_DIR=/usr/local/src/app +COPY --from=internal-base ${APP_SRC_DIR}/reefguide_img.so /reefguide_img.so + +ENTRYPOINT ["bash"] diff --git a/Dockerfile_sysimage b/Dockerfile.sysimage similarity index 97% rename from Dockerfile_sysimage rename to Dockerfile.sysimage index 329ec6d..b15f5b3 100644 --- a/Dockerfile_sysimage +++ b/Dockerfile.sysimage @@ -1,5 +1,5 @@ # Worker container with sysimage -# docker build -t reefguide-img-worker -f Dockerfile_sysimage . +# docker build -t reefguide-img-worker -f Dockerfile.sysimage . # Following advice found in this Discourse thread: # https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 From 312c7fad97059f67c7d9a2e84fa7231a26ee26fb Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 15 Jul 2025 12:58:44 +1000 Subject: [PATCH 05/13] Documentation, consolidation, single stage build Signed-off-by: Peter Baker --- .dockerignore | 1 + .github/workflows/PublishDockerImage.yml | 1 - .gitignore | 1 + Dockerfile | 92 +++++++++++------------- Dockerfile.build_sysimage | 41 ++++++----- Dockerfile.sysimage | 75 ------------------- README.md | 41 +++++++++++ launch_sysimage.sh | 49 +++++++++++++ 8 files changed, 158 insertions(+), 143 deletions(-) delete mode 100644 Dockerfile.sysimage create mode 100755 launch_sysimage.sh diff --git a/.dockerignore b/.dockerignore index 88bc099..d295451 100644 --- a/.dockerignore +++ b/.dockerignore @@ -6,3 +6,4 @@ docs/ Dockerfile .gitignore data +launch_sysimage.sh diff --git a/.github/workflows/PublishDockerImage.yml b/.github/workflows/PublishDockerImage.yml index 0a2dfe4..56c7759 100644 --- a/.github/workflows/PublishDockerImage.yml +++ b/.github/workflows/PublishDockerImage.yml @@ -60,7 +60,6 @@ jobs: uses: docker/build-push-action@v6 with: context: . - target: app-src # Specifies which stage of the Dockerfile to build push: true # Pushes the image to the registry tags: ${{ steps.meta.outputs.tags }} # Uses the tags generated in the metadata step labels: ${{ steps.meta.outputs.labels }} # Uses the labels generated in the metadata step diff --git a/.gitignore b/.gitignore index 08346a9..5be80ec 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ sandbox/* data .env +sysimages diff --git a/Dockerfile b/Dockerfile index b32870a..9176e56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,30 @@ -# Worker container without sysimage +#============================================================================== +# ReefGuide Worker Base Image +#============================================================================== +# This Dockerfile creates a Julia-based container image for the ReefGuide Worker +# application. It provides a complete Julia environment with the ReefGuideWorker +# package installed from source as a development dependency. +# +# Usage: +# Build> docker build -t reefguide-worker . +# Run> docker build -t reefguide-worker . +#============================================================================== # See https://hub.docker.com/_/julia for valid versions. ARG JULIA_VERSION="1.11.6" -#------------------------------------------------------------------------------ -# internal-base build target: julia with OS updates and an empty @app -# Julia environment prepared for use. NOT intended for standalone use. -#------------------------------------------------------------------------------ -FROM ${BASE_IMAGE} AS internal-base +FROM ${BASE_IMAGE} -# Record the actual base image used from the FROM command as label in the compiled image +# Redeclare +ARG BASE_IMAGE + +# Record the actual base image used from the FROM command as label in the +# compiled image LABEL org.opencontainers.image.base.name=${BASE_IMAGE} -# Update all pre-installed OS packages (to get security updates) -# and add a few extra utilities + +# Update all pre-installed OS packages (to get security updates) and add a few +# extra utilities RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ --mount=target=/var/cache/apt,type=cache,sharing=locked \ apt-get update \ @@ -33,32 +44,36 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ && rm -rf /var/lib/apt/lists/* # Tweak the JULIA_DEPOT_PATH setting so that our shared environments will end up -# in a user-agnostic location, not in ~/.julia => /root/.julia which is the default. -# See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH -# This allows apps derived from this image to drop privileges and run as non-root -# user accounts, but still activate environments configured by this dockerfile. +# in a user-agnostic location, not in ~/.julia => /root/.julia which is the +# default. See +# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_DEPOT_PATH +# This allows apps derived from this image to drop privileges and run as +# non-root user accounts, but still activate environments configured by this +# dockerfile. ENV JULIA_DEPOT_PATH=/usr/local/share/julia -ENV PRJ_PATH=/usr/local/share/julia/environments/app + +# Ensure the @app environment is in the load path for Julia, so that apps +# derived from this image can access any packages installed to there. (See +# https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_LOAD_PATH) +ENV JULIA_LOAD_PATH="@:@app:@v#.#:@stdlib" + +# This tells Julia's package manager to use the CLI installation of Git rather +# than an internal lib version - this works better with auth for example in +# CI/CD environments ENV JULIA_PKG_USE_CLI_GIT=true -# Coerce Julia to build across multiple targets -# Generic targets taken from: cpu_targets taken from: +# Coerce Julia to build across multiple targets. See: # https://docs.julialang.org/en/v1/devdocs/sysimg/#Specifying-multiple-system-image-targets -ENV JULIA_CPU_TARGET=generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1) - -# Alternate set that was found to initially alleviate issues on AWS at expense of very -# long build times. -# ENV JULIA_CPU_TARGET=x86_64;haswell;skylake;skylake-avx512;tigerlake +# Alternate set that was found to initially alleviate excessive +# (re)precompilation issues on AWS at expense of very long build times... +# x86_64;haswell;skylake;skylake-avx512;tigerlake +ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" -# Prepare an empty @app Julia environment for derived images to use - this is created in the shared depot path +# Prepare an empty @app Julia environment for derived images to use - this is +# created in the shared depot path RUN mkdir -p "${JULIA_DEPOT_PATH}" && \ chmod 0755 "${JULIA_DEPOT_PATH}" -# Ensure the @app environment is in the load path for Julia, so that apps derived -# from this image can access any packages installed to there. -# (See https://docs.julialang.org/en/v1/manual/environment-variables/#JULIA_LOAD_PATH) -ENV JULIA_LOAD_PATH="@:@app:@v#.#:@stdlib" - # Copy project and manifest - includes Manifest-v1.11 etc COPY Project.toml Manifest*.toml ./ @@ -68,36 +83,13 @@ RUN julia --project=@app \ # Install ReefGuideWorker from source and configure it as a development # package in the @app shared environment. -# Should be v speedy if the .toml file is unchanged, because all the -# dependencies *should* already be installed. COPY ./src src RUN julia --project=@app -e \ 'using Pkg; \ - Pkg.add("PackageCompiler"); \ Pkg.develop(PackageSpec(path=pwd())); \ Pkg.instantiate(); \ Pkg.precompile();' -# Pkg.add(name="MKL_jll", version="2024.2.0"); \ - -# Run Julia commands by default as the container launches. -# Derived applications should override the command. -ENTRYPOINT ["julia", "--project=@app"] - -#------------------------------------------------------------------------------ -# app-src build target: installs directly from source files in this repo. -#------------------------------------------------------------------------------ -FROM internal-base AS app-src -ENV JULIA_DEPOT_PATH=/usr/local/share/julia - -ENV APP_ENV_DIR=${JULIA_DEPOT_PATH}/environments/app \ - APP_SRC_DIR=/usr/local/src/app \ - JULIA_PKG_USE_CLI_GIT=true - -# Expect to include the prepped data at /data/app and the config at -# /data/.config.toml -VOLUME ["/data/app"] - # By default, drops the user into a julia shell with ReefGuideWorker activated ENTRYPOINT ["julia", "--project=@app", "-e"] diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index cda3954..c394edc 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -1,22 +1,33 @@ -# Build process to generate sysimage for ReefGuideWorker. -# docker build -t reefguide-img-worker -f Dockerfile.build_sysimage . - +#============================================================================== +# ReefGuide Worker: Sysimage Builder +#============================================================================== +# Builds a Julia system image for faster ReefGuide Worker startup times. +# The system image is compiled using PackageCompiler.jl and can be extracted +# from the resulting container. +# # Following advice found in this Discourse thread: # https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 - +# # Also found this relevant issue: # https://github.com/JuliaLang/PackageCompiler.jl/issues/743 - -# docker build --target export-sysimage -t my-sysimage . -# docker run --target app-src --env-file .env -it -v [src dir]:/data/app reefguide-worker +# +# Build: +# docker build --target export-sysimage -f Dockerfile.build_sysimage -t reefguide-sysimage . +# +# Extract sysimage: +# docker create --name temp-sysimage reefguide-sysimage +# docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so +# docker rm temp-sysimage +#============================================================================== ARG JULIA_VERSION="1.11.5" FROM julia:${JULIA_VERSION}-bookworm AS internal-base -# Since 1.9.0 Julia, the CPU target is set to "native" by default. -# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine -# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET -# to the same value used by the generic julia binaries, see +# Since 1.9.0 Julia, the CPU target is set to "native" by default. This settings +# avoids the need to compile the Julia packages for the specific CPU +# architecture of the host machine Make sure the image can be used on any x86_64 +# machine by setting JULIA_CPU_TARGET to the same value used by the generic +# julia binaries, see # https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" @@ -26,7 +37,6 @@ ENV JULIA_PATH=${JULIA_DIR} ENV JULIA_DEPOT_PATH=usr/local/share/julia ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app ENV APP_SRC_DIR=/usr/local/src/app -ENV SYSIMAGE_SRC_DIR=/usr/local/src/app_sysimage ENV JULIA_PKG_USE_CLI_GIT=true # Update all pre-installed OS packages (to get security updates) @@ -64,14 +74,11 @@ RUN julia --project=@app -e \ # Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation # https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 -ENV JULIA_NUM_PRECOMPILE_TAKS=2 +ENV JULIA_NUM_PRECOMPILE_TASKS=1 RUN julia --project=@app -t 1,1 -e 'include("src/sysimage.jl")' # Export Julia sysimage to host filesystem # From project root # docker build --target export-sysimage -t reefguide-sysimage -f sandbox/smaller_sysimage/Dockerfile . FROM scratch AS export-sysimage -ENV APP_SRC_DIR=/usr/local/src/app -COPY --from=internal-base ${APP_SRC_DIR}/reefguide_img.so /reefguide_img.so - -ENTRYPOINT ["bash"] +COPY --from=internal-base /usr/local/src/app/reefguide_img.so /reefguide_img.so diff --git a/Dockerfile.sysimage b/Dockerfile.sysimage deleted file mode 100644 index b15f5b3..0000000 --- a/Dockerfile.sysimage +++ /dev/null @@ -1,75 +0,0 @@ -# Worker container with sysimage -# docker build -t reefguide-img-worker -f Dockerfile.sysimage . - -# Following advice found in this Discourse thread: -# https://discourse.julialang.org/t/creating-a-docker-base-image-for-faster-deployments/121165/2 - -ARG JULIA_VERSION="1.11.5" -FROM julia:${JULIA_VERSION}-bookworm AS internal-base - -# Since 1.9.0 Julia, the CPU target is set to "native" by default. -# This settings avoids the need to compile the Julia packages for the specific CPU architecture of the host machine -# Make sure the image can be used on any x86_64 machine by setting JULIA_CPU_TARGET -# to the same value used by the generic julia binaries, see -# https://github.com/JuliaCI/julia-buildkite/blob/4b6932992f7985af71fc3f73af77abf4d25bd146/utilities/build_envs.sh#L23-L31 -ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,base(1);x86-64-v4,-rdrnd,base(1);znver4,-rdrnd,base(1)" - -ENV JULIA_VERSION=1.11.5 -ENV JULIA_DIR=/usr/local/julia -ENV JULIA_PATH=${JULIA_DIR} -ENV JULIA_DEPOT_PATH=usr/local/share/julia -ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app -ENV APP_SRC_DIR=/usr/local/src/app - -# Update all pre-installed OS packages (to get security updates) -# and add a few extra utilities -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - apt-get update \ - && apt-get -y upgrade \ - && apt-get install --no-install-recommends -y \ - git \ - openssl \ - libssl-dev \ - g++ \ - curl \ - ca-certificates \ - && apt-get clean \ - && apt-get autoremove --purge \ - && rm -rf /var/lib/apt/lists/* - -# Setup shared environment and add packages -RUN mkdir -p "${JULIA_DEPOT_PATH}" \ - && chmod 0755 "${JULIA_DEPOT_PATH}" - -WORKDIR "${APP_SRC_DIR}" - -COPY Project.toml Manifest*.toml ./ -COPY src/ src/ - -# Expect to include the prepped data at /data/app and the config at -# /data/.config.toml -VOLUME ["/data", "/data/app"] - -# # By default, drops the user into a julia shell with ReefGuideWorker activated -ENV JULIA_DIR=/usr/local/julia -ENV JULIA_PATH=${JULIA_DIR} -ENV JULIA_DEPOT_PATH=usr/local/share/julia - -# If dependencies draw in specific Artifacts (additional and lazily downloaded -# dependencies), then the environment using sysimages must also contain those Artifacts. -# So we recreate the conditions here. -# https://github.com/JuliaLang/PackageCompiler.jl/issues/743 -RUN julia --project=@app -e \ - 'using Pkg; \ - Pkg.develop(PackageSpec(path=pwd())); \ - Pkg.instantiate(); \ - Pkg.precompile();' - -ENTRYPOINT [ \ - "julia", "--project=@app", \ - "-J", "/data/reefguide_img.so", \ - "--sysimage-native-code=yes", \ - "-e"] - -CMD ["using ReefGuideWorker; ReefGuideWorker.start_worker()"] diff --git a/README.md b/README.md index de0f99a..b4eb920 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,47 @@ This worker template integrates with the ReefGuide ecosystem, connecting to the docker run --env-file .env worker ``` +#### Building the sysimage + +To build, using docker, a current sysimage + +``` +docker build --target export-sysimage -t sysimage . +``` + +Then copy the file out: + +``` +docker create --name temp-sysimage sysimage +docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so +docker rm temp-sysimage +``` + +#### Running the worker with the sysimage + +The main Dockerfile, once built, can be run in a sysimage mode. + +You need to + +1. mount a volume containing the sysimage file (as above) +2. override the entrypoint and cmd to run using the sysimage + +The script `./launch_sysimage.sh` does this for you + +``` +./launch_sysimage.sh +``` + +So for example, if you have a file `sysimage.so` in the folder `sysimages`, you would run + +``` +./run-worker.sh ./sysimages sysimage.so worker +``` + +**NOTE**: the `./` relative path is important in the first argument. + +Where worker was the `-t` argument when building the main dockerfile. + ## How It Works ### Core Architecture diff --git a/launch_sysimage.sh b/launch_sysimage.sh new file mode 100755 index 0000000..aea2fdd --- /dev/null +++ b/launch_sysimage.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Script to run the ReefGuide worker container +# Usage: ./run-worker.sh + +set -e + +# Check if required arguments are provided +if [ $# -lt 3 ]; then + echo "Usage: $0 " + echo "Example: $0 ./sysimages reefguide_img.so worker" + exit 1 +fi + +SYSIMAGE_DIR="$1" +SYSIMAGE_NAME="$2" +CONTAINER_NAME="$3" + +# Check if sysimage directory exists +if [ ! -d "$SYSIMAGE_DIR" ]; then + echo "Error: System image directory not found at $SYSIMAGE_DIR" + exit 1 +fi + +# Check if sysimage file exists in the directory +if [ ! -f "$SYSIMAGE_DIR/$SYSIMAGE_NAME" ]; then + echo "Error: System image file $SYSIMAGE_NAME not found in $SYSIMAGE_DIR" + exit 1 +fi + +echo "Starting ReefGuide worker container..." +echo "System image directory: $SYSIMAGE_DIR" +echo "System image file: $SYSIMAGE_NAME" +echo "Worker container name: $CONTAINER_NAME" + +# Run the container +docker run \ + -v "$SYSIMAGE_DIR:/data" \ + --entrypoint julia \ + ${CONTAINER_NAME} \ + --project=@app \ + -J "/data/$SYSIMAGE_NAME" \ + --sysimage-native-code=yes \ + -e "using ReefGuideWorker; ReefGuideWorker.start_worker()" + +echo "Container $CONTAINER_NAME started successfully!" +echo "To view logs: docker logs $CONTAINER_NAME" +echo "To stop: docker stop $CONTAINER_NAME" +echo "To remove: docker rm $CONTAINER_NAME" From ed6678d950332e2104cea92dc56fbc2e8a30a69a Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Tue, 15 Jul 2025 14:31:05 +1000 Subject: [PATCH 06/13] Fixing path and adding gdal dep Signed-off-by: Peter Baker --- Dockerfile.build_sysimage | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index c394edc..bc6a7d6 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -34,7 +34,7 @@ ENV JULIA_CPU_TARGET="generic;sandybridge,-xsaveopt,clone_all;haswell,-rdrnd,bas ENV JULIA_VERSION=1.11.5 ENV JULIA_DIR=/usr/local/julia ENV JULIA_PATH=${JULIA_DIR} -ENV JULIA_DEPOT_PATH=usr/local/share/julia +ENV JULIA_DEPOT_PATH=/usr/local/share/julia ENV APP_ENV_PATH=${JULIA_DEPOT_PATH}/environments/app ENV APP_SRC_DIR=/usr/local/src/app ENV JULIA_PKG_USE_CLI_GIT=true @@ -52,6 +52,9 @@ RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ g++ \ curl \ ca-certificates \ + gdal-bin \ + libgdal-dev \ + libfftw3-dev \ && apt-get clean \ && apt-get autoremove --purge \ && rm -rf /var/lib/apt/lists/* From 6329f36decdf9315e4b217aded7e7fd2201ca254 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Wed, 16 Jul 2025 09:21:05 +1000 Subject: [PATCH 07/13] Removing cache mounting - adding .env file to launch script (for now) Signed-off-by: Peter Baker --- Dockerfile | 4 +--- Dockerfile.build_sysimage | 4 +--- launch_sysimage.sh | 1 + 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9176e56..211c2a6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,9 +25,7 @@ LABEL org.opencontainers.image.base.name=${BASE_IMAGE} # Update all pre-installed OS packages (to get security updates) and add a few # extra utilities -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - apt-get update \ +RUN apt-get update \ && apt-get -y upgrade \ && apt-get install --no-install-recommends -y \ git \ diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index bc6a7d6..eb99c75 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -41,9 +41,7 @@ ENV JULIA_PKG_USE_CLI_GIT=true # Update all pre-installed OS packages (to get security updates) # and add a few extra utilities -RUN --mount=target=/var/lib/apt/lists,type=cache,sharing=locked \ - --mount=target=/var/cache/apt,type=cache,sharing=locked \ - apt-get update \ +RUN apt-get update \ && apt-get -y upgrade \ && apt-get install --no-install-recommends -y \ git \ diff --git a/launch_sysimage.sh b/launch_sysimage.sh index aea2fdd..24066fb 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -36,6 +36,7 @@ echo "Worker container name: $CONTAINER_NAME" # Run the container docker run \ -v "$SYSIMAGE_DIR:/data" \ + --env-file=.env \ --entrypoint julia \ ${CONTAINER_NAME} \ --project=@app \ From 6865583f7870781b32554e59b46e620fafacbe19 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Thu, 17 Jul 2025 12:03:31 +1000 Subject: [PATCH 08/13] Quick hack on launch sysimage scripts - and bumping version of ReefGuide to v0.1.5 Signed-off-by: Peter Baker --- Project.toml | 2 +- launch_sysimage.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 6c7bb0f..8388416 100644 --- a/Project.toml +++ b/Project.toml @@ -33,6 +33,6 @@ Logging = "1.11.0" Minio = "0.2.2" PrecompileSignatures = "3.0.3" Random = "1.11.0" -ReefGuide = "0.1.2" +ReefGuide = "0.1.5" SentryIntegration = "0.7.1" Serialization = "1.11.0" diff --git a/launch_sysimage.sh b/launch_sysimage.sh index 24066fb..c6e3e1f 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -35,12 +35,13 @@ echo "Worker container name: $CONTAINER_NAME" # Run the container docker run \ - -v "$SYSIMAGE_DIR:/data" \ + -v "$SYSIMAGE_DIR:/sysimage" \ + -v "./data:/data" \ --env-file=.env \ --entrypoint julia \ ${CONTAINER_NAME} \ --project=@app \ - -J "/data/$SYSIMAGE_NAME" \ + -J "/sysimage/$SYSIMAGE_NAME" \ --sysimage-native-code=yes \ -e "using ReefGuideWorker; ReefGuideWorker.start_worker()" From 19769ea14320767a253c2192069ebf017b6c5117 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 11:48:48 +1000 Subject: [PATCH 09/13] Some minor changes Signed-off-by: Peter Baker --- Dockerfile.build_sysimage | 5 ++--- README.md | 4 ++-- launch_sysimage.sh | 1 + 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile.build_sysimage b/Dockerfile.build_sysimage index eb99c75..82438b4 100644 --- a/Dockerfile.build_sysimage +++ b/Dockerfile.build_sysimage @@ -67,7 +67,7 @@ COPY Project.toml Manifest*.toml ./ COPY src/ src/ # Build sysimage -RUN julia --project=@app -e \ +RUN julia -t auto --project=@app -e \ 'using Pkg; \ Pkg.add("PackageCompiler"); \ Pkg.develop(PackageSpec(path=pwd())); \ @@ -75,8 +75,7 @@ RUN julia --project=@app -e \ # Reduce number of tasks/threads to avoid heavy memory use during sysimage compilation # https://github.com/JuliaLang/PackageCompiler.jl/issues/1031#issuecomment-2823054267 -ENV JULIA_NUM_PRECOMPILE_TASKS=1 -RUN julia --project=@app -t 1,1 -e 'include("src/sysimage.jl")' +RUN julia --project=@app -t auto -e 'include("src/sysimage.jl")' # Export Julia sysimage to host filesystem # From project root diff --git a/README.md b/README.md index b4eb920..d5050d9 100644 --- a/README.md +++ b/README.md @@ -90,13 +90,13 @@ This worker template integrates with the ReefGuide ecosystem, connecting to the To build, using docker, a current sysimage ``` -docker build --target export-sysimage -t sysimage . +docker build --target export-sysimage -f Dockerfile.build_sysimage -t sysimage . ``` Then copy the file out: ``` -docker create --name temp-sysimage sysimage +docker create --name temp-sysimage sysimage bash docker cp temp-sysimage:/reefguide_img.so ./reefguide_img.so docker rm temp-sysimage ``` diff --git a/launch_sysimage.sh b/launch_sysimage.sh index c6e3e1f..cfda47c 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -37,6 +37,7 @@ echo "Worker container name: $CONTAINER_NAME" docker run \ -v "$SYSIMAGE_DIR:/sysimage" \ -v "./data:/data" \ + --network host \ --env-file=.env \ --entrypoint julia \ ${CONTAINER_NAME} \ From 123a6fe88575ce0bd6d71e18ff53ea33c840abd8 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 11:54:19 +1000 Subject: [PATCH 10/13] Adding ReefGuide debug to dist Signed-off-by: Peter Baker --- .env.dist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env.dist b/.env.dist index add792a..f2618b6 100644 --- a/.env.dist +++ b/.env.dist @@ -14,7 +14,7 @@ WORKER_USERNAME=worker@email.com WORKER_PASSWORD=password # Debug -JULIA_DEBUG=ReefGuideWorker +JULIA_DEBUG=ReefGuideWorker,ReefGuide # Configuration for the worker POLL_INTERVAL_MS=5000 From f231d90d054987018dfb590e2906389b08b1e82d Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 12:12:02 +1000 Subject: [PATCH 11/13] Threads in launch script Signed-off-by: Peter Baker --- launch_sysimage.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/launch_sysimage.sh b/launch_sysimage.sh index cfda47c..90c5f13 100755 --- a/launch_sysimage.sh +++ b/launch_sysimage.sh @@ -42,6 +42,7 @@ docker run \ --entrypoint julia \ ${CONTAINER_NAME} \ --project=@app \ + -t auto \ -J "/sysimage/$SYSIMAGE_NAME" \ --sysimage-native-code=yes \ -e "using ReefGuideWorker; ReefGuideWorker.start_worker()" From c562a51ce58ab11dbf9d49ef74fa1a68c4e2a4cc Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Fri, 18 Jul 2025 12:15:12 +1000 Subject: [PATCH 12/13] Bump to v0.1.6 ReefGuide.jl Signed-off-by: Peter Baker --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 8388416..10b5702 100644 --- a/Project.toml +++ b/Project.toml @@ -33,6 +33,6 @@ Logging = "1.11.0" Minio = "0.2.2" PrecompileSignatures = "3.0.3" Random = "1.11.0" -ReefGuide = "0.1.5" +ReefGuide = "0.1.6" SentryIntegration = "0.7.1" Serialization = "1.11.0" From 3669ee5cfd4e7b4bdc2d9511c425b7f9c7477b79 Mon Sep 17 00:00:00 2001 From: Peter Baker Date: Mon, 11 Aug 2025 15:29:49 +1000 Subject: [PATCH 13/13] WIP Signed-off-by: Peter Baker --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 5be80ec..3483db6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,6 @@ sandbox/* data .env +.env.* +!.env.dist sysimages