From 7bf79727cf842bd384bf5eca844a1e10e81c430a Mon Sep 17 00:00:00 2001 From: Luigi Montoya Date: Fri, 24 Jul 2026 11:54:45 -0600 Subject: [PATCH 1/5] Update Ubuntu APT sources for Network Isolation --- docker/test-only/ubuntu/Dockerfile | 3 ++- docker/ubuntu/Dockerfile.msopenjdk-11-jdk | 1 + docker/ubuntu/Dockerfile.msopenjdk-17-jdk | 1 + docker/ubuntu/Dockerfile.msopenjdk-21-jdk | 1 + docker/ubuntu/Dockerfile.msopenjdk-25-jdk | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docker/test-only/ubuntu/Dockerfile b/docker/test-only/ubuntu/Dockerfile index 336ab18..3601941 100644 --- a/docker/test-only/ubuntu/Dockerfile +++ b/docker/test-only/ubuntu/Dockerfile @@ -13,7 +13,8 @@ LABEL "Support"="Microsoft OpenJDK Support " ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get -qq update && \ +RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \ + apt-get -qq update && \ apt-get -qq upgrade && \ apt-get -qq install --no-install-recommends tzdata ca-certificates fontconfig locales apt-transport-https wget binutils && \ echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen && \ diff --git a/docker/ubuntu/Dockerfile.msopenjdk-11-jdk b/docker/ubuntu/Dockerfile.msopenjdk-11-jdk index 6186008..c2b7999 100644 --- a/docker/ubuntu/Dockerfile.msopenjdk-11-jdk +++ b/docker/ubuntu/Dockerfile.msopenjdk-11-jdk @@ -10,6 +10,7 @@ ARG ms_package="msopenjdk-11 jaz" ARG PKGS="tzdata ca-certificates fontconfig locales apt-transport-https wget" RUN DEBIAN_FRONTEND=noninteractive && \ + sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \ apt-get -qq update && \ apt-get -qq upgrade && \ apt-get -qq install --no-install-recommends ${PKGS} && \ diff --git a/docker/ubuntu/Dockerfile.msopenjdk-17-jdk b/docker/ubuntu/Dockerfile.msopenjdk-17-jdk index 7693bce..fc1b3df 100644 --- a/docker/ubuntu/Dockerfile.msopenjdk-17-jdk +++ b/docker/ubuntu/Dockerfile.msopenjdk-17-jdk @@ -10,6 +10,7 @@ ARG ms_package="msopenjdk-17 jaz" ARG PKGS="tzdata ca-certificates fontconfig locales apt-transport-https wget binutils" RUN DEBIAN_FRONTEND=noninteractive && \ + sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \ apt-get -qq update && \ apt-get -qq upgrade && \ apt-get -qq install --no-install-recommends ${PKGS} && \ diff --git a/docker/ubuntu/Dockerfile.msopenjdk-21-jdk b/docker/ubuntu/Dockerfile.msopenjdk-21-jdk index 1bba671..64716a4 100644 --- a/docker/ubuntu/Dockerfile.msopenjdk-21-jdk +++ b/docker/ubuntu/Dockerfile.msopenjdk-21-jdk @@ -10,6 +10,7 @@ ARG ms_package="msopenjdk-21 jaz" ARG PKGS="tzdata ca-certificates fontconfig locales apt-transport-https wget binutils" RUN DEBIAN_FRONTEND=noninteractive && \ + sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \ apt-get -qq update && \ apt-get -qq upgrade && \ apt-get -qq install --no-install-recommends ${PKGS} && \ diff --git a/docker/ubuntu/Dockerfile.msopenjdk-25-jdk b/docker/ubuntu/Dockerfile.msopenjdk-25-jdk index a484146..72da5e4 100644 --- a/docker/ubuntu/Dockerfile.msopenjdk-25-jdk +++ b/docker/ubuntu/Dockerfile.msopenjdk-25-jdk @@ -10,6 +10,7 @@ ARG ms_package="msopenjdk-25 jaz" ARG PKGS="tzdata ca-certificates fontconfig locales apt-transport-https wget binutils" RUN DEBIAN_FRONTEND=noninteractive && \ + sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \ apt-get -qq update && \ apt-get -qq upgrade && \ apt-get -qq install --no-install-recommends ${PKGS} && \ From d03cbcef2ddcf14f8eaa0f490cfa22d89e3eecfe Mon Sep 17 00:00:00 2001 From: Luigi Montoya Date: Fri, 24 Jul 2026 12:57:10 -0600 Subject: [PATCH 2/5] QEMU diagnostics --- .devops/build.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.devops/build.yml b/.devops/build.yml index a45b49d..c567ad7 100644 --- a/.devops/build.yml +++ b/.devops/build.yml @@ -102,6 +102,49 @@ extends: echo "##vso[task.setvariable variable=REGISTRIES]$REGISTRIES" displayName: Set REGISTRIES variable + # TEMPORARY: collect QEMU/binfmt state of the build agent to diagnose the + # arm64 emulated "libc-bin ... exit status 139" (SIGSEGV) failure. Remove + # once the emulator is updated. Never fails the build (always()/exit 0). + - task: AzureCLI@2 + displayName: "QEMU/binfmt diagnostics (temporary)" + condition: always() + inputs: + azureSubscription: "JEG-Infrastructure" + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + set +e + echo "===== HOST =====" + uname -a + (source /etc/os-release && echo "OS: $PRETTY_NAME") + echo "===== binfmt_misc registrations (host kernel) =====" + ls -la /proc/sys/fs/binfmt_misc/ 2>/dev/null + for f in /proc/sys/fs/binfmt_misc/qemu-*; do + [ -e "$f" ] || continue + echo "-- $f --"; cat "$f" + done + echo "===== host qemu binaries + versions =====" + ls -la /usr/bin/qemu-*-static /usr/bin/qemu-* 2>/dev/null | grep -i qemu || echo "no qemu binaries in /usr/bin" + for b in /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64; do + [ -x "$b" ] && { echo "-- $b --"; "$b" --version 2>/dev/null | head -2; } + done + echo "===== binfmt.d configs =====" + ls -la /etc/binfmt.d /usr/lib/binfmt.d 2>/dev/null + cat /etc/binfmt.d/* /usr/lib/binfmt.d/* 2>/dev/null + echo "===== installed qemu packages =====" + (rpm -qa 2>/dev/null | grep -i qemu) || (dpkg -l 2>/dev/null | grep -i qemu) || echo "none found" + echo "===== docker / buildx =====" + docker version --format 'server={{.Server.Version}}' 2>/dev/null + docker buildx version 2>/dev/null + echo "===== qemu bundled inside mirrored moby/buildkit image =====" + az acr login -n junipercontainerregistry >/dev/null 2>&1 + docker run --rm --entrypoint sh junipercontainerregistry.azurecr.io/mirror/moby/buildkit -c ' + ls -la /usr/bin/ 2>/dev/null | grep -i qemu || echo "no qemu bundled in buildkit image"; + for b in /usr/bin/qemu-aarch64 /usr/bin/qemu-aarch64-static; do [ -x "$b" ] && { echo "-- $b --"; "$b" --version 2>/dev/null | head -1; }; done + ' 2>/dev/null || echo "could not inspect buildkit image" + echo "===== END DIAGNOSTICS =====" + exit 0 + - task: AzureCLI@2 retryCountOnTaskFailure: 2 inputs: From 672a3c3d9556d11c2641324d6e775078a3e3c66a Mon Sep 17 00:00:00 2001 From: Luigi Montoya Date: Fri, 24 Jul 2026 13:50:19 -0600 Subject: [PATCH 3/5] Add temporary QEMU registration for aarch64 emulator to fix arm64 segfaults --- .devops/build.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.devops/build.yml b/.devops/build.yml index c567ad7..cc4d8c8 100644 --- a/.devops/build.yml +++ b/.devops/build.yml @@ -145,6 +145,49 @@ extends: echo "===== END DIAGNOSTICS =====" exit 0 + # TEMPORARY: register a modern QEMU aarch64 emulator on the host BEFORE the + # buildx builder is created, to validate the fix for the intermittent arm64 + # "libc-bin ... exit status 139" (SIGSEGV) segfault caused by the old emulator + # the buildkit builder otherwise registers. buildkit defers to an already + # registered emulator, so this should make the arm64 build reliable. Uses the + # AzureLinux repo on packages.microsoft.com (already reachable) - no Docker Hub + # / mirror needed. Once proven, this moves to the agent image + # (linux-install-basic-tools) and is removed here. + - task: AzureCLI@2 + displayName: "Register modern QEMU (temporary validation)" + condition: always() + inputs: + azureSubscription: "JEG-Infrastructure" + scriptType: "bash" + scriptLocation: "inlineScript" + inlineScript: | + set -e + if [[ "$(uname -m)" != "x86_64" ]]; then + echo "Host is $(uname -m), not x86_64 - nothing to emulate. Skipping." + exit 0 + fi + echo "Host binfmt BEFORE:" + cat /proc/sys/fs/binfmt_misc/qemu-aarch64 2>/dev/null || echo " (qemu-aarch64 not registered)" + echo "Installing modern qemu-user-static from AzureLinux repo (packages.microsoft.com)" + sudo tdnf install -y qemu-user-static + /usr/bin/qemu-aarch64-static --version | head -1 + # Standard systemd binfmt.d config. The magic/mask is just the ELF header + # signature for a little-endian aarch64 executable (e_machine 0xb7 = AARCH64); + # it is the same line used by upstream qemu-binfmt-conf.sh / docker binfmt. + # Written ONCE here; systemd-binfmt (the OS tool) registers it. F = fix_binary + # (pin interpreter so it works inside the buildkit build container). + echo "Writing /etc/binfmt.d/qemu-aarch64.conf and registering via systemd-binfmt" + sudo mkdir -p /etc/binfmt.d + sudo tee /etc/binfmt.d/qemu-aarch64.conf >/dev/null <<'EOF' + :qemu-aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-aarch64-static:F + EOF + # Prefer the OS tool; if systemd-binfmt is unavailable, feed the same + # single-source config file straight to the kernel (identical line format). + sudo systemctl restart systemd-binfmt.service 2>/dev/null \ + || grep -v '^#' /etc/binfmt.d/qemu-aarch64.conf | sudo tee /proc/sys/fs/binfmt_misc/register >/dev/null + echo "Host binfmt AFTER (expect interpreter=/usr/bin/qemu-aarch64-static, flags with F):" + cat /proc/sys/fs/binfmt_misc/qemu-aarch64 + - task: AzureCLI@2 retryCountOnTaskFailure: 2 inputs: From c1e241dc36a28250f5cec25bba5e833bf91fb8fa Mon Sep 17 00:00:00 2001 From: Luigi Montoya Date: Fri, 24 Jul 2026 14:33:37 -0600 Subject: [PATCH 4/5] Use ubuntu for building instead of Azurelinux --- .devops/build.yml | 56 +++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/.devops/build.yml b/.devops/build.yml index cc4d8c8..9ecd7e1 100644 --- a/.devops/build.yml +++ b/.devops/build.yml @@ -84,7 +84,7 @@ extends: jobs: - job: build_internal pool: - name: JEG-azurelinux-x64-release + name: JEG-ubuntu22.04-x64-release os: linux strategy: matrix: ${{ parameters.jobs }} @@ -145,48 +145,42 @@ extends: echo "===== END DIAGNOSTICS =====" exit 0 - # TEMPORARY: register a modern QEMU aarch64 emulator on the host BEFORE the - # buildx builder is created, to validate the fix for the intermittent arm64 - # "libc-bin ... exit status 139" (SIGSEGV) segfault caused by the old emulator - # the buildkit builder otherwise registers. buildkit defers to an already - # registered emulator, so this should make the arm64 build reliable. Uses the - # AzureLinux repo on packages.microsoft.com (already reachable) - no Docker Hub - # / mirror needed. Once proven, this moves to the agent image - # (linux-install-basic-tools) and is removed here. + # TEMPORARY: ensure a fixed QEMU aarch64 emulator is registered on the Ubuntu + # 22.04 agent BEFORE the buildx builder is created, to fix the arm64 + # "libc-bin ... exit status 139" (SIGSEGV) segfault. Root cause: a QEMU ELF-loader + # regression (bug #1913 / LP #2072564) present in QEMU 8.1.1..9.1.x that crashes + # ldconfig.real when emulating aarch64. Ubuntu jammy ships qemu-user-static 6.2, + # which PREDATES the regression and registers with the F (fix_binary) flag, so + # buildkit defers to it. This step just ensures it is present/registered and + # prints the state. Once proven, this moves to the agent image and is removed here. - task: AzureCLI@2 - displayName: "Register modern QEMU (temporary validation)" + displayName: "Ensure fixed QEMU (temporary validation)" condition: always() inputs: azureSubscription: "JEG-Infrastructure" scriptType: "bash" scriptLocation: "inlineScript" inlineScript: | - set -e + set +e if [[ "$(uname -m)" != "x86_64" ]]; then echo "Host is $(uname -m), not x86_64 - nothing to emulate. Skipping." exit 0 fi echo "Host binfmt BEFORE:" cat /proc/sys/fs/binfmt_misc/qemu-aarch64 2>/dev/null || echo " (qemu-aarch64 not registered)" - echo "Installing modern qemu-user-static from AzureLinux repo (packages.microsoft.com)" - sudo tdnf install -y qemu-user-static - /usr/bin/qemu-aarch64-static --version | head -1 - # Standard systemd binfmt.d config. The magic/mask is just the ELF header - # signature for a little-endian aarch64 executable (e_machine 0xb7 = AARCH64); - # it is the same line used by upstream qemu-binfmt-conf.sh / docker binfmt. - # Written ONCE here; systemd-binfmt (the OS tool) registers it. F = fix_binary - # (pin interpreter so it works inside the buildkit build container). - echo "Writing /etc/binfmt.d/qemu-aarch64.conf and registering via systemd-binfmt" - sudo mkdir -p /etc/binfmt.d - sudo tee /etc/binfmt.d/qemu-aarch64.conf >/dev/null <<'EOF' - :qemu-aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/bin/qemu-aarch64-static:F - EOF - # Prefer the OS tool; if systemd-binfmt is unavailable, feed the same - # single-source config file straight to the kernel (identical line format). - sudo systemctl restart systemd-binfmt.service 2>/dev/null \ - || grep -v '^#' /etc/binfmt.d/qemu-aarch64.conf | sudo tee /proc/sys/fs/binfmt_misc/register >/dev/null - echo "Host binfmt AFTER (expect interpreter=/usr/bin/qemu-aarch64-static, flags with F):" - cat /proc/sys/fs/binfmt_misc/qemu-aarch64 + # Best-effort: install/register qemu-user-static (6.2 on jammy). No-op if the + # agent image already bakes it. Repoint apt to the Azure Ubuntu mirror first, + # since agents cannot reach archive.ubuntu.com (same blackhole that broke the + # image builds); azure.archive.ubuntu.com is reachable from Azure agents. + sudo sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list 2>/dev/null + sudo apt-get update -qq 2>/dev/null + sudo apt-get install -y -qq qemu-user-static binfmt-support 2>/dev/null + sudo systemctl restart systemd-binfmt.service 2>/dev/null + echo "qemu-aarch64-static version:" + /usr/bin/qemu-aarch64-static --version 2>/dev/null | head -1 || echo " (qemu-aarch64-static not found on host)" + echo "Host binfmt AFTER (expect a registration with flags containing F):" + cat /proc/sys/fs/binfmt_misc/qemu-aarch64 2>/dev/null || echo " (qemu-aarch64 STILL not registered)" + exit 0 - task: AzureCLI@2 retryCountOnTaskFailure: 2 @@ -247,7 +241,7 @@ extends: - job: build_public dependsOn: wait_for_validation pool: - name: JEG-azurelinux-x64-release + name: JEG-ubuntu22.04-x64-release os: linux strategy: matrix: ${{ parameters.jobs }} From 6097c99df5d660032d6b09db7c21f653329d6f96 Mon Sep 17 00:00:00 2001 From: Luigi Montoya Date: Fri, 24 Jul 2026 15:11:29 -0600 Subject: [PATCH 5/5] Remove QEMU diagnostics and validation tasks --- .devops/build.yml | 80 ----------------------------------------------- 1 file changed, 80 deletions(-) diff --git a/.devops/build.yml b/.devops/build.yml index 9ecd7e1..4fb291f 100644 --- a/.devops/build.yml +++ b/.devops/build.yml @@ -102,86 +102,6 @@ extends: echo "##vso[task.setvariable variable=REGISTRIES]$REGISTRIES" displayName: Set REGISTRIES variable - # TEMPORARY: collect QEMU/binfmt state of the build agent to diagnose the - # arm64 emulated "libc-bin ... exit status 139" (SIGSEGV) failure. Remove - # once the emulator is updated. Never fails the build (always()/exit 0). - - task: AzureCLI@2 - displayName: "QEMU/binfmt diagnostics (temporary)" - condition: always() - inputs: - azureSubscription: "JEG-Infrastructure" - scriptType: "bash" - scriptLocation: "inlineScript" - inlineScript: | - set +e - echo "===== HOST =====" - uname -a - (source /etc/os-release && echo "OS: $PRETTY_NAME") - echo "===== binfmt_misc registrations (host kernel) =====" - ls -la /proc/sys/fs/binfmt_misc/ 2>/dev/null - for f in /proc/sys/fs/binfmt_misc/qemu-*; do - [ -e "$f" ] || continue - echo "-- $f --"; cat "$f" - done - echo "===== host qemu binaries + versions =====" - ls -la /usr/bin/qemu-*-static /usr/bin/qemu-* 2>/dev/null | grep -i qemu || echo "no qemu binaries in /usr/bin" - for b in /usr/bin/qemu-aarch64-static /usr/bin/qemu-aarch64; do - [ -x "$b" ] && { echo "-- $b --"; "$b" --version 2>/dev/null | head -2; } - done - echo "===== binfmt.d configs =====" - ls -la /etc/binfmt.d /usr/lib/binfmt.d 2>/dev/null - cat /etc/binfmt.d/* /usr/lib/binfmt.d/* 2>/dev/null - echo "===== installed qemu packages =====" - (rpm -qa 2>/dev/null | grep -i qemu) || (dpkg -l 2>/dev/null | grep -i qemu) || echo "none found" - echo "===== docker / buildx =====" - docker version --format 'server={{.Server.Version}}' 2>/dev/null - docker buildx version 2>/dev/null - echo "===== qemu bundled inside mirrored moby/buildkit image =====" - az acr login -n junipercontainerregistry >/dev/null 2>&1 - docker run --rm --entrypoint sh junipercontainerregistry.azurecr.io/mirror/moby/buildkit -c ' - ls -la /usr/bin/ 2>/dev/null | grep -i qemu || echo "no qemu bundled in buildkit image"; - for b in /usr/bin/qemu-aarch64 /usr/bin/qemu-aarch64-static; do [ -x "$b" ] && { echo "-- $b --"; "$b" --version 2>/dev/null | head -1; }; done - ' 2>/dev/null || echo "could not inspect buildkit image" - echo "===== END DIAGNOSTICS =====" - exit 0 - - # TEMPORARY: ensure a fixed QEMU aarch64 emulator is registered on the Ubuntu - # 22.04 agent BEFORE the buildx builder is created, to fix the arm64 - # "libc-bin ... exit status 139" (SIGSEGV) segfault. Root cause: a QEMU ELF-loader - # regression (bug #1913 / LP #2072564) present in QEMU 8.1.1..9.1.x that crashes - # ldconfig.real when emulating aarch64. Ubuntu jammy ships qemu-user-static 6.2, - # which PREDATES the regression and registers with the F (fix_binary) flag, so - # buildkit defers to it. This step just ensures it is present/registered and - # prints the state. Once proven, this moves to the agent image and is removed here. - - task: AzureCLI@2 - displayName: "Ensure fixed QEMU (temporary validation)" - condition: always() - inputs: - azureSubscription: "JEG-Infrastructure" - scriptType: "bash" - scriptLocation: "inlineScript" - inlineScript: | - set +e - if [[ "$(uname -m)" != "x86_64" ]]; then - echo "Host is $(uname -m), not x86_64 - nothing to emulate. Skipping." - exit 0 - fi - echo "Host binfmt BEFORE:" - cat /proc/sys/fs/binfmt_misc/qemu-aarch64 2>/dev/null || echo " (qemu-aarch64 not registered)" - # Best-effort: install/register qemu-user-static (6.2 on jammy). No-op if the - # agent image already bakes it. Repoint apt to the Azure Ubuntu mirror first, - # since agents cannot reach archive.ubuntu.com (same blackhole that broke the - # image builds); azure.archive.ubuntu.com is reachable from Azure agents. - sudo sed -i 's|http://archive.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g; s|http://security.ubuntu.com/ubuntu|http://azure.archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list 2>/dev/null - sudo apt-get update -qq 2>/dev/null - sudo apt-get install -y -qq qemu-user-static binfmt-support 2>/dev/null - sudo systemctl restart systemd-binfmt.service 2>/dev/null - echo "qemu-aarch64-static version:" - /usr/bin/qemu-aarch64-static --version 2>/dev/null | head -1 || echo " (qemu-aarch64-static not found on host)" - echo "Host binfmt AFTER (expect a registration with flags containing F):" - cat /proc/sys/fs/binfmt_misc/qemu-aarch64 2>/dev/null || echo " (qemu-aarch64 STILL not registered)" - exit 0 - - task: AzureCLI@2 retryCountOnTaskFailure: 2 inputs: