From c60fb12e361977b177abd93de250361965e009f4 Mon Sep 17 00:00:00 2001 From: Abanoub Doss Date: Mon, 27 Jul 2026 19:52:38 -0500 Subject: [PATCH] test: retry docker compose up when the fixture image pull fails --- src/iceberg/test/util/docker_compose_util.cc | 36 ++++++++++++++++++-- src/iceberg/test/util/docker_compose_util.h | 4 +-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/src/iceberg/test/util/docker_compose_util.cc b/src/iceberg/test/util/docker_compose_util.cc index 6baa178a7e..8d78eea8ab 100644 --- a/src/iceberg/test/util/docker_compose_util.cc +++ b/src/iceberg/test/util/docker_compose_util.cc @@ -19,10 +19,24 @@ #include "iceberg/test/util/docker_compose_util.h" +#include +#include +#include +#include + #include "iceberg/test/util/cmd_util.h" namespace iceberg { +namespace { + +// Compose pulls the fixture image from Docker Hub on every run, and that request +// intermittently times out on CI runners. +constexpr int kUpMaxAttempts = 3; +constexpr std::chrono::seconds kUpRetryBackoff{5}; + +} // namespace + DockerCompose::DockerCompose(std::string project_name, std::filesystem::path docker_compose_dir) : project_name_(std::move(project_name)), @@ -31,8 +45,26 @@ DockerCompose::DockerCompose(std::string project_name, DockerCompose::~DockerCompose() { Down(); } void DockerCompose::Up() { - auto cmd = BuildDockerCommand({"up", "-d", "--wait", "--timeout", "60"}); - return cmd.RunCommand("docker compose up"); + for (int attempt = 1;; ++attempt) { + try { + auto cmd = BuildDockerCommand({"up", "-d", "--wait", "--timeout", "60"}); + cmd.RunCommand("docker compose up"); + return; + } catch (const std::runtime_error&) { + if (attempt >= kUpMaxAttempts) { + throw; + } + std::println(stderr, "[INFO] docker compose up attempt {}/{} failed, retrying", + attempt, kUpMaxAttempts); + try { + Down(); + } catch (const std::runtime_error&) { + // Nothing worth reporting: the retry re-runs up either way, and the + // original failure is what surfaces if attempts run out. + } + std::this_thread::sleep_for(kUpRetryBackoff * attempt); + } + } } void DockerCompose::Down() { diff --git a/src/iceberg/test/util/docker_compose_util.h b/src/iceberg/test/util/docker_compose_util.h index 63928eb8c8..fc003b3a05 100644 --- a/src/iceberg/test/util/docker_compose_util.h +++ b/src/iceberg/test/util/docker_compose_util.h @@ -46,8 +46,8 @@ class DockerCompose { /// \brief Get the docker project name. const std::string& project_name() const { return project_name_; } - /// \brief Executes 'docker-compose up' to start services. - /// \note May throw an exception if the services fail to start. + /// \brief Executes 'docker-compose up' to start services, retrying on failure. + /// \note May throw an exception if every attempt fails to start the services. void Up(); /// \brief Executes 'docker-compose down' to stop and remove services.