From e8c167032b3b20a9e1aaa4f6e0424147390bc32d Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Mon, 3 Aug 2026 15:55:18 +0200 Subject: [PATCH] add option to forbid the compute worker from pulling the competition image automatically --- compute_worker/compute_worker.py | 114 +++++++++++------- .../Compute-Worker-Management---Setup.md | 4 + 2 files changed, 72 insertions(+), 46 deletions(-) diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py index 243dcd16d..edc407f93 100644 --- a/compute_worker/compute_worker.py +++ b/compute_worker/compute_worker.py @@ -112,6 +112,7 @@ def to_bool(val): CODALAB_IGNORE_CLEANUP_STEP = to_bool(get("CODALAB_IGNORE_CLEANUP_STEP")) WORKER_BUNDLE_URL_REWRITE = get("WORKER_BUNDLE_URL_REWRITE", "").strip() + COMPETITION_IMAGE_PULL = to_bool(get("COMPETITION_IMAGE_PULL", "True")) # ----------------------------------------------- @@ -645,38 +646,55 @@ def _update_status(self, status, extra_information=None): def _get_container_image(self, image_name): logger.info("Running pull for image: {}".format(image_name)) retries, max_retries = (0, 3) - while retries < max_retries: + if Settings.COMPETITION_IMAGE_PULL: + while retries < max_retries: + try: + with Progress() as progress: + resp = client.pull(image_name, stream=True, decode=True) + for line in resp: + if isinstance(line, dict) and line.get("error"): + raise DockerImagePullException(line["error"]) + show_progress(line, progress) + break # Break if the loop is successful to exit "with Progress() as progress" + + except (docker.errors.APIError, Exception) as pull_error: + retries += 1 + if retries >= max_retries: + logger.error( + "There was a problem pulling the image : " + str(pull_error) + ) + # Prepare data to be sent to submissions api + docker_pull_fail_data = { + "type": "Docker_Image_Pull_Fail", + "error_message": pull_error, + "is_scoring": self.is_scoring, + } + # Send data to be written to ingestion logs + self._update_submission(docker_pull_fail_data) + # Send error through web socket to the frontend + asyncio.run(self._send_data_through_socket(str(pull_error))) + raise DockerImagePullException( + f"Pull for {image_name} failed! Check the logs for more information" + ) + else: + logger.warning("Failed. Retrying in 5 seconds...") + time.sleep(5) # Wait 5 seconds before retrying + else: + logger.info("COMPETITION_IMAGE_PULL is set to False, using local image if it exists") try: - with Progress() as progress: - resp = client.pull(image_name, stream=True, decode=True) - for line in resp: - if isinstance(line, dict) and line.get("error"): - raise DockerImagePullException(line["error"]) - show_progress(line, progress) - break # Break if the loop is successful to exit "with Progress() as progress" - - except (docker.errors.APIError, Exception) as pull_error: - retries += 1 - if retries >= max_retries: - logger.error( - "There was a problem pulling the image : " + str(pull_error) - ) - # Prepare data to be sent to submissions api - docker_pull_fail_data = { - "type": "Docker_Image_Pull_Fail", - "error_message": pull_error, - "is_scoring": self.is_scoring, - } - # Send data to be written to ingestion logs - self._update_submission(docker_pull_fail_data) - # Send error through web socket to the frontend - asyncio.run(self._send_data_through_socket(str(pull_error))) - raise DockerImagePullException( - f"Pull for {image_name} failed! Check the logs for more information" - ) + if client.inspect_image(image_name): + logger.warning("Image found, continuing") else: - logger.warning("Failed. Retrying in 5 seconds...") - time.sleep(5) # Wait 5 seconds before retrying + logger.error("Image not found, aborting") + except Exception as e: + raise DockerImagePullException(f"Pull for {image_name} failed! COMPETITION_IMAGE_PULL is set to False, make sure the image is available locally") + docker_pull_fail_data = { + "type": "Docker_Image_Pull_Fail", + "error_message": "COMPETITION_IMAGE_PULL set to False but image is not present locally", + "is_scoring": self.is_scoring, + } + self._update_submission(docker_pull_fail_data) + async def _send_data_through_socket(self, error_message): """ @@ -847,23 +865,27 @@ def _create_container( # Creating container # COMPETITION_CONTAINER_NETWORK_DISABLED: Disable or not the competition container access to Internet (False by default) # HTTP and HTTPS proxy for the competition container if needed - container = client.create_container( - self.container_image, - name=container_name, - host_config=host_config, - detach=False, - volumes=volumes_host, - command=command, - working_dir="/app/program", - environment=[ - "PYTHONUNBUFFERED=1", - "http_proxy=" + Settings.COMPETITION_CONTAINER_HTTP_PROXY, - "https_proxy=" + Settings.COMPETITION_CONTAINER_HTTPS_PROXY, - ], - network_disabled=Settings.COMPETITION_CONTAINER_NETWORK_DISABLED, - ) + try: + container = client.create_container( + self.container_image, + name=container_name, + host_config=host_config, + detach=False, + volumes=volumes_host, + command=command, + working_dir="/app/program", + environment=[ + "PYTHONUNBUFFERED=1", + "http_proxy=" + Settings.COMPETITION_CONTAINER_HTTP_PROXY, + "https_proxy=" + Settings.COMPETITION_CONTAINER_HTTPS_PROXY, + ], + network_disabled=Settings.COMPETITION_CONTAINER_NETWORK_DISABLED, + ) - logger.debug("Created container: " + str(container)) + logger.debug("Created container: " + str(container)) + except Exception as e: + logger.error(f"Error {e}") + raise SubmissionException(str(e)) return container diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md index 5735af914..66489e832 100644 --- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md +++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md @@ -59,6 +59,10 @@ HOST_DIRECTORY=/codabench CONTAINER_ENGINE_EXECUTABLE=docker #USE_GPU=True #GPU_DEVICE=nvidia.com/gpu=all +# If set to False, the compute worker will never pull for the +# competition image, the image will need to be downloaded +# manually on the host before running submissions. True by default +#COMPETITION_IMAGE_PULL=True ####################################################################### # Network #