Skip to content

Forbid the compute worker from pulling competition images #2486

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ObadaS wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

Forbid the compute worker from pulling competition images #2486

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
114 changes: 68 additions & 46 deletions compute_worker/compute_worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ def to_bool(val):
CODALAB_IGNORE_CLEANUP_STEP = to_bool(get("CODALAB_IGNORE_CLEANUP_STEP"))

WORKER_BUNDLE_URL_REWRITE = get("WORKER_BUNDLE_URL_REWRITE", "").strip()
COMPETITION_IMAGE_PULL = to_bool(get("COMPETITION_IMAGE_PULL", "True"))


# -----------------------------------------------
Expand Down Expand Up @@ -645,38 +646,55 @@ def _update_status(self, status, extra_information=None):
def _get_container_image(self, image_name):
logger.info("Running pull for image: {}".format(image_name))
retries, max_retries = (0, 3)
while retries < max_retries:
if Settings.COMPETITION_IMAGE_PULL:
while retries < max_retries:
try:
with Progress() as progress:
resp = client.pull(image_name, stream=True, decode=True)
for line in resp:
if isinstance(line, dict) and line.get("error"):
raise DockerImagePullException(line["error"])
show_progress(line, progress)
break # Break if the loop is successful to exit "with Progress() as progress"

except (docker.errors.APIError, Exception) as pull_error:
retries += 1
if retries >= max_retries:
logger.error(
"There was a problem pulling the image : " + str(pull_error)
)
# Prepare data to be sent to submissions api
docker_pull_fail_data = {
"type": "Docker_Image_Pull_Fail",
"error_message": pull_error,
"is_scoring": self.is_scoring,
}
# Send data to be written to ingestion logs
self._update_submission(docker_pull_fail_data)
# Send error through web socket to the frontend
asyncio.run(self._send_data_through_socket(str(pull_error)))
raise DockerImagePullException(
f"Pull for {image_name} failed! Check the logs for more information"
)
else:
logger.warning("Failed. Retrying in 5 seconds...")
time.sleep(5) # Wait 5 seconds before retrying
else:
logger.info("COMPETITION_IMAGE_PULL is set to False, using local image if it exists")
try:
with Progress() as progress:
resp = client.pull(image_name, stream=True, decode=True)
for line in resp:
if isinstance(line, dict) and line.get("error"):
raise DockerImagePullException(line["error"])
show_progress(line, progress)
break # Break if the loop is successful to exit "with Progress() as progress"

except (docker.errors.APIError, Exception) as pull_error:
retries += 1
if retries >= max_retries:
logger.error(
"There was a problem pulling the image : " + str(pull_error)
)
# Prepare data to be sent to submissions api
docker_pull_fail_data = {
"type": "Docker_Image_Pull_Fail",
"error_message": pull_error,
"is_scoring": self.is_scoring,
}
# Send data to be written to ingestion logs
self._update_submission(docker_pull_fail_data)
# Send error through web socket to the frontend
asyncio.run(self._send_data_through_socket(str(pull_error)))
raise DockerImagePullException(
f"Pull for {image_name} failed! Check the logs for more information"
)
if client.inspect_image(image_name):
logger.warning("Image found, continuing")
else:
logger.warning("Failed. Retrying in 5 seconds...")
time.sleep(5) # Wait 5 seconds before retrying
logger.error("Image not found, aborting")
except Exception as e:
raise DockerImagePullException(f"Pull for {image_name} failed! COMPETITION_IMAGE_PULL is set to False, make sure the image is available locally")
docker_pull_fail_data = {
"type": "Docker_Image_Pull_Fail",
"error_message": "COMPETITION_IMAGE_PULL set to False but image is not present locally",
"is_scoring": self.is_scoring,
}
self._update_submission(docker_pull_fail_data)


async def _send_data_through_socket(self, error_message):
"""
Expand Down Expand Up @@ -847,23 +865,27 @@ def _create_container(
# Creating container
# COMPETITION_CONTAINER_NETWORK_DISABLED: Disable or not the competition container access to Internet (False by default)
# HTTP and HTTPS proxy for the competition container if needed
container = client.create_container(
self.container_image,
name=container_name,
host_config=host_config,
detach=False,
volumes=volumes_host,
command=command,
working_dir="/app/program",
environment=[
"PYTHONUNBUFFERED=1",
"http_proxy=" + Settings.COMPETITION_CONTAINER_HTTP_PROXY,
"https_proxy=" + Settings.COMPETITION_CONTAINER_HTTPS_PROXY,
],
network_disabled=Settings.COMPETITION_CONTAINER_NETWORK_DISABLED,
)
try:
container = client.create_container(
self.container_image,
name=container_name,
host_config=host_config,
detach=False,
volumes=volumes_host,
command=command,
working_dir="/app/program",
environment=[
"PYTHONUNBUFFERED=1",
"http_proxy=" + Settings.COMPETITION_CONTAINER_HTTP_PROXY,
"https_proxy=" + Settings.COMPETITION_CONTAINER_HTTPS_PROXY,
],
network_disabled=Settings.COMPETITION_CONTAINER_NETWORK_DISABLED,
)

logger.debug("Created container: " + str(container))
logger.debug("Created container: " + str(container))
except Exception as e:
logger.error(f"Error {e}")
raise SubmissionException(str(e))

return container

Expand Down
4 changes: 4 additions & 0 deletions ...tation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,10 @@ HOST_DIRECTORY=/codabench
CONTAINER_ENGINE_EXECUTABLE=docker
#USE_GPU=True
#GPU_DEVICE=nvidia.com/gpu=all
# If set to False, the compute worker will never pull for the
# competition image, the image will need to be downloaded
# manually on the host before running submissions. True by default
#COMPETITION_IMAGE_PULL=True

#######################################################################
# Network #
Expand Down
Loading