From 9029ac3e3350431403071774cb7046e0458b33cd Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Wed, 24 Jun 2026 12:51:01 +0200
Subject: [PATCH 01/20] HITL enabled and working, needs polish and test

---
 compute_worker/compute_worker.py              | 50 ++++++++++++++++++-
 src/apps/api/serializers/competitions.py      |  6 ++-
 src/apps/competitions/models.py               |  1 +
 src/apps/competitions/tasks.py                | 13 +++++
 .../editor/_competition_details.tag           | 18 +++++++
 5 files changed, 85 insertions(+), 3 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 243dcd16d..75a3540c3 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -314,6 +314,9 @@ def run_wrapper(run_args):
         run.prepare()
         run.start()
         if run.is_scoring:
+            if run.human_in_the_loop:
+                run.wait_for_human_validation()
+                run._update_status(SubmissionStatus.FINISHED)
             run.push_scores()
         run.push_output()
     except DockerImagePullException as e:
@@ -470,6 +473,7 @@ def __init__(self, run_args):
         self.prediction_result = run_args["prediction_result"]
         self.scoring_result = run_args.get("scoring_result")
         self.execution_time_limit = run_args["execution_time_limit"]
+        self.human_in_the_loop = run_args.get("human_in_the_loop", False)
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -1445,11 +1449,55 @@ def start(self):
                 )
                 # Raise so upstream marks failed immediately
                 raise SubmissionException("Child task failed or non-zero return code")
-            self._update_status(SubmissionStatus.FINISHED)
+
+            if not self.human_in_the_loop:
+                self._update_status(SubmissionStatus.FINISHED)
 
         else:
             self._update_status(SubmissionStatus.SCORING)
 
+    def wait_for_human_validation(self):
+        container_output_dir = self.output_dir
+        host_output_dir = self._get_host_path(self.output_dir)
+
+        scores_path = os.path.join(host_output_dir, "scores.json")
+        if not os.path.exists(os.path.join(container_output_dir, "scores.json")):
+            scores_path = os.path.join(host_output_dir, "scores.txt")
+
+        approved_container = os.path.join(container_output_dir, "hitl_approved")
+        rejected_container = os.path.join(container_output_dir, "hitl_rejected")
+        approved_host = os.path.join(host_output_dir, "hitl_approved")
+        rejected_host = os.path.join(host_output_dir, "hitl_rejected")
+
+        logger.info("=" * 60)
+        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
+        logger.info("Inspect the scores file:")
+        logger.info(f"  cat {scores_path}")
+        logger.info(f"To approve : touch {approved_host}")
+        logger.info(f"To reject  : touch {rejected_host}")
+        logger.info("=" * 60)
+
+        poll_interval = 3
+        max_wait = 60 * 60 * 24
+
+        elapsed = 0
+        while elapsed < max_wait:
+            if os.path.exists(approved_container):  # ← poll le chemin conteneur
+                logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
+                return
+            if os.path.exists(rejected_container):  # ← poll le chemin conteneur
+                raise SubmissionException(
+                    f"HITL: scores rejected by the compute node operator "
+                    f"(submission {self.submission_id})"
+                )
+            time.sleep(poll_interval)
+            elapsed += poll_interval
+
+        raise SubmissionException(
+            f"HITL: 24h timeout reached without validation "
+            f"(submission {self.submission_id})"
+        )
+
     def push_scores(self):
         """This is only ran at the end of the scoring step"""
         # POST to some endpoint:
diff --git a/src/apps/api/serializers/competitions.py b/src/apps/api/serializers/competitions.py
index a0fb14fda..3a566446e 100644
--- a/src/apps/api/serializers/competitions.py
+++ b/src/apps/api/serializers/competitions.py
@@ -272,7 +272,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def validate_phases(self, phases):
@@ -417,7 +418,8 @@ class Meta:
             'contact_email',
             'report',
             'whitelist_emails',
-            'forum_enabled'
+            'forum_enabled',
+            'enable_human_in_the_loop'
         )
 
     def get_leaderboards(self, instance):
diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py
index f919ae432..164e66536 100644
--- a/src/apps/competitions/models.py
+++ b/src/apps/competitions/models.py
@@ -89,6 +89,7 @@ class Competition(models.Model):
 
     # If true, forum is enabled (default=True)
     forum_enabled = models.BooleanField(default=True)
+    enable_human_in_the_loop = models.BooleanField(default=False)
 
     def __str__(self):
         return f"competition-{self.title}-{self.pk}-{self.competition_type}"
diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 3d7d8abcb..1f424b4ac 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -58,6 +58,7 @@
     "contact_email",
     "fact_sheet",
     "forum_enabled",
+    "enable_human_in_the_loop"
 ]
 
 TASK_FIELDS = [
@@ -166,6 +167,7 @@ def _send_to_compute_worker(submission, is_scoring):
         ),
         "id": submission.pk,
         "is_scoring": is_scoring,
+        "human_in_the_loop": submission.phase.competition.enable_human_in_the_loop,
     }
 
     if (
@@ -260,6 +262,17 @@ def _send_to_compute_worker(submission, is_scoring):
             submission.queue = effective_queue
             submission.save(update_fields=["queue"])
 
+    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+        time_limit = 60 * 60 * 25
+
+    if (
+        submission.phase.competition.queue
+    ):  # if the competition is running on a custom queue, not the default queue
+        submission.queue = submission.phase.competition.queue
+        run_args["execution_time_limit"] = (
+            submission.phase.execution_time_limit
+        )  # use the competition time limit
+        submission.save(update_fields=["queue"])
     if submission.status == Submission.SUBMITTING:
         submission.status = Submission.SUBMITTED
         submission.save(update_fields=["status"])
diff --git a/src/static/riot/competitions/editor/_competition_details.tag b/src/static/riot/competitions/editor/_competition_details.tag
index 199edd404..da135fef0 100644
--- a/src/static/riot/competitions/editor/_competition_details.tag
+++ b/src/static/riot/competitions/editor/_competition_details.tag
@@ -211,6 +211,22 @@
             </sup>
         </div>
 
+        <!--  Human in the Loop  -->
+        <div class="field">
+            <label>Human in the Loop</label>
+            <div class="ui checkbox">
+                <label>Enable Human in the Loop validation</label>
+                <input type="checkbox" ref="enable_human_in_the_loop" onchange="{form_updated}">
+            </div>
+            <sup>
+                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform"
+                        data-inverted=""
+                        data-position="bottom center">
+                    <i class="help icon circle"></i>
+                </span>
+            </sup>
+        </div>
+
         <!--  Public submissions  -->
         <div class="field">
             <label>Public Submissions</label>
@@ -314,6 +330,7 @@
             self.data["auto_run_submissions"] = self.refs.auto_run_submissions.checked
             self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked
             self.data["forum_enabled"] = self.refs.forum_enabled.checked
+            self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked
             self.data["make_programs_available"] = self.refs.make_programs_available.checked
             self.data["make_input_data_available"] = self.refs.make_input_data_available.checked
             self.data["docker_image"] = $(self.refs.docker_image).val()
@@ -452,6 +469,7 @@
             self.refs.auto_run_submissions.checked = competition.auto_run_submissions
             self.refs.can_participants_make_submissions_public.checked = competition.can_participants_make_submissions_public
             self.refs.forum_enabled.checked = competition.forum_enabled
+            self.refs.enable_human_in_the_loop.checked = competition.enable_human_in_the_loop
             self.refs.make_programs_available.checked = competition.make_programs_available
             self.refs.make_input_data_available.checked = competition.make_input_data_available
             $(self.refs.docker_image).val(competition.docker_image)

From 72a876354536841159ef1bb3daba8a9573a5003c Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 29 Jun 2026 15:23:32 +0200
Subject: [PATCH 02/20] disable HITL for public CWs

---
 compute_worker/compute_worker.py                     |  4 ++--
 src/apps/competitions/tasks.py                       | 11 ++++++++---
 .../competitions/editor/_competition_details.tag     | 12 +++++++++++-
 3 files changed, 21 insertions(+), 6 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 75a3540c3..1d1a7ea11 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -1482,10 +1482,10 @@ def wait_for_human_validation(self):
 
         elapsed = 0
         while elapsed < max_wait:
-            if os.path.exists(approved_container):  # ← poll le chemin conteneur
+            if os.path.exists(approved_container):
                 logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
                 return
-            if os.path.exists(rejected_container):  # ← poll le chemin conteneur
+            if os.path.exists(rejected_container):
                 raise SubmissionException(
                     f"HITL: scores rejected by the compute node operator "
                     f"(submission {self.submission_id})"
diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 1f424b4ac..4251e2592 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -157,6 +157,11 @@ def _get_user_group_queues(user, competition):
 
 
 def _send_to_compute_worker(submission, is_scoring):
+    hitl_active = (
+        submission.phase.competition.enable_human_in_the_loop
+        and submission.queue is not None
+    )
+
     run_args = {
         "user_pk": submission.owner.pk,
         "submissions_api_url": settings.SUBMISSIONS_API_URL,
@@ -167,7 +172,7 @@ def _send_to_compute_worker(submission, is_scoring):
         ),
         "id": submission.pk,
         "is_scoring": is_scoring,
-        "human_in_the_loop": submission.phase.competition.enable_human_in_the_loop,
+        "human_in_the_loop": hitl_active,
     }
 
     if (
@@ -251,7 +256,7 @@ def _send_to_compute_worker(submission, is_scoring):
     logger.info(run_args)
 
     # Pad timelimit so worker has time to cleanup
-    time_padding = 60 * 20  # 20 minutes
+    time_padding = 60 * 20
     time_limit = submission.phase.execution_time_limit + time_padding
 
     effective_queue = submission.queue or submission.phase.competition.queue
@@ -262,7 +267,7 @@ def _send_to_compute_worker(submission, is_scoring):
             submission.queue = effective_queue
             submission.save(update_fields=["queue"])
 
-    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+    if is_scoring and hitl_active:
         time_limit = 60 * 60 * 25
 
     if (
diff --git a/src/static/riot/competitions/editor/_competition_details.tag b/src/static/riot/competitions/editor/_competition_details.tag
index da135fef0..25afa4099 100644
--- a/src/static/riot/competitions/editor/_competition_details.tag
+++ b/src/static/riot/competitions/editor/_competition_details.tag
@@ -219,12 +219,17 @@
                 <input type="checkbox" ref="enable_human_in_the_loop" onchange="{form_updated}">
             </div>
             <sup>
-                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform"
+                <span data-tooltip="If checked, the compute worker will pause after scoring and wait for a manual validation before sending scores to the platform. Only active on private queues."
                         data-inverted=""
                         data-position="bottom center">
                     <i class="help icon circle"></i>
                 </span>
             </sup>
+            <div ref="hitl_warning" class="ui yellow message" style="display:none; margin-top: 0.5em;">
+                <i class="exclamation triangle icon"></i>
+                Human in the Loop is only active on <strong>private queues</strong>.
+                It will have no effect until a private queue is selected for this competition or its participant groups.
+            </div>
         </div>
 
         <!--  Public submissions  -->
@@ -331,6 +336,11 @@
             self.data["can_participants_make_submissions_public"] = self.refs.can_participants_make_submissions_public.checked
             self.data["forum_enabled"] = self.refs.forum_enabled.checked
             self.data["enable_human_in_the_loop"] = self.refs.enable_human_in_the_loop.checked
+            const hitlChecked = self.refs.enable_human_in_the_loop.checked
+            const hasPrivateQueue = !!(self.data["queue"] && self.data["queue"] !== "")
+            if (self.refs.hitl_warning) {
+                self.refs.hitl_warning.style.display = (hitlChecked && !hasPrivateQueue) ? 'block' : 'none'
+            }
             self.data["make_programs_available"] = self.refs.make_programs_available.checked
             self.data["make_input_data_available"] = self.refs.make_input_data_available.checked
             self.data["docker_image"] = $(self.refs.docker_image).val()

From d8862a0f064cccd8b845131cb01b7839658a7117 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 30 Jun 2026 10:56:34 +0200
Subject: [PATCH 03/20] HITL secute on public CW + UI improvement

---
 src/apps/competitions/tasks.py                  | 17 +++++++++++++++++
 .../competitions/detail/submission_manager.tag  |  8 ++++++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 4251e2592..35bbebd32 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -375,6 +375,23 @@ def _run_submission(submission_pk, task_pks=None, is_scoring=False):
     )
     submission = qs.get(pk=submission_pk)
 
+    # ── HUMAN IN THE LOOP SECURE (private CW only)
+    if is_scoring and submission.phase.competition.enable_human_in_the_loop:
+        if submission.queue is None:
+            logger.error(
+                f"HITL: submission {submission_pk} rejected — "
+                f"Human in the Loop is enabled but no private queue is configured "
+                f"(competition or participant group)."
+            )
+            submission.status = Submission.FAILED
+            submission.status_details = (
+                "This competition requires Human in the Loop validation, but no "
+                "private compute queue is configured. Contact the organizer."
+            )
+            submission.save(update_fields=["status", "status_details"])
+            return
+    # ── HITL SECURE (private CW only)
+
     if submission.is_specific_task_re_run:
         # Should only be one task for a specified task submission
         tasks = Task.objects.filter(pk__in=task_pks)
diff --git a/src/static/riot/competitions/detail/submission_manager.tag b/src/static/riot/competitions/detail/submission_manager.tag
index 3f41a69eb..37530345c 100644
--- a/src/static/riot/competitions/detail/submission_manager.tag
+++ b/src/static/riot/competitions/detail/submission_manager.tag
@@ -107,7 +107,8 @@
                 <td class="right aligned collapsing">
                     { submission.status }
                     <sup data-tooltip="{submission.status_details}">
-                        <i if="{submission.status === 'Failed'}" class="failed question circle icon"></i>
+                        <i if="{submission.status === 'Failed' && !is_hitl_failure(submission)}" class="failed question circle icon"></i>
+                        <i if="{submission.status === 'Failed' && is_hitl_failure(submission)}" class="orange lock icon"></i>
                     </sup>
                     <sup data-tooltip="An organizer will run your submission soon">
                         <i if="{submission.status === 'Submitting' && !submission.auto_run}"
@@ -311,7 +312,10 @@
             self.update()
         }
 
-        
+        self.is_hitl_failure = function (submission) {
+            return !!(submission.status_details && submission.status_details.indexOf('Human in the Loop') !== -1)
+        }
+
         self.update_submissions = function (filters) {
             self.loading = true
             self.update()

From 25b978e913f02d1274c6479c4ae4ac338ff3d132 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Wed, 8 Jul 2026 16:55:58 +0200
Subject: [PATCH 04/20] fix after rebase + detailed result + CW check of HITL
 enabled

---
 compute_worker/compute_worker.py | 84 +++++++++++++++++++++++++++++---
 1 file changed, 78 insertions(+), 6 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 1d1a7ea11..54461b2d1 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -112,6 +112,9 @@ def to_bool(val):
     CODALAB_IGNORE_CLEANUP_STEP = to_bool(get("CODALAB_IGNORE_CLEANUP_STEP"))
 
     WORKER_BUNDLE_URL_REWRITE = get("WORKER_BUNDLE_URL_REWRITE", "").strip()
+    HUMAN_IN_THE_LOOP = (
+        get("HUMAN_IN_THE_LOOP", "false").lower() == "true"
+    )
 
 
 # -----------------------------------------------
@@ -313,12 +316,20 @@ def run_wrapper(run_args):
     try:
         run.prepare()
         run.start()
+
         if run.is_scoring:
             if run.human_in_the_loop:
                 run.wait_for_human_validation()
-                run._update_status(SubmissionStatus.FINISHED)
-            run.push_scores()
-        run.push_output()
+                run.send_final_detailed_results()
+                run.push_scores()
+                run.push_output()
+            else:
+                run.push_scores()
+                run.push_output()
+        else:
+            run.push_output()
+
+        run._update_status(SubmissionStatus.FINISHED)
     except DockerImagePullException as e:
         msg = str(e).strip()
         if msg:
@@ -473,7 +484,17 @@ def __init__(self, run_args):
         self.prediction_result = run_args["prediction_result"]
         self.scoring_result = run_args.get("scoring_result")
         self.execution_time_limit = run_args["execution_time_limit"]
+        # ----- HITL ------
         self.human_in_the_loop = run_args.get("human_in_the_loop", False)
+        compute_hitl = Settings.HUMAN_IN_THE_LOOP
+        if self.human_in_the_loop != compute_hitl:
+            raise SubmissionException(
+                "Task rejected because the Site Worker and Compute Worker "
+                "do not have the same HUMAN_IN_THE_LOOP configuration "
+                f"(task={self.human_in_the_loop}, "
+                f"compute_worker={Settings.HUMAN_IN_THE_LOOP})."
+            )
+
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -517,6 +538,13 @@ def __init__(self, run_args):
     async def watch_detailed_results(self):
         """Watches files alongside scoring + program containers, currently only used
         for detailed_results.html"""
+
+        if self.human_in_the_loop:
+            logger.info(
+                "HITL enabled: skipping detailed_results streaming"
+            )
+            return
+
         if not self.detailed_results_url:
             return
         file_path = self.get_detailed_results_file_path()
@@ -600,6 +628,49 @@ async def send_detailed_results(self, file_path):
             logger.exception(e)
             return
 
+    def send_final_detailed_results(self):
+        if not self.detailed_results_url:
+            return
+
+        file_path = self.get_detailed_results_file_path()
+
+        if not file_path:
+            logger.info("No detailed_results.html found")
+            return
+
+        logger.info(
+            f"Uploading final detailed results {file_path} - {self.detailed_results_url}"
+        )
+
+        self._put_file(
+            self.detailed_results_url,
+            file=file_path,
+            content_type="text/html",
+        )
+
+        websocket_url = f"{self.websocket_url}?kind=detailed_results"
+
+        try:
+            websocket = asyncio.run(
+                asyncio.wait_for(
+                    websockets.connect(websocket_url),
+                    timeout=30.0,
+                )
+            )
+
+            asyncio.run(
+                websocket.send(
+                    json.dumps(
+                        {
+                            "kind": "detailed_result_update",
+                        }
+                    )
+                )
+            )
+
+        except Exception as e:
+            logger.exception(e)
+
     def _get_stdout_stderr_file_names(self, run_args):
         # run_args should be the run_args argument passed to __init__ from the run_wrapper.
         if not self.is_scoring:
@@ -1312,9 +1383,10 @@ def start(self):
                 )
 
             # During scoring we watch for detailed results
-            tasks.append(
-                self.watch_detailed_results()
-            )
+            if not self.human_in_the_loop:
+                tasks.append(
+                    self.watch_detailed_results()
+                )
         else:
             # During ingestion we run ingestion program directory and submission directory
             tasks.extend([

From 120bd96139713daff82028224c4e867fe50f04c1 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Thu, 9 Jul 2026 15:32:32 +0200
Subject: [PATCH 05/20] HITL improvement + Websocket fixed + more logs

---
 compute_worker/compute_worker.py | 48 ++++++++++++++++++++++++--------
 1 file changed, 37 insertions(+), 11 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 54461b2d1..05e4b1fd1 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -311,10 +311,19 @@ def rewrite_bundle_url_if_needed(url):
 def run_wrapper(run_args):
     # We need to convert the UUID given by celery into a byte like object otherwise things will break
     run_args.update(secret=str(run_args["secret"]))
+
     logger.info(f"Received run arguments: \n {colorize_run_args(json.dumps(run_args))}")
+    logger.info(
+        "HITL configuration : "
+        f"task={run_args.get('human_in_the_loop', False)} "
+        f"compute_worker={Settings.HUMAN_IN_THE_LOOP}"
+    )
+
     run = Run(run_args)
+
     try:
         run.prepare()
+        run.validate_hitl_configuration()
         run.start()
 
         if run.is_scoring:
@@ -486,15 +495,6 @@ def __init__(self, run_args):
         self.execution_time_limit = run_args["execution_time_limit"]
         # ----- HITL ------
         self.human_in_the_loop = run_args.get("human_in_the_loop", False)
-        compute_hitl = Settings.HUMAN_IN_THE_LOOP
-        if self.human_in_the_loop != compute_hitl:
-            raise SubmissionException(
-                "Task rejected because the Site Worker and Compute Worker "
-                "do not have the same HUMAN_IN_THE_LOOP configuration "
-                f"(task={self.human_in_the_loop}, "
-                f"compute_worker={Settings.HUMAN_IN_THE_LOOP})."
-            )
-
         # stdout and stderr
         self.stdout, self.stderr, self.ingestion_stdout, self.ingestion_stderr = (
             self._get_stdout_stderr_file_names(run_args)
@@ -628,6 +628,13 @@ async def send_detailed_results(self, file_path):
             logger.exception(e)
             return
 
+        finally:
+            if websocket is not None:
+                try:
+                    await websocket.close()
+                except Exception as e:
+                    logger.exception(e)
+
     def send_final_detailed_results(self):
         if not self.detailed_results_url:
             return
@@ -1357,6 +1364,15 @@ def prepare(self):
         self._get_container_image(self.container_image)
         self._update_status(SubmissionStatus.RUNNING)
 
+    def validate_hitl_configuration(self):
+        if self.human_in_the_loop != Settings.HUMAN_IN_THE_LOOP:
+            raise SubmissionException(
+                "Task rejected because the Site Worker and Compute Worker "
+                "do not have the same HUMAN_IN_THE_LOOP configuration "
+                f"(task={self.human_in_the_loop}, "
+                f"compute_worker={Settings.HUMAN_IN_THE_LOOP})."
+            )
+
     def start(self):
 
         logger.info(f"Preparing to run: {ProgramKind.SCORING_PROGRAM if self.is_scoring else ProgramKind.INGESTION_PROGRAM}")
@@ -1505,9 +1521,15 @@ def start(self):
             # Check if scoring program failed
             # We have can have 2 or 3 gathered tasks: 3 gathered tasks in case when `ingestion_only_during_scoring` is True, 2 otherwise
             if self.ingestion_only_during_scoring:
-                program_results, _, _ = task_results
+                if self.human_in_the_loop:
+                    program_results, _ = task_results
+                else:
+                    program_results, _, _ = task_results
             else:
-                program_results, _ = task_results
+                if self.human_in_the_loop:
+                    (program_results,) = task_results
+                else:
+                    program_results, _ = task_results
             # Gather returns either normal values or exception instances when return_exceptions=True
             had_async_exc = isinstance(
                 program_results, BaseException
@@ -1553,6 +1575,10 @@ def wait_for_human_validation(self):
         max_wait = 60 * 60 * 24
 
         elapsed = 0
+
+        logger.info(
+            "Waiting for human validation..."
+        )
         while elapsed < max_wait:
             if os.path.exists(approved_container):
                 logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")

From 1b5ac5038d2e3e965bf96051ca451c6b7c211bd4 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Fri, 10 Jul 2026 16:04:38 +0200
Subject: [PATCH 06/20] submission status: awaiting validation

---
 compute_worker/compute_worker.py | 3 +++
 src/apps/competitions/models.py  | 2 ++
 2 files changed, 5 insertions(+)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 05e4b1fd1..84fcf1825 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -134,6 +134,7 @@ class SubmissionStatus:
     SUBMITTED = "Submitted"
     PREPARING = "Preparing"
     RUNNING = "Running"
+    AWAITING_VALIDATION = "Awaiting validation"
     SCORING = "Scoring"
     FINISHED = "Finished"
     FAILED = "Failed"
@@ -144,6 +145,7 @@ class SubmissionStatus:
         SUBMITTED,
         PREPARING,
         RUNNING,
+        AWAITING_VALIDATION,
         SCORING,
         FINISHED,
         FAILED,
@@ -328,6 +330,7 @@ def run_wrapper(run_args):
 
         if run.is_scoring:
             if run.human_in_the_loop:
+                run._update_status(SubmissionStatus.AWAITING_VALIDATION)
                 run.wait_for_human_validation()
                 run.send_final_detailed_results()
                 run.push_scores()
diff --git a/src/apps/competitions/models.py b/src/apps/competitions/models.py
index 164e66536..97d7ff057 100644
--- a/src/apps/competitions/models.py
+++ b/src/apps/competitions/models.py
@@ -438,6 +438,7 @@ class Submission(models.Model):
     PREPARING = "Preparing"
     RUNNING = "Running"
     SCORING = "Scoring"
+    AWAITING_VALIDATION = "Awaiting validation"
     CANCELLED = "Cancelled"
     FINISHED = "Finished"
     FAILED = "Failed"
@@ -449,6 +450,7 @@ class Submission(models.Model):
         (PREPARING, "Preparing"),
         (RUNNING, "Running"),
         (SCORING, "Scoring"),
+        (AWAITING_VALIDATION, "Awaiting validation"),
         (CANCELLED, "Cancelled"),
         (FINISHED, "Finished"),
         (FAILED, "Failed"),

From d50bfb4daf80ebfbb92fcba3dbe5066e31c6ece2 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 20 Jul 2026 16:23:26 +0200
Subject: [PATCH 07/20] adding HITL documentation

---
 .../Running_a_benchmark/Competition-HITL.md   | 280 ++++++++++++++++++
 .../_attachments/doc_HITL_1.png               | Bin 0 -> 17180 bytes
 documentation/zensical.toml                   |   3 +-
 3 files changed, 282 insertions(+), 1 deletion(-)
 create mode 100644 documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
 create mode 100644 documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png

diff --git a/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
new file mode 100644
index 000000000..b45049032
--- /dev/null
+++ b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
@@ -0,0 +1,280 @@
+# Human-in-the-Loop (HITL)
+
+## Introduction
+
+The **Human-in-the-Loop (HITL)** feature introduces a manual validation step into the Codabench submission workflow.
+
+When enabled, the Compute Worker executes the submission normally but **does not immediately publish the scoring results**. Instead, execution pauses after the scoring program has completed and waits for a human operator to explicitly approve or reject the submission.
+
+This mechanism is intended for competitions where the scoring results must be reviewed before being published, such as:
+
+* Medical AI competitions
+* Security-sensitive evaluations
+* Competitions involving confidential data
+* Any evaluation requiring manual verification from compute worker side
+
+---
+
+## Objectives
+
+The HITL feature guarantees:
+
+* the submission is fully executed before any result is published;
+* no score appears on the leaderboard before validation;
+* no detailed results are available before validation;
+* no output archive is sent back to the Site Worker before validation;
+* the operator can approve or reject the submission directly from the Compute Worker.
+
+---
+
+## Scope
+
+The HITL feature applies **only to private Compute Workers**.
+
+Public Compute Workers **must never execute HITL jobs**, as waiting for manual validation would permanently block shared workers.
+
+---
+
+## Activation
+
+### Competition configuration
+
+Each competition exposes the following option:
+
+```
+Enable Human-in-the-Loop
+```
+
+When enabled, every submission routed to a private Compute Worker is executed in HITL mode.
+![image1](_attachments/doc_HITL_1.png)
+
+---
+
+### Compute Worker configuration
+
+Each Compute Worker may enable HITL support using its `.env` file.
+
+``` title="Option"
+HUMAN_IN_THE_LOOP=true
+```
+
+If the variable is omitted, the Compute Worker behaves as a standard worker.
+The absence of the variable is interpreted as **False**, ensuring full backward compatibility with existing Compute Workers.
+
+---
+
+## Safety checks
+
+To prevent configuration errors, the Compute Worker verifies that both sides agree on the HITL configuration.
+The Site Worker includes the following field in every Celery task.
+
+Before starting the execution, the Compute Worker compares:
+
+* task `human_in_the_loop` Codabench side.
+* local `.env` variable `HUMAN_IN_THE_LOOP` Compute worker side.
+
+---
+
+### Mismatched configuration
+
+The submission is immediately rejected.
+The Compute Worker updates the submission status to **Failed** and reports the configuration mismatch.
+
+``` title="Example logs:"
+Task rejected because the Site Worker and Compute Worker
+do not share the same HUMAN_IN_THE_LOOP configuration
+(task=True, compute_worker=False)
+```
+
+This prevents accidental execution of HITL competitions on non-HITL workers.
+
+---
+
+## Submission lifecycle
+
+### Standard workflow
+
+When HITL is disabled, the workflow remains unchanged.
+
+During scoring:
+
+* scores are uploaded immediately;
+* detailed results are streamed continuously;
+* output files are returned immediately.
+
+---
+
+### HITL workflow
+
+When HITL is enabled:
+
+The scoring program executes normally.
+After scoring completes, the Compute Worker pauses before publishing any result.
+
+---
+
+## Awaiting validation status
+
+A new submission status is introduced:
+
+This status indicates:
+
+* scoring has completed successfully;
+* all output files are available locally;
+* publication is waiting for manual approval.
+
+This status allows users and organizers to distinguish between:
+
+* submissions that are still executing;
+* submissions waiting for human validation.
+
+---
+
+## Manual validation
+
+Once execution finishes, the Compute Worker logs instructions similar to:
+``` title="Compute worker terminal command (exemple with docker)"
+docker compose logs -f [container name]
+```
+
+``` title="HITL validation display on CW terminal"
+============================================================
+HUMAN IN THE LOOP — submission 42
+
+Inspect the scores file:
+
+cat /host/output/scores.json
+
+To approve:
+touch /host/output/hitl_approved
+
+To reject:
+touch /host/output/hitl_rejected
+============================================================
+```
+
+The operator inspects the generated files before deciding.
+This needs to be done inside of the compute worker host machine (not inside of the CW terminal).
+
+---
+
+### Approval
+
+The Compute Worker immediately resumes execution.
+The following artifacts are published:
+
+* detailed results;
+* scores;
+* output archive;
+* logs.
+
+The submission status becomes "Finished"
+
+---
+
+### Rejection
+
+After rejection from operator, the compute worker immediately aborts the submission.
+
+The submission status becomes "Failed"
+
+No scoring results are published.
+
+---
+
+### Timeout
+
+If no decision is received within 24 hours, the submission automatically fails.
+The Compute Worker reports a timeout error.
+
+---
+
+## Publication policy
+
+The fundamental principle of HITL is:
+
+> **Nothing leaves the Compute Worker before validation.**
+
+When HITL is enabled:
+
+| Artifact                                | Before validation | After approval |
+| --------------------------------------- | ----------------- | -------------- |
+| scores.json                             | ❌                 | ✅              |
+| leaderboard score                       | ❌                 | ✅              |
+| detailed_results.html                   | ❌                 | ✅              |
+| detailed results websocket notification | ❌                 | ✅              |
+| output archive                          | ❌                 | ✅              |
+| execution logs                          | Local only        | ✅              |
+
+The Compute Worker keeps every generated artifact locally until approval.
+
+---
+
+## Detailed Results behaviour
+
+### Without HITL
+
+```mermaid
+flowchart TD
+    A[Scoring] --> B[Watch detailed results]
+    B --> C[Upload detailed_results.html]
+    C --> D[Notify frontend]
+```
+
+### With HITL
+
+```mermaid
+flowchart TD
+    A[Scoring] --> B[Store detailed_results.html locally]
+    B --> C[Awaiting validation]
+    C --> D[Operator approval]
+    D --> E[Upload detailed_results.html]
+    E --> F[Notify frontend]
+```
+
+---
+
+## Output behaviour
+
+Prediction files, scoring outputs and logs are generated normally.
+However, publication is postponed until validation succeeds.
+This guarantees that no evaluation result becomes visible before manual approval.
+
+---
+
+## Error handling
+
+The following situations are handled explicitly.
+
+| Situation                                            | Behaviour                       |
+| ---------------------------------------------------- | ------------------------------- |
+| HITL enabled on competition and Compute Worker       | Execution pauses for validation |
+| HITL disabled everywhere                             | Standard workflow               |
+| HITL mismatch between Site Worker and Compute Worker | Submission fails immediately    |
+| Operator rejects submission                          | Submission fails                |
+| Validation timeout                                   | Submission fails                |
+| Approval received                                    | Results are published normally  |
+
+---
+
+## Design principles
+
+The HITL implementation follows four principles:
+
+1. **Backward compatibility**
+
+   * Existing deployments require no configuration changes.
+
+2. **Safety**
+
+   * Configuration mismatches are detected before execution.
+
+3. **Isolation**
+
+   * HITL is supported only on private Compute Workers.
+
+4. **Atomic publication**
+
+   * No artifact is published before manual approval.
+
+These principles ensure that sensitive competitions can safely introduce manual validation without impacting the standard Codabench execution workflow.
diff --git a/documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png b/documentation/docs/Organizers/Running_a_benchmark/_attachments/doc_HITL_1.png
new file mode 100644
index 0000000000000000000000000000000000000000..866ec86ddf52c3cb50931a4d30dbb41420d3dd37
GIT binary patch
literal 17180
zcmb8XWl&vF*QFc53GVK0f#B{MEV#S7I|PT|?jGFT-4E{W?(Wt*-~08wRbBUXb@PLw
zPMuBF-e;{f=NRLeCsa;G3=Rex=F_K7a1!Fezdn8XtO5L84g~@H#&=lN`t(U~M?zRY
z(KY=n16);610#s<`-P2*i#AGYHB>7X|M!YNYk|JL<%>>dUFX10n{&24+to$A?f@31
zJEp+G$d8Hdg%i97o*rlQ_e(VU2bCLML{NT&!qoeSU;>E58I(i~N%Y!%y}<~;%a^VC
ziq&tgPd_SjT7y;^ZF&a>DP~XZ&gW1@66t*zo9uQV60!9(Zay-CJs!@tWOrnFYy?6q
z!^s<JO~!<mrpe?xwee;v)bNz7tQPfISPsTB0$;B8F7MzvX`Tn82vl<#?4G8{UdQDu
zde$qzt=C%mJ3r!Y&;xXQ<@@eWQ-0W{aYN%}aBeML#VvL?uuT_8g^g#VKYsSUJ0_FK
z;DolZvC)1XNn&_?SzBpzUvc|*=ixid;`N%AnbCS!o-1wg`RH(7&l8JUjGePgs;Z-n
z+9DC{86Ew}b#|7<M*GU3;||8@EOetk5FQ#+Q9-Fz^4A$#?U>DGy>E_2Wpi&YKjZ!F
z#f6ub(|H*}?cQ;`Ti|5U-tiS%yUlfcWh$L3W9De4@N&cF!z}6Pa_egn0-ghZk&@k2
z4>*s@Xrs-BYg)^Q4!ck0gVf7Prw@Tl7H6QB$HV5;&R~r9y5A**0cy3^^)9rJO1Uy#
zXdJ0P!OxV*dJ7j-RBABUi3N7g!|&=#^}E4YuJ@-{A03WQh2Zt(`COoQYTtnf?5^b!
z<nb)7!5Nm5`4V}&at*F*bu=8uqgCi~JoOxbw-(dc?97^ilbS_Di-oeFI0eWv=Yz3~
zuOs$GlXVg#inKPN&^v>H>``Dr_su2~`0vs^!h=eU&eF_^FC?WI0a>m$193@Ks-5^H
zMMrhGcqcC|7vB=+Qf3PEw3?9#TFae`n!e$Hl$)BHIaFlg$YnP7hS0WkZ3@ey6X`TS
zbiw2iIJs>`YL=^20<B|0!SK2I;8M&M%a}@*Ws}u!e6&H8+U?s0t+rQw&P(GN8CC9c
zlDM3%PzIV}S9Mg@an%Md4dTFxdYobWv#74FPOQ~(Yinr~B@%{SlN>wQBAxO_vG`YT
zQ%CS1Ec!NFTEph%7RgXDBT3LV=Xs5RVg(aiK06o^(H8xQbFItk`<~Fa!lgeQ9tRD`
zVQ0P#Qi3AebNLo*noW-2Yb_65oj3b6cNnW}zrWcz4>VS8R9Q^$Hmv?I9R01$<8i1q
zn<?ojlfhe>xKv6z7`)hK<dA(P7s!8-K(7E{GNIFV+I&+r?F9~xyBLK^^($94%lq}^
zaEc_E)d?oG(R6rE{L2BSJ=ik>W<Xd{Wlyye&Nm#+WE!R#+Lzn&Kx(Cu?c1dr)fz{R
zNIb4TbEPUo?AWa40re+r*B??z<@{+4XT)c1-~!)c7h_pkjn6n$R~B`r{J^V96hBA8
z-@8k1ZW881;IPrx7>x|%&Xgp4J*WIkhT#`fvj@6ziOCgtYOP6NX@!~2ml}NWdA^m%
zK6}6G{2<cH6{h8dOvs)q)gfqoI2SvfE%whSgoP$mi@ZNho-UM~csX8eF10s#4aQlh
zv`9KxoPuW5?~^+z6u91{y3|x+|J+<e`?Ew9lu+B(-s%#=t!7{GHxiErbaTRH`dhz0
zEYGX8K|Gcny2bfqe8!~|6j(Oy`E<RNXg-}psfMi6>D}L4Yhnki?*{S@vyIOm3J1b_
zKSqtfm`ujQLv%-P(80PxKGz6`qU5Pu@GiD~e9}<a)ghHi5++hSp^z+l%K}Tfy#02X
zHv6bwYdD<CrnRs<jmYI}NvmD+O(KrGSJ#Ltu2lCZwm$@oM!D+7F{gl?KmmlhY*zdV
z{?u%4Y%vr~<iE~{&6#N;`xa27GI74TOYs06dZB^0;S_;g@46<LKvU~_?f(8{KG#h_
zJE|`^^5f1zw(W``Z+AG(qEWt81}~mU0X@jIe0*@S8oV$0fDX#<4|q4!x5EutmQDg&
zhj{#^9+5wrSw73PHVLVFcH4bhdu^{@DT7|!865F-r4@LkQ<-s-p&4AxSM^Gr;!jM`
zr|wSX^(MRCs~Q^gN?D8C!0Bk~R<aEUM<QtDVLFf+t=D!hMpK-?bSMZhP*Isl=gYLB
zxZQ59KS*Wjy!<r3pqh@Rvf!+$(5gtk4OpHPN~a~#jeLbfs2w`4aM|v&GX>Fl+@HRF
zz|_;@v8nY0sQm6qF!t%z`>zKso>-nx&mz`5JRvDPVWrWi|9pESUQkA7KE6Mc5|^aW
zsmBAm8G=Obei9dr_;P=?7sB}@Q(T-uDDXuX%-lUr$9dFLB{F@Z)g_6>pHw0af=d<y
zd3!)QRewBv2{(SZ)`U$o3cFw04Y{XE?ytDKQH+U*j}nJG$Ju(l0(h3Wo$a4&10tQ<
zX`_)C8W(GHAI|Y*r{nFJ!Kl+T9=GcN<ZU#Nl0u9J8Y+!)dcV!kra;?Re;9_|Y{(XE
z)o^^8-?+(OB;M(C4sTscUl>MIXA}Y7zyev#xCSB^iCRMfwbDSiS}mi=J*}nuXXF|~
zI$9XL)21uS(7mCUuRI=i`%eyYy&Bp40#=G?;)r~o?e{?Wx!cyR*`9u+lC$7cxx0$Q
ze8#7)qshDx9#qjjj<?Bn>KMXuH(yu-UvAF^*K#$r#I}R~T&pM^oSbrF>IH&@`|0N?
zR)uNrX4m9&p<6|H4xTKO_YcYEHD6<KyHbihCQ&W~wR_QwraUk_Bpxp2@VvMWi<O%7
zOuV`9rbEK<C&|d1qGR#u$CrGvdY!Pt6>Vy=weia>(R5lPMGYxa>HPcvB3yPk#H9MS
z{LH~X{LFrYP%%;8gh`<MbiW8&f(s!2{2fwG2;~QjSd9iI@clD!1Mm)V{yt?yux_xB
zMHoNA<WEQ%0*GKJg!(kE^1`J4!GywKd2JOmQ1U4Yl{$J`e&CCpHVoW6JTc5>zr61j
zyQgAwY|z0Jbn6zJY}T712cz)2F>*H2Q=X-=h`!qIz2o>HVoiSH&fs*`?UqiUMnkx~
z*?jlncD?)!d7Q#(V4S_h2~NcH{m)|TWC;W|%<ja*k(MBAgsIcfwGrnxB+c%_$+x*S
zGXr$Z6>6nT;6Wjj2?5IhC(OxWp=y#|qnhdGLpfmpWZtTt;Z%^4soR0MoBBny%y#dj
zAV<sf?zq|ar>!i|B4ZwhP*8Pt{62FLC!8%l5^I^xMDxr~IHRsfL)fR2$`8Gfed%=B
zv)$00bokKhCoy<}AVOhQQ6vp}ynqF!?7N<A_sJ@+73KGWYV}?iGV;HgW`dV;e+UOz
zt%y$-{&JhssV{cVYPQ}xU{1iDEa=spP@>Xnf<4_}=&JYyoh(!W-4#w;V+Q5g-T7MU
z-L!r%nk0(Q(csyshTcj89U?KL<j71-J|0^;$rm_6ojhadWNdqk!)upJZd-gN)FgVX
zU0Pyn(u62on4i=Q@!G9MBZ);ioxHfXxIJ`DD-G6Lvn+IHryNH)){=OVbVBFgYmS`p
z<^^uG<RHK1@f*q%()_CmjY8ImkH;$#2>8P&eG%9^*J@%+X4_A^UeAWM8}aF}4UA5I
za6s|&p~G?H_UQh^fpoWtd2dtRp`j=Py%OMN0}*(=FO)hJwDT2e2DUG=ETIk3V-G#(
z;FDWf&dWEjW_(}?j7FoBzlsQLgyO*c2-&{wF*x8eM>h)H6bJHS;g>9`-f*5R*UghJ
z&&DU6+q5Q*xp9C{1+Yk%qq%~aE)r?wzvWA!4~A^fYgQxA{wdO~|Kq&03DQvLh%YFT
zIa6<|WQ~M(-8m}O@;Qh+D^hF6hCv&Ra(6n}1o}mY5hk(7boWkzlI-M@q8QKji}knB
zmqZ>9cql<b1E%VH$cgcB8MRz*gpYv;9J>^M1HIq|Jp+xvA1AT(HEaCX4nn)iF^4x8
z3?xE9;fGVT7|P}MDBSLk6<V!9Za4b|<LPdmnx7SlAq}^Bz>}Gbu+@t|B+PP+Bp^EH
z%Sj|h$K!x4dLfPscmDZuEm)TaG)x|6+Qin%p5Q2g?^QlN0q7;3MNhH<SuJ<RIVMr(
zO&$*w&b-j25#@A|xIB^jbn2I{p(y0KSJU~6%^nErZC?H<Ka4R~8x(@?X>~2;tu}um
zkpv<1J~<q?W|o$;W%77{B$z2=Qo}R2Ty(o%ec-GqnD=<xZz+GK@%l#%@9as{pK4~x
z@G2uaK@QHVHtM0kP+^D+MxJrt*z$wa)QOEi@$Q{32$&()do-qnQ?D>+G=jiN%?mpN
z?zzsdgeZFwS*yiy3zDpCRi&c=2Q+A03g(|($E49{2|w2oZxH_Lev(4@bTOSc=-M?V
zU--yiNS*v+yGXvm@`}gnnR&>{>0&BwCP?A=d{VF&RBX1MttUi}0>cl!+wK1{6i<cn
z2ZNzI;cI?Vv3faYk6>`5OdCD8ZxPdSlOB)kg5zP~tF_PD9Y!jv#ds%;Vo|m*1zMKW
zT*=(6X9hN<6cac|JeksOqe`A6+#b*0^=jwFQ}R*bD4Y^azW7%@qj!(+x(mOc#yXG3
z{XXik<4;yr*5%tH*-Y+z;rMxTY&^((joh2Rbv&PQ>V!yU8(7r-41dwC_QV=b=NRjx
zf?N?_m|7-H^+QYn+E$fv^+3^MBFp8vVslRVZ8K8p`}_OF7N=6@X5eY*_k|!qA&{1V
z!=MY7D0-XKTO};Uj_}9Du$9RTEIvXbSBaYrr7;j}4@86kYRay5C6}M~Jsg)&vvqVs
z@37>a=Qm=ss+{iSPGOX9Vt#lFLuTiuF`JX@^W)anFh|qN%S$L`I-QPJTHC%P2A$h|
zGi2Uh$YN*JR3E3n6<U;v_~k2gA}V_)r>n!z7Rr_Tjwh-1d2S<#bViL3d(3(BUqjvb
z1*vV|Qh#=XJ%#gm-E)yk+Da^?Sy`1WzVR;wrZJoS0XnT-#P|#A?p(4eS{?jKolfH<
z*nG3XjT;R5eccqo<S%(00u4qZ=vH!mv?|EUUN84oj&P)PfMgVNP2sho_KABu67-%L
zwpRF7Av9k#2pdJ%Xomii(;7pG%Jc0x;zM@Rra!O_USQ0kMh3bjT{4k6aFSmFxA9`N
zf9!2%Ffzx<0+QKuGCGbSsE$C0*LJOL&u}Y&P6K``u<QxgI6GawZ>pW%bgQf6dd>DK
zlZ+Z_YBX8byet==;0bv9LYb+y3dyhj(i<a*`^c2XKEa|_x9$$b^nxDFK}*RPblTX4
zgi-1+27}U@*Hdi*=u_EQgaK%I>J8S`)|5qMGJdILzz&*3Sq22Dh`6cEYE$<{yjkL_
zQ>tu6%bo38s?Gx*GAc5r7R`A!a<I)t=i6%ZcyHzp6M9cM0hTqz_4ucx)Un;Hzxymk
zIr*|(%m>(%d(SRti_n;F@CdU$$*Ikx+T;GgMc3HA`BLZ(Zb5Zf%x2SIsiO15cpkz;
zoebb#B~&%)?YnCB&i}Q}jm$&#g`o3182gy4WX}dt(1=nT4<S*4i#RT~>nEDEwWSua
z2j3TvlIE7&MSL`&FBw5ORQV;%_mV|&iCC~SHk)~v3i`EHm*B84SmG~tf<sAsd>STd
zwZ?PaLt;;dQ$nk4ZgB*EPmgof3Os+KE|DV@K&r*vr->(j|IC+;YMFC1&)RUf(8xxw
z)6zEsS^P-`ZLjrb8mm=NWZ19zI1Sy>5>?ZzR_CDhi%N(;$q0Ddn1wPKhGFUI5Ne9L
z5oO)c_j%QrY}SiipGp24#v#UBio}CKZyQ|Y2CjaiNP<(VlvUynPkk|LCg600>J5fd
z+V)QCze!)VX|P=SYDJN|C*7g8AWu@|Hbd?lWx3oI`jACr00AYaIB@v6rVr#I>Sv*M
zRU3v3eO9H{izwev`YEm4DuI9I=<_?5thQCQ<>GgYbPg}<{xEdqFMRQII$A?aBsw}-
z>pV{nhl~a>V&xZ`x)WpfU%+8>|H707pda7z85Y65s7fW#jtlhop`$|~%qlWyRN%Ha
zo%uoBoov@%GsvW#q94D>{mlqyvCoI04L3CU#kUY5i2i<?QfqsAC_h^w4@WN)oa^-h
zug>=Q4mbNxwnQF;>CAw9f(gFk$$IZ*1M#o*x=ws*O4fcWCIZOzjD1!o>|cMyp#?%f
zm`0WXJgqLd#u3==GwbahkYZwDXp@vd-fz#~t?$N0#cFCRf{0*SF8!3_G}!H7uOTg!
zr{t3HP!<anaCTbiQS%sJ0*@Jte{g(*B>qmd_FxlEhd(Fq`aiA!ueZls{ih+E`q%vl
zKw)Dss{Ma;f&Uk+<L@D?n)er%WG`2QwvSH&hyl7eQ(sWX0~q#Ve!O^zeF{%re17hE
z-4;nE;P`e^Yoh0T12KBO5u8#;0a#Bx!lTKT(izwtOAcKWI=qjl44<=L_O{ZP(7Yjg
zSP&zY)>=b2Z0PmN)S+uBWWO*V(x7{ykY)>50X`KXZG6#fFJyAN7mjOYjk}j*qMyr1
z;-&d&K<qc-7G&~j7LP~7Y6z~{p&p-8^^M!3ErsG5>gZTaoG0VqFFF$P#885&L=SmP
z>!TN|K}*dJ-}&588Q}@Oyv8-gR=D1s!B*bmEy6@v2dG;bjHg7XzP)_T;+5w$b7eDw
zshlt0QYd@0IWs{5+rr4bB_Ak=2;nf`9F@)5m*545-G1rmzkK6zkMPH<S)@kcYfupD
zQ>}~kaR=lvL50C4&C<VKoQSc(N~BDiN6+~JACZr@gFy}EqGS$H3rgEEp_H(}OR7(`
z9XkXAVZ&t1*QGZI_{bi~D5cn(DJc3>tJ4jHHr7<STiu<6A^pxwM&8xP>SoDyRw6Tq
z>--2`)rSt#IzIs>#%DfnLg$hF@12g&Q@(_B8aHn*h=uA(&cZ$`EpWVvd;cD#Qpj2b
zoj$EL*oJ2F<=x-w1E5$Hh3kzZnum{PG!dLd35*lh<?P;_sIWU7leV?D6F~{?UVDKQ
z;bT+Q+Cr02C<sdo<SY$Jbd#4dxIBrqdEALfd41kTiEr-5aN3V<(+Xx@i$JZ7HjD})
zoP6>OVNmv`hm3VK7Fb(W;7XZ(gooL*YRH<z^ScuA??xqE`lV${$qK8F8w6i7mZ0&F
zfy%8H(+qe<>yFRTGg+AKS44mK2O9)AR&?52eXD9L4&d4AYW%CAOSr)WCI<`C$vn%-
zd><~54Bk6KKw!ukH#>v6&L`-dWt&=!^x?zp9uKx>$4X~52vHc8-A|8?n{%Z}%jE2I
z4j=DcXZBI}JUxJ~@{M&uBAH&RErj76FAG}QVO>QDw=SdA<pRR(=h2*7WY%}2Vuix}
zjXJtza<xK7TmoK9{Ee;qvz5g<GZdLj_U;$=kMp&Ol<w1qmxqfc?=$Qy1>wi%I~oED
zxdTz<tH7HBC9-Qx8_x0@z;Dg;Q3FR0*PDXvsq8CtFnNzGer=Rb;jh%|TdeZ!Y(JW7
z!erF_xj&ZSuhD3&Kel}L>anrzWsAiMcEF%Luq71CXv@!MI2?<XX~W&@W@BXKFH69;
z4Vt5Q%uStFt2G+DMmvAp_;??bKBLb_w>@2yPfklC(SChQ=CtDc*N%-nb$GcG*>rfi
zK3)>>Qg_!iRQDS;Qu#(69oLtdo=w8wu-S-2;&M#3s>q_(L}l?_doJIvHh=Ww$`Q5S
zq0=W<M*1LpY}AUm{LFlHSQ$%Yl}8K9f?8fwo;rURJ(_8wRiafyVm4d;2GqeW+Hi)z
zN#_=q3z&(8N;fVt$;7_D64v7Bq|g&OEsoFwk<NRd(EY}`TCvXeudaNK-ipI1r4}cp
z)XhykZ-FU4O!Be=!NG_)x65fY+*`xLg&m9tpLWN>io|tC=5_enN(0+#FN@<SmOwM!
zvlms{v&BCmv)BWXNVKX=9_U@&JhlS<Fk5dl<G5U`4n&@<xN#u2N+mJq4OZ~5yFJq~
z8G*pq_8O=1&F9S7-HyKvM5c8~d)J5x`O$VKG4NEmTzIeBq|TaE@t@HUl880uRw7qn
zO4h*>a0C&Hgqa^Gm1trWj;9M{dB4@a<b!flCh#e>3cihJc7`lD9O&i0z~Hi15$O5S
z$|I|nC?2*I$2NUzc!lGzB_71p9|NYL`~A7U0|vd`pOW8+R99<tb|e!%w}(U>UM~ZK
zo<9>D?oXGtPPu{2Yb8IC9aT>EVbNj3kW1~ZZ&8riv%gSTyOW~R3r9lwd)H#t6CE3|
zIGPfuWIh&sr8On>L&Q=^^mrQ9vRThhJ}2{~QF9jaL}t@}LBBi$4JG2bcLIag8?c!z
z`!=4g4-PI&CNhJXe9ZSh@@Y+$RcQAr#}l~N*jjr90f!P%M{Rj;L}l{)A@{m7Rzywa
zDfzs(z~|a<c#b1qMfp)aAaJHocB`Iq!+Oo*l6MH(X1b2b`E+qy`4reG(bP)jiz|(S
z>Fjp8`JhVS(AbjL<$^WQ<+`ILCqxykg91=Xhr<iH#)==J)>T8@uFXvx1)|cF14(i*
zIZHc>=vNS-vt>X6R;_WKVBnwS4a0gT%MaP;5uaDgTi3?a0;fb;wc7%V*9L8E?Zb^q
z1Pyy=WO73S(ok{RmFp`4ZVGl&jd_q)POG(Rt9}2aOYfw27SA2ZsWwk*Y3`d^gAH7S
zBkH$G?T+mWz~MD5jA@EZ+sm_Wmg|orHiBt)`c)|5f1()re8ZO!6BkP^HL#JHX%o%u
z1_LGN*w-kh_Z|siU_T5i_xZ8HCMF{cVQA*&_J&%4G8jeD7LLOe1paV%6(?Q<R^wcr
z>Uu-It;L5-AsOy^wcWk=S2`)Nb66*1aHCL|7@s8E`|2Ck+rVGK6V+P0YkvxPbZ<W}
z=x9a*QCwb6C_e93-I4Srw;Kq+LoHhy*cs&WcW8{y3KP!*Q$x(9;VlUi>t#%Rdm!LZ
z6^&xU`s!8f^_tx2HTM+<Yj-FG<)1MnVRStBqQebu9hh_hx_F%4A&bQ2{f4k-&|VHU
zI6KJ^t$V&+j}i!j0q?v-Eb_<xveM_n`*5igqobH`b(TtDa%vb_$q)0VzvsHUa*=%k
zDxLZ_4u`#g&2In6W+|11<l%>v7HJLb0&oAI<c|yPZ*nii8kP8w_#8d$)X#S~T{j=U
zJ~JLSR|6fNk=|fW`ckc~nObI`I$|yzh-^TB3C)4g<LXO?*$i<>4nqTVi}WM8Y~)b3
zFS6gw!FY~A@cpSlKGb7B?AD1Cr(rVe^W9mwa6SW*F=(`7O@qxQ^%fk4hK8n0r!(=Z
z4(cTgd<~OXiafQe2+x<H1i5@kCB-sDjv=R^gYuJy0(>-Ug#!KY)n-giL#P&~qtSPx
z!Fj7ogorC>K3J!BKM`9ytf2d|<&q6Ew~XBWKUbdiXp<cW<0)g^OeJ8VEiHnGxm+Do
zu`i+J_W2?r)pIyF0yh;`4@(Z4<zHxfE`rPJ(Wn*6Lf*eZ2qJn@Y%5YC+@K6zS$%wd
zjHiw$3i9UD*gG)jLBd4=%6`77qg0-O{5P-aJ2(eUCZNwM@@@8!otL}id`1I{84{ab
zeXTfMZHENhJbVWr+k=XU{NKBNh*Xb~D!+L2fo`%}d~#EcCXu0co-U08XZT~gUnquH
zWOH*eHJIitjxlvH)BPD@foGkF)(Puw|BcIt7*#e649E{{On;MrGVPzP$?!QUg@=Eo
zlkIiAzi|^zN9WL2)iexb{gLlW444;V!CFQiiNxH`@geG28N>)a_qco}Zt_EHPvJHr
zhP6s8Q)4vPrzMq$jW4`dt^O($%x=&$BDeegVl)2oK%tMCMW<Gmn4%CQVG6CDIiGhY
z^wo${Jb^f+Th{c~dO(2Fo@6q+o@X@BUXFzW4mwfUB&ALS4ht0X8<53#p1Kbc=TRC-
zo|t%qKmnnoAKgKfT#F*~PrJ16!|-#j`suZd5eiC9ni0mP&rd=<$qVvHE*I<iEAw=(
zd~iJOJI_wc=5kRt@XN}@J;YK+$@68}fqZCrstvoGP^FXRfK7NT>XODJ&2WGrz%djs
zl$WPKvCANbxYb>wiB)Ghg?#pIhCt+MN;#U$7_qaK#~9Y?a^lDJaBfj3vdakpiN(eD
z3i-R)ey?WPMrC&-DJ!nw*B2#e5cS&WvJdtY6Thfi;~`0M<o=ije~9P5tS4)TxTeCO
zYgRtqx6kUB+11@Wx88uJ&HD|sY43yFskBfD2y|kVNK4gvdoo&>-JN~4to8QWmTbVg
zhU@=>E*j)$re~mp1*E>SD+9GA6LN*lbeTHUZTZ^g6~E>7S8I$UiM*DXxn(SEM}mSR
z(Lgs>X6&w)pIWgfd!*Eyj(o6hTNQ`5hvJEN)R}GmP{?N5&cu3SP$y^2BwIXY`=P{h
z^5|r)a(5<pNjcrTz#+=cLs(6C^~3PWrf$fwyf7n(@H?ipkOZ}MRTwz{p%~awjiI2g
zr(2cf(lq~xX0vC?gWb&l96JK`&_SVxC4Bj5G@}=1JdFnS!FcA}c%G)Yql_Apk??u?
z5~FdQMbY&jmOfw&TB^;kPy<e-i6#8mYmN{^9^iy}(mLu)@rcal5*=pdlGg^|9H%Q3
z{v2NjhC1q);ZON7#FFa^3QT>k%WT(zwXQg+F`2MdRVo=;s5FfcRBZ=UrpSymn4NX)
z{any}X(;E`df=RP5b##!WN3kRe7jg}&Rb(`{!w(Q(O_zP$$2EGmO3`(1A^A7v@;OP
z3$7`<IJS(dsedhgb$+v?^4(7PQD_IahT|PlyjAVTL8RgVRBKO<&?t77mq><6k)C2o
zC&x9eQ-X_2O9mtJRzpSj_xsW~wTs{ZsnS;JjiP`6x4V6Wwti=^T8*x~5hee4=R!zp
zwCvx?8??RImqu-%?85S}S#62%euK=l$y#5w?DB_z-wa4DRj%xxw_35gSmznP%nrNB
z{;XPONF`Gm9FpojQy>F3XD2sF(rT`y+2NisQ}A#y6C7uK9LuJ|%k6p#mcnE*&?PXw
zSZ&ehj30`?!q{lF(VY_aa(9-Ou^{%;L^X)5<xe(}XfV7jvIUy`DXlCf2FYx>2=#)1
zFWCNkr)Og`xQCwl)R(!|Dqm|XEdwiS7P8!6H@j~&lka@Sx!B>=de4?GNv+ugm8D^N
z&L!05^O5QGR)<GT0TLD5tJnfUc}Yc$T(>ZEPd;|e2GN6Y8Q06r&--t!pJq%(1y2|4
z9<djdi?K~#6bl^jL?f{yGLDz!?opQso|IXlZPtKjShLd&79BnCwjz#xIF?MTUoxSt
zQ=#DJa67`OItNhO>!p^|-xYK77nMuRM_AhTe=Ix2ufN9RX~t06>5s!2%NIreiPqjf
zG6E$kA-v+tFv_rbzQE&M9c~G=UENBdN+2MSMg0wq{B{2`lN+3Yo7?tqvYUU9A0a9t
z0#2QSy{6MOdBoPpscjH|0lwk?aPpbrF@BycRlw29x7m14Uk*a%T3%edt`m*Gaj>oi
zvNJ;C@MLw^v{#3b_&v!`BB~6JodM}D`9${kO!18gOh(j-mGb^?Ryl#Gf8$9aUaR@M
zJqOEIo{5E1LaDS`_Vb3Re?n7?#iI$I%-GdiuR$zSXc<_9fEli})F*#8V+wPs*6-)o
zn`o)GKvFKCA*!ygH;N%`R9LWHlg|+ZA5CZEQM_S6D~zy0I|m87tzRE7WcSVROm>J{
z%BHhdz&QgA0c^ebi`GW;d22Qxdq@fuQkj>epVVNhkoeo~8m;5VCH(NXoVx`=X|;Oc
z@iGwEWHc*u&3GiMuqk9L={>Yc6?$i>tPfWz`E(vSZ+5$PsT8gx>Zw_+R)=nirzL>+
zNHUSO|3LO3X`d4-S9ZhO6Bo?LP5b+Yj^$cgiqA~@WVr`IMfL~qu}!e*{-!ni^=565
zT2|xzF}O?49>Z#-UNNjv61=xGQ)tP7$k+CY`vT<8@(7|>XD#v7iPo8rlApc^Cc>&s
zrM%Uyna?jU<9z*9l?`4nk)F>Mfvj_Xx(=sDlqbU^BRMU;G8Z`huoE2h>O*a$)l~8)
zWR)hfQpe$jJI(z3D3(Tr{_F!ad8!Ntw<Qrl039}z!itaCwFu^CW2aG%aHwjT#A=0h
zbI;fcH7nmiTo!}KpQ?lYybyNi0Q!8%g!w-b;x{%M@Rr@NhgdBHrkqY@%dIX9)XHTc
zKTIZc_bvGv1;rn<<X#KaiR-l1eyiyl32R=l;TNrD@_O|Q^IWW6!OD3(U8~^~!N7Au
zw5UkMQx<24=Kh2L)mwNA(3(#v6+O<V=O0KqWSB1fCY6lGghi+8E=&2*Wb&n17z+8?
z`RrYwc-rWa;${VWXw+Xq@8|ot2uQk0x0|@*bJ2c<u9~t9R*A0@G&NKDuenF3leq<Q
z_1S3d5W^G+#X!W%<YF2P+gGS=G67<-c_}%^v*naPi{62;)%5(b)|>q?5#b!K#M(Mx
z*BoO;vQ*}Pg4bfTN-0;8+s~CIst1q(0enSH=5r-uyt?WI*?%4tQc!Z5FTM#Z&~LiU
zbg<}4dT%&et?1vUa}+5q{n3Em-GPVg&hXKmBL<ae@&j6GGqc@2&m$@m7dLNX_89rS
zsv2nsPq+LvV3s@Ic+P2#bULu?$66P-(XrR#CXvfDy!bk|+tu-&6d;>LUWjrc#3e5u
z&9&A}yHE{7p@QReS<I$?zD(MvcJfB+%%LU+65WW&_(-LdnMgMq0|>#8BrW^l-DDhv
z%%ODNJQ%dL>HfhG5__`4`Kp>BTyEp1W@Jj;daD)ma#&xW+Q7n?2S(qO<7|56RS;MO
zXL~R2CKmmw$A+a-OSf^Xu(1`CJZGwDI`U+ZONmQ6y6C(i%$;<bW<kk$lc+;o`FMX}
zm<-N6A&McBQua1UOPvE;1)<>kGuP)R0`JdNo^CgGC`r~$E+7&9se1rZ$gy0q_iYeC
zKT}5ABJ0FB^9j3_MpKzGmbRtlMwOEu(Bg8bA>>sW8@t*WAR?ah7f(DbR-gb*1bgo)
zeK3P3G4xPHhX){HbB{UC&MsnY#w^<5#bW~}{xDmC=+z1WZ-&1kgm&eNzI0UN9X$h0
z|23vN^LVZ+K3={s$f81;_Kb*1aaW&YEyjSLTlj1C!+%iV{owC#Y_W$Wtoi&iRRhtQ
zi-5qkbVZ5;3WaT@ABh<J&22o@uV))JSnKtcP%<g>o!cXY)s;S*R_{cc9${!VFJvR3
z1lp&P;>OSIyZmTCl%}PMYq3!Lwfj+6s@myz5uYp!W9~(>&6TL{4&obbA1^hehezbB
zYzik{${d`OH-jJTXBeiYBFI)<-&c00Yf>PoV_FO#+nBG}qp32VlT@YVyLhxSVAR_-
z{40|Iv6Zj!a86y76!M@|YV1^JwE&N#_ueCddudwq#e25j;xQ!V^Z@qIA;PjqpVRYE
zu4uzEmBoV|k~NJLunt*Dh$@hO15qnPGk3Gtmvw|jMTLw$n<p5Tb9fbPmX+cjJDp}p
zHEW+6ccL*07&PkeykLah`4%SSQVSlN^;g}rdGI$K%t<dYmcK9bl-q+*6Fd_06<Yp&
zPdC*Fb>44THpH<aQhQ9Ei*j|&ru`KuAZ2Ax%N=an6z}lb%wKsD_;v6;d2sULX4y58
zqce%9B>crDH;bwYDx%k9ulHxI717W&5I*Co%4X{|og14K_HhJln=JvJWB?zC{btvy
zu4d?)K!~IqpT==UwegudlzbMulBJ%vE?>vlB6JoE8q(bNKSolSMA(Yt+>`FHBPxgi
z8+dskbZ~Lvc`kuUwk&zxQ_p}R>HHyj^eoa#GW|W&L3RVMq60UAa(Xohg-LpzBpZt-
zF*L?B$UF4-Op$*gNedu)bL`9iIIm=Ac-PVz{fS$S4ki%t9eeG1p}Ht2bMe`nbJyqb
zpW%EMu;lR}@Q6YxtSGEw`L;35cysxi5K6GFXNSz^DaZg9P9YokC;sWlJWU9|`%QZX
zqrT@=C+`d>>8ByfBLR}i2V<4i?_w7g5KqVsK4aH8-I=4g#uHxcRT;ADZOm=Rqh{QS
zA9~OLOp(jEfAGpA{<zQZyXcQML1^c@UsNDCoM9;DU?d==h&Xk<FF*Ob#M4I<NoK>F
z@y>gj#gDy-sRFiq$h7C%AI#RbU-WJoD}*J;;+E|v^E(%q7Rq0BI{1NWsZso6Z99Ar
zvH<awMun0~?Mt`v1$<MX40Dv52T6fKp%l0+MbiEF;n%aX&5)w-?uX^JmWSPo<alw$
zc(;7OQ2!|%?sTH-z5mTqAwNjv1|3KgP&F=EoZiY-3T2TjHb4EtK3PV&bx4#e@n1WW
zYJWgJ<}n3{S!VyWpE17V8elY;5DZ5f`DY3nJzjmSK093`yrt5D(2guenuE4?#m({q
zzcP3Qd*^d*ZLq8W9@(pXhmY8)vh(+iTAU6lg{_xuGI{5}3|f9Xf|br6U19co@^sU=
zSo)FJ63yxV_)?nh9<RwW8-1#+=pCUbJ1@_0mvh4p=n2r(oR3>fxO_oTn;!<v--K{S
zY5+?f(3w!=J0~iAj%nddfpogwr<9&E`MnQX9!;$(jsgr@2DZr;D`ATtb^c3vVy|g$
zq*lvah3;^V<gs<EeuS6=wP|cf4*6XAqiK*oqA<9}T9xTulNnJuh7|}ok<6=&c)bKi
zB~XwYmKA|0;e?G&nf4Kt20ja!%+(Tn!*Zz!2O^f#_u}InQe53o0MXB1^z``}%k>U>
zKMKF+^xi5ti*JI?wD#kDhJIh6>3dtyx4=d#G-0tG@&;_htv=0qWJcwRTL&svHvqBe
zUVJ2%hRDKZ*)%eN%&OT#%z(~XJidPCK~OgkWtqCO`0kNyUSJ~2vEq*vfm=Wv4BWf`
zG6TNn&O11%Y=LL>(N-J_jpxllu<RA`t0;|rU)KeHU%&Fz(~XH{v%T62m-48pcoHw(
zuY#XwC3kE#%K?C;yVY0x0|@nOCByc|QVE-{Cz=3=(goyI?fJyCxYYP-ffBz52JgiG
z<pD{dnXiT_DcFs&HGENBBS*0R&mo^KAu$Lr!)^CR&HjnmEwbD_04FkIl%&1^bYFIY
zmhM)OYBAG5MAq+*n2UMw$qwZ*Rd|5hiBX#St*3Wimh$>kLYyWX^bPCj5eAhOO2Hl|
zk4+3Z{{(FpZTeR{vA?)L8p0!b2F)!k0>z-y9tpr02FJHp?C!hY_W?)chF-hrx7a%m
zRtT5N`6j^lx}Ur}KW+j{O_R%ocJzyQ7BA0$Y;wqKv4TaRn}^5FbG^mu6*RHS>(URC
zO;$H#PEMzu>K%$|uLEnqRYT9<;@AREWY_-?#ltJa9iC4;of~FfeBawttm8(K=!;di
zCc5XG7V;fhDrrsN9LEiZ-A`_18~#HRA48u3o~GT=wn+OK1lxoh$7J({;?zp1N(Cj`
zvn=1U%N{6aC(pp=Y1)4zbX*OCbL00QJ(S-Sa8Q=|rtUJv`vmeP&}a}T?9<X_3Qyp#
z=^0pr@p`>nU8MXdpgB1OPUDLCij5+@uVgX^PanrD=K7<HcD|`kDwSFy7QxEE2~K0b
zJLDUTK#;322rt=YbMs28R--?k-V8Dy%WlM1^AnykpNnCaKz;|BFg#fF(cr=Z3tTSu
zZ(kvAds7@R7Lk1b`cmVh_F!D7**>xDyd>dVwZoDMmZF@`>-i_@FQn<-VECJsI-3c$
zOxj81dDh3jUqV$Xw7W6(T82{TxDqRE53DM+YO!hN8pLlUP(2(kf`H_*`f=dxgdrTC
z2LcF>yT@Tw;Tg2K#V86RY3qbJ_g;t_r83*xu0-)rGGNuNE44d&5Xd^1efCP`0G<OW
zvV_p&LYwEQ$;-=2zDAMiXBH!@kI0qF7&sB9p5n;;>~Nf=KnA-Vu0mncRt0ibkKUo9
zrIA&Hg2}iZ0OKvT82xa*SjO7s6oNnyJY}!8I~<Zf<#P=wj38iRowQF(Or-rDS$<k=
zIwgp%5GEL?8vQF@5^vj^n_GJmU^5o0%wXa1dHuq%ew27|ozU(4zmb6&VS8J<?e?x&
zLvAsxv*iZ9X)EOY#g0)Dor_5hgHj2zXap|Sb{qfR=4rT%PN&{NrWfrt7Lzf5WhT!p
zt$7L5e<0$KgynrY@2`DPI9%v}4z<016<J>h-5FGRvhgK^8t5(3Z<SLEH590~It!7;
zVaxRjj;yM!MYwzZsJr<6c(s$Ke6~#OnxGYWAF7*FXnZG8|1%8T8JC}A3YW<;c^-K#
zkamj2Vu7l35+V$p_P0_#jICqUpbzKUqo02OtMzIVuA)>7F`0V3y{<*7CKNU(@#ZA;
z*I!fl_X<m+F?6#Q$0OGd*(_e@IxQIDX(oUn=*gGDU=(b|NTh;g|LfKwH+eCc<6osR
zD0~jSta31}l_0grx0>x~|L%Bt*jFAoTRday#-oA;AH%6ct%S9z=PRARitO<~pxCi_
zxRfN1m`g2+wgQ;-JIfIor;TgvE(DxT=g5Y2A6vbUm!4->yA830;*kJ$NWu;PhxH@o
zl-g}zKoDVS`}E&5u;fp8-QnrIM}WN7Xi5F8CJyKr)fXR_565A^dH+IJc-3zVbn`YX
zjlOndl<aI?`?TDim!I(7&4K}GvAMcbOEf;S2f)_egYnk~I4s4lUd<B;9s>Rq7J-rS
z4%~Tz;Uf8<gnyi9ML-Z7*66!~0AhIXa1baIg(6aWGRQLw086Je1~g3Gcf_Iqfw`>P
zm?EiWXjmMxq5M_NRSM298FR9e05HZtA90h>9y|XK;L5o9sGX03v?QO+!?#u!--ate
zu_or1pK;`H#9#0(aL5*K_8GeD_Ux;E%~t`|jHYt;*i2xgFK8>X(<i;j>4e+|@byU^
zA>4W8PIM!#iP(S~jmzbty9F-I_IcFoT$%je0b~A~mfe1Lbhx7n2no&7W&rzrUc>ls
zlIYZ(y-YaB&ty*VI}N-3U}U)_ywi7!{coi5$~0<IyEA|JHTj1KR`cUfUestb)V2VT
zo6O|`A4?`%0Y+6jkF5pZzo2-k($rG`09KfJ1x6_r8*Lb3$)tmP4#tdJM1U*;5ec8q
zJA5inbo(Z0Vknb`i(9)*7naj_G`V|xh770*rmKIc_D5490IQc+`_&b*^0f-c{R|s}
zt;vuO?IUQl+n~fEaYeaGJTp7HT=tkG5@|Vr%79Ofvhq*a>1?c<E;kV|l@@scGZS<E
z6@1o$`ee!FcpBBF@D5l=CGV9$vZxDCnBSC%B~O6QI7<usbFpGEQ!7VNahnKu&`@lC
z_T-}7j#?d6uI!-rO>wKb?-%g9mIAVntWSUg+k?u+#x2(|38WkPe@khIB~i#?ce(Q4
zI|&jrb}qQxzIzRfA|8MTxVuF|wYEg=u0zZYE-ks)hP+UOxz3kyU~<^Ar*qbc^#p`&
zZZ2Zq;V=WO=s+)}O*Ax6%DYkSg4zB78gTz7Ebjk8VRXVV<2xw!8&;kG$gVI1o6TyM
zANa@i=~C_E>k&$nwb(EuK+EBq;>PHNPH8W+yX|n|1MQPefVYG3F+)3bvS#x~5YRxX
zF4cOl1;OEsnq`_+(O`eRnC_QyeYuB=ve{;JQjZ^I*GH`;%Ta>h2Dn^;W6OH5z0nrO
zl$z_*&`lUWL~>KbHo%U&>3VN}xfOYX@ZM3Kxh#qgI=SAHD$**fkZ01z`I`4=Ih+eu
zI(a=lcU`v*Y!bnGZX=PQIIhC`I0jj3Rh5dqJjW+Yx2hBDxe|TAjwGTN8>XU3Y}~Em
z@ViQ3*W)&seM|_)`>|X!Sf%&o&TO`dh}5sT!pKULfRUeHqCLleA}*Xs0NN81l<Li!
z{+@?25GB<6>~z(80*Q-@=Y}O&DWYtx*hL8s2avo<rYw{0dUiaBDC9EwQpsS65JZL5
zCR8V%A_8&nozGT%1^glGz3JhYM|IcR+=MifYF0QK%>4gK#1YjnI;IQW`rB_{^(L8?
zoxMsDU%$#Fn)(4;w`inpcmAw8bHR;2B!VkGT%D|esRBfjPQ-8}>XR}&F8l94Xf#1D
zq7rGWcs0gjppTjCWoAqcBXp?BMQq}_o#tN+tBG`Hgitww5T%8(ic8~_q{I`lUGubX
z*&BjZ768}DXmg&PHG(BBc1QEwY@zaEqqEe{D)jm8q_?}9NX|X!QqbSL)BWjJnp8N<
z%KS16I{0;tp1-@h|8Irb5u-&pfJMR^OJmu(-B@dJ-s)~1W~m0eZS4vG0A6fxqWO?z
zp)m5N)F?pxgGwDjpTT8@!lT{kow1O&5$_uFUTFNQJe5UmG4~Uvi~Y8y2{vW7;fu4C
ze3_|UJiU@%nr3QRQ0k}*9+&RwA|rRFXLC(Sab3^>xKlcpi>27u@tj<duJ>%I2BgVE
zIzaqUjtDDh;Ls<D!kv`z_lTTvOLoujvveOOTa{5e^vJ+<<Eiiib2JkNB1ZgsO1ZA`
z+EKI-iFBz~cJswruo&Z62udl3g5^m*%MDu=Ma}bo_px)skj%%BHFLJo2#1fMB?Ni|
zY-dg}&{4s#u7Hu+->Sg%Ye$<V>CC@x(hu}TPuVB3D~r`$Y0mfwIC!;y+kfWNlta9G
z+m%G6g-j-ey5k|^Ak#8t>UU1`4^duGPN41qGK({A5WJvTtQpRKGs>1#n9Jj9tu9-W
zpvX!Fyu6A{Ab<^7*l5!GI+6F17C9`ZQXGB06Ox!j-!c)k&k25422Sb`iR<2pP~u^8
zh-f17DSNn5jPAg4$$^ks7a){z`o_oMR~nwso<;Az>+El;_PM>Ea=iayA<cKqNei9s
zl}lWUNNcngg+iv%0>(SUk}9+~jGh5F1M|l`gGTRDSQ1B`5Wjfau)Q-L4}?@wYiZg!
z58o$&YTI{DTGgiLhY#_mykI!I+zheYSZ)!RGL6od#zy=6*37Ci6~AN_3sziiM=)UO
zP7es)+B{z7H|=7@r|hCf{OyC$evN@5@m6knkr9#S&r5JAzh0=<UkC=m<Y~3Cx&qU8
z1O^mj1*p1<bw)VA2^94C?dAG5I&uV&6fu6pYT>*+IXAak(Ob!?e5%pL2bae?Hp&60
zaB@kZL0j~_i-T-H_SZ3g>l0<IHPt2}Wx3Sg|HG!vVn&@tw53d=75)62=DYi;Dy@r7
zAOZoOuI|S>6kWV&Tx#2ha+&a&`yKly2Nj9~9HT^hZbM9XJdS~-_>;Nfe05FPYL5xt
z(X38HpcHnWYjMF@r!g4<Oeo4fq}up0WUJTqeU&5g;FuF;B9o_on{4E}-L&DUVg)Jn
z#P8^f*cx>O;^{eyh0^TFPu}p#<(kD>K@|(Y$VrvmjPdKs#h1IXvI9w0O25`95_N3>
zq(qa5ni{zRBsc)cX9PgP3l$+CVNgu@%dgmVsNZU_KwqA14n<if3~>FAIboEJpD{|Y
z7w6|IgBav%z?3W0{ipLsD09a8s0OL3Nuhs3xL1{zSvB$DH#2Sbg+?seZQGXM3UwR+
zJY$Z<!g;#G=|$(q0O0{qI^|S!5z<#kxNThRx;N{!*4jIXkzZC)2%|t^!(Hk51OrN?
z2`W{oJS>-`|7NZ&pQuaCg^VUNDB-qzXq<O@8b6#g{p@(O95J&E23Vo}|4hkLW-E}}
z@bmr2gqgSXdYi9sD6(A&i`8<tLQbQ-tuQW`F<XTW>{XLru2Wh=n`J06S^dm)u`Z=_
zsZOVFi?xd?p`Zr2VZPN$0~+97Pdz_Sepl}h6yc>%PZ~dJcV`FMOteE+eD0=+hp$iP
z_6Oq;z?f!^ITXkFm+725lu9Y1pXF4@G=AzbYBh#?_W9|KcZwqshb4)5H)DC8AAyP@
zwn~M8`M$J{Fz`xp)+{YSa5yYIlnaa@7<{}Fg|xHrepDvo>6Aco4oJhH7kH@roCjbY
za7ex)eS27x2LUf$Bf@^~h)H*okW_Jj%g6T%z-eb`m%RK2b0QF{Yx5(aoxoOW&}wyB
z*hl!wlr1^Y+w1RRGMXGSBO-zpeq5_nZ=O(-xNN7O3p7Smjg1Gvu^o-iY{Q`ypc3Fj
zVDFKdA&*$>$uq|CI`lnRn3#|R(hQb`NYrAl9P4cakUhzv=&sFPOA&fd@afKA)a&=%
zDd@y0p?deR7Q%%or~4T=AwNN7_@^WZT91e0Vc&MyEWT=u-=5Na51)u71SNByuV%#j
zU#OJJ*iE1t>Z|polBQ!6)|u9pwZAG!)}PEyy%(I(tG6k>yo}zM;F^+Jq<{{?TAF=c
zFJTt!zyt6tkzvlBOSZE;0`3Junf+-{{E!2ugBt{lihMQE4a<*M%{slGdgwBKGh5Ck
z6Ns8SD{#o~;i`AG?QhMI0Zh7kVrO9F79t?=-XkmHzA+cTtYad7tC=j;KQ2if0aCIZ
zP=VxJNyr6519K%1MY4O`hyQU<6s$jH`DUC^U<sP7jQ119{!=0urpey&G8Yjy`9W$Z
zfHdoci=0EnAQW(*W-;3YtpZ`#A2%mMK<z_K@n_4lLPW6(P)Z_equ8COY89k)z%bg*
z5jHV(T^}%7B|q8YNAL`+Vl$hceoXxpjxGn|qlR!*L$j|`6^{P$`oojRQwtxSp0D$T
z%SmO)z+}lWgjty;romHFtMgHrRw4k2=1T@T&sc7a>r{oLS;phJgqPB2Fp=i!LG#8M
z&1<mC4Mb*rk{Y>w?TWo%ZW|OCIT7^JHzthWMEaFtN+@NuvCeb4lLwh49>Wc;lD>;Q
zMFy}{zVn5U_AldFtCqapr<m%E&ir5ZSfN4JImT_|epZwX*%~=9?9oct+AR-<#i`48
z&DZ}A?{4lr!>I85%mYFb5HPxK9S0`R;^Kde&=LwHROcAL)3iKP6ZQN8j-Yju@_6(9
z>4UHm)8cDFX{cgaZ*Wm35@1+2C)-C<le!6?$Ndo63K%3LrmX|J6V66pfprQ@WG|-8
zzDA?Ay$LL<1UQ+XruI$xZ!ECG2n8#vxm_?%m51Bd+fO%HiF5t12Wo9YQ7*F3A0tWl
zxL$L(E^^*&leh=n{*X=jL??-<=I1mjSJIU@kJJ2nB9$!1OvZIIVGBU65JO}@u2^B&
zKPZ{-w_=5i3EC<pdLlLT4VaCdR$t~wzybKug;mX2PjS<)>nMG|(CR<S37r8n#O?VA
zMXOYS)*^d^t?+!G{cJv?A@RA{>jnLSAvc=wyW6xdy0X0+oq{)rG&IikLyl7xM#fw5
zi=elxePc9boa_(6-J5pLqu9*J3?3Kd=0K(@8WQe*yIk2V@IY-Qa2-`DliMvGRlTQ3
zKTVhg8khwyL(|l1p~;H+2hXR4Oa6AD>10&<Y<PBezIULxz#Iw}L#QX{`AZrT8Ba(c
zD-pE84ba??z{KIyd5%Bi&imVkpC??jpA<0M*$oU&GwQv%&K9YKmAB5C{knnUvRYR3
z{(dMx+<|jh9kkCr%yz<Q4#sM;+$UWpoyJzJltN?>#DCHtC-B#bT3o0d>U^VfB4$TA
zjg>e8hTKpNp?i(1v0A;^w<w9<q%iW`=>BVrv?)l~KncMfe^h!5h*K_YJYJL+ONn=O
zC%gcjGpsJc$;cmXmw?x`Yas#@Ii97isixQ!h5z%~?pOa$Ylv$S1s`KAW&{Rsonna&
z?={YfS@B0=Nk6bFj(z?G|8d{*mBNWvEtz9={QuzkdoPjdga@<UwMHZOrd)Ftvzt4P
zz|hwP&oLvvycfVqA!_@zM3L0C^*on}P=v2fB&HscHr1QY^#sCVxMKeowXX});&hQi
zo1b+0HSfI%)iOKnJg_7mZmvmqt4Cy!Ks@FY8stt0D~oZRHYF{Br1VVwK}m;!>MRM?
zQe5yIylMzROaR)?Q&*vI;`hj8&SJJ7_^y8*c@+_XifLqAYAlbUT)4H&F@BsHxY>}G
z;Jg6;P@#Btf*m#XN6#Jyq*zh0B(DPJ#<*6Ceoug12nrWGWO_*B#X4~fv{Xv;-VUen
zw^lsLY~k)kz_Tz&Pi3__Ml}SbizprT#xNO`@Sprr$SC7IP7xHemY@6*N8V}{<(s>>
zGZ?W~A%r$}N5+;wzbi$S1ME7~BqX(zfx{iiK~BJtsK2nIZ2_FQ#*1txl%EPlKYENX
zQ<=@X1f-}5S_Z!g5;qwx*^@PVoy^w8L%aPs+Ahkxz{Cya@?R_VPAyF4e+~*1l&Cqm
zJ)Gm=;^8SF7AC`JgA*XFmYIvwSZD~l6rnSe_^|+!<eLTUgu-IGypg!!`b*P?TDzT7
zXRIP?p+Jy?v)xjMU(Nvd@>~W0X!W1a#_2oeSdmU$2O(6?MR>Rg%|9s$e8B@CVFf3}
t;Q{>YzlKHs7lHbJHevezchLvrI`w7&lUg1=@U0i0Bt&F{D+G1@{s(nG?`Z%4

literal 0
HcmV?d00001

diff --git a/documentation/zensical.toml b/documentation/zensical.toml
index e69af2918..a2b056c92 100644
--- a/documentation/zensical.toml
+++ b/documentation/zensical.toml
@@ -36,7 +36,8 @@ nav = [
             {"Queue Management" = "Organizers/Running_a_benchmark/Queue-Management.md"},
             {"Compute Worker Management & Setup" = "Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md"},
             {"Compute Worker Management with Podman" = "Organizers/Running_a_benchmark/Compute-worker-installation-with-Podman.md"},
-            {"Server Status" = "Organizers/Running_a_benchmark/Server-status-page.md"}
+            {"Server Status" = "Organizers/Running_a_benchmark/Server-status-page.md"},
+            {"Human in the loop" = "Organizers/Running_a_benchmark/Competition-HITL.md"}
         ]}
     ]},
     {"Developers" = [

From 9ca592aee6c295348e3e5832976d12530afa0bbc Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 21 Jul 2026 13:30:27 +0200
Subject: [PATCH 08/20] HITL documentation reviwed

---
 .../Running_a_benchmark/Competition-HITL.md   | 74 ++-----------------
 1 file changed, 8 insertions(+), 66 deletions(-)

diff --git a/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
index b45049032..5acdf4c07 100644
--- a/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
+++ b/documentation/docs/Organizers/Running_a_benchmark/Competition-HITL.md
@@ -19,10 +19,10 @@ This mechanism is intended for competitions where the scoring results must be re
 
 The HITL feature guarantees:
 
-* the submission is fully executed before any result is published;
-* no score appears on the leaderboard before validation;
-* no detailed results are available before validation;
-* no output archive is sent back to the Site Worker before validation;
+* the submission is fully executed before any result is published.
+* no score appears on the leaderboard before validation.
+* no detailed results are available before validation.
+* no output archive is sent back to instance.
 * the operator can approve or reject the submission directly from the Compute Worker.
 
 ---
@@ -31,8 +31,6 @@ The HITL feature guarantees:
 
 The HITL feature applies **only to private Compute Workers**.
 
-Public Compute Workers **must never execute HITL jobs**, as waiting for manual validation would permanently block shared workers.
-
 ---
 
 ## Activation
@@ -41,10 +39,6 @@ Public Compute Workers **must never execute HITL jobs**, as waiting for manual v
 
 Each competition exposes the following option:
 
-```
-Enable Human-in-the-Loop
-```
-
 When enabled, every submission routed to a private Compute Worker is executed in HITL mode.
 ![image1](_attachments/doc_HITL_1.png)
 
@@ -163,9 +157,9 @@ This needs to be done inside of the compute worker host machine (not inside of t
 The Compute Worker immediately resumes execution.
 The following artifacts are published:
 
-* detailed results;
-* scores;
-* output archive;
+* detailed results.
+* scores.
+* output archive.
 * logs.
 
 The submission status becomes "Finished"
@@ -186,27 +180,7 @@ No scoring results are published.
 
 If no decision is received within 24 hours, the submission automatically fails.
 The Compute Worker reports a timeout error.
-
----
-
-## Publication policy
-
-The fundamental principle of HITL is:
-
-> **Nothing leaves the Compute Worker before validation.**
-
-When HITL is enabled:
-
-| Artifact                                | Before validation | After approval |
-| --------------------------------------- | ----------------- | -------------- |
-| scores.json                             | ❌                 | ✅              |
-| leaderboard score                       | ❌                 | ✅              |
-| detailed_results.html                   | ❌                 | ✅              |
-| detailed results websocket notification | ❌                 | ✅              |
-| output archive                          | ❌                 | ✅              |
-| execution logs                          | Local only        | ✅              |
-
-The Compute Worker keeps every generated artifact locally until approval.
+The compute worker is enable to run another submission while waiting for HITL approval.
 
 ---
 
@@ -234,14 +208,6 @@ flowchart TD
 
 ---
 
-## Output behaviour
-
-Prediction files, scoring outputs and logs are generated normally.
-However, publication is postponed until validation succeeds.
-This guarantees that no evaluation result becomes visible before manual approval.
-
----
-
 ## Error handling
 
 The following situations are handled explicitly.
@@ -254,27 +220,3 @@ The following situations are handled explicitly.
 | Operator rejects submission                          | Submission fails                |
 | Validation timeout                                   | Submission fails                |
 | Approval received                                    | Results are published normally  |
-
----
-
-## Design principles
-
-The HITL implementation follows four principles:
-
-1. **Backward compatibility**
-
-   * Existing deployments require no configuration changes.
-
-2. **Safety**
-
-   * Configuration mismatches are detected before execution.
-
-3. **Isolation**
-
-   * HITL is supported only on private Compute Workers.
-
-4. **Atomic publication**
-
-   * No artifact is published before manual approval.
-
-These principles ensure that sensitive competitions can safely introduce manual validation without impacting the standard Codabench execution workflow.

From dbe05d5c37c35d0ce53a28e93d9f723a2a228ebd Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 21 Jul 2026 13:50:45 +0200
Subject: [PATCH 09/20] HITL in cw documentation

---
 .../Running_a_benchmark/Compute-Worker-Management---Setup.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md
index 5735af914..fc0576c1c 100644
--- a/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md
+++ b/documentation/docs/Organizers/Running_a_benchmark/Compute-Worker-Management---Setup.md
@@ -59,7 +59,7 @@ HOST_DIRECTORY=/codabench
 CONTAINER_ENGINE_EXECUTABLE=docker
 #USE_GPU=True
 #GPU_DEVICE=nvidia.com/gpu=all
-
+#HUMAN_IN_THE_LOOP=False
 #######################################################################
 #                       Network                                       #
 #######################################################################
@@ -72,6 +72,8 @@ By default, the competition container created by the compute worker has access t
 
 If the VM hosting the compute worker is behind a proxy, and you want to allow the competition container to access internet, you will also need to set the proxy for the competition container to use, in which case you can use `COMPETITION_CONTAINER_HTTP_PROXY`
 
+To control the scoring output before sending it to the instance, you can use the human in the loop feature the check the scoring file (json or txt) before sending anything to the instance, see: [HITL documentation](https://docs.codabench.org/latest/Organizers/Running_a_benchmark/Competition-HITL/)
+
 !!! note
     - The broker URL is a unique identifier of the job queue that the worker should listen to. To create a queue or obtain the broker URL of an existing queue, you can refer to [Queue Management](Queue-Management.md) docs page.
 
@@ -204,6 +206,7 @@ The folder `$HOST_DIRECTORY/data`, usually `/codabench/data`, is shared between
 
 
 !!! tip "If you simply wish to set up some compute workers to increase the computing power of your benchmark, you don't need to scroll this page any further."
+
 ---
 
 ## Building compute worker

From 38a0251a0233ba2dc524e0639e5b358cac6aef7a Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Thu, 23 Jul 2026 11:46:09 +0200
Subject: [PATCH 10/20] site worker modified to return submission status failed
 if hitl enabled on public CW

---
 src/apps/competitions/tasks.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 35bbebd32..3f43d69ea 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -162,6 +162,25 @@ def _send_to_compute_worker(submission, is_scoring):
         and submission.queue is not None
     )
 
+    if (
+        submission.phase.competition.enable_human_in_the_loop
+        and submission.queue is None
+    ):
+        submission.status = Submission.FAILED
+        submission.status_details = (
+            "This competition requires Human-in-the-Loop (HITL), "
+            "but the submission was routed to a public compute worker. "
+            "HITL is only supported on private compute workers."
+        )
+        submission.save(update_fields=["status", "status_details"])
+
+        logger.error(
+            "Submission %s rejected: HITL requires a private compute worker.",
+            submission.id,
+        )
+
+        return
+
     run_args = {
         "user_pk": submission.owner.pk,
         "submissions_api_url": settings.SUBMISSIONS_API_URL,

From 090487d33755e651109b5b4372611c5a5945ec7c Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Thu, 23 Jul 2026 17:33:38 +0200
Subject: [PATCH 11/20] in progress: detailed results check and validation

---
 compute_worker/compute_worker.py | 73 ++++++++++++--------------------
 src/apps/competitions/tasks.py   |  1 -
 2 files changed, 26 insertions(+), 48 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 84fcf1825..d2f388a35 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -332,7 +332,12 @@ def run_wrapper(run_args):
             if run.human_in_the_loop:
                 run._update_status(SubmissionStatus.AWAITING_VALIDATION)
                 run.wait_for_human_validation()
-                run.send_final_detailed_results()
+                if run.pending_detailed_results:
+                    asyncio.run(
+                        run.send_detailed_results(
+                            run.pending_detailed_results
+                        )
+                    )
                 run.push_scores()
                 run.push_output()
             else:
@@ -515,6 +520,7 @@ def __init__(self, run_args):
         self.reference_data = run_args.get("reference_data")
         self.ingestion_only_during_scoring = run_args.get("ingestion_only_during_scoring")
         self.detailed_results_url = run_args.get("detailed_results_url")
+        self.pending_detailed_results = None
 
         self.ingestion_program_exit_code = None
         self.ingestion_program_elapsed_time = None
@@ -563,7 +569,10 @@ async def watch_detailed_results(self):
                 new_time = os.path.getmtime(file_path)
                 if new_time != last_modified_time:
                     last_modified_time = new_time
-                    await self.send_detailed_results(file_path)
+                    if self.human_in_the_loop:
+                        self.pending_detailed_results = file_path
+                    else:
+                        await self.send_detailed_results(file_path)
             else:
                 logger.info(time.time() - start)
                 if time.time() - start > expiration_seconds:
@@ -576,7 +585,10 @@ async def watch_detailed_results(self):
         else:
             # make sure we always send the final version of the file
             if file_path:
-                await self.send_detailed_results(file_path)
+                if self.human_in_the_loop:
+                    self.pending_detailed_results = file_path
+                else:
+                    await self.send_detailed_results(file_path)
 
     def push_logs(self):
         """Upload any collected logs, even in case of crash.
@@ -638,49 +650,6 @@ async def send_detailed_results(self, file_path):
                 except Exception as e:
                     logger.exception(e)
 
-    def send_final_detailed_results(self):
-        if not self.detailed_results_url:
-            return
-
-        file_path = self.get_detailed_results_file_path()
-
-        if not file_path:
-            logger.info("No detailed_results.html found")
-            return
-
-        logger.info(
-            f"Uploading final detailed results {file_path} - {self.detailed_results_url}"
-        )
-
-        self._put_file(
-            self.detailed_results_url,
-            file=file_path,
-            content_type="text/html",
-        )
-
-        websocket_url = f"{self.websocket_url}?kind=detailed_results"
-
-        try:
-            websocket = asyncio.run(
-                asyncio.wait_for(
-                    websockets.connect(websocket_url),
-                    timeout=30.0,
-                )
-            )
-
-            asyncio.run(
-                websocket.send(
-                    json.dumps(
-                        {
-                            "kind": "detailed_result_update",
-                        }
-                    )
-                )
-            )
-
-        except Exception as e:
-            logger.exception(e)
-
     def _get_stdout_stderr_file_names(self, run_args):
         # run_args should be the run_args argument passed to __init__ from the run_wrapper.
         if not self.is_scoring:
@@ -1569,7 +1538,17 @@ def wait_for_human_validation(self):
         logger.info("=" * 60)
         logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
         logger.info("Inspect the scores file:")
-        logger.info(f"  cat {scores_path}")
+        logger.info(f"cat {scores_path}")
+
+        if self.detailed_results_url:
+            detailed_results = self.get_detailed_results_file_path()
+
+            if detailed_results and os.path.exists(detailed_results):
+                logger.info("")
+                logger.info("Detailed results")
+                logger.info(f"  {detailed_results}")
+                logger.info("  (Open this file in your browser to review the HTML report)")
+
         logger.info(f"To approve : touch {approved_host}")
         logger.info(f"To reject  : touch {rejected_host}")
         logger.info("=" * 60)
diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py
index 3f43d69ea..220e93677 100644
--- a/src/apps/competitions/tasks.py
+++ b/src/apps/competitions/tasks.py
@@ -178,7 +178,6 @@ def _send_to_compute_worker(submission, is_scoring):
             "Submission %s rejected: HITL requires a private compute worker.",
             submission.id,
         )
-
         return
 
     run_args = {

From 0ff24454f1a179f43fc2443ccd484490cfe9e82c Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Fri, 24 Jul 2026 16:45:01 +0200
Subject: [PATCH 12/20] feature OK / detailed result added to validation
 process

---
 compute_worker/compute_worker.py | 92 ++++++++++++++++++++++++--------
 1 file changed, 71 insertions(+), 21 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index d2f388a35..e1eb54c29 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -3,6 +3,10 @@
 import hashlib
 import json
 import os
+import functools
+import http.server
+import socketserver
+import threading
 import traceback
 import shutil
 import signal
@@ -521,6 +525,8 @@ def __init__(self, run_args):
         self.ingestion_only_during_scoring = run_args.get("ingestion_only_during_scoring")
         self.detailed_results_url = run_args.get("detailed_results_url")
         self.pending_detailed_results = None
+        self.hitl_http_server = None
+        self.hitl_http_thread = None
 
         self.ingestion_program_exit_code = None
         self.ingestion_program_elapsed_time = None
@@ -548,12 +554,6 @@ async def watch_detailed_results(self):
         """Watches files alongside scoring + program containers, currently only used
         for detailed_results.html"""
 
-        if self.human_in_the_loop:
-            logger.info(
-                "HITL enabled: skipping detailed_results streaming"
-            )
-            return
-
         if not self.detailed_results_url:
             return
         file_path = self.get_detailed_results_file_path()
@@ -618,6 +618,40 @@ def get_detailed_results_file_path(self):
             if html_files:
                 return html_files[0]
 
+    def start_hitl_http_server(self):
+        if not self.pending_detailed_results:
+            return
+        root = os.path.dirname(self.pending_detailed_results)
+        logger.info(
+            "Starting temporary HTTP server for HITL review (%s)",
+            root,
+        )
+        handler = functools.partial(
+            http.server.SimpleHTTPRequestHandler,
+            directory=root,
+        )
+        self.hitl_http_server = socketserver.TCPServer(
+            ("127.0.0.1", 8765),
+            handler,
+        )
+        self.hitl_http_thread = threading.Thread(
+            target=self.hitl_http_server.serve_forever,
+            daemon=True,
+        )
+        self.hitl_http_thread.start()
+
+    def stop_hitl_http_server(self):
+        if self.hitl_http_server is None:
+            return
+
+        logger.info("Stopping HITL HTTP server")
+
+        self.hitl_http_server.shutdown()
+        self.hitl_http_server.server_close()
+
+        self.hitl_http_server = None
+        self.hitl_http_thread = None
+
     async def send_detailed_results(self, file_path):
         logger.info(
             f"Updating detailed results {file_path} - {self.detailed_results_url}"
@@ -1535,44 +1569,59 @@ def wait_for_human_validation(self):
         approved_host = os.path.join(host_output_dir, "hitl_approved")
         rejected_host = os.path.join(host_output_dir, "hitl_rejected")
 
+        detailed_results = None
+        if self.detailed_results_url:
+            detailed_results = self.get_detailed_results_file_path()
+            if detailed_results:
+                self.pending_detailed_results = detailed_results
+                self.start_hitl_http_server()
+
         logger.info("=" * 60)
         logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
         logger.info("Inspect the scores file:")
         logger.info(f"cat {scores_path}")
 
-        if self.detailed_results_url:
-            detailed_results = self.get_detailed_results_file_path()
-
-            if detailed_results and os.path.exists(detailed_results):
-                logger.info("")
-                logger.info("Detailed results")
-                logger.info(f"  {detailed_results}")
-                logger.info("  (Open this file in your browser to review the HTML report)")
+        if detailed_results and os.path.exists(detailed_results):
+            logger.info("")
+            logger.info("Inspect the detailed results:")
+            logger.info("")
+            logger.info("Create an SSH tunnel from your workstation:")
+            logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
+            logger.info("")
+            logger.info("Then open in your browser:")
+            logger.info(
+                "http://127.0.0.1:8765/%s",
+                os.path.basename(detailed_results),
+            )
 
+        logger.info("")
         logger.info(f"To approve : touch {approved_host}")
         logger.info(f"To reject  : touch {rejected_host}")
         logger.info("=" * 60)
+        logger.info("Waiting for human validation...")
 
         poll_interval = 3
         max_wait = 60 * 60 * 24
-
         elapsed = 0
 
-        logger.info(
-            "Waiting for human validation..."
-        )
         while elapsed < max_wait:
             if os.path.exists(approved_container):
-                logger.info(f"HITL: submission {self.submission_id} approved, sending scores.")
+                logger.info(
+                    f"HITL: submission {self.submission_id} approved, sending results."
+                )
+                self.stop_hitl_http_server()
                 return
+
             if os.path.exists(rejected_container):
+                self.stop_hitl_http_server()
                 raise SubmissionException(
-                    f"HITL: scores rejected by the compute node operator "
-                    f"(submission {self.submission_id})"
+                    f"HITL: submission {self.submission_id} rejected by the compute node operator."
                 )
+
             time.sleep(poll_interval)
             elapsed += poll_interval
 
+        self.stop_hitl_http_server()
         raise SubmissionException(
             f"HITL: 24h timeout reached without validation "
             f"(submission {self.submission_id})"
@@ -1650,6 +1699,7 @@ def push_output(self):
             self._put_dir(self.scoring_result, self.output_dir)
 
     def clean_up(self):
+        self.stop_hitl_http_server()
         if Settings.CODALAB_IGNORE_CLEANUP_STEP:
             logger.warning(
                 f"CODALAB_IGNORE_CLEANUP_STEP mode enabled, ignoring clean up of: {self.root_dir}"

From 40361389163791d0ef4a3fb2c701d59f321cfd41 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 27 Jul 2026 14:43:00 +0200
Subject: [PATCH 13/20] hitl added to competiton admin

---
 src/apps/competitions/admin.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py
index 69bc44229..f9a5c0e1e 100644
--- a/src/apps/competitions/admin.py
+++ b/src/apps/competitions/admin.py
@@ -212,6 +212,7 @@ class CompetitionExpansion(admin.ModelAdmin):
                     "submissions_count",
                     "participants_count",
                     "created_when",
+                    "enable_human_in_the_loop"
                 ]
             },
         ),

From 18693f1bede19baae68850d7ec2259b757975fc4 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 27 Jul 2026 14:45:06 +0200
Subject: [PATCH 14/20] admin: moving htil in checkbox section

---
 src/apps/competitions/admin.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py
index f9a5c0e1e..6f2d32bde 100644
--- a/src/apps/competitions/admin.py
+++ b/src/apps/competitions/admin.py
@@ -212,7 +212,6 @@ class CompetitionExpansion(admin.ModelAdmin):
                     "submissions_count",
                     "participants_count",
                     "created_when",
-                    "enable_human_in_the_loop"
                 ]
             },
         ),
@@ -234,6 +233,7 @@ class CompetitionExpansion(admin.ModelAdmin):
                     "can_participants_make_submissions_public",
                     "is_featured",
                     "forum_enabled",
+                    "enable_human_in_the_loop"
                 ]
             },
         ),

From 529bb1f60ea88e46db34029e57ae418328e9d2eb Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Mon, 27 Jul 2026 15:41:31 +0200
Subject: [PATCH 15/20] options added for detailed result check and predictions
 not sent if sub rejected

---
 compute_worker/compute_worker.py | 30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index e1eb54c29..22c637881 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -335,13 +335,18 @@ def run_wrapper(run_args):
         if run.is_scoring:
             if run.human_in_the_loop:
                 run._update_status(SubmissionStatus.AWAITING_VALIDATION)
-                run.wait_for_human_validation()
+                if not run.wait_for_human_validation():
+                    raise SubmissionException(
+                        f"HITL: submission {run.submission_id} rejected by the compute node operator."
+                    )
+
                 if run.pending_detailed_results:
                     asyncio.run(
                         run.send_detailed_results(
                             run.pending_detailed_results
                         )
                     )
+
                 run.push_scores()
                 run.push_output()
             else:
@@ -1582,17 +1587,26 @@ def wait_for_human_validation(self):
         logger.info(f"cat {scores_path}")
 
         if detailed_results and os.path.exists(detailed_results):
-            logger.info("")
             logger.info("Inspect the detailed results:")
-            logger.info("")
+            logger.info("HTML report location:")
+            logger.info(f"{detailed_results}")
+            logger.info("Option 1 (recommended): Preview without copying the file")
             logger.info("Create an SSH tunnel from your workstation:")
             logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
-            logger.info("")
             logger.info("Then open in your browser:")
             logger.info(
                 "http://127.0.0.1:8765/%s",
                 os.path.basename(detailed_results),
             )
+            host_detailed_results = self._get_host_path(detailed_results)
+            logger.info("HTML report location:")
+            logger.info("%s", host_detailed_results)
+            logger.info("")
+            logger.info("Option 2: Copy the HTML report")
+            logger.info(
+                "cat %s",
+                host_detailed_results,
+            )
 
         logger.info("")
         logger.info(f"To approve : touch {approved_host}")
@@ -1610,20 +1624,18 @@ def wait_for_human_validation(self):
                     f"HITL: submission {self.submission_id} approved, sending results."
                 )
                 self.stop_hitl_http_server()
-                return
+                return True
 
             if os.path.exists(rejected_container):
                 self.stop_hitl_http_server()
-                raise SubmissionException(
-                    f"HITL: submission {self.submission_id} rejected by the compute node operator."
-                )
+                return False
 
             time.sleep(poll_interval)
             elapsed += poll_interval
 
         self.stop_hitl_http_server()
         raise SubmissionException(
-            f"HITL: 24h timeout reached without validation "
+            f"HITL: 24h timeout reached without validation"
             f"(submission {self.submission_id})"
         )
 

From 1f5f02eaa815f82573bdac90c89041ea72d3f07a Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 28 Jul 2026 10:11:33 +0200
Subject: [PATCH 16/20] cw logs conf modified to display colors (private cw
 compatible)

---
 compute_worker/compute_worker.py | 114 +++++++++++++++++++++++--------
 1 file changed, 85 insertions(+), 29 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 22c637881..fbff4ace7 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -34,11 +34,41 @@
 from billiard.exceptions import SoftTimeLimitExceeded
 
 from logs_loguru import configure_logging, colorize_run_args
-
 logger = logging.getLogger(__name__)
 
 sys.path.append("/app/src/settings/")
 
+# Colors for logs (method below)
+RESET = "\033[0m"
+BOLD = "\033[1m"
+
+RED = "\033[31m"
+GREEN = "\033[32m"
+YELLOW = "\033[33m"
+CYAN = "\033[36m"
+
+
+def colorize(text, color="", bold=False):
+    """
+    Colorize any log message using ANSI escape sequences.
+    Compatible with logging, Loguru, Docker and SSH terminals, public and private CWs.
+    """
+    prefix = ""
+
+    if bold:
+        prefix += BOLD
+
+    if color == "red":
+        prefix += RED
+    elif color == "green":
+        prefix += GREEN
+    elif color == "yellow":
+        prefix += YELLOW
+    elif color == "cyan":
+        prefix += CYAN
+
+    return f"{prefix}{text}{RESET}"
+
 
 # -----------------------------------------------
 # Settings
@@ -1581,38 +1611,59 @@ def wait_for_human_validation(self):
                 self.pending_detailed_results = detailed_results
                 self.start_hitl_http_server()
 
-        logger.info("=" * 60)
-        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
-        logger.info("Inspect the scores file:")
-        logger.info(f"cat {scores_path}")
+        logger.info(colorize("=" * 60, "cyan", bold=True))
+        logger.info(colorize(f"HUMAN IN THE LOOP — submission {self.submission_id}", "cyan", bold=True))
+
+        logger.info(colorize("Inspect the scores file:", bold=True))
+        logger.info(colorize(f"cat {scores_path}", "green"))
 
         if detailed_results and os.path.exists(detailed_results):
-            logger.info("Inspect the detailed results:")
-            logger.info("HTML report location:")
-            logger.info(f"{detailed_results}")
-            logger.info("Option 1 (recommended): Preview without copying the file")
+            host_detailed_results = self._get_host_path(detailed_results)
+
+            logger.info("")
+            logger.info(
+                colorize(
+                    "Option 1 (recommended): Preview without copying the file",
+                    "yellow",
+                )
+            )
+
             logger.info("Create an SSH tunnel from your workstation:")
-            logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
+            logger.info(
+                colorize(
+                    "ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>",
+                    "green",
+                )
+            )
+
             logger.info("Then open in your browser:")
             logger.info(
-                "http://127.0.0.1:8765/%s",
-                os.path.basename(detailed_results),
+                colorize(
+                    f"http://127.0.0.1:8765/{os.path.basename(detailed_results)}",
+                    "green",
+                )
             )
-            host_detailed_results = self._get_host_path(detailed_results)
+
+            logger.info("")
             logger.info("HTML report location:")
-            logger.info("%s", host_detailed_results)
+            logger.info(colorize(host_detailed_results, "green"))
+
             logger.info("")
-            logger.info("Option 2: Copy the HTML report")
             logger.info(
-                "cat %s",
-                host_detailed_results,
+                colorize(
+                    "Option 2: Display the HTML directly in the terminal",
+                    "yellow",
+                )
             )
+            logger.info(colorize(f"cat {host_detailed_results}", "green"))
 
         logger.info("")
-        logger.info(f"To approve : touch {approved_host}")
-        logger.info(f"To reject  : touch {rejected_host}")
-        logger.info("=" * 60)
-        logger.info("Waiting for human validation...")
+        logger.info(colorize("Validation commands:", bold=True))
+        logger.info(colorize(f"Approve : touch {approved_host}", "green"))
+        logger.info(colorize(f"Reject  : touch {rejected_host}", "red"))
+
+        logger.info(colorize("=" * 60, "cyan", bold=True))
+        logger.info(colorize("Waiting for human validation...", "yellow"))
 
         poll_interval = 3
         max_wait = 60 * 60 * 24
@@ -1621,21 +1672,26 @@ def wait_for_human_validation(self):
         while elapsed < max_wait:
             if os.path.exists(approved_container):
                 logger.info(
-                    f"HITL: submission {self.submission_id} approved, sending results."
+                    colorize(
+                        f"HITL: submission {self.submission_id} approved, sending results.",
+                        "green",
+                        bold=True,
+                    )
                 )
-                self.stop_hitl_http_server()
-                return True
-
-            if os.path.exists(rejected_container):
-                self.stop_hitl_http_server()
-                return False
 
+                logger.info(
+                    colorize(
+                        f"HITL: submission {self.submission_id} rejected by the compute node operator.",
+                        "red",
+                        bold=True,
+                    )
+                )
             time.sleep(poll_interval)
             elapsed += poll_interval
 
         self.stop_hitl_http_server()
         raise SubmissionException(
-            f"HITL: 24h timeout reached without validation"
+            f"HITL: 24h timeout reached without validation "
             f"(submission {self.submission_id})"
         )
 

From 3b677cbc365bea09bceeb91bbf87dc53359d9cd9 Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Tue, 28 Jul 2026 11:12:31 +0200
Subject: [PATCH 17/20] cw code for hitl clean

---
 compute_worker/compute_worker.py | 116 ++++++++-----------------------
 1 file changed, 28 insertions(+), 88 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index fbff4ace7..b3a1590dc 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -34,41 +34,11 @@
 from billiard.exceptions import SoftTimeLimitExceeded
 
 from logs_loguru import configure_logging, colorize_run_args
+
 logger = logging.getLogger(__name__)
 
 sys.path.append("/app/src/settings/")
 
-# Colors for logs (method below)
-RESET = "\033[0m"
-BOLD = "\033[1m"
-
-RED = "\033[31m"
-GREEN = "\033[32m"
-YELLOW = "\033[33m"
-CYAN = "\033[36m"
-
-
-def colorize(text, color="", bold=False):
-    """
-    Colorize any log message using ANSI escape sequences.
-    Compatible with logging, Loguru, Docker and SSH terminals, public and private CWs.
-    """
-    prefix = ""
-
-    if bold:
-        prefix += BOLD
-
-    if color == "red":
-        prefix += RED
-    elif color == "green":
-        prefix += GREEN
-    elif color == "yellow":
-        prefix += YELLOW
-    elif color == "cyan":
-        prefix += CYAN
-
-    return f"{prefix}{text}{RESET}"
-
 
 # -----------------------------------------------
 # Settings
@@ -1611,59 +1581,34 @@ def wait_for_human_validation(self):
                 self.pending_detailed_results = detailed_results
                 self.start_hitl_http_server()
 
-        logger.info(colorize("=" * 60, "cyan", bold=True))
-        logger.info(colorize(f"HUMAN IN THE LOOP — submission {self.submission_id}", "cyan", bold=True))
-
-        logger.info(colorize("Inspect the scores file:", bold=True))
-        logger.info(colorize(f"cat {scores_path}", "green"))
+        logger.info("=" * 60)
+        logger.info(f"HUMAN IN THE LOOP — submission {self.submission_id}")
+        logger.info("Inspect scoring:")
+        logger.info(f"cat {scores_path}")
+        logger.info("")
 
         if detailed_results and os.path.exists(detailed_results):
-            host_detailed_results = self._get_host_path(detailed_results)
-
-            logger.info("")
-            logger.info(
-                colorize(
-                    "Option 1 (recommended): Preview without copying the file",
-                    "yellow",
-                )
-            )
-
+            logger.info("Inspect detailed results:")
+            logger.info("Option 1: Preview without copying the file")
             logger.info("Create an SSH tunnel from your workstation:")
+            logger.info("ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>")
+            logger.info("Then open in your browser:",)
             logger.info(
-                colorize(
-                    "ssh -L 8765:127.0.0.1:8765 operator@<compute-worker>",
-                    "green",
-                )
-            )
-
-            logger.info("Then open in your browser:")
-            logger.info(
-                colorize(
-                    f"http://127.0.0.1:8765/{os.path.basename(detailed_results)}",
-                    "green",
-                )
+                "http://127.0.0.1:8765/%s",
+                os.path.basename(detailed_results),
             )
-
-            logger.info("")
-            logger.info("HTML report location:")
-            logger.info(colorize(host_detailed_results, "green"))
-
-            logger.info("")
+            host_detailed_results = self._get_host_path(detailed_results)
+            logger.info("Option 2: Copy the HTML report")
             logger.info(
-                colorize(
-                    "Option 2: Display the HTML directly in the terminal",
-                    "yellow",
-                )
+                "cat %s",
+                host_detailed_results,
             )
-            logger.info(colorize(f"cat {host_detailed_results}", "green"))
 
         logger.info("")
-        logger.info(colorize("Validation commands:", bold=True))
-        logger.info(colorize(f"Approve : touch {approved_host}", "green"))
-        logger.info(colorize(f"Reject  : touch {rejected_host}", "red"))
-
-        logger.info(colorize("=" * 60, "cyan", bold=True))
-        logger.info(colorize("Waiting for human validation...", "yellow"))
+        logger.info(f"To approve : touch {approved_host}")
+        logger.info(f"To reject  : touch {rejected_host}")
+        logger.info("=" * 60)
+        logger.info("Waiting for human validation...")
 
         poll_interval = 3
         max_wait = 60 * 60 * 24
@@ -1672,26 +1617,21 @@ def wait_for_human_validation(self):
         while elapsed < max_wait:
             if os.path.exists(approved_container):
                 logger.info(
-                    colorize(
-                        f"HITL: submission {self.submission_id} approved, sending results.",
-                        "green",
-                        bold=True,
-                    )
+                    f"HITL: submission {self.submission_id} approved, sending results."
                 )
+                self.stop_hitl_http_server()
+                return True
+
+            if os.path.exists(rejected_container):
+                self.stop_hitl_http_server()
+                return False
 
-                logger.info(
-                    colorize(
-                        f"HITL: submission {self.submission_id} rejected by the compute node operator.",
-                        "red",
-                        bold=True,
-                    )
-                )
             time.sleep(poll_interval)
             elapsed += poll_interval
 
         self.stop_hitl_http_server()
         raise SubmissionException(
-            f"HITL: 24h timeout reached without validation "
+            f"HITL: 24h timeout reached without validation"
             f"(submission {self.submission_id})"
         )
 

From 0a9cea20abb2b3fc5a7091b23a307f58d38be250 Mon Sep 17 00:00:00 2001
From: didayolo <adrien.pavao@gmail.com>
Date: Tue, 4 Aug 2026 15:37:05 +0200
Subject: [PATCH 18/20] Add migration files

---
 ...ition_enable_human_in_the_loop_and_more.py | 38 +++++++++++++++++++
 1 file changed, 38 insertions(+)
 create mode 100644 src/apps/competitions/migrations/0062_competition_enable_human_in_the_loop_and_more.py

diff --git a/src/apps/competitions/migrations/0062_competition_enable_human_in_the_loop_and_more.py b/src/apps/competitions/migrations/0062_competition_enable_human_in_the_loop_and_more.py
new file mode 100644
index 000000000..fcb4779d3
--- /dev/null
+++ b/src/apps/competitions/migrations/0062_competition_enable_human_in_the_loop_and_more.py
@@ -0,0 +1,38 @@
+# Generated by Django 5.2.15 on 2026-08-04 13:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("competitions", "0061_competition_participant_groups"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="competition",
+            name="enable_human_in_the_loop",
+            field=models.BooleanField(default=False),
+        ),
+        migrations.AlterField(
+            model_name="submission",
+            name="status",
+            field=models.CharField(
+                choices=[
+                    ("None", "None"),
+                    ("Submitting", "Submitting"),
+                    ("Submitted", "Submitted"),
+                    ("Preparing", "Preparing"),
+                    ("Running", "Running"),
+                    ("Scoring", "Scoring"),
+                    ("Awaiting validation", "Awaiting validation"),
+                    ("Cancelled", "Cancelled"),
+                    ("Finished", "Finished"),
+                    ("Failed", "Failed"),
+                ],
+                default="Submitting",
+                max_length=128,
+            ),
+        ),
+    ]

From fd301d1ecd59001d1e9d29098deb1a136f5907bb Mon Sep 17 00:00:00 2001
From: didayolo <adrien.pavao@gmail.com>
Date: Fri, 7 Aug 2026 14:26:20 +0200
Subject: [PATCH 19/20] Simplify code

---
 compute_worker/compute_worker.py | 12 +++---------
 1 file changed, 3 insertions(+), 9 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index b3a1590dc..19b8288e9 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -346,16 +346,10 @@ def run_wrapper(run_args):
                             run.pending_detailed_results
                         )
                     )
-
-                run.push_scores()
-                run.push_output()
-            else:
-                run.push_scores()
-                run.push_output()
-        else:
-            run.push_output()
-
+            run.push_scores()
+        run.push_output()
         run._update_status(SubmissionStatus.FINISHED)
+
     except DockerImagePullException as e:
         msg = str(e).strip()
         if msg:

From c31df9889b230870a2e582b2bf508adb3d6a3ddf Mon Sep 17 00:00:00 2001
From: Idir Chikhoune <idir.chikhoune@lisn.fr>
Date: Fri, 7 Aug 2026 16:47:30 +0200
Subject: [PATCH 20/20] test fix detailed result path

---
 compute_worker/compute_worker.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/compute_worker/compute_worker.py b/compute_worker/compute_worker.py
index 19b8288e9..769c31643 100644
--- a/compute_worker/compute_worker.py
+++ b/compute_worker/compute_worker.py
@@ -1559,6 +1559,21 @@ def wait_for_human_validation(self):
         container_output_dir = self.output_dir
         host_output_dir = self._get_host_path(self.output_dir)
 
+        detailed_results = None
+        host_detailed_results = None
+
+        if self.detailed_results_url:
+            detailed_results = self.get_detailed_results_file_path()
+
+            if detailed_results:
+                self.pending_detailed_results = detailed_results
+                self.start_hitl_http_server()
+
+                host_detailed_results = os.path.join(
+                    host_output_dir,
+                    os.path.basename(detailed_results),
+                )
+
         scores_path = os.path.join(host_output_dir, "scores.json")
         if not os.path.exists(os.path.join(container_output_dir, "scores.json")):
             scores_path = os.path.join(host_output_dir, "scores.txt")
@@ -1593,10 +1608,7 @@ def wait_for_human_validation(self):
             )
             host_detailed_results = self._get_host_path(detailed_results)
             logger.info("Option 2: Copy the HTML report")
-            logger.info(
-                "cat %s",
-                host_detailed_results,
-            )
+            logger.info("cat %s", host_detailed_results)
 
         logger.info("")
         logger.info(f"To approve : touch {approved_host}")