From b96781ebe7915039b5fa6a2c4a1355711c6f3993 Mon Sep 17 00:00:00 2001 From: Oskaril-data Date: Sun, 21 Jun 2026 14:12:10 +0300 Subject: [PATCH 1/5] WIP: push resolution sets to git in a single commit New single-task Cloud Run job `func-push-resolution-sets` gathers all resolution sets from the bucket and pushes them in one commit, fixing the race condition from parallel resolve tasks pushing independently. Still needs end-to-end testing in a GCP dev project. Refs #148 --- Makefile | 5 +- src/helpers/git.py | 9 ++++ src/nightly_update_workflow/manager/main.py | 16 +++++- src/nightly_update_workflow/worker/main.py | 13 ++++- src/orchestration/_io.py | 51 +++++++++++++++++-- .../func_push_resolution_sets/Makefile | 46 +++++++++++++++++ .../func_push_resolution_sets/main.py | 27 ++++++++++ .../requirements.txt | 13 +++++ 8 files changed, 173 insertions(+), 7 deletions(-) create mode 100644 src/orchestration/func_push_resolution_sets/Makefile create mode 100644 src/orchestration/func_push_resolution_sets/main.py create mode 100644 src/orchestration/func_push_resolution_sets/requirements.txt diff --git a/Makefile b/Makefile index b65e2f67..3cf0772e 100644 --- a/Makefile +++ b/Makefile @@ -107,7 +107,7 @@ orchestration: nightly-worker-job nightly-manager-job compress_buckets metadata: tag-questions validate-questions -resolve: resolve-forecasts +resolve: resolve-forecasts push-resolution-sets leaderboards: leaderboard-tournament leaderboard-baseline leaderboard-preliminary @@ -205,6 +205,9 @@ validate-questions: resolve-forecasts: $(MAKE) -C src/orchestration/func_resolve || echo "* $@" >> $(MAKE_FAILURE_LOG) +push-resolution-sets: + $(MAKE) -C src/orchestration/func_push_resolution_sets || echo "* $@" >> $(MAKE_FAILURE_LOG) + naive-and-dummy-forecasters: $(MAKE) -C src/base_eval/naive_and_dummy_forecasters || echo "* $@" >> $(MAKE_FAILURE_LOG) diff --git a/src/helpers/git.py b/src/helpers/git.py index d6d8b6f3..7211a21c 100644 --- a/src/helpers/git.py +++ b/src/helpers/git.py @@ -84,6 +84,15 @@ def clone_and_push_files( shutil.copy(source, full_destination_path, follow_symlinks=False) repo.index.add([destination]) + # Skip the commit/push if none of the staged files differ from what's already in the repo. + # This avoids empty commits: the resolution-set push job re-adds every resolution set every + # night, but on nights where nothing changed there is nothing to commit. + if not repo.index.diff(repo.head.commit): + logger.info(f"No changes to push to {repo_url}; skipping commit.") + os.remove(tmp_key_file_path) + shutil.rmtree(local_repo_dir, ignore_errors=True) + return + error_encountered = False author = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL) committer = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL) diff --git a/src/nightly_update_workflow/manager/main.py b/src/nightly_update_workflow/manager/main.py index c6835a6f..25d19488 100644 --- a/src/nightly_update_workflow/manager/main.py +++ b/src/nightly_update_workflow/manager/main.py @@ -181,7 +181,7 @@ def main(): dict_to_use=dict_to_use_naive_and_dummy_forecasters, task_count=1 ) - # Block on resolve forecasts before launching leaderboards + # Block on resolve forecasts before pushing resolution sets to git cloud_run.block_and_check_job_result( operation=operation_resolve_forecasts, name=dict_to_use_resolve_forecasts, @@ -189,6 +189,20 @@ def main(): timeout=timeout_resolve_forecasts, ) + # Push all resolution sets to git in a single commit, now that every (parallel) resolve task + # has finished uploading its resolution set to the bucket. Done as its own job so that only + # one process clones and pushes to the dataset repo, avoiding a race condition. + dict_to_use_push_resolution_sets = "push_resolution_sets" + operation_push_resolution_sets = call_worker( + dict_to_use=dict_to_use_push_resolution_sets, + task_count=1, + ) + cloud_run.block_and_check_job_result( + operation=operation_push_resolution_sets, + name=dict_to_use_push_resolution_sets, + exit_on_error=False, + ) + # Launch leaderboard jobs in parallel dict_to_use_leaderboards = "leaderboards" timeout_leaderboards = cloud_run.timeout_1h * 4 diff --git a/src/nightly_update_workflow/worker/main.py b/src/nightly_update_workflow/worker/main.py index 68cc3b89..02f47c7d 100644 --- a/src/nightly_update_workflow/worker/main.py +++ b/src/nightly_update_workflow/worker/main.py @@ -17,6 +17,15 @@ ("func-resolve-forecasts", True, cloud_run.timeout_1h * 3, 50), ] ] + + +push_resolution_sets = [ + [ + ("func-push-resolution-sets", True, cloud_run.timeout_1h, 1), + ] +] + + leaderboards = [ [ ("func-leaderboard-tournament", True, cloud_run.timeout_1h * 4, 1), @@ -139,7 +148,8 @@ def main(): Env variables: CLOUD_RUN_TASK_INDEX: automatically set by Cloud Run Jobs - DICT_TO_USE: one of `fetch_and_update`, `metadata`, `resolve_forecasts`, `leaderboards`. + DICT_TO_USE: one of `fetch_and_update`, `metadata`, `resolve_forecasts`, + `push_resolution_sets`, `leaderboards`. """ dict_mapping = { "fetch_and_update": get_fetch_and_update(), @@ -147,6 +157,7 @@ def main(): "create_question_set": get_create_question_set(), "publish_question_set_make_llm_baseline": get_publish_question_set_make_llm_baseline(), "resolve_forecasts": resolve_forecasts, + "push_resolution_sets": push_resolution_sets, "leaderboards": leaderboards, "naive_and_dummy_forecasters": get_naive_and_dummy_forecasters(), "website": website, diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py index f2f7f1ac..72c4651c 100644 --- a/src/orchestration/_io.py +++ b/src/orchestration/_io.py @@ -223,9 +223,13 @@ def upload_hash_mapping(raw_json: str, source_name: str) -> None: def upload_resolution_set(df: pd.DataFrame, forecast_due_date: str, question_set_filename: str): - """Upload resolution set to GCS and push to git.""" - from helpers import git # noqa: E402 + """Upload resolution set to GCS. + The resolution set is only uploaded to the bucket here. Pushing the resolution sets to git + happens later in a single commit via `push_all_resolution_sets`, run as its own Cloud Run job + once all (parallel) resolution tasks have finished. This avoids the race condition that + occurred when each parallel task cloned and pushed to the git repository independently. + """ basename = f"{forecast_due_date}_resolution_set.json" local_filename = f"/tmp/{basename}" df = df[["id", "source", "direction", "resolution_date", "resolved_to", "resolved"]] @@ -248,14 +252,53 @@ def upload_resolution_set(df: pd.DataFrame, forecast_due_date: str, question_set ) logger.info(f"Uploaded Resolution File {local_filename} to {upload_folder}.") + +def push_all_resolution_sets() -> None: + """Push every resolution set in the bucket to the git dataset repo in a single commit. + + The parallel resolution tasks each upload their resolution set to `PUBLIC_RELEASE_BUCKET` + (see `upload_resolution_set`). This function gathers all of those files and pushes them to + git in one commit, so only a single process ever clones and pushes to the repository. This + removes the race condition that arose when each parallel task pushed independently. + """ + from helpers import git # noqa: E402 + + if env.RUNNING_LOCALLY: + logger.info("Running locally; not pushing resolution sets to git.") + return + + folder = "datasets/resolution_sets" + blob_names = gcp.storage.list_with_prefix( + bucket_name=env.PUBLIC_RELEASE_BUCKET, + prefix=folder, + ) + + files = {} + for blob_name in blob_names: + basename = os.path.basename(blob_name) + if not basename.endswith("_resolution_set.json"): + continue + local_filename = f"/tmp/{basename}" + gcp.storage.download( + bucket_name=env.PUBLIC_RELEASE_BUCKET, + filename=blob_name, + local_filename=local_filename, + ) + files[local_filename] = f"{folder}/{basename}" + + if not files: + logger.warning(f"No resolution sets found under {folder}; nothing to push.") + return + mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL") mirrors = [mirrors] if mirrors else [] git.clone_and_push_files( repo_url=keys.API_GITHUB_DATASET_REPO_URL, - files={local_filename: f"{upload_folder}/{basename}"}, - commit_message=f"resolution set: automatic update for {question_set_filename}.", + files=files, + commit_message="resolution sets: automatic update.", mirrors=mirrors, ) + logger.info(f"Pushed {len(files)} resolution sets to git in a single commit.") def upload_processed_forecast_file(data: dict, forecast_due_date: str, filename: str): diff --git a/src/orchestration/func_push_resolution_sets/Makefile b/src/orchestration/func_push_resolution_sets/Makefile new file mode 100644 index 00000000..df4facf4 --- /dev/null +++ b/src/orchestration/func_push_resolution_sets/Makefile @@ -0,0 +1,46 @@ +all : + $(MAKE) clean + $(MAKE) deploy + +.PHONY : all clean deploy + +UPLOAD_DIR = upload + +.gcloudignore: + cp -r $(ROOT_DIR)src/helpers/.gcloudignore . + +Dockerfile: $(ROOT_DIR)src/helpers/Dockerfile.template + sed \ + -e 's/REGION/$(CLOUD_DEPLOY_REGION)/g' \ + -e 's/STACK/google-22-full/g' \ + -e 's/PYTHON_VERSION/python312/g' \ + $< > Dockerfile + +NUM_CPUS = 1 + +deploy : main.py .gcloudignore requirements.txt Dockerfile + mkdir -p $(UPLOAD_DIR) + cp -r $(ROOT_DIR)utils $(UPLOAD_DIR)/ + cp -r $(ROOT_DIR)src/helpers $(UPLOAD_DIR)/helpers + cp -r $(ROOT_DIR)src/sources $(UPLOAD_DIR)/sources + mkdir -p $(UPLOAD_DIR)/orchestration + cp $(ROOT_DIR)src/orchestration/__init__.py $(UPLOAD_DIR)/orchestration/ + cp $(ROOT_DIR)src/orchestration/_io.py $(UPLOAD_DIR)/orchestration/ + cp $(ROOT_DIR)src/_fb_types.py $(UPLOAD_DIR)/ + cp $(ROOT_DIR)src/_schemas.py $(UPLOAD_DIR)/ + cp $^ $(UPLOAD_DIR)/ + gcloud run jobs deploy \ + func-push-resolution-sets \ + --project $(CLOUD_PROJECT) \ + --region $(CLOUD_DEPLOY_REGION) \ + --tasks 1 \ + --task-timeout 1h \ + --memory 2Gi \ + --cpu $(NUM_CPUS) \ + --max-retries 0 \ + --service-account $(QUESTION_BANK_BUCKET_SERVICE_ACCOUNT) \ + --set-env-vars $(DEFAULT_CLOUD_FUNCTION_ENV_VARS),NUM_CPUS=$(NUM_CPUS) \ + --source $(UPLOAD_DIR) + +clean : + rm -rf $(UPLOAD_DIR) .gcloudignore Dockerfile diff --git a/src/orchestration/func_push_resolution_sets/main.py b/src/orchestration/func_push_resolution_sets/main.py new file mode 100644 index 00000000..694e0a36 --- /dev/null +++ b/src/orchestration/func_push_resolution_sets/main.py @@ -0,0 +1,27 @@ +"""Push all resolution sets to the git dataset repo in a single commit. + +This runs as its own Cloud Run job after the (parallel) resolve-forecasts tasks have finished. +Each resolve task uploads its resolution set to the bucket only; this job gathers all of them +and pushes them to git in a single commit, so that only one process ever clones and pushes to +the dataset repository. This removes the race condition that occurred when each parallel task +pushed independently. See `orchestration._io.push_all_resolution_sets`. +""" + +import logging +from typing import Any + +from helpers import decorator +from orchestration import _io + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +@decorator.log_runtime +def driver(_: Any) -> None: + """Push all resolution sets to git in a single commit.""" + _io.push_all_resolution_sets() + + +if __name__ == "__main__": + driver(None) diff --git a/src/orchestration/func_push_resolution_sets/requirements.txt b/src/orchestration/func_push_resolution_sets/requirements.txt new file mode 100644 index 00000000..4c503990 --- /dev/null +++ b/src/orchestration/func_push_resolution_sets/requirements.txt @@ -0,0 +1,13 @@ +google-cloud-storage +google-cloud-secret-manager +pandas>=2.2.2,<3.0 +tqdm +gcsfs==2025.7.0 +GitPython +scipy +termcolor +slack_sdk +pandera +pytz +python-dateutil +backoff \ No newline at end of file From 3b75380194663146393ede239a93a1f6bfc128f0 Mon Sep 17 00:00:00 2001 From: Oskaril-data Date: Tue, 30 Jun 2026 11:47:52 +0300 Subject: [PATCH 2/5] fix: don't log a resolution-set push when nothing was committed clone_and_push_files now returns whether it pushed, and push_all_resolution_sets only logs the push count when a commit was actually made. Avoids a misleading 'Pushed N resolution sets' log on nights where the empty-commit skip applied. --- src/helpers/git.py | 8 +++++--- src/orchestration/_io.py | 5 +++-- .../func_push_resolution_sets/requirements.txt | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/helpers/git.py b/src/helpers/git.py index 7211a21c..e1decbfe 100644 --- a/src/helpers/git.py +++ b/src/helpers/git.py @@ -57,7 +57,7 @@ def clone_and_push_files( files: Dict[str, str], commit_message: str, mirrors: Optional[List[str]] = None, -) -> None: +) -> bool: """Clone a Git repository, add/update files, commit, and push to origin and optional mirrors. Args: @@ -68,7 +68,8 @@ def clone_and_push_files( If None, attempts to load from secrets. Returns: - None. Exits with status 1 if an error is encountered while pushing. + bool: True if a commit was pushed, False if there was nothing to commit. + Exits with status 1 if an error is encountered while pushing. """ if not mirrors: mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL") @@ -91,7 +92,7 @@ def clone_and_push_files( logger.info(f"No changes to push to {repo_url}; skipping commit.") os.remove(tmp_key_file_path) shutil.rmtree(local_repo_dir, ignore_errors=True) - return + return False error_encountered = False author = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL) @@ -118,6 +119,7 @@ def clone_and_push_files( sys.exit(1) logger.info(f"Pushed to {repo_url} with commit message: {commit_message}") + return True def clone_commit_and_push( diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py index 72c4651c..5673f742 100644 --- a/src/orchestration/_io.py +++ b/src/orchestration/_io.py @@ -292,13 +292,14 @@ def push_all_resolution_sets() -> None: mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL") mirrors = [mirrors] if mirrors else [] - git.clone_and_push_files( + pushed = git.clone_and_push_files( repo_url=keys.API_GITHUB_DATASET_REPO_URL, files=files, commit_message="resolution sets: automatic update.", mirrors=mirrors, ) - logger.info(f"Pushed {len(files)} resolution sets to git in a single commit.") + if pushed: + logger.info(f"Pushed {len(files)} resolution sets to git in a single commit.") def upload_processed_forecast_file(data: dict, forecast_due_date: str, filename: str): diff --git a/src/orchestration/func_push_resolution_sets/requirements.txt b/src/orchestration/func_push_resolution_sets/requirements.txt index 4c503990..4d431fb3 100644 --- a/src/orchestration/func_push_resolution_sets/requirements.txt +++ b/src/orchestration/func_push_resolution_sets/requirements.txt @@ -10,4 +10,4 @@ slack_sdk pandera pytz python-dateutil -backoff \ No newline at end of file +backoff From 5b2f577416ec745bf5d4b3fdf0a5258cb0397509 Mon Sep 17 00:00:00 2001 From: Oskaril-data Date: Tue, 30 Jun 2026 12:36:45 +0300 Subject: [PATCH 3/5] docs: tighten comments and docstrings for the resolution-set push --- src/helpers/git.py | 4 +--- src/nightly_update_workflow/manager/main.py | 5 ++--- src/orchestration/_io.py | 12 ++++-------- src/orchestration/func_push_resolution_sets/main.py | 8 ++------ 4 files changed, 9 insertions(+), 20 deletions(-) diff --git a/src/helpers/git.py b/src/helpers/git.py index e1decbfe..bdaa34b8 100644 --- a/src/helpers/git.py +++ b/src/helpers/git.py @@ -85,9 +85,7 @@ def clone_and_push_files( shutil.copy(source, full_destination_path, follow_symlinks=False) repo.index.add([destination]) - # Skip the commit/push if none of the staged files differ from what's already in the repo. - # This avoids empty commits: the resolution-set push job re-adds every resolution set every - # night, but on nights where nothing changed there is nothing to commit. + # Skip empty commits: nothing to push if the staged files match HEAD. if not repo.index.diff(repo.head.commit): logger.info(f"No changes to push to {repo_url}; skipping commit.") os.remove(tmp_key_file_path) diff --git a/src/nightly_update_workflow/manager/main.py b/src/nightly_update_workflow/manager/main.py index 25d19488..dc608f94 100644 --- a/src/nightly_update_workflow/manager/main.py +++ b/src/nightly_update_workflow/manager/main.py @@ -189,9 +189,8 @@ def main(): timeout=timeout_resolve_forecasts, ) - # Push all resolution sets to git in a single commit, now that every (parallel) resolve task - # has finished uploading its resolution set to the bucket. Done as its own job so that only - # one process clones and pushes to the dataset repo, avoiding a race condition. + # Push all resolution sets in a single dedicated job now that the parallel resolve tasks have + # finished, so only one process pushes to the dataset repo (avoids a race condition). dict_to_use_push_resolution_sets = "push_resolution_sets" operation_push_resolution_sets = call_worker( dict_to_use=dict_to_use_push_resolution_sets, diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py index 5673f742..c0425056 100644 --- a/src/orchestration/_io.py +++ b/src/orchestration/_io.py @@ -225,10 +225,8 @@ def upload_hash_mapping(raw_json: str, source_name: str) -> None: def upload_resolution_set(df: pd.DataFrame, forecast_due_date: str, question_set_filename: str): """Upload resolution set to GCS. - The resolution set is only uploaded to the bucket here. Pushing the resolution sets to git - happens later in a single commit via `push_all_resolution_sets`, run as its own Cloud Run job - once all (parallel) resolution tasks have finished. This avoids the race condition that - occurred when each parallel task cloned and pushed to the git repository independently. + Only uploads to the bucket; resolution sets are pushed to git later by + `push_all_resolution_sets`. """ basename = f"{forecast_due_date}_resolution_set.json" local_filename = f"/tmp/{basename}" @@ -256,10 +254,8 @@ def upload_resolution_set(df: pd.DataFrame, forecast_due_date: str, question_set def push_all_resolution_sets() -> None: """Push every resolution set in the bucket to the git dataset repo in a single commit. - The parallel resolution tasks each upload their resolution set to `PUBLIC_RELEASE_BUCKET` - (see `upload_resolution_set`). This function gathers all of those files and pushes them to - git in one commit, so only a single process ever clones and pushes to the repository. This - removes the race condition that arose when each parallel task pushed independently. + Run as one job rather than having each parallel resolve task push its own set, which + avoids the race condition of concurrent pushes to the repository. """ from helpers import git # noqa: E402 diff --git a/src/orchestration/func_push_resolution_sets/main.py b/src/orchestration/func_push_resolution_sets/main.py index 694e0a36..be370306 100644 --- a/src/orchestration/func_push_resolution_sets/main.py +++ b/src/orchestration/func_push_resolution_sets/main.py @@ -1,10 +1,6 @@ -"""Push all resolution sets to the git dataset repo in a single commit. +"""Cloud Run job: push all resolution sets to the git dataset repo in a single commit. -This runs as its own Cloud Run job after the (parallel) resolve-forecasts tasks have finished. -Each resolve task uploads its resolution set to the bucket only; this job gathers all of them -and pushes them to git in a single commit, so that only one process ever clones and pushes to -the dataset repository. This removes the race condition that occurred when each parallel task -pushed independently. See `orchestration._io.push_all_resolution_sets`. +See `orchestration._io.push_all_resolution_sets`. """ import logging From 394bfa092d53ff98f27ad1070e1f594018c7f87f Mon Sep 17 00:00:00 2001 From: Oskaril-data Date: Tue, 30 Jun 2026 13:59:06 +0300 Subject: [PATCH 4/5] build: deploy func-push-resolution-sets via orchestration_upload.mk Upstream moved orchestration jobs to the shared orchestration_upload.mk staging macro and made utils a pip dependency (fri-utils). Update the resolution-set push job's Makefile to match instead of copying the removed utils submodule by hand. --- .../func_push_resolution_sets/Makefile | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/src/orchestration/func_push_resolution_sets/Makefile b/src/orchestration/func_push_resolution_sets/Makefile index df4facf4..75786745 100644 --- a/src/orchestration/func_push_resolution_sets/Makefile +++ b/src/orchestration/func_push_resolution_sets/Makefile @@ -5,6 +5,8 @@ all : .PHONY : all clean deploy UPLOAD_DIR = upload +ROOT_DIR ?= $(abspath ../../..)/ +include $(ROOT_DIR)orchestration_upload.mk .gcloudignore: cp -r $(ROOT_DIR)src/helpers/.gcloudignore . @@ -16,30 +18,19 @@ Dockerfile: $(ROOT_DIR)src/helpers/Dockerfile.template -e 's/PYTHON_VERSION/python312/g' \ $< > Dockerfile -NUM_CPUS = 1 - deploy : main.py .gcloudignore requirements.txt Dockerfile - mkdir -p $(UPLOAD_DIR) - cp -r $(ROOT_DIR)utils $(UPLOAD_DIR)/ - cp -r $(ROOT_DIR)src/helpers $(UPLOAD_DIR)/helpers - cp -r $(ROOT_DIR)src/sources $(UPLOAD_DIR)/sources - mkdir -p $(UPLOAD_DIR)/orchestration - cp $(ROOT_DIR)src/orchestration/__init__.py $(UPLOAD_DIR)/orchestration/ - cp $(ROOT_DIR)src/orchestration/_io.py $(UPLOAD_DIR)/orchestration/ - cp $(ROOT_DIR)src/_fb_types.py $(UPLOAD_DIR)/ - cp $(ROOT_DIR)src/_schemas.py $(UPLOAD_DIR)/ - cp $^ $(UPLOAD_DIR)/ + $(stage-orchestration-upload) gcloud run jobs deploy \ func-push-resolution-sets \ --project $(CLOUD_PROJECT) \ --region $(CLOUD_DEPLOY_REGION) \ --tasks 1 \ + --parallelism 1 \ --task-timeout 1h \ --memory 2Gi \ - --cpu $(NUM_CPUS) \ --max-retries 0 \ --service-account $(QUESTION_BANK_BUCKET_SERVICE_ACCOUNT) \ - --set-env-vars $(DEFAULT_CLOUD_FUNCTION_ENV_VARS),NUM_CPUS=$(NUM_CPUS) \ + --set-env-vars $(DEFAULT_CLOUD_FUNCTION_ENV_VARS) \ --source $(UPLOAD_DIR) clean : From cc5147563c505d8cecb7852cc0cb14836bc6441b Mon Sep 17 00:00:00 2001 From: Oskaril-data Date: Thu, 9 Jul 2026 12:06:55 +0300 Subject: [PATCH 5/5] fix: keep git mirror in sync on reruns and surface rejected pushes Push origin and mirrors on every run so a mirror that fell behind on an earlier partial failure catches up, while still skipping empty commits. Check push results with `raise_if_error()`, which GitPython does not raise on by default, so a rejected push now fails the job instead of being reported as success. --- src/helpers/git.py | 32 ++++++++++--------- src/orchestration/_io.py | 6 ++-- .../requirements.txt | 3 -- 3 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/helpers/git.py b/src/helpers/git.py index bdaa34b8..9944bad6 100644 --- a/src/helpers/git.py +++ b/src/helpers/git.py @@ -68,8 +68,9 @@ def clone_and_push_files( If None, attempts to load from secrets. Returns: - bool: True if a commit was pushed, False if there was nothing to commit. - Exits with status 1 if an error is encountered while pushing. + bool: True if a new commit was created, False if HEAD was unchanged. + Origin and mirrors are pushed either way. Exits with status 1 if an + error is encountered while pushing. """ if not mirrors: mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL") @@ -85,26 +86,26 @@ def clone_and_push_files( shutil.copy(source, full_destination_path, follow_symlinks=False) repo.index.add([destination]) - # Skip empty commits: nothing to push if the staged files match HEAD. - if not repo.index.diff(repo.head.commit): - logger.info(f"No changes to push to {repo_url}; skipping commit.") - os.remove(tmp_key_file_path) - shutil.rmtree(local_repo_dir, ignore_errors=True) - return False + # Skip empty commits, but always push so a lagging mirror catches up. + has_changes = bool(repo.index.diff(repo.head.commit)) + if not has_changes: + logger.info(f"No new commit for {repo_url}; syncing remotes to HEAD.") error_encountered = False author = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL) committer = Actor("ForecastBench bot", constants.BENCHMARK_EMAIL) ssh_env = {"GIT_SSH_COMMAND": f"ssh -i {tmp_key_file_path} -o StrictHostKeyChecking=no"} try: - repo.index.commit(commit_message, author=author, committer=committer) + if has_changes: + repo.index.commit(commit_message, author=author, committer=committer) origin = repo.remote(name="origin") - origin.push(env=ssh_env) + # A rejected push does not raise in GitPython; surface it explicitly. + origin.push(env=ssh_env).raise_if_error() for index, mirror_url in enumerate(mirrors): mirror = repo.create_remote(f"mirror_{index}", url=mirror_url) - mirror.push(env=ssh_env) + mirror.push(env=ssh_env).raise_if_error() repo.delete_remote(mirror.name) - logger.info(f"Pushed to {mirror_url} (mirror) with commit message: {commit_message}") + logger.info(f"Pushed to {mirror_url} (mirror)") except Exception as e: error_encountered = True message = e.message if hasattr(e, "message") else str(e) @@ -116,15 +117,16 @@ def clone_and_push_files( if error_encountered: sys.exit(1) - logger.info(f"Pushed to {repo_url} with commit message: {commit_message}") - return True + if has_changes: + logger.info(f"Pushed to {repo_url} with commit message: {commit_message}") + return has_changes def clone_commit_and_push( files: Dict[str, str], commit_message: str, ) -> None: - """Upload files files to Cloud Storage and push updates to Git. + """Push files to the git dataset repository. Args: files (Dict[str, str]): Mapping of local file paths to their git location. diff --git a/src/orchestration/_io.py b/src/orchestration/_io.py index a07c54a8..de3519ff 100644 --- a/src/orchestration/_io.py +++ b/src/orchestration/_io.py @@ -276,7 +276,7 @@ def push_all_resolution_sets() -> None: folder = "datasets/resolution_sets" blob_names = gcp.storage.list_with_prefix( bucket_name=env.PUBLIC_RELEASE_BUCKET, - prefix=folder, + prefix=f"{folder}/", ) files = {} @@ -298,13 +298,13 @@ def push_all_resolution_sets() -> None: mirrors = keys.get_secret_that_may_not_exist("HUGGING_FACE_REPO_URL") mirrors = [mirrors] if mirrors else [] - pushed = git.clone_and_push_files( + committed = git.clone_and_push_files( repo_url=keys.API_GITHUB_DATASET_REPO_URL, files=files, commit_message="resolution sets: automatic update.", mirrors=mirrors, ) - if pushed: + if committed: logger.info(f"Pushed {len(files)} resolution sets to git in a single commit.") diff --git a/src/orchestration/func_push_resolution_sets/requirements.txt b/src/orchestration/func_push_resolution_sets/requirements.txt index 4d431fb3..23e06c69 100644 --- a/src/orchestration/func_push_resolution_sets/requirements.txt +++ b/src/orchestration/func_push_resolution_sets/requirements.txt @@ -1,10 +1,7 @@ google-cloud-storage google-cloud-secret-manager pandas>=2.2.2,<3.0 -tqdm -gcsfs==2025.7.0 GitPython -scipy termcolor slack_sdk pandera