From 9f315d818120fd34345343ecd01d75e2c00dda85 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 25 Jun 2026 21:33:24 +0300 Subject: [PATCH 1/7] feat: make keys lazy on load --- src/helpers/keys.py | 57 +++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/src/helpers/keys.py b/src/helpers/keys.py index 0e9c162a..1df1dfd6 100644 --- a/src/helpers/keys.py +++ b/src/helpers/keys.py @@ -1,9 +1,39 @@ -"""utils for key-related tasks in llm-benchmark.""" +"""utils for key-related tasks in llm-benchmark. + +Secrets are resolved lazily on first attribute access (PEP 562 module ``__getattr__``) +and memoized. This ensures that merely *importing* this module performs no network/Secret Manager call. +""" from google.cloud import secretmanager from . import env +# Public attribute name -> Secret Manager secret name. The two differ only for GOOGLE/GEMINI. +_SECRET_NAMES = { + # LLM + "API_KEY_ANTHROPIC": "API_KEY_ANTHROPIC", + "API_KEY_OPENAI": "API_KEY_OPENAI", + "API_KEY_TOGETHERAI": "API_KEY_TOGETHERAI", + "API_KEY_GOOGLE": "API_KEY_GEMINI", + "API_KEY_MISTRAL": "API_KEY_MISTRAL", + "API_KEY_XAI": "API_KEY_XAI", + # QUESTION DATASET SOURCES + "API_EMAIL_ACLED": "API_EMAIL_ACLED", + "API_PASSWORD_ACLED": "API_PASSWORD_ACLED", + "API_KEY_FRED": "API_KEY_FRED", + # QUESTION MARKET SOURCES + "API_KEY_METACULUS": "API_KEY_METACULUS", + "API_KEY_POLYMARKET": "API_KEY_POLYMARKET", + "API_KEY_INFER": "API_KEY_INFER", + # WORKFLOW BOT + "API_SLACK_BOT_NOTIFICATION": "API_SLACK_BOT_NOTIFICATION", + "API_SLACK_BOT_CHANNEL": "API_SLACK_BOT_CHANNEL", + # GITHUB + "API_GITHUB_DATASET_REPO_URL": "API_GITHUB_DATASET_REPO_URL", +} + +_cache: dict = {} + def get_secret(secret_name, version_id="latest"): """ @@ -27,19 +57,16 @@ def get_secret_that_may_not_exist(secret_name, version_id="latest"): return None -# QUESTION DATASET SOURCES -API_EMAIL_ACLED = get_secret(secret_name="API_EMAIL_ACLED") -API_PASSWORD_ACLED = get_secret(secret_name="API_PASSWORD_ACLED") -API_KEY_FRED = get_secret("API_KEY_FRED") - -# QUESTION MARKET SOURCES -API_KEY_METACULUS = get_secret(secret_name="API_KEY_METACULUS") -API_KEY_POLYMARKET = get_secret("API_KEY_POLYMARKET") -API_KEY_INFER = get_secret("API_KEY_INFER") +def __getattr__(name): + """Lazily resolve and memoize ``API_*`` secrets on first access (PEP 562).""" + secret_name = _SECRET_NAMES.get(name) + if secret_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + if name not in _cache: + _cache[name] = get_secret(secret_name) + return _cache[name] -# WORKFLOW BOT -API_SLACK_BOT_NOTIFICATION = get_secret(secret_name="API_SLACK_BOT_NOTIFICATION") -API_SLACK_BOT_CHANNEL = get_secret(secret_name="API_SLACK_BOT_CHANNEL") -# GITHUB -API_GITHUB_DATASET_REPO_URL = get_secret(secret_name="API_GITHUB_DATASET_REPO_URL") +def __dir__(): + """Expose the lazily-resolved secret names to ``dir()``/autocomplete.""" + return sorted(set(globals()) | set(_SECRET_NAMES)) From 1993f1f46cce954a779ac6040c507897abed8c05 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 25 Jun 2026 21:34:21 +0300 Subject: [PATCH 2/7] feat: make env vars lazy on load --- src/helpers/env.py | 66 ++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 22 deletions(-) diff --git a/src/helpers/env.py b/src/helpers/env.py index d3ad9100..73fa6030 100644 --- a/src/helpers/env.py +++ b/src/helpers/env.py @@ -1,25 +1,47 @@ -"""Environment variables.""" +"""Environment variables. + +Values are read from ``os.environ`` lazily on each attribute access (PEP 562 module +``__getattr__``) rather than snapshotted at import. +""" import os -PROJECT_ID = os.environ.get("CLOUD_PROJECT") -CLOUD_DEPLOY_REGION = os.environ.get("CLOUD_DEPLOY_REGION") -QUESTION_BANK_BUCKET = os.environ.get("QUESTION_BANK_BUCKET") -QUESTION_SETS_BUCKET = os.environ.get("QUESTION_SETS_BUCKET") -FORECAST_SETS_BUCKET = os.environ.get("FORECAST_SETS_BUCKET") -FORECAST_SETS_TRANSCRIPTS_BUCKET = os.environ.get("FORECAST_SETS_TRANSCRIPTS_BUCKET") -PROCESSED_FORECAST_SETS_BUCKET = os.environ.get("PROCESSED_FORECAST_SETS_BUCKET") -PUBLIC_RELEASE_BUCKET = os.environ.get("PUBLIC_RELEASE_BUCKET") -WEBSITE_BUCKET = os.environ.get("WEBSITE_BUCKET") -WEBSITE_STAGING_ASSETS_BUCKET = os.environ.get("WEBSITE_STAGING_ASSETS_BUCKET") -LLM_BASELINE_DOCKER_IMAGE_NAME = os.environ.get("LLM_BASELINE_DOCKER_IMAGE_NAME") -LLM_BASELINE_DOCKER_REPO_NAME = os.environ.get("LLM_BASELINE_DOCKER_REPO_NAME") -LLM_BASELINE_PUB_SUB_TOPIC_NAME = os.environ.get("LLM_BASELINE_PUB_SUB_TOPIC_NAME") -LLM_BASELINE_STAGING_BUCKET = os.environ.get("LLM_BASELINE_STAGING_BUCKET") -LLM_BASELINE_SERVICE_ACCOUNT = os.environ.get("LLM_BASELINE_SERVICE_ACCOUNT") -LLM_BASELINE_STAGING_BUCKET = os.environ.get("LLM_BASELINE_STAGING_BUCKET") -LLM_BASELINE_NEWS_BUCKET = os.environ.get("LLM_BASELINE_NEWS_BUCKET") -NUM_CPUS = int(os.environ.get("NUM_CPUS", 1)) -RUNNING_LOCALLY = bool(int(os.environ.get("RUNNING_LOCALLY", False))) -BUCKET_MOUNT_POINT = os.environ.get("BUCKET_MOUNT_POINT", "") -WORKSPACE_BUCKET = os.environ.get("WORKSPACE_BUCKET") +# Names read as plain strings (``None`` if unset). +_STR_VARS = { + "PROJECT_ID": "CLOUD_PROJECT", + "CLOUD_DEPLOY_REGION": "CLOUD_DEPLOY_REGION", + "QUESTION_BANK_BUCKET": "QUESTION_BANK_BUCKET", + "QUESTION_SETS_BUCKET": "QUESTION_SETS_BUCKET", + "FORECAST_SETS_BUCKET": "FORECAST_SETS_BUCKET", + "FORECAST_SETS_TRANSCRIPTS_BUCKET": "FORECAST_SETS_TRANSCRIPTS_BUCKET", + "PROCESSED_FORECAST_SETS_BUCKET": "PROCESSED_FORECAST_SETS_BUCKET", + "PUBLIC_RELEASE_BUCKET": "PUBLIC_RELEASE_BUCKET", + "WEBSITE_BUCKET": "WEBSITE_BUCKET", + "WEBSITE_STAGING_ASSETS_BUCKET": "WEBSITE_STAGING_ASSETS_BUCKET", + "LLM_BASELINE_DOCKER_IMAGE_NAME": "LLM_BASELINE_DOCKER_IMAGE_NAME", + "LLM_BASELINE_DOCKER_REPO_NAME": "LLM_BASELINE_DOCKER_REPO_NAME", + "LLM_BASELINE_PUB_SUB_TOPIC_NAME": "LLM_BASELINE_PUB_SUB_TOPIC_NAME", + "LLM_BASELINE_STAGING_BUCKET": "LLM_BASELINE_STAGING_BUCKET", + "LLM_BASELINE_SERVICE_ACCOUNT": "LLM_BASELINE_SERVICE_ACCOUNT", + "LLM_BASELINE_NEWS_BUCKET": "LLM_BASELINE_NEWS_BUCKET", + "WORKSPACE_BUCKET": "WORKSPACE_BUCKET", +} + + +def __getattr__(name): + """Read environment variables lazily on each access (PEP 562).""" + if name in _STR_VARS: + return os.environ.get(_STR_VARS[name]) + if name == "NUM_CPUS": + return int(os.environ.get("NUM_CPUS", 1)) + if name == "RUNNING_LOCALLY": + return bool(int(os.environ.get("RUNNING_LOCALLY", False))) + if name == "BUCKET_MOUNT_POINT": + return os.environ.get("BUCKET_MOUNT_POINT", "") + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + """Expose the lazily-read environment variable names to ``dir()``/autocomplete.""" + extra = {"NUM_CPUS", "RUNNING_LOCALLY", "BUCKET_MOUNT_POINT"} + return sorted(set(globals()) | set(_STR_VARS) | extra) From 4dc87ce66d576429e0a3294af889eab265570a7a Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Thu, 25 Jun 2026 21:36:36 +0300 Subject: [PATCH 3/7] feat: add random seed to `_question_level_bootstrap` to make it testable/reproducible --- src/leaderboard/main.py | 55 ++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/src/leaderboard/main.py b/src/leaderboard/main.py index 0962dc9a..c9ddba05 100644 --- a/src/leaderboard/main.py +++ b/src/leaderboard/main.py @@ -2620,12 +2620,38 @@ def score_models( return df_leaderboard, question_fixed_effects +def _question_level_bootstrap(df: pd.DataFrame, random_state=None) -> pd.DataFrame: + """Resample question_pks with replacement for one bootstrap replicate of a group. + + Args: + df (pd.DataFrame): Rows for one ``(forecast_due_date, source)`` group. + random_state: Seed / ``RandomState`` for reproducible resampling. ``None`` (the default) + draws from fresh entropy, matching the historical non-deterministic behavior. + + Returns: + pd.DataFrame: Rows for the resampled questions, with ``question_pk`` made unique per draw. + """ + questions = df["question_pk"].drop_duplicates() + questions_bs = questions.sample(frac=1, replace=True, random_state=random_state) + sample = questions_bs.to_frame(name="question_pk") + sample["draw"] = sample.groupby("question_pk").cumcount() + retval = pd.merge(sample, df, on="question_pk", how="left") + # `question_pk` must be overwritten with a unique id in case it was sampled more than once. + # This ensures that `two_way_fixed_effects()` treats each drawn question separately (instead + # of treating multiple draws as one question). + retval["question_pk"] = ( + retval["question_pk"].astype(str) + "_sim_id_" + retval["draw"].astype(str) + ) + return retval.drop(columns=["draw"]) + + @decorator.log_runtime def generate_simulated_leaderboards( df: pd.DataFrame, primary_scoring_func: Callable[[pd.DataFrame], pd.DataFrame], market_question_adjustment: MarketQuestionAdjustment, N: int = N_REPLICATES, + seed: int | None = None, ) -> pd.DataFrame: """Generate simulated leaderboards by bootstrap sampling. @@ -2636,6 +2662,9 @@ def generate_simulated_leaderboards( market_question_adjustment (MarketQuestionAdjustment): How to estimate market question effects. N (int): Number of bootstrap replicates to generate. + seed (int | None): Base seed for reproducible bootstrap draws. When set, replicate ``i`` + uses ``seed + i`` so each loky child process is deterministic independent of process + RNG state; ``None`` (the default) preserves the historical non-deterministic behavior. Returns: pd.DataFrame: Simulated scores with each column representing a replicate. @@ -2646,30 +2675,16 @@ def generate_simulated_leaderboards( df = df.copy() - def question_level_bootstrap(df: pd.DataFrame) -> pd.DataFrame: - questions = df["question_pk"].drop_duplicates() - questions_bs = questions.sample(frac=1, replace=True) - sample = questions_bs.to_frame(name="question_pk") - sample["draw"] = sample.groupby("question_pk").cumcount() - retval = pd.merge( - sample, - df, - on="question_pk", - how="left", - ) - # `question_pk` must be overwritten with a unique id in case it was sampled more than once. - # This ensures that `two_way_fixed_effects()` treats each drawn question separately (instead - # of treating multiple draws as one question). - retval["question_pk"] = ( - retval["question_pk"].astype(str) + "_sim_id_" + retval["draw"].astype(str) - ) - return retval.drop(columns=["draw"]) - def bootstrap_and_score(idx): logger.info(f"[replicate {idx+1}/{N}] starting...") + # Per-replicate RandomState so each loky child is deterministic when `seed` is set. + random_state = None if seed is None else np.random.RandomState(seed + idx) df_bs = ( df.groupby(["forecast_due_date", "source"]) - .apply(question_level_bootstrap, include_groups=False) + .apply( + lambda group: _question_level_bootstrap(group, random_state=random_state), + include_groups=False, + ) .reset_index() ) try: From d0914f15d119a90b68e3912fe6ea296f16b96ee9 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Fri, 26 Jun 2026 10:48:44 +0300 Subject: [PATCH 4/7] feat: add random sample to `human_sample_questions` to allow reproducibility --- src/curate_questions/create_question_set/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/curate_questions/create_question_set/main.py b/src/curate_questions/create_question_set/main.py index 746d6a7b..e107a72f 100644 --- a/src/curate_questions/create_question_set/main.py +++ b/src/curate_questions/create_question_set/main.py @@ -179,19 +179,24 @@ def process_questions( return processed_questions -def human_sample_questions(values: dict, n_single: int) -> pd.DataFrame: +def human_sample_questions( + values: dict, n_single: int, rng: random.Random | None = None +) -> pd.DataFrame: """Get questions for the human question set by sampling from LLM questions. Args: values (dict): Source data dict containing "dfq" DataFrame n_single (int): Number of questions to sample + rng (random.Random | None): Seeded RNG for reproducible sampling. ``None`` (the default) + uses the module-global ``random``. Returns dfq (pd.DataFrame): Randomly sampled questions """ dfq = values["dfq"].copy() indices_to_sample_from = dfq.index.tolist() - indices = random.sample(indices_to_sample_from, min(n_single, len(indices_to_sample_from))) + sampler = rng or random + indices = sampler.sample(indices_to_sample_from, min(n_single, len(indices_to_sample_from))) return dfq.loc[indices] From f50ab48ec67a5e21b82c9cde05f590d26e553e68 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 4 Jul 2026 10:57:36 +0300 Subject: [PATCH 5/7] feat: lazify fred fetch params --- src/questions/fred/fetch/main.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/src/questions/fred/fetch/main.py b/src/questions/fred/fetch/main.py index 88970734..c9cbc947 100644 --- a/src/questions/fred/fetch/main.py +++ b/src/questions/fred/fetch/main.py @@ -29,10 +29,19 @@ logger = logging.getLogger(__name__) SOURCE = "fred" -PARAMS = { - "api_key": keys.API_KEY_FRED, - "file_type": "json", -} + + +def _fred_params() -> dict: + """Build the FRED API params at call time. + + Reading ``keys.API_KEY_FRED`` here rather than at module scope keeps importing this job free of + any Secret Manager access (the offline-import contract). + """ + return { + "api_key": keys.API_KEY_FRED, + "file_type": "json", + } + # FRED throttles each API key at ~2 requests/second (120/minute); exceeding it # returns HTTP 429. Space requests out to stay safely under that limit. @@ -295,14 +304,15 @@ def fetch_all(dfq, FRED_QUESTIONS_NAMES): logger.info(f"# of combined questions: {len(combined_questions.keys())}") ids_to_delete = [] + params = _fred_params() # fetch release, series, and background for series_id in combined_questions: combined_questions[series_id]["release"] = fetch_all_releases( - PARAMS, series_id=series_id, single=True + params, series_id=series_id, single=True )[0] - combined_questions[series_id]["series"] = fetch_all_series(PARAMS, series_id=series_id) + combined_questions[series_id]["series"] = fetch_all_series(params, series_id=series_id) combined_questions[series_id]["observations"] = fetch_all_observations( - PARAMS, series_id=series_id + params, series_id=series_id ) if not combined_questions[series_id]["observations"]: ids_to_delete.append(series_id) From b677eb1d020d60db9f26cbfeab14fd3002ec7198 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 4 Jul 2026 11:10:05 +0300 Subject: [PATCH 6/7] feat: add random sample/state to `create_question_set` --- .../create_question_set/main.py | 68 ++++++++++++++----- 1 file changed, 50 insertions(+), 18 deletions(-) diff --git a/src/curate_questions/create_question_set/main.py b/src/curate_questions/create_question_set/main.py index e107a72f..73c9df11 100644 --- a/src/curate_questions/create_question_set/main.py +++ b/src/curate_questions/create_question_set/main.py @@ -15,7 +15,6 @@ import json import logging import os -import random import sys from collections.abc import Callable from copy import deepcopy @@ -23,6 +22,7 @@ from enum import Enum from fractions import Fraction +import numpy as np import pandas as pd from tqdm import tqdm from utils import gcp @@ -94,15 +94,19 @@ def process_questions( single_generation_func: Callable, show_plots: bool, question_set_target: QuestionSetTarget, + random_state=None, ) -> dict: """Sample from `questions` to get the number of questions needed. Args: questions (dict): Source questions keyed by source name, each with a "dfq" DataFrame to_questions (dict): Allocation info keyed by source with "num_questions_to_sample" - single_generation_func (Callable): Sampling function taking (values, n) and returning DataFrame + single_generation_func (Callable): Sampling function taking (values, n, random_state) and + returning a DataFrame show_plots (bool): Whether to display distribution plots question_set_target (QuestionSetTarget): Target question set ("llm" or "human") + random_state: Seed/``np.random.RandomState`` threaded to ``single_generation_func`` for + reproducibility. ``None`` (default) samples without a fixed seed. Returns processed_questions (dict): Deep copy of questions with sampled DataFrames @@ -118,7 +122,7 @@ def process_questions( df_available = values["dfq"].copy() # Sample questions for this source - values["dfq"] = single_generation_func(values, num_single) + values["dfq"] = single_generation_func(values, num_single, random_state) df_sampled = values["dfq"] num_found += len(df_sampled) @@ -179,25 +183,21 @@ def process_questions( return processed_questions -def human_sample_questions( - values: dict, n_single: int, rng: random.Random | None = None -) -> pd.DataFrame: +def human_sample_questions(values: dict, n_single: int, random_state=None) -> pd.DataFrame: """Get questions for the human question set by sampling from LLM questions. Args: values (dict): Source data dict containing "dfq" DataFrame n_single (int): Number of questions to sample - rng (random.Random | None): Seeded RNG for reproducible sampling. ``None`` (the default) - uses the module-global ``random``. + random_state: Seed/``np.random.RandomState`` for reproducible sampling (anything + ``DataFrame.sample`` accepts). ``None`` (default) samples without a fixed seed. Returns dfq (pd.DataFrame): Randomly sampled questions """ dfq = values["dfq"].copy() - indices_to_sample_from = dfq.index.tolist() - sampler = rng or random - indices = sampler.sample(indices_to_sample_from, min(n_single, len(indices_to_sample_from))) - return dfq.loc[indices] + n = min(n_single, len(dfq)) + return dfq.sample(n=n, random_state=random_state) def get_bin_label(bin_config: dict, bin_type: str) -> str: @@ -748,7 +748,21 @@ def add_line_chart( fig.show() -def stratified_sample_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFrame: +def _as_random_state(random_state): + """Normalize a seed/RandomState/None to a RandomState (or None). + + Returns ``None`` unchanged (preserving unseeded behavior) and a ``RandomState`` unchanged, but + promotes a bare int seed to a ``RandomState`` so successive ``.sample`` calls in a loop + *decorrelate* (advancing one shared generator) instead of reusing the same seed per iteration. + """ + if random_state is None or isinstance(random_state, np.random.RandomState): + return random_state + return np.random.RandomState(random_state) + + +def stratified_sample_questions( + dfq: pd.DataFrame, n_target: int, random_state=None +) -> pd.DataFrame: """Sample questions using stratified sampling to achieve target distribution. This ensures we get the desired distribution regardless of source data skew. @@ -756,10 +770,14 @@ def stratified_sample_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFram Args: dfq (pd.DataFrame): DataFrame with bin_weight column and composite bins n_target (int): Number of questions to sample + random_state: Seed/``np.random.RandomState`` for reproducible per-bin sampling (anything + ``DataFrame.sample`` accepts). ``None`` (default) samples without a fixed seed. Thread a + single ``RandomState`` instance through to decorrelate the successive per-bin draws. Returns result (pd.DataFrame): Sampled questions """ + random_state = _as_random_state(random_state) if len(dfq) == 0 or n_target == 0: return pd.DataFrame() @@ -816,7 +834,7 @@ def stratified_sample_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFram for bin_name, n_samples in bin_samples.items(): if n_samples > 0: bin_df = dfq_weighted[dfq_weighted["composite_bin"] == bin_name] - sampled = bin_df.sample(n=n_samples, replace=False) + sampled = bin_df.sample(n=n_samples, replace=False, random_state=random_state) sampled_dfs.append(sampled) if not sampled_dfs: @@ -824,7 +842,7 @@ def stratified_sample_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFram return pd.concat(sampled_dfs, ignore_index=True) -def sample_market_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFrame: +def sample_market_questions(dfq: pd.DataFrame, n_target: int, random_state=None) -> pd.DataFrame: """Sample market questions using multi-dimensional binning strategy. Ensures balanced sampling across market probability values and time horizons. @@ -832,6 +850,8 @@ def sample_market_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFrame: Args: dfq (pd.DataFrame): Market questions n_target (int): Number of questions to sample + random_state: Seed/``np.random.RandomState`` threaded to ``stratified_sample_questions`` + for reproducibility. ``None`` (default) samples without a fixed seed. Returns df_result (pd.DataFrame): Sampled questions @@ -845,6 +865,7 @@ def sample_market_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFrame: df_result = stratified_sample_questions( dfq=dfq, n_target=n_target, + random_state=random_state, ) df_result = df_result.drop( columns=[ @@ -857,7 +878,7 @@ def sample_market_questions(dfq: pd.DataFrame, n_target: int) -> pd.DataFrame: return df_result -def llm_sample_questions(values: dict, n_single: int) -> pd.DataFrame: +def llm_sample_questions(values: dict, n_single: int, random_state=None) -> pd.DataFrame: """Generate questions for the LLM question set. For market questions: Sample using binning strategy. @@ -866,23 +887,26 @@ def llm_sample_questions(values: dict, n_single: int) -> pd.DataFrame: Args: values (dict): Source data dict containing "dfq" DataFrame n_single (int): Number of questions to sample + random_state: Seed/``np.random.RandomState`` threaded to the market/category samplers for + reproducibility. ``None`` (default) samples without a fixed seed. Returns df (pd.DataFrame): Sampled questions """ dfq = values["dfq"].copy() source = dfq["source"].iloc[0] + random_state = _as_random_state(random_state) if source in question_curation.MARKET_SOURCES: # Use binning-based sampling for market questions - return sample_market_questions(dfq, n_single) + return sample_market_questions(dfq, n_single, random_state=random_state) else: # Use existing category-based sampling for data sources allocation = allocate_across_categories(num_questions=n_single, dfq=dfq) dfs = [] for key, value in allocation.items(): - dfs.append(dfq[dfq["category"] == key].sample(value)) + dfs.append(dfq[dfq["category"] == key].sample(value, random_state=random_state)) return pd.concat(dfs, ignore_index=True) @@ -1211,6 +1235,12 @@ def driver(_: None) -> None: ) HUMAN_QUESTIONS.update(human_questions_of_question_type) + # Optional reproducibility: when QUESTION_SET_SEED is set, thread a single RandomState through + # both sampling passes so the published set is deterministic. Unset (the default) preserves the + # historical unseeded behaviour. + seed_env = os.getenv("QUESTION_SET_SEED") + random_state = np.random.RandomState(int(seed_env)) if seed_env not in (None, "") else None + # Sample questions logger.info("LLM SET") LLM_QUESTIONS = process_questions( @@ -1219,6 +1249,7 @@ def driver(_: None) -> None: single_generation_func=llm_sample_questions, show_plots=env.RUNNING_LOCALLY, question_set_target=QuestionSetTarget.LLM, + random_state=random_state, ) logger.info("HUMAN SET") @@ -1228,6 +1259,7 @@ def driver(_: None) -> None: single_generation_func=human_sample_questions, show_plots=False, question_set_target=QuestionSetTarget.HUMAN, + random_state=random_state, ) write_questions(LLM_QUESTIONS, question_set_target=QuestionSetTarget.LLM) From 7de71bda356edf3ec38fd63c89200999f84e73c6 Mon Sep 17 00:00:00 2001 From: Nikolay Petrov Date: Sat, 4 Jul 2026 11:12:13 +0300 Subject: [PATCH 7/7] fix: replace `print` with `logger.debug` --- src/resolve/_prepare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/resolve/_prepare.py b/src/resolve/_prepare.py index 732d1907..253378db 100644 --- a/src/resolve/_prepare.py +++ b/src/resolve/_prepare.py @@ -119,7 +119,7 @@ def check_and_prepare_forecast_file( .query('_merge == "left_only"') .drop("_merge", axis=1) ) - print(dropped_rows) + logger.debug(f"Dropped duplicate rows:\n{dropped_rows}") msg = f"Duplicate Rows encountered in {organization} forecast file." logger.error(msg) raise ValueError(msg)