From 067519ab26b1d9b64b0b751cdd614c59778d69fa Mon Sep 17 00:00:00 2001 From: drozado Date: Thu, 2 Jul 2026 17:47:01 +1200 Subject: [PATCH] fix(metadata): give gpt-5-mini reasoning headroom for tagging and validation `gpt-5-mini` is a reasoning model, so a 50-token cap with default (medium) reasoning returns `status=incomplete (max_output_tokens)` and every question silently fell back to `category="Other"` / `valid_question=True`. Set reasoning effort to `minimal` and raise the output budgets so tagging and validation complete. --- src/helpers/metadata_llm.py | 1 + src/metadata/tag_questions/main.py | 2 +- src/metadata/validate_questions/main.py | 2 +- src/tests/test_model_request_params.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/helpers/metadata_llm.py b/src/helpers/metadata_llm.py index 867e325f..7eb6f3da 100644 --- a/src/helpers/metadata_llm.py +++ b/src/helpers/metadata_llm.py @@ -32,5 +32,6 @@ def get_metadata_model_response(prompt: str, max_output_tokens: int) -> str: return _get_metadata_model_run().get_response( prompt=prompt, max_output_tokens=max_output_tokens, + reasoning={"effort": "minimal"}, safety_identifier=get_openai_safety_identifier(), ) diff --git a/src/metadata/tag_questions/main.py b/src/metadata/tag_questions/main.py index e661cdd3..16df9aed 100644 --- a/src/metadata/tag_questions/main.py +++ b/src/metadata/tag_questions/main.py @@ -34,7 +34,7 @@ async def _get_category_single(index, row, semaphore): response = await asyncio.to_thread( metadata_llm.get_metadata_model_response, prompt=prompt, - max_output_tokens=50, + max_output_tokens=512, ) category = response.strip('"').strip("'").strip(" ").strip(".") logger.info( diff --git a/src/metadata/validate_questions/main.py b/src/metadata/validate_questions/main.py index 224d62b1..c1035828 100644 --- a/src/metadata/validate_questions/main.py +++ b/src/metadata/validate_questions/main.py @@ -32,7 +32,7 @@ async def _validate_single_question(index, question, semaphore): response = await asyncio.to_thread( metadata_llm.get_metadata_model_response, prompt=prompt, - max_output_tokens=500, + max_output_tokens=1024, ) if "Classification:" not in response: logger.error(f"'Classification:' not in response for: {question}") diff --git a/src/tests/test_model_request_params.py b/src/tests/test_model_request_params.py index 5a8ba43f..e9d89b6b 100644 --- a/src/tests/test_model_request_params.py +++ b/src/tests/test_model_request_params.py @@ -138,6 +138,7 @@ def fake_get_model_run(model_run_key): "Classify this question.", { "max_output_tokens": 123, + "reasoning": {"effort": "minimal"}, "safety_identifier": "forecastbench-safety-id", }, ),