From d8b8ca9ee0764fb3e80958c0a314708b5e0317ec Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Sun, 4 Oct 2026 15:53:01 -0500 Subject: [PATCH 1/3] Checkpoint unfinished extraction reasoning default fix --- docs/ai_configuration.md | 9 ++-- tests/recipes/wrangles/test_extract.py | 19 +++---- tests/test_ai_config.py | 4 +- tests/test_openai_extract_ai.py | 70 ++++++++++++++++++++++---- tests/test_wrangles.py | 24 +++------ wrangles/ai_defaults.yml | 2 + wrangles/extract.py | 33 +++--------- wrangles/recipe_wrangles/extract.py | 6 +-- 8 files changed, 93 insertions(+), 74 deletions(-) diff --git a/docs/ai_configuration.md b/docs/ai_configuration.md index d525b165..63ece62e 100644 --- a/docs/ai_configuration.md +++ b/docs/ai_configuration.md @@ -100,8 +100,11 @@ For extraction, a saved definition's model retains its existing precedence over the caller's `model`. Both tuning and runtime defaults are resolved for that selected model before applying explicit caller overrides. Explicit reasoning takes precedence over saved reasoning, which takes precedence -over configured reasoning. Declared reasoning and verbosity enums are checked -against the requested value; unsupported values are warned about and omitted. +over configured reasoning. Extraction defaults to `none` for every model, +including unlisted and future models. Reasoning is always sent to the provider; +unsupported settings produce a provider error instead of silently applying the +provider default. Declared verbosity enums are checked against the requested +value; unsupported verbosity values are warned about and omitted. Recipe reasoning includes `max`. Saved `ReasoningEffort` remains `none|low` to preserve compatibility with existing editors that use that narrower contract. @@ -168,7 +171,7 @@ resolves configuration once per operation and uses a private transport for its individual rows. Generation remains unreleased. Its operation keeps `low` reasoning; extraction -keeps `none` where supported. Its existing direct-Python and recipe strictness +keeps `none` for every model. Its existing direct-Python and recipe strictness defaults are represented by `strict` and `recipe_strict`, respectively. Extraction uses the operation's `defaults` directly. There is no profile registry diff --git a/tests/recipes/wrangles/test_extract.py b/tests/recipes/wrangles/test_extract.py index 03c765ea..f02e0abe 100644 --- a/tests/recipes/wrangles/test_extract.py +++ b/tests/recipes/wrangles/test_extract.py @@ -4265,6 +4265,8 @@ def test_ai_o3_mini(self): wrangles: - extract.ai: model: o3-mini + reasoning: + effort: low api_key: ${OPENAI_API_KEY} timeout: 60 retries: 2 @@ -4324,13 +4326,8 @@ def test_ai_reasoning_and_verbosity(self): ]) assert matches >= 1 - def test_ai_pre_gpt5_reasoning_and_verbosity_ignored(self, caplog): - """ - Test extract.ai with a pre-gpt5 model that does not support - reasoning/verbosity. The recipe should still complete - successfully, ignoring those settings with a warning - rather than failing the request. - """ + def test_ai_pre_gpt5_reasoning_rejected_and_verbosity_ignored(self, caplog): + """The recipe reports provider errors instead of dropping reasoning.""" with caplog.at_level(logging.WARNING, logger="wrangles.extract"): df = wrangles.recipe.run( """ @@ -4354,12 +4351,8 @@ def test_ai_pre_gpt5_reasoning_and_verbosity_ignored(self, caplog): "data": ["wrench 25mm", "6m cable"], }) ) - matches = sum([ - df['length'][0] == '25mm', - df['length'][1] == '6m', - ]) - assert matches >= 1 - assert "Ignoring 'reasoning' parameter" in caplog.text + assert all('OpenAI API error' in value and 'reasoning' in value for value in df['length']) + assert "Ignoring 'reasoning' parameter" not in caplog.text assert "Ignoring 'verbosity' parameter" in caplog.text def test_ai_invalid_model_fails_recipe(self): diff --git a/tests/test_ai_config.py b/tests/test_ai_config.py index 59a6ff3c..88ed23cf 100644 --- a/tests/test_ai_config.py +++ b/tests/test_ai_config.py @@ -315,7 +315,7 @@ def test_custom_model_and_endpoint_are_preserved(monkeypatch, tmp_path): assert policy["model"] == "unlisted-model" assert policy["endpoints"]["responses"] == "https://custom.example/responses" assert policy["default_concurrency"] == 32 - assert "reasoning" not in policy + assert policy["reasoning"] == {"effort": "none"} assert "text" not in policy @@ -329,7 +329,7 @@ def test_snapshot_inherits_model_options_with_exact_overrides(monkeypatch, tmp_p } use_config(config, monkeypatch, tmp_path) policy = ai_config.resolve("extract.ai", model="gpt-6-luna-2026-09-25") - assert policy["reasoning"] == {"effort": "high"} + assert policy["reasoning"] == {"effort": "none"} assert policy["text"]["verbosity"] == "low" assert ai_config.model_capabilities(policy["model"])["reasoning_none"] is False assert ai_config.model_capabilities("gpt-6-luna-2026-09-26")["reasoning_none"] is True diff --git a/tests/test_openai_extract_ai.py b/tests/test_openai_extract_ai.py index 67d2922c..78a12718 100644 --- a/tests/test_openai_extract_ai.py +++ b/tests/test_openai_extract_ai.py @@ -71,7 +71,7 @@ def post(**kwargs): assert result == {"length": "25mm"} assert calls[0]["url"] == "https://api.openai.com/v1/responses" assert payload["model"] == "gpt-5-mini" - assert "reasoning" not in payload + assert payload["reasoning"] == {"effort": "none"} assert payload["text"]["verbosity"] == "low" assert payload["text"]["format"]["strict"] is True assert payload["store"] is True @@ -537,7 +537,7 @@ def test_field_examples_are_stable_system_content_for_legacy_protocol(monkeypatc assert "yellow grip" in stable_content -def test_extract_ai_omits_default_reasoning_for_non_reasoning_models(monkeypatch): +def test_extract_ai_sends_default_reasoning_for_non_reasoning_models(monkeypatch): calls = [] body = { "output": [ @@ -574,7 +574,7 @@ def post(**kwargs): payload = calls[0]["json"] assert result == {"fruits": ["bananas", "lemons"]} - assert "reasoning" not in payload + assert payload["reasoning"] == {"effort": "none"} assert "verbosity" not in payload["text"] @@ -777,7 +777,7 @@ def test_extract_chat_tuning_uses_saved_model_and_caller_precedence( {"reasoning": {"effort": "none"}, "verbosity": "low"}, {"reasoning_effort": "none", "verbosity": "low"}, ]) -def test_extract_chat_omits_unsupported_tuning(monkeypatch, caplog, explicit): +def test_extract_chat_sends_reasoning_and_omits_unsupported_verbosity(monkeypatch, caplog, explicit): calls = [] monkeypatch.setattr( extract._openai._requests, "post", @@ -789,9 +789,9 @@ def test_extract_chat_omits_unsupported_tuning(monkeypatch, caplog, explicit): output={"length": {"type": "string"}}, protocol="chat_completions", threads=1, cache=False, **explicit, ) == {"length": "25mm"} - assert "reasoning_effort" not in calls[0]["json"] + assert calls[0]["json"]["reasoning_effort"] == "none" assert "verbosity" not in calls[0]["json"] - assert "Ignoring reasoning effort" in caplog.text + assert "Ignoring reasoning effort" not in caplog.text assert "Ignoring 'verbosity' parameter" in caplog.text @@ -1076,7 +1076,7 @@ def test_extract_uses_selected_catalog_model_defaults( assert extract.ai("wrench 25mm", **settings) == {"length": "25mm"} payload = calls[0]["json"] assert payload["model"] == "catalog-model" - assert payload["reasoning"] == {"effort": "low" if explicit_tuning else "medium"} + assert payload["reasoning"] == {"effort": "low" if explicit_tuning else "none"} assert payload["text"]["verbosity"] == ("medium" if explicit_tuning else "high") assert payload["max_output_tokens"] == (128 if explicit_tuning else 256) finally: @@ -1100,10 +1100,10 @@ def save(): @pytest.mark.parametrize("reasoning,verbosity,expected_reasoning,expected_verbosity", [ (None, None, {"effort": "none"}, "medium"), ({"effort": "low"}, "high", {"effort": "low"}, "high"), - ({"effort": "high"}, "medium", None, "medium"), + ({"effort": "high"}, "medium", {"effort": "high"}, "medium"), (None, "low", {"effort": "none"}, None), ]) -def test_extract_checks_each_requested_enum_value( +def test_extract_sends_reasoning_and_checks_verbosity_enum( extraction_config, monkeypatch, reasoning, verbosity, expected_reasoning, expected_verbosity, ): config, save = extraction_config @@ -2227,3 +2227,55 @@ def test_legacy_chat_completions_uses_same_result_cache(monkeypatch): assert result == [{"length": "25mm"}, {"length": "25mm"}] assert len(calls) == 1 + + +@pytest.mark.parametrize("model", [ + "gpt-6-luna", "gpt-6-luna-2026-09-25", "future-model", "gpt-4o-mini", +]) +@pytest.mark.parametrize("protocol", ["responses", "chat_completions"]) +@pytest.mark.parametrize("via_recipe", [False, True]) +@pytest.mark.parametrize("reasoning", [None, {"effort": "none"}, {"effort": "low"}]) +def test_extract_reasoning_always_reaches_provider(monkeypatch, model, protocol, via_recipe, reasoning): + calls = [] + monkeypatch.setattr( + extract._openai_responses._requests, "post", + lambda **kwargs: calls.append(kwargs) or _successful_extraction_response(protocol), + ) + settings = { + "model": model, "protocol": protocol, "api_key": "key", "threads": 1, + "output": {"length": {"type": "string"}}, "cache": False, + } + if reasoning is not None: + settings["reasoning"] = reasoning + if via_recipe: + result = recipe.run( + {"wrangles": [{"extract.ai": {"input": "data", **settings}}]}, + dataframe=pd.DataFrame({"data": ["wrench 25mm"]}), + ) + assert result["length"].tolist() == ["25mm"] + else: + assert extract.ai("wrench 25mm", **settings) == {"length": "25mm"} + payload = calls[0]["json"] + expected = reasoning or {"effort": "none"} + if protocol == "responses": + assert payload["reasoning"] == expected + else: + assert payload["reasoning_effort"] == expected["effort"] + + +@pytest.mark.parametrize("reasoning", [None, {"effort": "low"}]) +def test_extract_reports_reasoning_rejection_without_fallback(monkeypatch, reasoning): + calls = [] + message = "Unsupported parameter: reasoning.effort" + def post(**kwargs): + calls.append(kwargs) + return _requests_response({"error": {"message": message}}, status_code=400) + monkeypatch.setattr(extract._openai_responses._requests, "post", post) + result = extract.ai( + "wrench 25mm", "key", model="future-model", + output={"length": {"type": "string"}}, reasoning=reasoning, + threads=1, retries=2, cache=False, + ) + assert len(calls) == 1 + assert calls[0]["json"]["reasoning"] == (reasoning or {"effort": "none"}) + assert message in result["length"] diff --git a/tests/test_wrangles.py b/tests/test_wrangles.py index 91617523..30a32581 100644 --- a/tests/test_wrangles.py +++ b/tests/test_wrangles.py @@ -456,7 +456,7 @@ def test_extract_ai_properties_list(): def test_extract_ai_pre_gpt5_model(): """ Test using python api for extract.ai - with a model older than gpt-5 (no reasoning/verbosity support) + with a model that rejects the default reasoning setting """ result = wrangles.extract.ai( "yellow square", @@ -471,22 +471,11 @@ def test_extract_ai_pre_gpt5_model(): retries=2 ) - assert ( - 'Colors' in result and - isinstance(result['Colors'], str) - ) + assert 'OpenAI API error' in result['Colors'] + assert 'reasoning' in result['Colors'] def test_extract_ai_pre_gpt5_model_explicit_reasoning_and_verbosity(caplog): - """ - Test using python api for extract.ai - with a model older than gpt-5, explicitly passing - reasoning and verbosity parameters that are only - supported by gpt-5+ models. - - These parameters are unsupported by the OpenAI API for - older models, so they should be skipped (with a warning - logged) rather than causing the request to fail. - """ + """The provider rejects unsupported reasoning; verbosity is still omitted.""" with caplog.at_level(logging.WARNING, logger="wrangles.extract"): result = wrangles.extract.ai( "yellow square", @@ -503,8 +492,9 @@ def test_extract_ai_pre_gpt5_model_explicit_reasoning_and_verbosity(caplog): retries=2 ) - assert result == {'Colors': 'yellow'} - assert "Ignoring 'reasoning' parameter" in caplog.text + assert 'OpenAI API error' in result['Colors'] + assert 'reasoning' in result['Colors'] + assert "Ignoring 'reasoning' parameter" not in caplog.text assert "Ignoring 'verbosity' parameter" in caplog.text ### Format Split Tests diff --git a/wrangles/ai_defaults.yml b/wrangles/ai_defaults.yml index 236004f2..ecc30a48 100644 --- a/wrangles/ai_defaults.yml +++ b/wrangles/ai_defaults.yml @@ -215,6 +215,8 @@ operations: provider: openai protocol: responses defaults: + reasoning: + effort: none default_concurrency: 32 request_timeout_seconds: 12 retries: 1 diff --git a/wrangles/extract.py b/wrangles/extract.py index ee90fe7a..9d3463ee 100644 --- a/wrangles/extract.py +++ b/wrangles/extract.py @@ -216,7 +216,8 @@ def ai( :param strict: (Optional) Enable structured output strict mode. Dynamic object schemas \ automatically use non-strict mode and are validated locally. :param reasoning: (Optional) Responses API reasoning options. Defaults to {"effort": "none"} \ - for models that support disabling reasoning; otherwise omitted so the provider default applies. + for every model. Explicit caller, saved-model, and configuration settings override this default. \ + Reasoning is always sent; provider incompatibility is reported by the provider. :param verbosity: (Optional) Responses API text verbosity. Defaults to "low" \ for models that support low verbosity. :param provider: (Optional) AI provider. Currently only "openai" is supported. @@ -440,22 +441,9 @@ def ai( if reasoning is not None else saved_reasoning or policy.get("reasoning", {"effort": "none"}) ) - if _openai_responses.supports_reasoning(model): - effort = configured_reasoning.get("effort") - if _openai_responses.supports_reasoning_effort(model, effort): - payload["reasoning"] = configured_reasoning - else: - _LOG.warning( - "Ignoring reasoning effort %r: not supported by model '%s'; " - "the provider's default reasoning effort will apply.", - effort, - model, - ) - elif reasoning is not None or saved_reasoning is not None: - _LOG.warning( - "Ignoring 'reasoning' parameter: not supported by model '%s'", - model, - ) + # Do not infer reasoning support from a model name or catalog. New + # models must not silently fall back to the provider's reasoning default. + payload["reasoning"] = {"effort": "none", **configured_reasoning} if _openai_responses.supports_verbosity(model, configured_verbosity): payload["text"]["verbosity"] = configured_verbosity elif verbosity is not None or "verbosity" in explicit_text: @@ -559,16 +547,7 @@ def ai( else {"effort": explicit_effort} if explicit_effort is not None else saved_reasoning or policy.get("reasoning", {}) ) - effort = configured_reasoning.get("effort") - if effort is not None: - if _openai_responses.supports_reasoning_effort(model, effort): - kwargs["reasoning_effort"] = effort - else: - _LOG.warning( - "Ignoring reasoning effort %r: not supported by model '%s'; " - "the provider's default reasoning effort will apply.", - effort, model, - ) + kwargs["reasoning_effort"] = configured_reasoning.get("effort", "none") configured_verbosity = ( verbosity if verbosity is not None else policy.get("text", {}).get("verbosity") diff --git a/wrangles/recipe_wrangles/extract.py b/wrangles/recipe_wrangles/extract.py index 21413e3b..29e713dc 100644 --- a/wrangles/recipe_wrangles/extract.py +++ b/wrangles/recipe_wrangles/extract.py @@ -618,9 +618,9 @@ def ai( reasoning: type: object description: >- - Responses API reasoning controls. Set effort for reasoning-capable - models. The configured default is none when that model supports it; - otherwise the provider default applies. + Reasoning controls. Defaults to effort none for every model. Explicit + settings override the default and are always sent to the provider; + unsupported settings produce a provider error. properties: effort: type: string From b171d0f8c3ae0eea578bcfbcce0d5bb16727eea7 Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:02:59 -0500 Subject: [PATCH 2/3] Preserve reasoning overrides with universal extraction defaults --- docs/ai_configuration.md | 9 ++- tests/conftest.py | 28 ++++++++ tests/recipes/wrangles/test_extract.py | 12 ++-- .../recipes/wrangles/test_generate_config.py | 18 +++++ tests/test_ai_config.py | 8 +-- tests/test_openai_extract_ai.py | 65 +++++++++++++++---- tests/test_wrangles.py | 6 +- wrangles/ai_defaults.yml | 6 +- 8 files changed, 121 insertions(+), 31 deletions(-) create mode 100644 tests/conftest.py diff --git a/docs/ai_configuration.md b/docs/ai_configuration.md index 63ece62e..b63aac12 100644 --- a/docs/ai_configuration.md +++ b/docs/ai_configuration.md @@ -170,8 +170,13 @@ remains available through `extract.ai(protocol="chat_completions")`. Extraction resolves configuration once per operation and uses a private transport for its individual rows. -Generation remains unreleased. Its operation keeps `low` reasoning; extraction -keeps `none` for every model. Its existing direct-Python and recipe strictness +The packaged OpenAI model reasoning defaults are `none`. Extraction and +generation inherit those model settings without an operation-level reasoning +override, so configured model defaults remain effective. Extraction also sends +`none` when an unlisted model has no configured effort. Explicit caller and +saved-definition settings retain their precedence. + +Generation remains unreleased. Its existing direct-Python and recipe strictness defaults are represented by `strict` and `recipe_strict`, respectively. Extraction uses the operation's `defaults` directly. There is no profile registry diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..1e034376 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,28 @@ +"""Shared fixtures for integration tests.""" +from copy import deepcopy + +import pytest +from wrangles import ai_config, data + + +@pytest.fixture +def saved_extract_schema_model(monkeypatch): + """Fetch the real saved schema but select the configured test model locally. + + These tests exercise saved definitions and output shapes, not the legacy + model choice stored in shared test data. Do not write back to that data. + """ + fetch_model_content = data.model_content + + def model_content(*args, **kwargs): + content = deepcopy(fetch_model_content(*args, **kwargs)) + settings = content.setdefault("Settings", {}) + # Remove legacy aliases so an old model cannot win during normalization. + for key in list(settings): + normalized = "".join(char.lower() for char in key if char.isalnum()) + if normalized in {"gptmodel", "aimodel", "model"}: + del settings[key] + settings["GPTModel"] = ai_config.resolve("extract.ai", role="test")["model"] + return content + + monkeypatch.setattr(data, "model_content", model_content) diff --git a/tests/recipes/wrangles/test_extract.py b/tests/recipes/wrangles/test_extract.py index f02e0abe..4551d333 100644 --- a/tests/recipes/wrangles/test_extract.py +++ b/tests/recipes/wrangles/test_extract.py @@ -4411,7 +4411,7 @@ def test_ai_legacy_chat_completions_endpoint(self): ]) assert matches >= 1 - def test_model_id(self): + def test_model_id(self, saved_extract_schema_model): """ Test using extract.ai with a saved model """ @@ -4437,7 +4437,7 @@ def test_model_id(self): ('square' in df['Shapes'].values or 'circle' in df['Shapes'].values or 'diamond' in df['Shapes'].values) ) - def test_model_id_additional_properties(self): + def test_model_id_additional_properties(self, saved_extract_schema_model): """ Test non-explicitly passed properties, i.e. kwargs This is set to specify a type of integer for the items in the array @@ -4458,7 +4458,7 @@ def test_model_id_additional_properties(self): ) assert 3 in df['Numbers'][0] or 2 in df['Numbers'][0] - def test_model_id_named_output_single_column(self): + def test_model_id_named_output_single_column(self, saved_extract_schema_model): """ Test using a predefined model that specifies one output for all extracted data @@ -4486,7 +4486,7 @@ def test_model_id_named_output_single_column(self): 'Colors' in df['result'][0] ) - def test_model_id_named_output_multi_column(self): + def test_model_id_named_output_multi_column(self, saved_extract_schema_model): """ Test using a predefined model that specifies one output per model row @@ -4516,7 +4516,7 @@ def test_model_id_named_output_multi_column(self): ('square' in df['Shapes'].values or 'circle' in df['Shapes'].values) ) - def test_model_id_object_with_properties(self): + def test_model_id_object_with_properties(self, saved_extract_schema_model): """ Test a model_id that contains an output of type object that contains properties defined as JSON @@ -4538,7 +4538,7 @@ def test_model_id_object_with_properties(self): ) assert 'unit' in df['attributes'][0] and 'value' in df['attributes'][1] - def test_model_id_array_of_objects(self): + def test_model_id_array_of_objects(self, saved_extract_schema_model): """ Test a model_id that contains an output of type array that contains items as objects with properties defined as JSON diff --git a/tests/recipes/wrangles/test_generate_config.py b/tests/recipes/wrangles/test_generate_config.py index 09b696eb..83199298 100644 --- a/tests/recipes/wrangles/test_generate_config.py +++ b/tests/recipes/wrangles/test_generate_config.py @@ -49,6 +49,24 @@ def call_openai(api_key, payload, url, timeout, retries, previous_response_id=No return calls +@pytest.mark.parametrize("via_recipe", [False, True]) +@pytest.mark.parametrize("model", ["gpt-6-luna", "gpt-6-sol"]) +@pytest.mark.parametrize("configured_effort", [None, "high"]) +def test_generate_preserves_packaged_and_configured_model_reasoning( + generate_config, generation_calls, via_recipe, model, configured_effort +): + config, install = generate_config + if configured_effort is not None: + config["providers"]["openai"]["models"][model]["defaults"]["reasoning"] = { + "effort": configured_effort, + } + install() + _run_generate(via_recipe, model=model) + assert len(generation_calls) == 2 + for payload, _ in generation_calls: + assert payload["reasoning"] == {"effort": configured_effort or "none"} + + @pytest.mark.parametrize("via_recipe", [False, True]) @pytest.mark.parametrize("explicit_model", [None, "explicit-model"]) @pytest.mark.parametrize("configured_model", [None, "configured-model"]) diff --git a/tests/test_ai_config.py b/tests/test_ai_config.py index 88ed23cf..e17a7e2d 100644 --- a/tests/test_ai_config.py +++ b/tests/test_ai_config.py @@ -33,7 +33,7 @@ def test_packaged_operation_defaults_and_model_lifecycle(): generation = ai_config.resolve("generate.ai") assert extraction["model"] == generation["model"] == "gpt-6-luna" assert extraction["reasoning"] == {"effort": "none"} - assert generation["reasoning"] == {"effort": "low"} + assert generation["reasoning"] == {"effort": "none"} assert generation["strict"] is True assert generation["recipe_strict"] is False assert extraction["request_timeout_seconds"] == 12 @@ -315,7 +315,7 @@ def test_custom_model_and_endpoint_are_preserved(monkeypatch, tmp_path): assert policy["model"] == "unlisted-model" assert policy["endpoints"]["responses"] == "https://custom.example/responses" assert policy["default_concurrency"] == 32 - assert policy["reasoning"] == {"effort": "none"} + assert "reasoning" not in policy assert "text" not in policy @@ -329,7 +329,7 @@ def test_snapshot_inherits_model_options_with_exact_overrides(monkeypatch, tmp_p } use_config(config, monkeypatch, tmp_path) policy = ai_config.resolve("extract.ai", model="gpt-6-luna-2026-09-25") - assert policy["reasoning"] == {"effort": "none"} + assert policy["reasoning"] == {"effort": "high"} assert policy["text"]["verbosity"] == "low" assert ai_config.model_capabilities(policy["model"])["reasoning_none"] is False assert ai_config.model_capabilities("gpt-6-luna-2026-09-26")["reasoning_none"] is True @@ -407,7 +407,7 @@ def test_inherited_snapshot_defaults_are_validated(monkeypatch, tmp_path): (lambda c: c["providers"]["openai"]["models"]["gpt-6-luna"]["supported_values"].update({"reasoning.effort": True}), "list of enum values"), (lambda c: c["providers"]["openai"]["models"]["gpt-6-luna"]["supported_values"].update({"reasoning.effort": [True]}), "list of enum values"), (lambda c: c["providers"]["openai"]["models"]["gpt-6-luna"]["defaults"]["reasoning"].update(effort="invalid"), "must be one of"), - (lambda c: c["operations"]["generate.ai"]["defaults"]["reasoning"].update(effort="invalid"), "must be one of"), + (lambda c: c["operations"]["generate.ai"]["defaults"].update(reasoning={"effort": "invalid"}), "must be one of"), (lambda c: c["operations"]["generate.ai"]["defaults"].update(reasoning="low"), "reasoning must be an object"), (lambda c: c["operations"]["extract.ai"]["defaults"].update(default_concurrency=True), "must be an integer"), (lambda c: c["operations"]["extract.ai"]["defaults"].update(request_timeout_seconds=float("nan")), "positive finite"), diff --git a/tests/test_openai_extract_ai.py b/tests/test_openai_extract_ai.py index 78a12718..d000c0c3 100644 --- a/tests/test_openai_extract_ai.py +++ b/tests/test_openai_extract_ai.py @@ -1076,7 +1076,7 @@ def test_extract_uses_selected_catalog_model_defaults( assert extract.ai("wrench 25mm", **settings) == {"length": "25mm"} payload = calls[0]["json"] assert payload["model"] == "catalog-model" - assert payload["reasoning"] == {"effort": "low" if explicit_tuning else "none"} + assert payload["reasoning"] == {"effort": "low" if explicit_tuning else "medium"} assert payload["text"]["verbosity"] == ("medium" if explicit_tuning else "high") assert payload["max_output_tokens"] == (128 if explicit_tuning else 256) finally: @@ -2230,11 +2230,11 @@ def test_legacy_chat_completions_uses_same_result_cache(monkeypatch): @pytest.mark.parametrize("model", [ - "gpt-6-luna", "gpt-6-luna-2026-09-25", "future-model", "gpt-4o-mini", + "gpt-6-luna", "gpt-6-sol", "gpt-6-luna-2026-09-25", "future-model", "gpt-4o-mini", ]) @pytest.mark.parametrize("protocol", ["responses", "chat_completions"]) @pytest.mark.parametrize("via_recipe", [False, True]) -@pytest.mark.parametrize("reasoning", [None, {"effort": "none"}, {"effort": "low"}]) +@pytest.mark.parametrize("reasoning", [None, {}, {"effort": "none"}, {"effort": "low"}]) def test_extract_reasoning_always_reaches_provider(monkeypatch, model, protocol, via_recipe, reasoning): calls = [] monkeypatch.setattr( @@ -2264,18 +2264,61 @@ def test_extract_reasoning_always_reaches_provider(monkeypatch, model, protocol, @pytest.mark.parametrize("reasoning", [None, {"effort": "low"}]) -def test_extract_reports_reasoning_rejection_without_fallback(monkeypatch, reasoning): +@pytest.mark.parametrize("protocol", ["responses", "chat_completions"]) +@pytest.mark.parametrize("via_recipe", [False, True]) +def test_extract_reports_reasoning_rejection_without_fallback(monkeypatch, reasoning, protocol, via_recipe): calls = [] - message = "Unsupported parameter: reasoning.effort" + parameter = "reasoning.effort" if protocol == "responses" else "reasoning_effort" + message = f"Unsupported parameter: {parameter}" def post(**kwargs): calls.append(kwargs) return _requests_response({"error": {"message": message}}, status_code=400) monkeypatch.setattr(extract._openai_responses._requests, "post", post) - result = extract.ai( - "wrench 25mm", "key", model="future-model", - output={"length": {"type": "string"}}, reasoning=reasoning, - threads=1, retries=2, cache=False, - ) + settings = { + "api_key": "key", "model": "future-model", "protocol": protocol, + "output": {"length": {"type": "string"}}, + "threads": 1, "retries": 2, "cache": False, + } + if reasoning is not None: + settings["reasoning"] = reasoning + if via_recipe: + result = recipe.run( + {"wrangles": [{"extract.ai": {"input": "data", **settings}}]}, + dataframe=pd.DataFrame({"data": ["wrench 25mm"]}), + ).iloc[0].to_dict() + else: + result = extract.ai("wrench 25mm", **settings) assert len(calls) == 1 - assert calls[0]["json"]["reasoning"] == (reasoning or {"effort": "none"}) + expected = reasoning or {"effort": "none"} + if protocol == "responses": + assert calls[0]["json"]["reasoning"] == expected + else: + assert calls[0]["json"]["reasoning_effort"] == expected["effort"] assert message in result["length"] + + +@pytest.mark.parametrize("model_key", ["GPTModel", "AIModel", "Model"]) +def test_saved_schema_fixture_preserves_shared_definition(monkeypatch, request, model_key): + shared = { + "Settings": {model_key: "gpt-4o-mini", "AdditionalMessages": "Use source units."}, + "Columns": ["Find", "Type", "Description"], + "Data": [["length", "string", "Length in source units"]], + } + calls = [] + monkeypatch.setattr(extract._data, "model_content", lambda model_id: shared) + request.getfixturevalue("saved_extract_schema_model") + monkeypatch.setattr( + extract._openai_responses._requests, "post", + lambda **kwargs: calls.append(kwargs) or _successful_extraction_response(), + ) + assert extract.ai( + "wrench 25mm", "key", model_id="shared-test-model", threads=1, cache=False, + ) == {"length": "25mm"} + payload = calls[0]["json"] + assert payload["model"] == ai_config.resolve("extract.ai", role="test")["model"] + assert payload["reasoning"] == {"effort": "none"} + assert payload["text"]["format"]["schema"]["properties"]["length"]["description"] == "Length in source units" + assert "Use source units." in payload["instructions"] + assert shared["Settings"] == {model_key: "gpt-4o-mini", "AdditionalMessages": "Use source units."} + assert shared["Columns"] == ["Find", "Type", "Description"] + assert shared["Data"] == [["length", "string", "Length in source units"]] diff --git a/tests/test_wrangles.py b/tests/test_wrangles.py index 30a32581..dd294863 100644 --- a/tests/test_wrangles.py +++ b/tests/test_wrangles.py @@ -346,7 +346,7 @@ def test_embedding_list(): assert len(result[0]) == 1536 assert [round(float(x), 3) for x in result[0][:3]] == [0.007, -0.045, 0.025] -def test_extract_ai_model_id(): +def test_extract_ai_model_id(saved_extract_schema_model): """ Test using python api for extract.ai using a pre-created model_id @@ -363,7 +363,7 @@ def test_extract_ai_model_id(): isinstance(results['Colors'], list) ) -def test_extract_ai_model_id_list(): +def test_extract_ai_model_id_list(saved_extract_schema_model): """ Test using python api for extract.ai using a pre-created model_id with a list @@ -922,4 +922,4 @@ def test_compare_overlap_exact_match_custom(): Test compare.overlap with exact_match parameter """ result = wrangles.compare.overlap([['test', 'test']], exact_match='MATCH') - assert result == ['MATCH'] \ No newline at end of file + assert result == ['MATCH'] diff --git a/wrangles/ai_defaults.yml b/wrangles/ai_defaults.yml index ecc30a48..fb8247fc 100644 --- a/wrangles/ai_defaults.yml +++ b/wrangles/ai_defaults.yml @@ -31,7 +31,7 @@ providers: default_for: [] defaults: reasoning: - effort: low + effort: none text: verbosity: low supported_values: @@ -215,8 +215,6 @@ operations: provider: openai protocol: responses defaults: - reasoning: - effort: none default_concurrency: 32 request_timeout_seconds: 12 retries: 1 @@ -247,8 +245,6 @@ operations: retries: 1 strict: true recipe_strict: false - reasoning: - effort: low embeddings: provider: openai protocol: embeddings From f7d901359337bcb81f1cc0860fc6c6497c26faf2 Mon Sep 17 00:00:00 2001 From: Eric Hills <53243273+ebhills@users.noreply.github.com> Date: Fri, 9 Oct 2026 20:23:48 -0500 Subject: [PATCH 3/3] Accept scalar reasoning in extraction recipes and Python calls --- docs/ai_configuration.md | 5 ++ docs/extract_ai_user_guide.md | 14 ++-- tests/recipes/wrangles/test_main.py | 4 +- tests/test_openai_extract_ai.py | 110 +++++++++++++++++++++++++++- wrangles/extract.py | 14 +++- wrangles/recipe_wrangles/extract.py | 27 +++++-- 6 files changed, 151 insertions(+), 23 deletions(-) diff --git a/docs/ai_configuration.md b/docs/ai_configuration.md index b63aac12..54bc22a3 100644 --- a/docs/ai_configuration.md +++ b/docs/ai_configuration.md @@ -99,6 +99,11 @@ keeps `0.2`. Unset temperature uses the provider's default (`1.0` for Gemini 3). For extraction, a saved definition's model retains its existing precedence over the caller's `model`. Both tuning and runtime defaults are resolved for that selected model before applying explicit caller overrides. +Extraction accepts the simple recipe form `reasoning: none` and the equivalent +nested form `reasoning: {effort: none}`. Direct Python calls likewise accept +`reasoning="none"` or `reasoning={"effort": "none"}`. A scalar is normalized to +the object form before resolving overrides; nested objects retain their other +reasoning options. Explicit reasoning takes precedence over saved reasoning, which takes precedence over configured reasoning. Extraction defaults to `none` for every model, including unlisted and future models. Reasoning is always sent to the provider; diff --git a/docs/extract_ai_user_guide.md b/docs/extract_ai_user_guide.md index be0db405..3a0721fb 100644 --- a/docs/extract_ai_user_guide.md +++ b/docs/extract_ai_user_guide.md @@ -17,10 +17,13 @@ settings such as concurrency, cache, and the base prompt belong to the operation See [AI model configuration](ai_configuration.md) for the schema, caller coverage, override precedence, test-role selection, and version-1 compatibility. -The public parameters remain `reasoning: {effort: ...}` and -`verbosity: low | medium | high`. Supported values in the catalog describe the -model; they are separate from the value requested by a recipe. Existing saved -model validation and legacy model-family compatibility behavior are preserved. +Use `reasoning: none` or `reasoning: low` for a simple effort setting. The nested +form, such as `reasoning: {effort: none}`, remains supported and can include other +reasoning options. Both forms have the same precedence over saved and configured +defaults. Extraction always sends the effective reasoning setting; unsupported +settings produce a provider error. `verbosity: low | medium | high` remains +separate. Supported values in the catalog describe the model; they are separate +from the value requested by a recipe. Existing saved-model validation is preserved. For each model upgrade, verify capabilities against [OpenAI's model documentation](https://developers.openai.com/api/docs/models) @@ -336,8 +339,7 @@ wrangles: - Title - Technical Data api_key: ${OPENAI_API_KEY} - reasoning: - effort: low + reasoning: low instructions: - Extract values supported by the supplied product information. - Return null when a top-level attribute is absent. diff --git a/tests/recipes/wrangles/test_main.py b/tests/recipes/wrangles/test_main.py index 9842ffa7..b84a7263 100644 --- a/tests/recipes/wrangles/test_main.py +++ b/tests/recipes/wrangles/test_main.py @@ -9374,6 +9374,7 @@ def test_all_wrangle_docstrings_parse_as_yaml(self): def test_extract_ai_schema_documents_public_parameters_and_example_shape(self): import inspect + import jsonschema import yaml method = wrangles.recipe._recipe_wrangles.extract.ai @@ -9435,7 +9436,8 @@ def test_extract_ai_schema_documents_public_parameters_and_example_shape(self): definition["description"].strip() for definition in field_schema["properties"].values() ) - assert "effort" in properties["reasoning"]["properties"] + jsonschema.validate("none", properties["reasoning"]) + jsonschema.validate({"effort": "none"}, properties["reasoning"]) def test_extract_codes_schema_matches_microservice_params(self): import yaml diff --git a/tests/test_openai_extract_ai.py b/tests/test_openai_extract_ai.py index d000c0c3..597ca27c 100644 --- a/tests/test_openai_extract_ai.py +++ b/tests/test_openai_extract_ai.py @@ -732,6 +732,7 @@ def test_extract_chat_sends_catalog_tuning_defaults(monkeypatch, via_recipe, inf ("low", {"reasoning": {"effort": "high"}, "verbosity": "high"}, "high", "high"), ("low", {"reasoning_effort": "medium"}, "medium", "low"), ("low", {"reasoning": {"effort": "none"}, "reasoning_effort": "high"}, "none", "low"), + ("low", {"reasoning": "none", "reasoning_effort": "high"}, "none", "low"), ]) def test_extract_chat_tuning_uses_saved_model_and_caller_precedence( extraction_config, monkeypatch, saved_effort, explicit, expected_effort, expected_verbosity, @@ -1158,15 +1159,116 @@ def test_extract_rejects_invalid_text_and_schema_override(extraction_config, mon assert calls == [] -def test_extract_recipe_schema_accepts_max_reasoning(): +@pytest.mark.parametrize("effort", ["none", "minimal", "low", "medium", "high", "xhigh", "max"]) +def test_extract_recipe_schema_accepts_flat_and_nested_reasoning(effort): import jsonschema import yaml schema = yaml.safe_load(recipe._recipe_wrangles.extract.ai.__doc__) - jsonschema.validate({"effort": "max"}, schema["properties"]["reasoning"]) + jsonschema.validate(effort, schema["properties"]["reasoning"]) + jsonschema.validate({"effort": effort}, schema["properties"]["reasoning"]) assert extract._openai_responses.supports_reasoning_effort("gpt-6-luna", "max") +def test_generated_recipe_schema_preserves_reasoning_forms(tmp_path, monkeypatch): + import runpy + from pathlib import Path + import jsonschema + import yaml + + root = Path(__file__).resolve().parents[1] + (tmp_path / "recipe_base_schema.json").write_text( + (root / "schema/recipe_base_schema.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + monkeypatch.chdir(tmp_path) + runpy.run_path(str(root / "schema/generate_recipe_schema.py")) + schema = json.loads((tmp_path / "schema.json").read_text(encoding="utf-8")) + validator = jsonschema.Draft7Validator(schema) + settings = {"api_key": "test-key", "input": "data", "output": "length"} + for value in ("none", "low", {"effort": "none"}, {"effort": "low", "summary": "auto"}): + document = {"wrangles": [{"extract.ai": {**settings, "reasoning": value}}]} + validator.validate(yaml.safe_load(yaml.safe_dump(document))) + for value in (True, 1, [], "invalid", {"effort": "invalid"}): + with pytest.raises(jsonschema.ValidationError) as error: + validator.validate({"wrangles": [{"extract.ai": {**settings, "reasoning": value}}]}) + assert list(error.value.path) == ["wrangles", 0, "extract.ai", "reasoning"] + + +@pytest.mark.parametrize("protocol", ["responses", "chat_completions"]) +@pytest.mark.parametrize("via_recipe", [False, True]) +@pytest.mark.parametrize("effort", ["none", "low", "max"]) +@pytest.mark.parametrize("saved_effort", [None, "low"]) +def test_extract_flat_and_nested_reasoning_have_identical_precedence( + extraction_config, monkeypatch, protocol, via_recipe, effort, saved_effort, +): + config, save = extraction_config + config["operations"]["extract.ai"]["defaults"]["reasoning"] = {"effort": "high"} + save() + saved_settings = {"GPTModel": "gpt-6-luna"} + if saved_effort is not None: + saved_settings["ReasoningEffort"] = saved_effort + monkeypatch.setattr(extract._data, "model_content", lambda model_id: { + "Settings": saved_settings, + "Columns": ["Find", "Type"], "Data": [["length", "string"]], + }) + calls = [] + monkeypatch.setattr( + extract._openai_responses._requests, "post", + lambda **kwargs: calls.append(kwargs) or _successful_extraction_response(protocol), + ) + for reasoning in (effort, {"effort": effort}): + if via_recipe: + reasoning_yaml = ( + f"reasoning: {effort}" if isinstance(reasoning, str) + else f"reasoning:\n effort: {effort}" + ) + result = recipe.run( + f"""wrangles: +- extract.ai: + input: data + model_id: saved-definition + api_key: key + protocol: {protocol} + threads: 1 + cache: false + {reasoning_yaml} +""", + dataframe=pd.DataFrame({"data": ["wrench 25mm"]}), + ) + assert result["length"].tolist() == ["25mm"] + else: + assert extract.ai( + "wrench 25mm", "key", model_id="saved-definition", protocol=protocol, + reasoning=reasoning, threads=1, cache=False, + ) == {"length": "25mm"} + assert len(calls) == 2 + assert calls[0]["json"] == calls[1]["json"] + payload = calls[0]["json"] + if protocol == "responses": + assert payload["reasoning"] == {"effort": effort} + else: + assert payload["reasoning_effort"] == effort + + +@pytest.mark.parametrize("reasoning", [True, 1, []]) +@pytest.mark.parametrize("via_recipe", [False, True]) +def test_extract_rejects_invalid_reasoning_before_saved_model_lookup(monkeypatch, reasoning, via_recipe): + def unexpected_call(*args, **kwargs): + pytest.fail("Invalid reasoning must fail before a model lookup or provider call") + monkeypatch.setattr(extract._data, "model_content", unexpected_call) + monkeypatch.setattr(extract._openai_responses._requests, "post", unexpected_call) + settings = {"api_key": "key", "model_id": "saved-definition", "reasoning": reasoning} + with pytest.raises(ValueError, match="reasoning must be an effort string .* or an object"): + if via_recipe: + recipe.run( + {"wrangles": [{"extract.ai": {"input": "data", **settings}}]}, + dataframe=pd.DataFrame({"data": ["wrench 25mm"]}), + ) + else: + extract.ai("wrench 25mm", **settings) + + @pytest.mark.parametrize("protocol", ["responses", "chat_completions"]) @pytest.mark.parametrize("explicit", [False, True]) def test_extract_saved_model_resolves_runtime_defaults_and_warns_once( @@ -2263,7 +2365,7 @@ def test_extract_reasoning_always_reaches_provider(monkeypatch, model, protocol, assert payload["reasoning_effort"] == expected["effort"] -@pytest.mark.parametrize("reasoning", [None, {"effort": "low"}]) +@pytest.mark.parametrize("reasoning", [None, "low", {"effort": "low"}]) @pytest.mark.parametrize("protocol", ["responses", "chat_completions"]) @pytest.mark.parametrize("via_recipe", [False, True]) def test_extract_reports_reasoning_rejection_without_fallback(monkeypatch, reasoning, protocol, via_recipe): @@ -2289,7 +2391,7 @@ def post(**kwargs): else: result = extract.ai("wrench 25mm", **settings) assert len(calls) == 1 - expected = reasoning or {"effort": "none"} + expected = {"effort": reasoning} if isinstance(reasoning, str) else reasoning or {"effort": "none"} if protocol == "responses": assert calls[0]["json"]["reasoning"] == expected else: diff --git a/wrangles/extract.py b/wrangles/extract.py index 9d3463ee..efd3a01d 100644 --- a/wrangles/extract.py +++ b/wrangles/extract.py @@ -172,7 +172,7 @@ def ai( record_examples: _Union[dict, list] = None, url: str = None, strict: bool = None, - reasoning: dict = None, + reasoning: _Union[str, dict] = None, verbosity: str = None, provider: str = None, protocol: str = None, @@ -215,8 +215,9 @@ def ai( :param url: (Optional) Override the configured endpoint. :param strict: (Optional) Enable structured output strict mode. Dynamic object schemas \ automatically use non-strict mode and are validated locally. - :param reasoning: (Optional) Responses API reasoning options. Defaults to {"effort": "none"} \ - for every model. Explicit caller, saved-model, and configuration settings override this default. \ + :param reasoning: (Optional) Reasoning effort, such as "none" or "low", or an options object \ + such as {"effort": "none"}. Defaults to "none" for every model. \ + Explicit caller, saved-model, and configuration settings override this default. \ Reasoning is always sent; provider incompatibility is reported by the provider. :param verbosity: (Optional) Responses API text verbosity. Defaults to "low" \ for models that support low verbosity. @@ -279,8 +280,13 @@ def ai( raise ValueError("store must be true or false.") if verbosity is not None and verbosity not in {"low", "medium", "high"}: raise ValueError("verbosity must be 'low', 'medium', or 'high'.") + if isinstance(reasoning, str): + reasoning = {"effort": reasoning} if reasoning is not None and not isinstance(reasoning, dict): - raise ValueError("reasoning must be an object such as {'effort': 'none'}.") + raise ValueError( + "reasoning must be an effort string such as 'none' " + "or an object such as {'effort': 'none'}." + ) # Reject invalid explicit options before any saved-definition lookup. # Omitted values are resolved after the effective model is known. _validate_ai_runtime_settings( diff --git a/wrangles/recipe_wrangles/extract.py b/wrangles/recipe_wrangles/extract.py index 29e713dc..adc72e0b 100644 --- a/wrangles/recipe_wrangles/extract.py +++ b/wrangles/recipe_wrangles/extract.py @@ -616,15 +616,13 @@ def ai( type: string description: Separator used only when output_format is concatenate. Defaults to comma-space. reasoning: - type: object description: >- - Reasoning controls. Defaults to effort none for every model. Explicit - settings override the default and are always sent to the provider; - unsupported settings produce a provider error. - properties: - effort: - type: string - description: Amount of reasoning work requested from a compatible model. + Reasoning effort, such as none or low, or an object with effort and + other reasoning options. Both forms are equivalent for setting effort. + Defaults to none for every model. Explicit settings are always sent to + the provider; unsupported settings produce a provider error. + oneOf: + - type: string enum: - none - minimal @@ -633,6 +631,19 @@ def ai( - high - xhigh - max + - type: object + properties: + effort: + type: string + description: Amount of reasoning work requested from a compatible model. + enum: + - none + - minimal + - low + - medium + - high + - xhigh + - max verbosity: type: string description: >-