From 435fff2f114f664910a62aa41e79c1c1f2a53ae2 Mon Sep 17 00:00:00 2001 From: Austin Date: Sun, 26 Oct 2025 18:17:55 -0500 Subject: [PATCH 1/2] docs: add toolbox parity plan and validation checklist --- docs/PLAN_TOOLBOX.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/PLAN_TOOLBOX.md diff --git a/docs/PLAN_TOOLBOX.md b/docs/PLAN_TOOLBOX.md new file mode 100644 index 00000000..5fefa586 --- /dev/null +++ b/docs/PLAN_TOOLBOX.md @@ -0,0 +1,30 @@ +# Toolbox Parity Plan Stub + +## Planned Modules / Files +- `custom_components/extended_openai_conversation/conversation.py`: integrate orchestration, web search + MCP toggles. +- `custom_components/extended_openai_conversation/tools_orchestrator.py`: async loop handling for Responses vs Chat tool calls, depth caps. +- `custom_components/extended_openai_conversation/tools_builtin.py`: implement execute_service, add_automation, get_history, rest, scrape, composite, script, template. +- `custom_components/extended_openai_conversation/tools_web_search.py`: hosted web search adapter with reasoning vs chat paths. +- `custom_components/extended_openai_conversation/tools_mcp_bridge.py`: optional MCP client bridge with namespaced tools. +- `custom_components/extended_openai_conversation/config_flow.py`, `const.py`, `translations/en.json`: options UI for functions YAML + web search toggles with validation. +- `custom_components/extended_openai_conversation/openai_support.py`, `responses_adapter.py`, `model_capabilities.py`: routing hooks, schema helpers. +- `custom_components/extended_openai_conversation/manifest.json`: optional deps (e.g., `beautifulsoup4`) and metadata. +- `README.md`, `docs/PLAN_TOOLBOX.md`: documentation updates and manual test plan. + +## Key Risks & Mitigations +- Async safety / event loop blocking → use HA executor helpers, streaming HTTPX timeouts, lazy imports for heavy deps. +- Tool abuse of non-exposed entities → reuse HA exposed-entity checks and guard by integration allow-lists. +- External calls (REST/scrape/web search/MCP) hanging → enforce per-tool timeout, payload caps, structured error responses. +- MCP / optional deps missing → feature-flag, detect import errors, return user-facing warnings without failing setup. +- Responses API schema regressions → centralize payload construction and continuation handling, cover with manual tests. + +## Acceptance Tests Checklist +- [ ] Basic Assist Q&A without tools returns speech and continue flag as expected. +- [ ] `execute_service` on exposed entity executes; denied on non-exposed entity with clear message. +- [ ] `get_history` with small window returns summary/truncated data within caps. +- [ ] `rest` and `scrape` respect timeout/size limits and handle errors gracefully. +- [ ] `composite` chains at least two tools; exceeding depth limit yields safe stop message. +- [ ] Web search toggle off → no tool registration; on with reasoning model uses Responses tool continuation. +- [ ] Non-reasoning model attempts web search via Chat path; unsupported tool yields graceful degradation note. +- [ ] MCP bridge disabled → no effect; enabled with server available exposes namespaced tool and returns data. +- [ ] ConversationResult stays valid and continue_conversation logic unchanged for Assist. From 33e1e908cd04d1c88667ef93988d2a59207dfeed Mon Sep 17 00:00:00 2001 From: Austin Date: Sun, 26 Oct 2025 18:45:02 -0500 Subject: [PATCH 2/2] feat: full toolbox parity + web search + MCP bridge --- README.md | 24 +- .../config_flow.py | 91 ++++- .../extended_openai_conversation/const.py | 15 + .../conversation.py | 327 +++++++++++++++--- .../manifest.json | 5 +- .../extended_openai_conversation/strings.json | 8 + .../tools_builtin.py | 234 +++++++++++++ .../tools_mcp_bridge.py | 99 ++++++ .../tools_orchestrator.py | 198 +++++++++++ .../tools_web_search.py | 65 ++++ .../translations/en.json | 11 + 11 files changed, 1008 insertions(+), 69 deletions(-) create mode 100644 custom_components/extended_openai_conversation/tools_builtin.py create mode 100644 custom_components/extended_openai_conversation/tools_mcp_bridge.py create mode 100644 custom_components/extended_openai_conversation/tools_orchestrator.py create mode 100644 custom_components/extended_openai_conversation/tools_web_search.py diff --git a/README.md b/README.md index 6346d623..b2328830 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,9 @@ A maintained fork of **Extended OpenAI Conversation** for **Home Assistant** tha - No `temperature/top_p` on reasoning endpoints - **Chat Completions** for non‑reasoning models (sampling allowed) - **Options gear** (model, strategy, effort, max tokens, temp/top‑p, prompt) via `OptionsFlowWithReload` +- **Toolbox parity** with execute service / automation / history built-ins plus YAML-defined scripts, REST, scrape, template, composite helpers +- **Hosted web search** toggle mirroring the stock OpenAI agent (Responses API + graceful chat fallback) +- **Optional MCP bridge** that surfaces Home Assistant MCP servers as namespaced tools - **Azure OpenAI** support (Base URL + API version) - **Async‑safe client** using HA’s shared HTTPX to avoid blocking SSL CA loads @@ -34,11 +37,12 @@ A maintained fork of **Extended OpenAI Conversation** for **Home Assistant** tha ## Configure Open the integration card → **Configure** (gear): -- **Model** (default `gpt-5`) -- **Model strategy**: `auto` \| `force_chat_completions` \| `force_responses_api` -- **Use Responses API** (for non‑reasoning models when strategy is `auto`) -- **Reasoning effort**: `minimal` \| `low` \| `medium` \| `high` -- **Max tokens**, **Temperature**, **Top‑p**, **System prompt** +- **Model & Strategy** – choose default model plus routing strategy between Chat vs Responses API. +- **Reasoning effort & token limits** – effort hint for reasoning models, max completion/output tokens, temperature & top‑p (chat only). +- **Hosted Web Search** – enable OpenAI’s web search tool, set context size, optionally include approximate home location metadata. +- **Toolbox limits** – cap total tool calls per user turn. +- **MCP bridge** – opt‑in to surface any configured MCP servers (timeout + payload guardrails). +- **Functions (YAML)** – edit the toolbox definition (defaults include `execute_service`, `add_automation`, `get_history`); append your own `rest`, `scrape`, `script`, `template`, or `composite` functions. ### Recommended (long, smart chats) - Model: `gpt-5` @@ -48,6 +52,12 @@ Open the integration card → **Configure** (gear): - Max tokens: **800–1200** > With Responses API, *system text* goes into **`instructions`** and inputs use **`input_text`**. +## Toolbox YAML quick reference +- Default entries provide `execute_service`, `add_automation`, and `get_history` using the historical EOC schema. +- Append additional entries to expose `script`, `template`, `rest`, `scrape`, or `composite` actions. +- Each entry requires a `spec` (tool definition shown to the model) and `function` (executor metadata); the existing [upstream examples](https://github.com/jekalmin/extended_openai_conversation/tree/main/examples/function) remain compatible. +- Invalid YAML is rejected by the options flow and logged with context so setup continues safely. + ## Assist usage - **Settings → Voice Assistants** → set **Conversation agent** = *Extended OpenAI Conversation* - Use Assist (text/voice) or ESPHome satellites normally. @@ -58,6 +68,10 @@ Open the integration card → **Configure** (gear): - **`intent-failed` with `.as_dict`**: fixed in v1.4.1 (compat shim). :contentReference[oaicite:9]{index=9} - **Blocking SSL warning (`load_verify_locations`)**: fixed by using HA’s shared HTTPX client in v1.4.1. :contentReference[oaicite:10]{index=10} +## Web search & MCP notes +- Web search is sent only on Responses API routes. For chat-only models we log a notice and the agent replies without search context. +- MCP tools are surfaced when Home Assistant (or custom code) registers MCP servers under `hass.data["mcp_servers"]`; each tool call is sandboxed with timeout and payload limits. + Enable debug: ```yaml action: logger.set_level diff --git a/custom_components/extended_openai_conversation/config_flow.py b/custom_components/extended_openai_conversation/config_flow.py index cb1e0dd8..4e4cdcfe 100644 --- a/custom_components/extended_openai_conversation/config_flow.py +++ b/custom_components/extended_openai_conversation/config_flow.py @@ -5,6 +5,7 @@ from typing import Any import voluptuous as vol +import yaml from homeassistant import config_entries from homeassistant.config_entries import ( ConfigFlow, @@ -26,6 +27,14 @@ CONF_TOP_P, CONF_MAX_TOKENS, CONF_REASONING_EFFORT, + CONF_FUNCTIONS_YAML, + CONF_ENABLE_WEB_SEARCH, + CONF_SEARCH_CONTEXT_SIZE, + CONF_INCLUDE_HOME_LOCATION, + CONF_MAX_TOOL_CALLS, + CONF_ENABLE_MCP, + CONF_MCP_TIMEOUT, + CONF_MCP_MAX_PAYLOAD, DEFAULT_BASE_URL, DEFAULT_CHAT_MODEL, DEFAULT_MODEL_STRATEGY, @@ -34,7 +43,17 @@ DEFAULT_TOP_P, DEFAULT_MAX_TOKENS, DEFAULT_PROMPT, + DEFAULT_ENABLE_WEB_SEARCH, + DEFAULT_SEARCH_CONTEXT_SIZE, + DEFAULT_INCLUDE_HOME_LOCATION, + DEFAULT_MAX_TOOL_CALLS, + DEFAULT_ENABLE_MCP, + DEFAULT_MCP_TIMEOUT, + DEFAULT_MCP_MAX_PAYLOAD, ) +from .tools_builtin import build_default_functions_yaml + +DEFAULT_FUNCTIONS_YAML = build_default_functions_yaml() class ExtendedOpenAIConfigFlow(ConfigFlow, domain=DOMAIN): @@ -105,29 +124,81 @@ def async_get_options_flow(config_entry: config_entries.ConfigEntry) -> config_e class EOCOptionsFlow(OptionsFlowWithReload): """Options flow that auto-reloads on save.""" + def _validate_functions_yaml(self, functions_text: str | None) -> str | None: + """Ensure the provided YAML parses into a list/dict structure.""" + if not functions_text: + return "" + try: + data = yaml.safe_load(functions_text) if functions_text.strip() else None + except yaml.YAMLError as err: + raise vol.Invalid(f"Invalid YAML: {err}") from err + + if data is None: + return "" + if not isinstance(data, (list, dict)): + raise vol.Invalid("Functions YAML must be a list or mapping.") + return functions_text + async def async_step_init( self, user_input: dict[str, Any] | None = None ) -> ConfigFlowResult: if user_input is not None: + try: + user_input[CONF_FUNCTIONS_YAML] = self._validate_functions_yaml( + user_input.get(CONF_FUNCTIONS_YAML) + ) + except vol.Invalid as err: + errors = {"base": "invalid_functions_yaml"} + return self.async_show_form( + step_id="init", + data_schema=self._build_schema(self.config_entry.options, user_input), + errors=errors, + ) + return self.async_create_entry(data=user_input) # Build the form with suggested current values. opts = self.config_entry.options - schema = vol.Schema( + return self.async_show_form( + step_id="init", + data_schema=self._build_schema(opts), + ) + + def _build_schema( + self, opts: dict[str, Any], user_input: dict[str, Any] | None = None + ) -> vol.Schema: + """Build the options schema with defaults.""" + data = user_input or opts + return vol.Schema( { - vol.Optional(CONF_CHAT_MODEL, default=opts.get(CONF_CHAT_MODEL, DEFAULT_CHAT_MODEL)): str, - vol.Optional(CONF_MODEL_STRATEGY, default=opts.get(CONF_MODEL_STRATEGY, DEFAULT_MODEL_STRATEGY)): vol.In( + vol.Optional(CONF_CHAT_MODEL, default=data.get(CONF_CHAT_MODEL, DEFAULT_CHAT_MODEL)): str, + vol.Optional(CONF_MODEL_STRATEGY, default=data.get(CONF_MODEL_STRATEGY, DEFAULT_MODEL_STRATEGY)): vol.In( ["auto", "force_chat_completions", "force_responses_api"] ), - vol.Optional(CONF_USE_RESPONSES_API, default=opts.get(CONF_USE_RESPONSES_API, True)): bool, - vol.Optional(CONF_REASONING_EFFORT, default=opts.get(CONF_REASONING_EFFORT, "medium")): vol.In( + vol.Optional(CONF_USE_RESPONSES_API, default=data.get(CONF_USE_RESPONSES_API, DEFAULT_USE_RESPONSES_API)): bool, + vol.Optional(CONF_REASONING_EFFORT, default=data.get(CONF_REASONING_EFFORT, "medium")): vol.In( ["minimal", "low", "medium", "high"] ), - vol.Optional(CONF_TEMPERATURE, default=opts.get(CONF_TEMPERATURE, DEFAULT_TEMPERATURE)): vol.Coerce(float), - vol.Optional(CONF_TOP_P, default=opts.get(CONF_TOP_P, DEFAULT_TOP_P)): vol.Coerce(float), - vol.Optional(CONF_MAX_TOKENS, default=opts.get(CONF_MAX_TOKENS, DEFAULT_MAX_TOKENS)): vol.Coerce(int), - vol.Optional("prompt", default=opts.get("prompt", DEFAULT_PROMPT)): str, + vol.Optional(CONF_TEMPERATURE, default=data.get(CONF_TEMPERATURE, DEFAULT_TEMPERATURE)): vol.Coerce(float), + vol.Optional(CONF_TOP_P, default=data.get(CONF_TOP_P, DEFAULT_TOP_P)): vol.Coerce(float), + vol.Optional(CONF_MAX_TOKENS, default=data.get(CONF_MAX_TOKENS, DEFAULT_MAX_TOKENS)): vol.Coerce(int), + vol.Optional(CONF_ENABLE_WEB_SEARCH, default=data.get(CONF_ENABLE_WEB_SEARCH, DEFAULT_ENABLE_WEB_SEARCH)): bool, + vol.Optional(CONF_SEARCH_CONTEXT_SIZE, default=data.get(CONF_SEARCH_CONTEXT_SIZE, DEFAULT_SEARCH_CONTEXT_SIZE)): vol.All( + vol.Coerce(int), vol.Range(min=1, max=32) + ), + vol.Optional(CONF_INCLUDE_HOME_LOCATION, default=data.get(CONF_INCLUDE_HOME_LOCATION, DEFAULT_INCLUDE_HOME_LOCATION)): bool, + vol.Optional(CONF_MAX_TOOL_CALLS, default=data.get(CONF_MAX_TOOL_CALLS, DEFAULT_MAX_TOOL_CALLS)): vol.All( + vol.Coerce(int), vol.Range(min=1, max=16) + ), + vol.Optional(CONF_ENABLE_MCP, default=data.get(CONF_ENABLE_MCP, DEFAULT_ENABLE_MCP)): bool, + vol.Optional(CONF_MCP_TIMEOUT, default=data.get(CONF_MCP_TIMEOUT, DEFAULT_MCP_TIMEOUT)): vol.All( + vol.Coerce(int), vol.Range(min=1, max=120) + ), + vol.Optional(CONF_MCP_MAX_PAYLOAD, default=data.get(CONF_MCP_MAX_PAYLOAD, DEFAULT_MCP_MAX_PAYLOAD)): vol.All( + vol.Coerce(int), vol.Range(min=1024, max=65536) + ), + vol.Optional(CONF_FUNCTIONS_YAML, default=data.get(CONF_FUNCTIONS_YAML, DEFAULT_FUNCTIONS_YAML)): str, + vol.Optional("prompt", default=data.get("prompt", DEFAULT_PROMPT)): str, } ) - return self.async_show_form(step_id="init", data_schema=schema) diff --git a/custom_components/extended_openai_conversation/const.py b/custom_components/extended_openai_conversation/const.py index 57ed1453..65608afe 100644 --- a/custom_components/extended_openai_conversation/const.py +++ b/custom_components/extended_openai_conversation/const.py @@ -30,6 +30,14 @@ # Optional scaffolding (off by default) CONF_MEMORY_ENABLED = "memory_enabled" CONF_MEMORY_DEFAULT_NAMESPACE = "memory_default_namespace" +CONF_FUNCTIONS_YAML = "functions" +CONF_ENABLE_WEB_SEARCH = "enable_web_search" +CONF_SEARCH_CONTEXT_SIZE = "search_context_size" +CONF_INCLUDE_HOME_LOCATION = "include_home_location" +CONF_MAX_TOOL_CALLS = "max_tool_calls" +CONF_ENABLE_MCP = "enable_mcp_tools" +CONF_MCP_TIMEOUT = "mcp_timeout" +CONF_MCP_MAX_PAYLOAD = "mcp_max_payload" # Optional service SERVICE_QUERY_IMAGE = "query_image" @@ -47,3 +55,10 @@ DEFAULT_PROMPT = "" DEFAULT_MEMORY_ENABLED = False DEFAULT_MEMORY_DEFAULT_NAMESPACE = "default" +DEFAULT_ENABLE_WEB_SEARCH = False +DEFAULT_SEARCH_CONTEXT_SIZE = 8 +DEFAULT_INCLUDE_HOME_LOCATION = False +DEFAULT_MAX_TOOL_CALLS = 4 +DEFAULT_ENABLE_MCP = False +DEFAULT_MCP_TIMEOUT = 20 +DEFAULT_MCP_MAX_PAYLOAD = 16384 diff --git a/custom_components/extended_openai_conversation/conversation.py b/custom_components/extended_openai_conversation/conversation.py index cd9ee71d..44298913 100644 --- a/custom_components/extended_openai_conversation/conversation.py +++ b/custom_components/extended_openai_conversation/conversation.py @@ -2,15 +2,22 @@ from __future__ import annotations +import json +from types import SimpleNamespace import logging from typing import Any, Optional, Literal +from homeassistant.components import conversation as ha_conversation from homeassistant.components.conversation import ( ConversationEntity, ConversationEntityFeature, ChatLog, + ConversationInput, ) -from homeassistant.helpers import intent +from homeassistant.components.homeassistant.exposed_entities import ( + async_should_expose, +) +from homeassistant.helpers import entity_registry as er, intent from homeassistant.const import CONF_API_KEY # Try to import the real ConversationResult; otherwise provide a compatible shim. @@ -58,6 +65,7 @@ def as_dict(self) -> dict[str, Any]: ) from .model_capabilities import detect_model_capabilities from .responses_adapter import response_text_from_responses_result +from .tools_orchestrator import ToolExecutionContext, ToolError, ToolOrchestrator _LOGGER = logging.getLogger(__name__) @@ -136,58 +144,192 @@ async def _async_handle_message( ) sys_prompt = (options.get("prompt") or DEFAULT_PROMPT).strip() - user_text = user_input.text - - if use_responses: - # ✅ Correct Responses API schema: - # - System instructions go into "instructions" - # - Input content items use type "input_text" - payload: dict[str, Any] = { - "model": model, - "input": [ + orchestrator = ToolOrchestrator(self.hass, options) + context = ToolExecutionContext( + hass=self.hass, + user_input=user_input, + exposed_entities=self._collect_exposed_entities(), + ) + + try: + if use_responses: + text, cont = await self._handle_with_responses_api( + client=client, + model=model, + user_input=user_input, + sys_prompt=sys_prompt, + caps=caps, + options=options, + orchestrator=orchestrator, + context=context, + ) + else: + text, cont = await self._handle_with_chat_completions( + client=client, + model=model, + user_input=user_input, + sys_prompt=sys_prompt, + caps=caps, + options=options, + orchestrator=orchestrator, + context=context, + ) + except Exception as err: + _LOGGER.exception("Conversation handling failed: %s", err) + return _err(str(err), user_input.language) + + if not text: + text = "I'm sorry, I couldn't produce a response." + + return _ok( + text=text, + language=user_input.language, + conversation_id=user_input.conversation_id, + cont=cont, + ) + + def _collect_exposed_entities(self) -> list[dict[str, Any]]: + """Return all entities exposed to the conversation agent.""" + + registry = er.async_get(self.hass) + exposed: list[dict[str, Any]] = [] + for state in self.hass.states.async_all(): + entity_id = state.entity_id + if not async_should_expose(self.hass, ha_conversation.DOMAIN, entity_id): + continue + entry = registry.async_get(entity_id) + aliases = list(entry.aliases) if entry and entry.aliases else [] + exposed.append( + { + "entity_id": entity_id, + "name": state.name, + "state": state.state, + "aliases": aliases, + } + ) + return exposed + + async def _handle_with_responses_api( + self, + *, + client, + model: str, + user_input: ConversationInput, + sys_prompt: str, + caps, + options: dict[str, Any], + orchestrator: ToolOrchestrator, + context: ToolExecutionContext, + ) -> tuple[str, bool]: + """Execute the Responses API interaction with tool-calling.""" + + max_tokens = int(options.get(CONF_MAX_TOKENS) or DEFAULT_MAX_TOKENS) + reasoning_effort = options.get(CONF_REASONING_EFFORT) + + tools_payload = orchestrator.responses_tools + previous_response_id: str | None = None + tool_outputs: list[dict[str, Any]] | None = None + total_calls = 0 + first_request = True + + response = None + + while True: + request: dict[str, Any] = {"model": model} + if first_request: + request["input"] = [ { "role": "user", - "content": [{"type": "input_text", "text": user_text}], + "content": [{"type": "input_text", "text": user_input.text}], } - ], - } - if sys_prompt: - payload["instructions"] = sys_prompt + ] + if sys_prompt: + request["instructions"] = sys_prompt + else: + request["previous_response_id"] = previous_response_id + request["input"] = [] + if tool_outputs: + request["tool_outputs"] = tool_outputs - max_tokens = int(options.get(CONF_MAX_TOKENS) or DEFAULT_MAX_TOKENS) if max_tokens > 0: - payload["max_output_tokens"] = max_tokens - - if caps.is_reasoning: - effort = options.get(CONF_REASONING_EFFORT) - # OpenAI 1.x expects nested object: {"reasoning": {"effort": "low|medium|high|minimal"}} - payload["reasoning"] = {"effort": effort} - - try: - result = await client.responses.create(**payload) # type: ignore[arg-type] - text = response_text_from_responses_result(result) - cont = _should_continue(text) - return _ok( - text=text, - language=user_input.language, - conversation_id=user_input.conversation_id, - cont=cont, + request["max_output_tokens"] = max_tokens + if caps.is_reasoning and reasoning_effort: + request["reasoning"] = {"effort": reasoning_effort} + + if tools_payload: + request["tools"] = tools_payload + request["tool_choice"] = "auto" + request["max_tool_calls"] = orchestrator.max_calls() + + response = await client.responses.create(**request) + function_calls = _extract_responses_function_calls(response) + + if not function_calls: + text = response_text_from_responses_result(response) + if not text: + text = "I'm sorry, I wasn't able to produce a response." + return text, _should_continue(text) + + tool_outputs = [] + for call in function_calls: + tool_call_id = getattr(call, "call_id", None) or getattr(call, "id", "") + try: + arguments = json.loads(call.arguments or "{}") + except (TypeError, json.JSONDecodeError): + arguments = {} + + if total_calls >= orchestrator.max_calls(): + tool_outputs.append( + { + "tool_call_id": tool_call_id, + "output": "Tool call limit reached; skipping execution.", + } + ) + continue + + try: + tool_result = await orchestrator.execute_tool( + call.name, arguments, context + ) + except ToolError as err: + tool_result = f"Tool '{call.name}' failed: {err}" + except Exception as err: # pragma: no cover - defensive + tool_result = f"Tool '{call.name}' raised unexpected error: {err}" + + tool_outputs.append( + {"tool_call_id": tool_call_id, "output": tool_result} ) - except Exception as err: - _LOGGER.exception("Responses API failure: %s", err) - return _err(str(err), user_input.language) - - # Fallback: Chat Completions - messages = [] + total_calls += 1 + + previous_response_id = response.id + first_request = False + tool_outputs = None + + return "", False + + async def _handle_with_chat_completions( + self, + *, + client, + model: str, + user_input: ConversationInput, + sys_prompt: str, + caps, + options: dict[str, Any], + orchestrator: ToolOrchestrator, + context: ToolExecutionContext, + ) -> tuple[str, bool]: + """Execute Chat Completions with tool-calling loop.""" + + messages: list[dict[str, Any]] = [] if sys_prompt: messages.append({"role": "system", "content": sys_prompt}) - messages.append({"role": "user", "content": user_text}) + messages.append({"role": "user", "content": user_input.text}) kwargs: dict[str, Any] = {"model": model, "messages": messages} max_tokens = int(options.get(CONF_MAX_TOKENS) or DEFAULT_MAX_TOKENS) if max_tokens > 0: - # Some reasoning-capable chat models use max_completion_tokens kwargs["max_completion_tokens" if caps.is_reasoning else "max_tokens"] = max_tokens if caps.accepts_temperature: @@ -196,20 +338,78 @@ async def _async_handle_message( if (p := options.get(CONF_TOP_P)) is not None: kwargs["top_p"] = float(p) - try: - result = await client.chat.completions.create(**kwargs) # type: ignore[arg-type] - msg = result.choices[0].message - text = msg.content or "" - cont = _should_continue(text) - return _ok( - text=text, - language=user_input.language, - conversation_id=user_input.conversation_id, - cont=cont, + tool_specs = orchestrator.function_specs + if tool_specs: + kwargs["tools"] = [{"type": "function", "function": spec} for spec in tool_specs] + kwargs["tool_choice"] = "auto" + + if orchestrator.supports_web_search(): + _LOGGER.debug( + "Web search enabled, but Chat Completions route does not support hosted web search; skipping." ) - except Exception as err: - _LOGGER.exception("Chat Completions failure: %s", err) - return _err(str(err), user_input.language) + + total_calls = 0 + + while True: + result = await client.chat.completions.create(**kwargs) + choice = result.choices[0] + message = choice.message + finish_reason = choice.finish_reason + + if finish_reason in ("tool_calls", "function_call"): + messages.append(message.model_dump(exclude_none=True)) + pending_calls = list(message.tool_calls or []) + + if message.function_call: + pending_calls.append( + _to_tool_call(message.function_call) + ) + + if not pending_calls: + # No tool calls despite finish reason; break to avoid loop. + text = message.content or "" + return text, _should_continue(text) + + for call in pending_calls: + name = getattr(call.function, "name", None) + if not name: + continue + + try: + arguments = json.loads(call.function.arguments or "{}") + except (TypeError, json.JSONDecodeError): + arguments = {} + + if total_calls >= orchestrator.max_calls(): + tool_result = "Tool call limit reached; skipping execution." + else: + try: + tool_result = await orchestrator.execute_tool( + name, arguments, context + ) + except ToolError as err: + tool_result = f"Tool '{name}' failed: {err}" + except Exception as err: # pragma: no cover - defensive + tool_result = f"Tool '{name}' raised unexpected error: {err}" + else: + total_calls += 1 + + messages.append( + { + "role": "tool", + "tool_call_id": call.id, + "name": name, + "content": tool_result, + } + ) + + kwargs["messages"] = messages + continue + + text = message.content or "" + return text, _should_continue(text) + + return "", False def _default_options(self) -> dict[str, Any]: return { @@ -248,3 +448,24 @@ def _err(msg: str, language: Optional[str]) -> ConversationResult: def _should_continue(text: str) -> bool: return "?" in (text or "") + + +def _extract_responses_function_calls(response: Any) -> list[Any]: + output = getattr(response, "output", None) + if not output: + return [] + calls: list[Any] = [] + for item in output: + if getattr(item, "type", None) == "function_call": + calls.append(item) + return calls + + +def _to_tool_call(function_call: Any) -> SimpleNamespace: + return SimpleNamespace( + id=getattr(function_call, "id", ""), + function=SimpleNamespace( + name=getattr(function_call, "name", ""), + arguments=getattr(function_call, "arguments", ""), + ), + ) diff --git a/custom_components/extended_openai_conversation/manifest.json b/custom_components/extended_openai_conversation/manifest.json index 5cb59e3f..7b59391d 100644 --- a/custom_components/extended_openai_conversation/manifest.json +++ b/custom_components/extended_openai_conversation/manifest.json @@ -8,7 +8,10 @@ "iot_class": "cloud_polling", "integration_type": "service", "dependencies": ["conversation"], - "requirements": ["openai>=1.0.0,<2.0.0"], + "requirements": [ + "openai>=1.0.0,<2.0.0", + "beautifulsoup4>=4.12.0" + ], "version": "1.4.1", "loggers": ["custom_components.extended_openai_conversation"], "platforms": ["conversation"] diff --git a/custom_components/extended_openai_conversation/strings.json b/custom_components/extended_openai_conversation/strings.json index e37f722f..d06b10da 100644 --- a/custom_components/extended_openai_conversation/strings.json +++ b/custom_components/extended_openai_conversation/strings.json @@ -39,6 +39,14 @@ "temperature": "Temperature (non‑reasoning only)", "top_p": "Top‑p (non‑reasoning only)", "max_tokens": "Max output tokens", + "enable_web_search": "Enable hosted web search (Responses API)", + "search_context_size": "Web search context size", + "include_home_location": "Include approximate home location", + "max_tool_calls": "Max tool calls per turn", + "enable_mcp_tools": "Enable MCP tools", + "mcp_timeout": "MCP tool timeout (seconds)", + "mcp_max_payload": "MCP max payload (bytes)", + "functions": "Functions (YAML configuration)", "prompt": "System prompt (optional)" } } diff --git a/custom_components/extended_openai_conversation/tools_builtin.py b/custom_components/extended_openai_conversation/tools_builtin.py new file mode 100644 index 00000000..24e0b578 --- /dev/null +++ b/custom_components/extended_openai_conversation/tools_builtin.py @@ -0,0 +1,234 @@ +"""Helpers for loading function-call tools defined via YAML.""" + +from __future__ import annotations + +from dataclasses import dataclass +import logging +from typing import Any, Iterable, List + +import yaml + +from homeassistant.components import conversation +from homeassistant.core import HomeAssistant + +from .exceptions import FunctionLoadFailed, FunctionNotFound, InvalidFunction +from .helpers import get_function_executor, convert_to_template + +_LOGGER = logging.getLogger(__name__) + + +@dataclass +class FunctionTool: + """Runtime representation of a function-call tool.""" + + spec: dict[str, Any] + name: str + executor_type: str + function_config: dict[str, Any] + + async def async_execute( + self, + hass: HomeAssistant, + *, + arguments: dict[str, Any], + user_input: conversation.ConversationInput, + exposed_entities: list[dict[str, Any]], + ) -> Any: + executor = get_function_executor(self.executor_type) + return await executor.execute( + hass, + self.function_config, + arguments, + user_input, + exposed_entities, + ) + + +def _ensure_sequence(value: Any) -> list[dict[str, Any]]: + if value is None: + return [] + if isinstance(value, list): + return value + if isinstance(value, dict): + return [value] + raise FunctionLoadFailed("Functions YAML must be a list or dictionary") + + +def load_function_tools( + hass: HomeAssistant, + yaml_text: str | None, +) -> list[FunctionTool]: + """Parse the functions YAML into executable FunctionTool objects.""" + + if yaml_text is None: + return [] + + try: + parsed = yaml.safe_load(yaml_text) if yaml_text.strip() else None + except yaml.YAMLError as err: + raise FunctionLoadFailed(f"Invalid YAML: {err}") from err + + if parsed is None: + return [] + + tools: list[FunctionTool] = [] + for index, item in enumerate(_ensure_sequence(parsed)): + if not isinstance(item, dict): + _LOGGER.warning("Skipping function entry %s: expected mapping, got %s", index, type(item)) + continue + + spec = item.get("spec") + function_cfg = item.get("function") + + if not isinstance(spec, dict) or not isinstance(function_cfg, dict): + _LOGGER.warning("Skipping function entry %s: missing spec/function keys", index) + continue + + name = spec.get("name") + if not name or not isinstance(name, str): + _LOGGER.warning("Skipping function entry %s: spec.name missing or invalid", index) + continue + + fn_type = function_cfg.get("type") + if not fn_type or not isinstance(fn_type, str): + _LOGGER.warning("Skipping function entry %s (%s): function.type missing", index, name) + continue + + try: + executor = get_function_executor(fn_type) + parsed_config = executor.to_arguments(function_cfg) + except (InvalidFunction, FunctionNotFound) as err: + raise FunctionLoadFailed(f"Invalid function configuration for '{name}': {err}") from err + + # Ensure any template-like structures are bound to hass for execution. + convert_to_template( + parsed_config, + template_keys=[ + "data", + "event_data", + "target", + "service", + "payload_template", + "resource_template", + "value_template", + ], + hass=hass, + ) + + tools.append( + FunctionTool( + spec=spec, + name=name, + executor_type=fn_type, + function_config=parsed_config, + ) + ) + + return tools + + +DEFAULT_FUNCTIONS: list[dict[str, Any]] = [ + { + "spec": { + "name": "execute_service", + "description": "Call Home Assistant services on exposed devices. Use only after explicit user confirmation.", + "parameters": { + "type": "object", + "properties": { + "list": { + "type": "array", + "description": "List of services to call sequentially.", + "items": { + "type": "object", + "properties": { + "domain": { + "type": "string", + "description": "Domain of the service (e.g. light, switch).", + }, + "service": { + "type": "string", + "description": "Service name within the domain (e.g. turn_on).", + }, + "service_data": { + "type": "object", + "description": "Service data dictionary matching Home Assistant service schema.", + }, + }, + "required": ["domain", "service"], + }, + } + }, + "required": ["list"], + }, + }, + "function": {"type": "native", "name": "execute_service"}, + }, + { + "spec": { + "name": "add_automation", + "description": "Write a Home Assistant automation given YAML configuration text.", + "parameters": { + "type": "object", + "properties": { + "automation_config": { + "type": "string", + "description": "Full automation YAML body to append to automations.yaml.", + } + }, + "required": ["automation_config"], + }, + }, + "function": {"type": "native", "name": "add_automation"}, + }, + { + "spec": { + "name": "get_history", + "description": "Fetch state history for exposed entities within a time window.", + "parameters": { + "type": "object", + "properties": { + "entity_ids": { + "type": "array", + "items": {"type": "string"}, + "description": "Entity IDs to query. Must be exposed to the assistant.", + }, + "start_time": { + "type": "string", + "description": "ISO8601 timestamp for the beginning of the window. Defaults to 24h ago if omitted.", + }, + "end_time": { + "type": "string", + "description": "ISO8601 timestamp for the end of the window. Defaults to start_time + 24h.", + }, + "include_start_time_state": { + "type": "boolean", + "description": "Whether to include state at the exact start time.", + "default": True, + }, + "significant_changes_only": { + "type": "boolean", + "description": "If true, return only significant state changes.", + "default": True, + }, + "minimal_response": { + "type": "boolean", + "description": "If true, omit intermediate event data to reduce payload size.", + "default": True, + }, + "no_attributes": { + "type": "boolean", + "description": "If true, exclude attribute payloads.", + "default": True, + }, + }, + "required": ["entity_ids"], + }, + }, + "function": {"type": "native", "name": "get_history"}, + }, +] + + +def build_default_functions_yaml() -> str: + """Return the canonical YAML string for the default toolbox.""" + return yaml.safe_dump(DEFAULT_FUNCTIONS, sort_keys=False) diff --git a/custom_components/extended_openai_conversation/tools_mcp_bridge.py b/custom_components/extended_openai_conversation/tools_mcp_bridge.py new file mode 100644 index 00000000..d5797ae6 --- /dev/null +++ b/custom_components/extended_openai_conversation/tools_mcp_bridge.py @@ -0,0 +1,99 @@ +"""Minimal Model Context Protocol (MCP) bridge.""" + +from __future__ import annotations + +from dataclasses import dataclass +import logging +from typing import Any, Awaitable, Callable, List + +from homeassistant.core import HomeAssistant + +_LOGGER = logging.getLogger(__name__) + + +@dataclass +class McpBridgeConfig: + timeout: int + max_payload: int + + +@dataclass +class McpTool: + name: str + spec: dict[str, Any] + definition: dict[str, Any] + _executor: Callable[[dict[str, Any]], Awaitable[Any]] + + async def async_execute(self, arguments: dict[str, Any]) -> Any: + return await self._executor(arguments) + + +def load_mcp_tools(hass: HomeAssistant, config: McpBridgeConfig) -> List[McpTool]: + """Discover MCP tools exposed by the Home Assistant MCP integration. + + The current Home Assistant builds do not yet expose a formal API for listing MCP + servers. This helper keeps the plumbing ready while gracefully degrading when MCP + is unavailable. + """ + + servers = hass.data.get("mcp_servers") + if isinstance(servers, dict): + server_iter = servers.items() + elif isinstance(servers, list): + server_iter = ((str(index), item) for index, item in enumerate(servers)) + else: + server_iter = [] + + discovered: list[McpTool] = [] + saw_server = False + + for server_id, server in server_iter: + saw_server = True + if not server: + continue + tools = getattr(server, "tools", None) + if tools is None and isinstance(server, dict): + tools = server.get("tools") + if not tools: + continue + _LOGGER.debug("MCP bridge: no servers discovered") + for tool in tools: + name = tool.get("name") if isinstance(tool, dict) else getattr(tool, "name", None) + spec = tool.get("spec") if isinstance(tool, dict) else getattr(tool, "spec", None) + if not name or not spec: + continue + + definition = { + "type": "mcp", + "mcp": { + "server_label": server_id, + "type": "mcp", + "allowed_tools": [name], + }, + } + + async def _call_tool(arguments: dict[str, Any], *, server_ref=server, tool_name=name) -> Any: + call = getattr(server_ref, "async_call_tool", None) + if call is None and isinstance(server_ref, dict): + call = server_ref.get("async_call_tool") + if call is None: + raise RuntimeError("Server does not support async_call_tool") + if callable(call): + return await call(tool_name, arguments) + raise RuntimeError("async_call_tool is not callable") + + discovered.append( + McpTool( + name=name, + spec=spec, + definition=definition, + _executor=_call_tool, + ) + ) + + if not saw_server: + _LOGGER.debug("MCP bridge: no servers discovered") + elif not discovered: + _LOGGER.debug("MCP bridge: servers found but no compatible tools exposed") + + return discovered diff --git a/custom_components/extended_openai_conversation/tools_orchestrator.py b/custom_components/extended_openai_conversation/tools_orchestrator.py new file mode 100644 index 00000000..9df8d21d --- /dev/null +++ b/custom_components/extended_openai_conversation/tools_orchestrator.py @@ -0,0 +1,198 @@ +"""Central tool orchestration for Responses API and Chat Completions.""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass +from typing import Any, Callable + +from homeassistant.components import conversation +from homeassistant.core import HomeAssistant + +from .const import ( + CONF_ENABLE_MCP, + CONF_ENABLE_WEB_SEARCH, + CONF_FUNCTIONS_YAML, + CONF_INCLUDE_HOME_LOCATION, + CONF_MAX_TOOL_CALLS, + CONF_MCP_MAX_PAYLOAD, + CONF_MCP_TIMEOUT, + CONF_SEARCH_CONTEXT_SIZE, + DEFAULT_ENABLE_MCP, + DEFAULT_ENABLE_WEB_SEARCH, + DEFAULT_INCLUDE_HOME_LOCATION, + DEFAULT_MAX_TOOL_CALLS, + DEFAULT_MCP_MAX_PAYLOAD, + DEFAULT_MCP_TIMEOUT, + DEFAULT_SEARCH_CONTEXT_SIZE, +) +from .memory_tools import ( + MEMORY_SEARCH_NAME, + MEMORY_WRITE_NAME, + MEMORY_TOOL_SPECS, + MemoryServiceConfig, + build_memory_tool_definitions, + dispatch_memory_tool, + get_memory_service_config, + is_configured as memory_is_configured, +) +from .tools_builtin import FunctionTool, load_function_tools, build_default_functions_yaml +from .tools_web_search import WebSearchConfig, build_web_search_tool +from .tools_mcp_bridge import ( + McpBridgeConfig, + load_mcp_tools, + McpTool, +) + +_LOGGER = logging.getLogger(__name__) + + +@dataclass +class ToolExecutionContext: + """Context passed to tool executors.""" + + hass: HomeAssistant + user_input: conversation.ConversationInput + exposed_entities: list[dict[str, Any]] + + +class ToolError(Exception): + """Raised when a tool execution fails.""" + + +class ToolOrchestrator: + """Registry and execution engine for Home Assistant tool calls.""" + + def __init__( + self, + hass: HomeAssistant, + options: dict[str, Any], + ) -> None: + self.hass = hass + self.options = options + + self.max_tool_calls = int( + options.get(CONF_MAX_TOOL_CALLS, DEFAULT_MAX_TOOL_CALLS) + ) + self._function_tools = self._load_function_tools() + self._memory_config = get_memory_service_config(options) + self._memory_enabled = memory_is_configured(self._memory_config) + self._web_search_config = self._build_web_search_config() + self._mcp_tools = self._load_mcp_tools() + + def _load_function_tools(self) -> dict[str, FunctionTool]: + text = self.options.get(CONF_FUNCTIONS_YAML) + if text is None: + text = build_default_functions_yaml() + try: + tools = load_function_tools(self.hass, text) + except Exception as err: + _LOGGER.error("Unable to load toolbox functions: %s", err) + return {} + return {tool.name: tool for tool in tools} + + def _build_web_search_config(self) -> WebSearchConfig | None: + enabled = bool( + self.options.get(CONF_ENABLE_WEB_SEARCH, DEFAULT_ENABLE_WEB_SEARCH) + ) + if not enabled: + return None + + context_size = int( + self.options.get(CONF_SEARCH_CONTEXT_SIZE, DEFAULT_SEARCH_CONTEXT_SIZE) + ) + include_home = bool( + self.options.get( + CONF_INCLUDE_HOME_LOCATION, DEFAULT_INCLUDE_HOME_LOCATION + ) + ) + return WebSearchConfig( + enabled=enabled, + context_size=context_size, + include_home_location=include_home, + ) + + def _load_mcp_tools(self) -> dict[str, McpTool]: + if not bool(self.options.get(CONF_ENABLE_MCP, DEFAULT_ENABLE_MCP)): + return {} + config = McpBridgeConfig( + timeout=int(self.options.get(CONF_MCP_TIMEOUT, DEFAULT_MCP_TIMEOUT)), + max_payload=int( + self.options.get(CONF_MCP_MAX_PAYLOAD, DEFAULT_MCP_MAX_PAYLOAD) + ), + ) + tools = load_mcp_tools(self.hass, config=config) + return {tool.name: tool for tool in tools} + + @property + def function_specs(self) -> list[dict[str, Any]]: + """Function specs suitable for Chat Completions API.""" + specs: list[dict[str, Any]] = [tool.spec for tool in self._function_tools.values()] + if self._memory_enabled: + specs.extend(MEMORY_TOOL_SPECS) + return specs + + @property + def responses_tools(self) -> list[dict[str, Any]]: + """Tool definitions for the Responses API.""" + tools: list[dict[str, Any]] = [ + {"type": "function", "function": tool.spec} + for tool in self._function_tools.values() + ] + if self._memory_enabled: + tools.extend(build_memory_tool_definitions()) + tools.extend(tool.definition for tool in self._mcp_tools.values()) + if self._web_search_config and self._web_search_config.enabled: + tools.append(build_web_search_tool(self.hass, self._web_search_config)) + return tools + + def supports_web_search(self) -> bool: + return bool(self._web_search_config and self._web_search_config.enabled) + + def max_calls(self) -> int: + return max(1, self.max_tool_calls) + + async def execute_tool( + self, + name: str, + arguments: dict[str, Any], + context: ToolExecutionContext, + ) -> str: + """Execute a tool by name and return a stringified result.""" + + if tool := self._function_tools.get(name): + try: + result = await tool.async_execute( + context.hass, + arguments=arguments, + user_input=context.user_input, + exposed_entities=context.exposed_entities, + ) + except Exception as err: + raise ToolError(str(err)) from err + return _stringify_result(result) + + if self._memory_enabled and name in (MEMORY_SEARCH_NAME, MEMORY_WRITE_NAME): + result = await dispatch_memory_tool( + context.hass, self._memory_config, name, arguments + ) + return _stringify_result(result) + + if mcp_tool := self._mcp_tools.get(name): + try: + result = await mcp_tool.async_execute(arguments) + except Exception as err: + raise ToolError(f"MCP tool '{name}' failed: {err}") from err + return _stringify_result(result) + + raise ToolError(f"Unknown tool '{name}'") + + +def _stringify_result(result: Any) -> str: + if isinstance(result, str): + return result + try: + return json.dumps(result, ensure_ascii=False) + except Exception: + return str(result) diff --git a/custom_components/extended_openai_conversation/tools_web_search.py b/custom_components/extended_openai_conversation/tools_web_search.py new file mode 100644 index 00000000..7e5a317c --- /dev/null +++ b/custom_components/extended_openai_conversation/tools_web_search.py @@ -0,0 +1,65 @@ +"""Helpers to configure OpenAI hosted web search tool.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +from homeassistant.core import HomeAssistant + + +@dataclass +class WebSearchConfig: + enabled: bool + context_size: int + include_home_location: bool + + +def _context_size_label(value: int) -> str: + if value <= 4: + return "low" + if value <= 12: + return "medium" + return "high" + + +def _home_location_payload(hass: HomeAssistant) -> dict[str, Any] | None: + name = (hass.config.location_name or "").strip() + country = getattr(hass.config, "country", None) + region = None + timezone = hass.config.time_zone + + if not any([name, country, timezone]): + return None + + payload: dict[str, Any] = {"type": "approximate"} + if name: + payload["city"] = name + if region: + payload["region"] = region + if country: + payload["country"] = country + if timezone: + payload["timezone"] = timezone + return payload + + +def build_web_search_tool( + hass: HomeAssistant, config: WebSearchConfig +) -> dict[str, Any]: + """Return the tool definition for hosted web search.""" + + tool: dict[str, Any] = {"type": "web_search"} + payload: dict[str, Any] = { + "search_context_size": _context_size_label(config.context_size) + } + + if config.include_home_location: + location = _home_location_payload(hass) + if location: + payload["user_location"] = location + + if payload: + tool["web_search"] = payload + + return tool diff --git a/custom_components/extended_openai_conversation/translations/en.json b/custom_components/extended_openai_conversation/translations/en.json index 56ec6132..673ed6e5 100644 --- a/custom_components/extended_openai_conversation/translations/en.json +++ b/custom_components/extended_openai_conversation/translations/en.json @@ -27,6 +27,9 @@ } }, "options": { + "error": { + "invalid_functions_yaml": "Functions YAML must be a valid list or mapping." + }, "step": { "init": { "title": "EOC Options", @@ -38,6 +41,14 @@ "temperature": "Temperature (chat models only)", "top_p": "Top‑p (chat models only)", "max_tokens": "Max tokens", + "enable_web_search": "Enable hosted web search (Responses API)", + "search_context_size": "Web search context size", + "include_home_location": "Include approximate home location", + "max_tool_calls": "Max tool calls per turn", + "enable_mcp_tools": "Enable MCP tools", + "mcp_timeout": "MCP tool timeout (seconds)", + "mcp_max_payload": "MCP max payload (bytes)", + "functions": "Functions (YAML configuration)", "prompt": "System prompt" } }