diff --git a/.env.example b/.env.example index 663f31b3f..4cff4556a 100644 --- a/.env.example +++ b/.env.example @@ -64,6 +64,8 @@ POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS=60 # Generation and Embedding may use different providers; `powercontext config init` configures valid combinations. POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS=30 POWERCONTEXT_SERVER_INFERENCE_GENERATION_MAX_REQUESTS=2 +# Optional Pydantic AI model settings. Provider-specific request fields belong under `extra_body`. +# POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS='{"extra_body":{"chat_template_kwargs":{"enable_thinking":false}}}' POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_NORMALIZATION=unit POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_BATCH_SIZE=10 POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_TIMEOUT_SECONDS=30 diff --git a/docs/en/development/pydantic-ai-inference.md b/docs/en/development/pydantic-ai-inference.md index 85d8ac914..1d8a9ec1f 100644 --- a/docs/en/development/pydantic-ai-inference.md +++ b/docs/en/development/pydantic-ai-inference.md @@ -27,6 +27,17 @@ Source-to-Memory extraction: export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL="provider:model-name" ``` +Additional Pydantic AI model settings can be supplied as JSON. Provider-specific request fields belong under +`extra_body`; for example, a compatible OpenAI-style endpoint can disable Qwen thinking with: + +```bash +export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS='{"extra_body":{"chat_template_kwargs":{"enable_thinking":false}}}' +``` + +These settings are shared by the generation-backed pipelines, the optional LLM reranker, and the readiness probe. +PowerContext retains its own request bounds: readiness uses `max_tokens=1`, and reranking uses `temperature=0`. +Keep credentials and static headers in provider configuration rather than model settings. + Vector search needs the embedding model and its complete deployment profile: ```bash diff --git a/docs/en/docs/reference/configuration.md b/docs/en/docs/reference/configuration.md index 726c18a41..c0ee374e0 100644 --- a/docs/en/docs/reference/configuration.md +++ b/docs/en/docs/reference/configuration.md @@ -74,6 +74,7 @@ Server settings use the `POWERCONTEXT_SERVER_` prefix. | `POWERCONTEXT_SERVER_RUNTIME_MEMORY_RERANK_CANDIDATE_LIMIT` | `30` | Coarse candidate pool supplied to the reranker | | `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` | unset | Scheduler interval; unset disables scheduling | | `POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL` | unset | Pydantic AI model used by configured extraction, generation, Handoff, and reranking operations | +| `POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS` | `{}` | JSON object of Pydantic AI model settings shared by generation and reranking | | `POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS` | `30` | Timeout in seconds for one structured generation operation | | `POWERCONTEXT_SERVER_INFERENCE_GENERATION_MAX_REQUESTS` | `2` | Maximum provider requests for one structured generation operation, including retries | | `POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_MODEL` | unset | Pydantic AI embedding model; requires profile ID and dimension | @@ -127,6 +128,19 @@ command-line arguments, documentation, or Memory. Replace `provider:model-name` Pydantic AI. Scheduled extraction requires both a generation model and `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS`. An explicit Memory write does not require either. +For provider-specific request parameters, set model settings as one JSON object. For example, an OpenAI-compatible +endpoint that supports Qwen's thinking switch can receive `chat_template_kwargs.enable_thinking=false` through the +portable Pydantic AI `extra_body` setting: + +```bash +export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS='{"extra_body":{"chat_template_kwargs":{"enable_thinking":false}}}' +``` + +The Server applies these settings to extraction, Experience and Skill generation, Handoff generation, optional LLM +reranking, and the generation readiness probe. The readiness probe always overrides `max_tokens` to `1`, and reranking +always overrides `temperature` to `0`. Only settings supported by the selected Pydantic AI model and provider are +meaningful. Keep credentials and static headers in the selected provider's configuration rather than this JSON object. + The default `coding` extraction profile keeps cross-task work context such as preferences, decisions, constraints, expensive facts, and unfinished progress. Select `conversation` when the product must preserve independently answerable personal facts, relationships, events, exact dates, lists, and historical states from dialogue evidence: diff --git a/docs/zh/development/pydantic-ai-inference.md b/docs/zh/development/pydantic-ai-inference.md index 70b65463e..18097f660 100644 --- a/docs/zh/development/pydantic-ai-inference.md +++ b/docs/zh/development/pydantic-ai-inference.md @@ -26,6 +26,17 @@ Source-to-Memory extraction： export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL="provider:model-name" ``` +额外的 Pydantic AI model settings 可以通过 JSON 配置。provider-specific request field 应放在 `extra_body` +下面；例如，兼容的 OpenAI-style endpoint 可以这样关闭 Qwen thinking： + +```bash +export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS='{"extra_body":{"chat_template_kwargs":{"enable_thinking":false}}}' +``` + +generation-backed pipeline、可选的 LLM reranker 和 readiness probe 共用这些 settings。PowerContext 仍保留自身的 +请求边界：readiness 使用 `max_tokens=1`，rerank 使用 `temperature=0`。credential 与 static header 应放在 +provider 配置中，不要放入 model settings。 + vector search 需要 embedding model 和完整的 deployment profile： ```bash diff --git a/docs/zh/docs/reference/configuration.md b/docs/zh/docs/reference/configuration.md index 0cd6b416e..e7c6d3c4c 100644 --- a/docs/zh/docs/reference/configuration.md +++ b/docs/zh/docs/reference/configuration.md @@ -71,6 +71,7 @@ Server 配置使用 `POWERCONTEXT_SERVER_` 前缀。 | `POWERCONTEXT_SERVER_RUNTIME_MEMORY_RERANK_CANDIDATE_LIMIT` | `30` | 交给 reranker 的粗排候选池大小 | | `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS` | 未设置 | Scheduler 间隔；未设置即不启用 | | `POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL` | 未设置 | 配置的 extraction、generation、Handoff 和 rerank 操作共用的 Pydantic AI 模型 | +| `POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS` | `{}` | generation 与 rerank 共用的 Pydantic AI model settings JSON object | | `POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS` | `30` | 单次结构化 generation 操作的超时秒数 | | `POWERCONTEXT_SERVER_INFERENCE_GENERATION_MAX_REQUESTS` | `2` | 单次结构化 generation 操作最多发起的 provider 请求数，包含重试 | | `POWERCONTEXT_SERVER_INFERENCE_EMBEDDING_MODEL` | 未设置 | Pydantic AI embedding model；必须同时设置 profile ID 和 dimension | @@ -119,6 +120,19 @@ powercontext server run Memory。请把 `provider:model-name` 替换为 Pydantic AI 支持的模型标识。定时提取需要同时配置 generation model 和 `POWERCONTEXT_SERVER_RUNTIME_SCHEDULE_SECONDS`；显式 Memory 写入不需要这两项配置。 +provider-specific request parameter 使用一个 JSON object 配置。例如，OpenAI-compatible endpoint 支持 Qwen 的 +thinking switch 时，可以通过 Pydantic AI 的通用 `extra_body` setting 发送 +`chat_template_kwargs.enable_thinking=false`： + +```bash +export POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS='{"extra_body":{"chat_template_kwargs":{"enable_thinking":false}}}' +``` + +Server 会将这些 settings 用于 extraction、Experience 与 Skill generation、Handoff generation、可选的 LLM +rerank 以及 generation readiness probe。readiness probe 始终将 `max_tokens` 覆盖为 `1`，rerank 始终将 +`temperature` 覆盖为 `0`。只有所选 Pydantic AI model 与 provider 支持的 setting 才有意义。credential 和 +static header 应保留在所选 provider 的配置中，不要放入这个 JSON object。 + 默认的 `coding` 抽取 profile 保留跨任务工作上下文，例如偏好、决策、约束、昂贵事实和未完成进度。当产品 需要从对话证据中保留可独立回答的人物事实、关系、事件、精确日期、列表和历史状态时，可选择 `conversation`： diff --git a/src/powercontext/builtin/inference/pydantic_ai.py b/src/powercontext/builtin/inference/pydantic_ai.py index d12f41979..5bf569303 100644 --- a/src/powercontext/builtin/inference/pydantic_ai.py +++ b/src/powercontext/builtin/inference/pydantic_ai.py @@ -81,7 +81,7 @@ def __init__(self, code: str, detail: object | None = None) -> None: ) from pydantic_ai.messages import ModelRequest, UserPromptPart from pydantic_ai.models import Model, ModelRequestParameters - from pydantic_ai.settings import ModelSettings + from pydantic_ai.settings import ModelSettings, merge_model_settings from pydantic_ai.usage import RunUsage, UsageLimits from pydantic_core import PydanticSerializationError except ModuleNotFoundError as error: # pragma: no cover - exercised in a dependency-free environment @@ -272,6 +272,7 @@ async def probe_pydantic_ai_model( /, *, timeout_seconds: float, + model_settings: ModelSettings | None = None, ) -> None: """Send one minimal text request through an assembled generation model.""" @@ -281,7 +282,7 @@ async def probe_pydantic_ai_model( await asyncio.wait_for( model.request( [ModelRequest(parts=[UserPromptPart("Reply with one token.")])], - ModelSettings(max_tokens=1), + merge_model_settings(model_settings, ModelSettings(max_tokens=1)), ModelRequestParameters(), ), timeout=timeout_seconds, diff --git a/src/powercontext/builtin/runtime/composition.py b/src/powercontext/builtin/runtime/composition.py index b74668587..3867a28e9 100644 --- a/src/powercontext/builtin/runtime/composition.py +++ b/src/powercontext/builtin/runtime/composition.py @@ -19,7 +19,7 @@ from collections.abc import AsyncIterator from contextlib import AsyncExitStack, asynccontextmanager from pathlib import Path -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING, TypeVar, cast from pydantic import JsonValue from typing_extensions import override @@ -406,7 +406,7 @@ async def _generation_pipelines( from pydantic_ai.models import infer_model from pydantic_ai.models.instrumented import InstrumentedModel - from pydantic_ai.settings import ModelSettings + from pydantic_ai.settings import ModelSettings, merge_model_settings from powercontext.builtin.artifacts.experience import ( EXPERIENCE_GENERATION_INSTRUCTIONS, @@ -447,10 +447,15 @@ async def _generation_pipelines( provider_model = await resources.enter_async_context(infer_model(settings.generation_model)) model = provider_model if instrumentation is None else InstrumentedModel(provider_model, instrumentation) + model_settings = cast(ModelSettings, dict(settings.generation_model_settings)) or None async def probe_generation() -> None: # Readiness probing runs outside any operation span; keep it out of traces. - await probe_pydantic_ai_model(provider_model, timeout_seconds=READINESS_PROBE_TIMEOUT_SECONDS) + await probe_pydantic_ai_model( + provider_model, + timeout_seconds=READINESS_PROBE_TIMEOUT_SECONDS, + model_settings=model_settings, + ) limits = InferenceLimits( timeout_seconds=settings.generation_timeout_seconds, @@ -462,6 +467,7 @@ async def probe_generation() -> None: input_type=MemoryExtractionInput, output_type=MemoryExtractionOutput, limits=limits, + model_settings=model_settings, name="memory_extraction", ) experience_generator = PydanticAIStructuredGenerator( @@ -470,6 +476,7 @@ async def probe_generation() -> None: input_type=ExperienceIncubationInput, output_type=ExperienceIncubationOutput, limits=limits, + model_settings=model_settings, name="experience_incubation", ) explicit_experience_generator = PydanticAIStructuredGenerator( @@ -478,6 +485,7 @@ async def probe_generation() -> None: input_type=ArtifactGenerationInput, output_type=ExperienceGenerationOutput, limits=limits, + model_settings=model_settings, name="experience_generation", ) skill_generator = PydanticAIStructuredGenerator( @@ -486,6 +494,7 @@ async def probe_generation() -> None: input_type=ArtifactGenerationInput, output_type=SkillGenerationOutput, limits=limits, + model_settings=model_settings, name="skill_generation", ) handoff_generator = PydanticAIStructuredGenerator( @@ -494,6 +503,7 @@ async def probe_generation() -> None: input_type=HandoffGenerationInput, output_type=HandoffGenerationOutput, limits=limits, + model_settings=model_settings, name="handoff_generation", ) rerank_generator = ( @@ -503,7 +513,7 @@ async def probe_generation() -> None: input_type=MemoryRerankInput, output_type=MemoryRerankOutput, limits=limits, - model_settings=ModelSettings(temperature=0.0), + model_settings=merge_model_settings(model_settings, ModelSettings(temperature=0.0)), name="memory_rerank", ) if runtime.memory_rerank_enabled diff --git a/src/powercontext/builtin/runtime/config.py b/src/powercontext/builtin/runtime/config.py index be613fcc7..c41989927 100644 --- a/src/powercontext/builtin/runtime/config.py +++ b/src/powercontext/builtin/runtime/config.py @@ -19,7 +19,7 @@ from collections.abc import Mapping from typing import Any, Literal, Self -from pydantic import BaseModel, Field, field_validator, model_validator +from pydantic import BaseModel, Field, JsonValue, field_validator, model_validator from powercontext.builtin.artifacts.memory.prompts import MemoryExtractionProfile from powercontext.builtin.artifacts.skill import AgentSkillTarget, CodexSkillRoot @@ -51,6 +51,7 @@ class InferenceConfig(BaseModel): """Optional generation and embedding configuration.""" generation_model: str | None = None + generation_model_settings: dict[str, JsonValue] = Field(default_factory=dict) generation_timeout_seconds: float = Field(default=30.0, gt=0) generation_max_requests: int = Field(default=2, ge=1) embedding_model: str | None = None @@ -80,6 +81,19 @@ def validate_normalization(cls, value: object) -> object: raise ValueError("embedding normalization must be 'none' or 'unit'") # noqa: TRY003 return normalized + @field_validator("generation_model_settings") + @classmethod + def reserve_generation_headers(cls, value: dict[str, JsonValue]) -> dict[str, JsonValue]: + if "extra_headers" in value: + raise ValueError("configure credentials and static headers through the selected inference provider") # noqa: TRY003 + return value + + @model_validator(mode="after") + def validate_generation_model_settings(self) -> Self: + if self.generation_model_settings and self.generation_model is None: + raise ValueError("generation_model_settings requires generation_model") # noqa: TRY003 + return self + @model_validator(mode="after") def validate_embedding_profile(self) -> Self: values = (self.embedding_model, self.embedding_profile_id, self.embedding_dimension) diff --git a/tests/builtin/inference/test_pydantic_ai.py b/tests/builtin/inference/test_pydantic_ai.py index b2c205a86..cd76fd8b5 100644 --- a/tests/builtin/inference/test_pydantic_ai.py +++ b/tests/builtin/inference/test_pydantic_ai.py @@ -257,16 +257,30 @@ async def never_finishes(messages: list[ModelMessage], info: AgentInfo) -> Model def test_generation_readiness_probe_uses_one_bounded_text_request() -> None: - observed_max_tokens: list[int | None] = [] + observed_settings: list[dict[str, object] | None] = [] async def respond(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: assert messages - observed_max_tokens.append(None if info.model_settings is None else info.model_settings.get("max_tokens")) + observed_settings.append(None if info.model_settings is None else dict(info.model_settings)) return ModelResponse(parts=[TextPart("ok")]) - asyncio.run(probe_pydantic_ai_model(FunctionModel(respond), timeout_seconds=1)) + asyncio.run( + probe_pydantic_ai_model( + FunctionModel(respond), + timeout_seconds=1, + model_settings={ + "max_tokens": 100, + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + }, + ) + ) - assert observed_max_tokens == [1] + assert observed_settings == [ + { + "max_tokens": 1, + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + } + ] def test_generation_readiness_probe_maps_bad_provider_endpoint_without_leaking_body() -> None: diff --git a/tests/test_server.py b/tests/test_server.py index 9c541f11b..db6e8e9d0 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -103,6 +103,10 @@ def test_settings_load_server_environment(monkeypatch) -> None: monkeypatch.setenv("POWERCONTEXT_SERVER_RUNTIME_MEMORY_RERANK_CANDIDATE_LIMIT", "40") monkeypatch.setenv("POWERCONTEXT_SERVER_RUNTIME_EXPERIENCE_SCHEDULE_SECONDS", "45") monkeypatch.setenv("POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL", " test ") + monkeypatch.setenv( + "POWERCONTEXT_SERVER_INFERENCE_GENERATION_MODEL_SETTINGS", + '{"extra_body":{"chat_template_kwargs":{"enable_thinking":false}}}', + ) monkeypatch.setenv("POWERCONTEXT_SERVER_INFERENCE_GENERATION_TIMEOUT_SECONDS", "12.5") monkeypatch.setenv("POWERCONTEXT_SERVER_INFERENCE_GENERATION_MAX_REQUESTS", "4") monkeypatch.setenv("POWERCONTEXT_SERVER_MCP_ENABLED", "false") @@ -131,6 +135,9 @@ def test_settings_load_server_environment(monkeypatch) -> None: assert settings.runtime.memory_rerank_candidate_limit == 40 assert settings.runtime.experience_schedule_seconds == 45 assert settings.inference.generation_model == "test" + assert settings.inference.generation_model_settings == { + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}} + } assert settings.inference.generation_timeout_seconds == 12.5 assert settings.inference.generation_max_requests == 4 assert settings.mcp.enabled is False @@ -393,6 +400,39 @@ def test_server_factory_reports_database_and_configured_generation_readiness(tmp } +def test_server_factory_applies_generation_model_settings_to_readiness(monkeypatch, tmp_path) -> None: + observed_settings: list[dict[str, object] | None] = [] + + async def respond(_messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + observed_settings.append(None if info.model_settings is None else dict(info.model_settings)) + return ModelResponse(parts=[]) + + monkeypatch.setattr("pydantic_ai.models.infer_model", lambda _name: FunctionModel(respond)) + app = create_server_app( + settings=ServerSettings( + database=SQLiteConfig(url=f"sqlite+aiosqlite:///{tmp_path / 'runtime.db'}"), + inference=InferenceConfig( + generation_model="provider:test-model", + generation_model_settings={ + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + }, + ), + mcp=McpConfig(enabled=False), + ) + ) + + with TestClient(app) as client: + response = client.get("/health/ready") + + assert response.status_code == 200 + assert observed_settings == [ + { + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + "max_tokens": 1, + } + ] + + def test_server_factory_reports_generation_failure_as_degraded(monkeypatch, tmp_path) -> None: async def rate_limited(_messages: list[ModelMessage], _info: AgentInfo) -> ModelResponse: raise ModelHTTPError(429, "test-model", {"secret": "provider response"}) diff --git a/tests/test_server_generation.py b/tests/test_server_generation.py new file mode 100644 index 000000000..a37588a59 --- /dev/null +++ b/tests/test_server_generation.py @@ -0,0 +1,37 @@ +# Copyright (c) 2026 OceanBase. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from powercontext.builtin.runtime import InferenceConfig + + +def test_generation_model_settings_require_generation_model() -> None: + with pytest.raises(ValidationError, match="generation_model_settings requires generation_model"): + InferenceConfig( + generation_model_settings={ + "extra_body": {"chat_template_kwargs": {"enable_thinking": False}}, + } + ) + + +def test_generation_model_settings_reject_static_headers() -> None: + with pytest.raises(ValidationError, match="configure credentials and static headers"): + InferenceConfig( + generation_model="provider:model", + generation_model_settings={"extra_headers": {"Authorization": "Bearer secret"}}, + )