From 9342a84733ae4fa778bda6d56a1ba040db1b7f98 Mon Sep 17 00:00:00 2001 From: zeding Date: Mon, 2 Nov 2026 18:41:28 +0000 Subject: [PATCH 1/3] Python: Honor injected Foundry embedding clients without endpoint settings An injected text or image SDK client is the transport for that modality, so construction no longer requires a Foundry endpoint, API key, or credential. Ambient endpoint settings cannot replace or block that client. The other modality is created only when a request needs it and connection settings exist. Co-authored-by: ZedingZhang --- python/packages/foundry/README.md | 12 +- .../_embedding_client.py | 173 +++++++++++--- .../foundry/test_foundry_embedding_client.py | 218 +++++++++++++++++- 3 files changed, 362 insertions(+), 41 deletions(-) diff --git a/python/packages/foundry/README.md b/python/packages/foundry/README.md index f6be0f52062..f63c272563b 100644 --- a/python/packages/foundry/README.md +++ b/python/packages/foundry/README.md @@ -71,9 +71,15 @@ The client uses the project for authentication and converts a `https://.services.ai.azure.com/api/projects/` endpoint to the documented resource-scoped `https://.openai.azure.com/openai/v1/` model route. The existing `FOUNDRY_MODELS_ENDPOINT` and `FOUNDRY_MODELS_API_KEY` configuration remains available for Foundry Models inference endpoints. A Models endpoint is -required for image embedding models. If both project and Models endpoints are configured only through environment -variables, the Models endpoint is retained for backward compatibility; pass `project_endpoint` explicitly to select -the project OpenAI deployment. +required for image embedding models unless an `ImageEmbeddingsClient` is supplied. If both project and Models +endpoints are configured only through environment variables, the Models endpoint is retained for backward +compatibility; pass `project_endpoint` explicitly to select the project OpenAI deployment. + +A preconfigured `EmbeddingsClient` or `ImageEmbeddingsClient` can be passed as `text_client` or `image_client`. +The injected client is the transport for that modality and takes precedence over constructor connection settings, +environment settings, and framework defaults, so no endpoint, API key, credential, or Foundry environment is +required for it. `model` and `image_model` keep their usual explicit-argument, environment, then default +resolution. The other modality is created only when a request needs it and enough settings are available. ## Evaluations diff --git a/python/packages/foundry/agent_framework_foundry/_embedding_client.py b/python/packages/foundry/agent_framework_foundry/_embedding_client.py index 8050198ab30..e4cd6f54dd5 100644 --- a/python/packages/foundry/agent_framework_foundry/_embedding_client.py +++ b/python/packages/foundry/agent_framework_foundry/_embedding_client.py @@ -8,7 +8,7 @@ import sys from collections.abc import Mapping, Sequence from contextlib import suppress -from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypedDict, cast +from typing import TYPE_CHECKING, Any, ClassVar, Generic, TypedDict from urllib.parse import urlsplit, urlunsplit from agent_framework import ( @@ -59,6 +59,50 @@ def _get_openai_model_base_url(endpoint: str) -> str: return urlunsplit((parts.scheme, openai_netloc, "/openai/v1/", "", "")) +def _injected_client_endpoint(client: Any) -> str | None: + """Return the endpoint configured on an injected Azure AI Inference client. + + ``EmbeddingsClient`` and ``ImageEmbeddingsClient`` store the endpoint passed to + their constructor on ``_config.endpoint``. The pipeline base URL is an unresolved + ``{endpoint}`` template, so it is not a service URL. Non-string values, including + those produced by test doubles, are ignored. + """ + config = getattr(client, "_config", None) + endpoint = getattr(config, "endpoint", None) if config is not None else None + if not isinstance(endpoint, str): + return None + endpoint = endpoint.strip() + return endpoint or None + + +def _service_url_for_inference_clients( + *, + text_client: Any, + image_client: Any, + explicit_endpoint: str | None, + resolved_endpoint: str | None, +) -> str: + """Choose a service URL without letting ambient settings replace an injected transport.""" + if text_client is not None or image_client is not None: + for client in (text_client, image_client): + if client is None: + continue + if endpoint := _injected_client_endpoint(client): + return endpoint + return explicit_endpoint or "" + return resolved_endpoint or "" + + +_MISSING_TEXT_EMBEDDING_TRANSPORT = ( + "Text embeddings require an injected 'text_client' or enough Foundry Models settings to create one. " + "Set 'endpoint' or 'FOUNDRY_MODELS_ENDPOINT', and 'api_key', 'credential', or 'FOUNDRY_MODELS_API_KEY'." +) +_MISSING_IMAGE_EMBEDDING_TRANSPORT = ( + "Image embeddings require an injected 'image_client' or enough Foundry Models settings to create one. " + "Set 'endpoint' or 'FOUNDRY_MODELS_ENDPOINT', and 'api_key', 'credential', or 'FOUNDRY_MODELS_API_KEY'." +) + + class FoundryEmbeddingOptions(EmbeddingGenerationOptions, total=False): """Foundry-specific embedding options. @@ -138,12 +182,16 @@ class RawFoundryEmbeddingClient( Can also be set via environment variable FOUNDRY_MODELS_ENDPOINT. api_key: API key for authentication. Can also be set via environment variable FOUNDRY_MODELS_API_KEY. - text_client: Optional pre-configured ``EmbeddingsClient``. - image_client: Optional pre-configured ``ImageEmbeddingsClient``. + text_client: Optional pre-configured ``EmbeddingsClient``. When supplied, + it is the text-embedding transport: endpoint, API key, credential, + and Foundry environment settings are not required and do not replace it. + image_client: Optional pre-configured ``ImageEmbeddingsClient``. When supplied, + it is the image-embedding transport: endpoint, API key, credential, + and Foundry environment settings are not required and do not replace it. credential: Async Azure credential. Required when using ``project_endpoint`` without a ``project_client``. For a Foundry Models endpoint, an ``AzureKeyCredential`` is created from - ``api_key`` when needed. + ``api_key`` when a client still needs to be created. allow_preview: Enables preview opt-in on an internally created ``AIProjectClient``. default_headers: Additional HTTP headers for project OpenAI requests. @@ -215,6 +263,7 @@ def __init__( self._openai_client: AsyncOpenAI | None = None self._text_client: EmbeddingsClient | None = None self._image_client: ImageEmbeddingsClient | None = None + self._inference_client_kwargs: dict[str, Any] | None = None self.default_headers = ( {key: value for key, value in default_headers.items() if key != USER_AGENT_KEY} if default_headers else None ) @@ -262,7 +311,9 @@ def __init__( self._endpoint = _get_openai_model_base_url(str(self._openai_client.base_url)) self._openai_client.base_url = self._endpoint else: - if not resolved_models_endpoint: + # An injected SDK client is a complete transport. Connection settings are + # required only for a text or image operation that still needs a client created. + if not resolved_models_endpoint and text_client is None and image_client is None: raise ValueError( "Either 'project_endpoint', 'project_client', or 'endpoint' is required. " "Set a Foundry project endpoint via 'FOUNDRY_PROJECT_ENDPOINT' or a Foundry Models " @@ -275,24 +326,31 @@ def __init__( if credential is None and text_client is None and image_client is None: raise ValueError("Either 'api_key', 'credential', or pre-configured client(s) must be provided.") - client_kwargs: dict[str, Any] = { - "endpoint": resolved_models_endpoint, - "credential": credential, - } - if IS_TELEMETRY_ENABLED: - client_kwargs["user_agent"] = get_user_agent() + client_kwargs: dict[str, Any] | None = None + if resolved_models_endpoint and credential is not None: + client_kwargs = { + "endpoint": resolved_models_endpoint, + "credential": credential, + } + if IS_TELEMETRY_ENABLED: + client_kwargs["user_agent"] = get_user_agent() + self._text_client = text_client self._image_client = image_client - if credential is not None: - self._text_client = text_client or EmbeddingsClient( - **client_kwargs, - per_retry_policies=[create_feature_usage_policy()], - ) - self._image_client = image_client or ImageEmbeddingsClient( - **client_kwargs, - per_retry_policies=[create_feature_usage_policy()], - ) - self._endpoint = resolved_models_endpoint + self._inference_client_kwargs = client_kwargs + # Create missing clients immediately only when the caller supplied no transport. + # An injected client must not be replaced, and an unrelated or invalid ambient + # endpoint must not fail construction of the transport the caller already provided. + # The other modality is created later, when a request needs it and settings allow it. + if client_kwargs is not None and text_client is None and image_client is None: + self._ensure_text_client() + self._ensure_image_client() + self._endpoint = _service_url_for_inference_clients( + text_client=text_client, + image_client=image_client, + explicit_endpoint=endpoint, + resolved_endpoint=resolved_models_endpoint, + ) super().__init__(additional_properties=additional_properties) @@ -320,9 +378,48 @@ async def __aexit__(self, *args: Any) -> None: await self.close() def service_url(self) -> str: - """Get the URL of the service.""" + """Get the URL of the service. + + When a text or image SDK client is injected, the URL is taken from that client + when its endpoint can be read. An explicit ``endpoint`` argument is used if the + injected client does not expose one. Ambient endpoint settings are not substituted + for an injected transport; an unknown endpoint is reported as an empty string. + """ return self._endpoint or "" + def _ensure_text_client(self) -> EmbeddingsClient: + """Return the text embeddings transport, creating it from stored settings when needed.""" + if self._text_client is not None: + return self._text_client + client_kwargs = self._inference_client_kwargs + if client_kwargs is None: + raise ValueError(_MISSING_TEXT_EMBEDDING_TRANSPORT) + created = EmbeddingsClient( + **client_kwargs, + per_retry_policies=[create_feature_usage_policy()], + ) + self._text_client = created + return created + + def _ensure_image_client(self) -> ImageEmbeddingsClient: + """Return the image embeddings transport, creating it from stored settings when needed.""" + if self._image_client is not None: + return self._image_client + if self.project_client is not None or self._openai_client is not None: + raise ValueError( + "Image embeddings require a Foundry Models inference endpoint. " + "Configure 'endpoint' or 'FOUNDRY_MODELS_ENDPOINT' instead of a project endpoint." + ) + client_kwargs = self._inference_client_kwargs + if client_kwargs is None: + raise ValueError(_MISSING_IMAGE_EMBEDDING_TRANSPORT) + created = ImageEmbeddingsClient( + **client_kwargs, + per_retry_policies=[create_feature_usage_policy()], + ) + self._image_client = created + return created + async def get_embeddings( self, values: Sequence[Content | str], @@ -344,7 +441,9 @@ async def get_embeddings( Generated embeddings with usage metadata. Raises: - ValueError: If model is not provided or an unsupported content type is encountered. + ValueError: If model is not provided, an unsupported content type is encountered, + or the requested text or image operation has neither an injected transport nor + enough settings to create one. """ if not values: return GeneratedEmbeddings([], options=options) @@ -395,13 +494,11 @@ async def get_embeddings( embeddings: list[Embedding[list[float]] | None] = [None] * len(values) usage_details: UsageDetails = {} - image_client = self._image_client - if image_items and image_client is None: - raise ValueError( - "Image embeddings require a Foundry Models inference endpoint. " - "Configure 'endpoint' or 'FOUNDRY_MODELS_ENDPOINT' instead of a project endpoint." - ) - image_client = cast(ImageEmbeddingsClient, image_client) + # Resolve an image transport before any request so a mixed batch cannot send text + # and then fail to embed the images. + image_client: ImageEmbeddingsClient | None = None + if image_items: + image_client = self._ensure_image_client() # Embed text inputs. if text_items: @@ -456,8 +553,8 @@ async def get_embeddings( if openai_response.usage: usage_details["input_token_count"] = openai_response.usage.prompt_tokens usage_details["total_token_count"] = openai_response.usage.total_tokens - elif self._text_client is not None: - inference_response = await self._text_client.embed( + else: + inference_response = await self._ensure_text_client().embed( input=text_inputs, model=text_model, **common_kwargs, @@ -477,11 +574,9 @@ async def get_embeddings( usage_details["output_token_count"] = (usage_details.get("output_token_count") or 0) + ( getattr(inference_response.usage, "completion_tokens", 0) or 0 ) - else: - raise RuntimeError("No text embedding client is configured.") # Embed image inputs. - if image_items: + if image_client is not None: if not (image_model := opts.get("image_model") or self.image_model): raise ValueError("An image_model is required, either in the client or options, for image inputs.") image_inputs = [img for _, img in image_items] @@ -536,8 +631,12 @@ class FoundryEmbeddingClient( Can also be set via environment variable FOUNDRY_MODELS_ENDPOINT. api_key: API key for authentication. Can also be set via environment variable FOUNDRY_MODELS_API_KEY. - text_client: Optional pre-configured ``EmbeddingsClient``. - image_client: Optional pre-configured ``ImageEmbeddingsClient``. + text_client: Optional pre-configured ``EmbeddingsClient``. When supplied, + it is the text-embedding transport and does not require endpoint, API key, + credential, or Foundry environment settings. + image_client: Optional pre-configured ``ImageEmbeddingsClient``. When supplied, + it is the image-embedding transport and does not require endpoint, API key, + credential, or Foundry environment settings. credential: Async Azure credential. allow_preview: Enables preview opt-in on an internally created ``AIProjectClient``. diff --git a/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py b/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py index 91a0e994bf4..5895be2df8e 100644 --- a/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py +++ b/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py @@ -4,13 +4,14 @@ import os from collections.abc import Sequence +from pathlib import Path from typing import Any, cast from unittest.mock import ANY, AsyncMock, MagicMock, patch import pytest from agent_framework import Content, SecretString from agent_framework._telemetry import USER_AGENT_KEY, get_user_agent -from agent_framework.exceptions import IntegrationInvalidResponseException +from agent_framework.exceptions import IntegrationInvalidResponseException, SettingNotFoundError from azure.core.credentials import AzureKeyCredential from azure.identity.aio import AzureCliCredential @@ -547,6 +548,221 @@ async def test_image_model_sent_to_image_client( assert call_kwargs.kwargs["model"] == "image-model" +class TestInjectedEmbeddingClients: + """Injected SDK clients are complete transports and outrank connection settings.""" + + async def test_text_client_without_connection_settings(self) -> None: + """A text client and model are enough: no endpoint, key, credential, dotenv, or Foundry env.""" + text_client = AsyncMock() + text_client.embed = AsyncMock(return_value=_make_embed_response([[0.1, 0.2, 0.3]])) + with ( + patch.dict(os.environ, {}, clear=True), + patch("agent_framework_foundry._embedding_client.EmbeddingsClient") as text_factory, + patch("agent_framework_foundry._embedding_client.ImageEmbeddingsClient") as image_factory, + patch("azure.ai.projects.aio.AIProjectClient") as project_factory, + ): + client = RawFoundryEmbeddingClient(model="text-embedding-model", text_client=text_client) + result = await client.get_embeddings(["hello"]) + + assert result[0].vector == [0.1, 0.2, 0.3] + assert client.model == "text-embedding-model" + assert client.service_url() == "" + text_client.embed.assert_awaited_once() + assert text_client.embed.call_args.kwargs["model"] == "text-embedding-model" + text_factory.assert_not_called() + image_factory.assert_not_called() + project_factory.assert_not_called() + + async def test_image_client_without_connection_settings(self) -> None: + """An image client and model are enough under the same empty configuration.""" + image_client = AsyncMock() + image_client.embed = AsyncMock(return_value=_make_embed_response([[0.4, 0.5]])) + image = Content.from_data(data=b"\x89PNG", media_type="image/png") + with ( + patch.dict(os.environ, {}, clear=True), + patch("agent_framework_foundry._embedding_client.EmbeddingsClient") as text_factory, + patch("agent_framework_foundry._embedding_client.ImageEmbeddingsClient") as image_factory, + patch("azure.ai.projects.aio.AIProjectClient") as project_factory, + ): + client = FoundryEmbeddingClient(model="image-embedding-model", image_client=image_client) + result = await client.get_embeddings([image]) + + assert result[0].vector == [0.4, 0.5] + assert client.image_model == "image-embedding-model" + assert client.service_url() == "" + image_client.embed.assert_awaited_once() + assert image_client.embed.call_args.kwargs["model"] == "image-embedding-model" + text_factory.assert_not_called() + image_factory.assert_not_called() + project_factory.assert_not_called() + + def test_minimal_reproduction_constructs(self) -> None: + """The reported MagicMock reproduction no longer fails during construction.""" + with patch.dict(os.environ, {}, clear=True): + client = RawFoundryEmbeddingClient(model="text-embedding-model", text_client=MagicMock()) + + assert client.model == "text-embedding-model" + assert client.service_url() == "" + + async def test_invalid_endpoint_env_does_not_replace_or_block_injected_client(self, tmp_path: Path) -> None: + """Ambient and dotenv endpoints are ignored for an injected transport.""" + env_file = tmp_path / ".env" + env_file.write_text( + "FOUNDRY_MODELS_ENDPOINT=::::dotenv-not-a-url::::\nFOUNDRY_MODELS_API_KEY=dotenv-key\n", + encoding="utf-8", + ) + text_client = AsyncMock() + text_client.embed = AsyncMock(return_value=_make_embed_response([[0.7, 0.8]])) + text_client._config.endpoint = "https://injected.inference.ai.azure.com" + with ( + patch.dict( + os.environ, + { + "FOUNDRY_PROJECT_ENDPOINT": "not-a-project-endpoint", + "FOUNDRY_MODELS_ENDPOINT": "https://unrelated.example", + "FOUNDRY_MODELS_API_KEY": "env-key", + "FOUNDRY_EMBEDDING_MODEL": "env-model", + }, + clear=True, + ), + patch("agent_framework_foundry._embedding_client.EmbeddingsClient") as text_factory, + patch("agent_framework_foundry._embedding_client.ImageEmbeddingsClient") as image_factory, + patch("azure.ai.projects.aio.AIProjectClient") as project_factory, + ): + client = RawFoundryEmbeddingClient(text_client=text_client, env_file_path=str(env_file)) + result = await client.get_embeddings(["hello"]) + + assert result[0].vector == [0.7, 0.8] + assert client.model == "env-model" + assert client.service_url() == "https://injected.inference.ai.azure.com" + text_client.embed.assert_awaited_once() + text_factory.assert_not_called() + image_factory.assert_not_called() + project_factory.assert_not_called() + + def test_model_resolution_keeps_explicit_then_environment_order(self) -> None: + """Wrapper model settings still resolve independently of the injected transport.""" + with patch.dict( + os.environ, + { + "FOUNDRY_EMBEDDING_MODEL": "env-model", + "FOUNDRY_IMAGE_EMBEDDING_MODEL": "env-image-model", + "FOUNDRY_MODELS_ENDPOINT": "https://unrelated.example", + }, + clear=True, + ): + from_env = RawFoundryEmbeddingClient(text_client=AsyncMock()) + explicit = RawFoundryEmbeddingClient( + model="explicit-model", + image_model="explicit-image-model", + image_client=AsyncMock(), + ) + + assert from_env.model == "env-model" + assert from_env.image_model == "env-image-model" + assert explicit.model == "explicit-model" + assert explicit.image_model == "explicit-image-model" + + def test_injected_client_still_requires_a_model(self) -> None: + """Model resolution is unchanged: an injected transport does not invent a model.""" + with ( + patch.dict(os.environ, {}, clear=True), + pytest.raises(SettingNotFoundError, match="embedding_model"), + ): + RawFoundryEmbeddingClient(text_client=AsyncMock()) + + def test_service_url_prefers_injected_client_over_explicit_endpoint(self) -> None: + """A readable SDK endpoint outranks constructor and ambient endpoint settings.""" + text_client = AsyncMock() + image_client = AsyncMock() + image_client._config.endpoint = "https://image.inference.ai.azure.com" + with patch.dict( + os.environ, + {"FOUNDRY_MODELS_ENDPOINT": "https://env.example", "FOUNDRY_EMBEDDING_MODEL": "env-model"}, + clear=True, + ): + text_only = RawFoundryEmbeddingClient( + text_client=text_client, + endpoint="https://explicit.example", + ) + text_client._config.endpoint = "https://text.inference.ai.azure.com" + both = RawFoundryEmbeddingClient( + text_client=text_client, + image_client=image_client, + endpoint="https://explicit.example", + ) + + assert text_only.service_url() == "https://explicit.example" + assert both.service_url() == "https://text.inference.ai.azure.com" + + async def test_supplied_transport_factory_is_not_called_when_settings_can_create_the_other(self) -> None: + """Factories run only for a modality the caller did not already supply.""" + text_client = AsyncMock() + text_client.embed = AsyncMock(return_value=_make_embed_response([[0.1]])) + image = Content.from_data(data=b"\x89PNG", media_type="image/png") + with ( + patch.dict(os.environ, {}, clear=True), + patch("agent_framework_foundry._embedding_client.EmbeddingsClient") as text_factory, + patch("agent_framework_foundry._embedding_client.ImageEmbeddingsClient") as image_factory, + ): + image_factory.return_value.embed = AsyncMock(return_value=_make_embed_response([[0.9]])) + client = RawFoundryEmbeddingClient( + model="text-embedding-model", + endpoint="https://created.inference.ai.azure.com", + api_key="test-key", + text_client=text_client, + ) + text_factory.assert_not_called() + image_factory.assert_not_called() + + await client.get_embeddings(["hello"]) + text_factory.assert_not_called() + image_factory.assert_not_called() + + await client.get_embeddings([image]) + + text_factory.assert_not_called() + image_factory.assert_called_once_with( + endpoint="https://created.inference.ai.azure.com", + credential=ANY, + user_agent=get_user_agent(), + per_retry_policies=[ANY], + ) + assert image_factory.return_value.embed.call_args.kwargs["model"] == "text-embedding-model" + + async def test_missing_modality_raises_until_settings_can_create_it(self) -> None: + """A request raises only for the modality that has neither a client nor connection settings.""" + text_client = AsyncMock() + text_client.embed = AsyncMock(return_value=_make_embed_response([[0.1]])) + image_client = AsyncMock() + image_client.embed = AsyncMock(return_value=_make_embed_response([[0.4]])) + image = Content.from_data(data=b"\x89PNG", media_type="image/png") + with patch.dict(os.environ, {}, clear=True): + text_only = RawFoundryEmbeddingClient(model="text-embedding-model", text_client=text_client) + image_only = RawFoundryEmbeddingClient(model="image-embedding-model", image_client=image_client) + await text_only.get_embeddings(["hello"]) + await image_only.get_embeddings([image]) + with pytest.raises(ValueError, match="injected 'image_client'"): + await text_only.get_embeddings([image]) + with pytest.raises(ValueError, match="injected 'text_client'"): + await image_only.get_embeddings(["hello"]) + + text_client.embed.assert_awaited_once() + image_client.embed.assert_awaited_once() + + async def test_mixed_batch_fails_before_sending_text_when_image_transport_is_missing(self) -> None: + """A mixed request does not call the injected text client when image embeddings cannot be sent.""" + text_client = AsyncMock() + text_client.embed = AsyncMock(return_value=_make_embed_response([[0.1]])) + image = Content.from_data(data=b"\x89PNG", media_type="image/png") + with patch.dict(os.environ, {}, clear=True): + client = RawFoundryEmbeddingClient(model="text-embedding-model", text_client=text_client) + with pytest.raises(ValueError, match="injected 'image_client'"): + await client.get_embeddings(["hello", image]) + + text_client.embed.assert_not_called() + + class TestFoundryEmbeddingClient: """Tests for the telemetry-enabled Foundry embedding client.""" From b13fd4a2339298aa8b0e865b8e232d5e289cb857 Mon Sep 17 00:00:00 2001 From: Zeding Zhang Date: Tue, 3 Nov 2026 17:44:37 +0800 Subject: [PATCH 2/3] Python: Honor injected embedding clients over explicit project settings --- .../_embedding_client.py | 19 +++-- .../foundry/test_foundry_embedding_client.py | 76 +++++++++++++++++-- 2 files changed, 82 insertions(+), 13 deletions(-) diff --git a/python/packages/foundry/agent_framework_foundry/_embedding_client.py b/python/packages/foundry/agent_framework_foundry/_embedding_client.py index e4cd6f54dd5..a88d5f7d3e0 100644 --- a/python/packages/foundry/agent_framework_foundry/_embedding_client.py +++ b/python/packages/foundry/agent_framework_foundry/_embedding_client.py @@ -229,12 +229,13 @@ def __init__( ): api_key = None + has_injected_client = text_client is not None or image_client is not None explicit_project_source = project_client is not None or project_endpoint is not None explicit_inference_source = any(value is not None for value in (endpoint, api_key, text_client, image_client)) - if explicit_project_source and explicit_inference_source: + if not has_injected_client and explicit_project_source and explicit_inference_source: raise ValueError( "Foundry project embedding configuration cannot be combined with Foundry Models " - "'endpoint', 'api_key', 'text_client', or 'image_client' configuration." + "'endpoint' or 'api_key' configuration." ) settings = load_settings( @@ -254,8 +255,14 @@ def __init__( self.image_model: str = settings.get("image_embedding_model") or self.model # type: ignore[assignment] resolved_models_endpoint = settings.get("models_endpoint") or None resolved_project_endpoint = settings.get("project_endpoint") or None - use_project_client = explicit_project_source or ( - not explicit_inference_source and resolved_models_endpoint is None and resolved_project_endpoint is not None + # Injected SDK clients own their transport even when explicit project settings are also provided. + use_project_client = not has_injected_client and ( + explicit_project_source + or ( + not explicit_inference_source + and resolved_models_endpoint is None + and resolved_project_endpoint is not None + ) ) self.project_client: AIProjectClient | None = None @@ -269,10 +276,6 @@ def __init__( ) if use_project_client: - if text_client is not None or image_client is not None: - raise ValueError( - "'text_client' and 'image_client' cannot be used with 'project_endpoint' or 'project_client'." - ) if project_client is None: if not resolved_project_endpoint: raise ValueError( diff --git a/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py b/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py index 5895be2df8e..80a8fec72fb 100644 --- a/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py +++ b/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py @@ -415,13 +415,28 @@ def test_project_endpoint_requires_credential(self) -> None: with pytest.raises(ValueError, match="A token credential is required"): RawFoundryEmbeddingClient(credential=AzureKeyCredential("test-key")) - def test_explicit_project_and_inference_sources_raise(self) -> None: - """Explicit project and Models endpoint configuration cannot be combined.""" - with pytest.raises(ValueError, match="cannot be combined with Foundry Models"): + @pytest.mark.parametrize("project_source", ["project_endpoint", "project_client"]) + @pytest.mark.parametrize( + ("models_source", "models_value"), + [("endpoint", "https://test.inference.ai.azure.com"), ("api_key", "test-key")], + ) + def test_explicit_project_and_inference_sources_raise( + self, project_source: str, models_source: str, models_value: str + ) -> None: + """Explicit project and Models settings still conflict without an injected SDK client.""" + kwargs: dict[str, Any] = { + project_source: MagicMock() + if project_source == "project_client" + else "https://test.services.ai.azure.com/api/projects/test", + models_source: models_value, + } + with ( + patch.dict(os.environ, {}, clear=True), + pytest.raises(ValueError, match="cannot be combined with Foundry Models"), + ): RawFoundryEmbeddingClient( model="text-embedding-3-small", - project_client=MagicMock(), - endpoint="https://test.inference.ai.azure.com", + **kwargs, ) @pytest.mark.parametrize(("endpoint", "api_key"), [("", ""), (" ", " ")]) @@ -596,6 +611,57 @@ async def test_image_client_without_connection_settings(self) -> None: image_factory.assert_not_called() project_factory.assert_not_called() + @pytest.mark.parametrize("client_type", [RawFoundryEmbeddingClient, FoundryEmbeddingClient]) + @pytest.mark.parametrize("injected_client_name", ["text_client", "image_client"]) + @pytest.mark.parametrize("project_source", ["project_endpoint", "project_client"]) + @pytest.mark.parametrize( + "models_settings", + [{}, {"endpoint": "https://explicit.inference.ai.azure.com", "api_key": "test-key"}], + ids=["no-models-settings", "explicit-models-settings"], + ) + async def test_injected_client_takes_precedence_over_explicit_project_settings( + self, + client_type: type[RawFoundryEmbeddingClient[Any]], + injected_client_name: str, + project_source: str, + models_settings: dict[str, str], + mock_text_client: AsyncMock, + mock_image_client: AsyncMock, + ) -> None: + """Explicit project settings cannot block or replace an injected transport.""" + injected_client = mock_text_client if injected_client_name == "text_client" else mock_image_client + injected_client._config.endpoint = "https://injected.inference.ai.azure.com" + project_client = MagicMock() + kwargs: dict[str, Any] = { + project_source: project_client + if project_source == "project_client" + else "https://test.services.ai.azure.com/api/projects/test", + injected_client_name: injected_client, + **models_settings, + } + values: Sequence[Content | str] = ( + ["hello"] + if injected_client_name == "text_client" + else [Content.from_data(data=b"\x89PNG", media_type="image/png")] + ) + with ( + patch.dict(os.environ, {}, clear=True), + patch("agent_framework_foundry._embedding_client.EmbeddingsClient") as text_factory, + patch("agent_framework_foundry._embedding_client.ImageEmbeddingsClient") as image_factory, + patch("azure.ai.projects.aio.AIProjectClient") as project_factory, + ): + client = client_type(model="embedding-model", **kwargs) + result = await client.get_embeddings(values) + + expected_vector = [0.1, 0.2, 0.3] if injected_client_name == "text_client" else [0.4, 0.5, 0.6] + assert result[0].vector == expected_vector + assert client.service_url() == "https://injected.inference.ai.azure.com" + injected_client.embed.assert_awaited_once() + project_client.get_openai_client.assert_not_called() + text_factory.assert_not_called() + image_factory.assert_not_called() + project_factory.assert_not_called() + def test_minimal_reproduction_constructs(self) -> None: """The reported MagicMock reproduction no longer fails during construction.""" with patch.dict(os.environ, {}, clear=True): From 6d11349a8bb395ed817cd6a30c316538a2318b3e Mon Sep 17 00:00:00 2001 From: Zeding Zhang Date: Tue, 3 Nov 2026 18:04:08 +0800 Subject: [PATCH 3/3] Python: Preserve lazy project text embeddings with injected image clients --- python/packages/foundry/README.md | 2 + .../_embedding_client.py | 121 ++++++++------ .../foundry/test_foundry_embedding_client.py | 158 ++++++++++++++++++ 3 files changed, 227 insertions(+), 54 deletions(-) diff --git a/python/packages/foundry/README.md b/python/packages/foundry/README.md index f63c272563b..6849e542bb1 100644 --- a/python/packages/foundry/README.md +++ b/python/packages/foundry/README.md @@ -80,6 +80,8 @@ The injected client is the transport for that modality and takes precedence over environment settings, and framework defaults, so no endpoint, API key, credential, or Foundry environment is required for it. `model` and `image_model` keep their usual explicit-argument, environment, then default resolution. The other modality is created only when a request needs it and enough settings are available. +For example, an injected `image_client` can be combined with project configuration for text embeddings; +the project text transport is initialized only when a request includes text. ## Evaluations diff --git a/python/packages/foundry/agent_framework_foundry/_embedding_client.py b/python/packages/foundry/agent_framework_foundry/_embedding_client.py index a88d5f7d3e0..63e06aacd95 100644 --- a/python/packages/foundry/agent_framework_foundry/_embedding_client.py +++ b/python/packages/foundry/agent_framework_foundry/_embedding_client.py @@ -177,7 +177,7 @@ class RawFoundryEmbeddingClient( embedding deployments. Can also be set via environment variable FOUNDRY_PROJECT_ENDPOINT. project_client: An existing ``AIProjectClient``. If provided, its - OpenAI-compatible client is used for text embeddings. + OpenAI-compatible client is used for text embeddings unless ``text_client`` is supplied. endpoint: The Foundry inference endpoint URL. Can also be set via environment variable FOUNDRY_MODELS_ENDPOINT. api_key: API key for authentication. @@ -231,7 +231,7 @@ def __init__( has_injected_client = text_client is not None or image_client is not None explicit_project_source = project_client is not None or project_endpoint is not None - explicit_inference_source = any(value is not None for value in (endpoint, api_key, text_client, image_client)) + explicit_inference_source = endpoint is not None or api_key is not None if not has_injected_client and explicit_project_source and explicit_inference_source: raise ValueError( "Foundry project embedding configuration cannot be combined with Foundry Models " @@ -255,8 +255,8 @@ def __init__( self.image_model: str = settings.get("image_embedding_model") or self.model # type: ignore[assignment] resolved_models_endpoint = settings.get("models_endpoint") or None resolved_project_endpoint = settings.get("project_endpoint") or None - # Injected SDK clients own their transport even when explicit project settings are also provided. - use_project_client = not has_injected_client and ( + # Only an injected text client overrides project-backed text embeddings. + use_project_client = text_client is None and ( explicit_project_source or ( not explicit_inference_source @@ -265,54 +265,30 @@ def __init__( ) ) - self.project_client: AIProjectClient | None = None + self.project_client: AIProjectClient | None = project_client if use_project_client else None + self._project_endpoint = resolved_project_endpoint if use_project_client else None + self._project_credential = credential + self._project_allow_preview = allow_preview + self._project_default_headers = default_headers self._owns_project_client = False self._openai_client: AsyncOpenAI | None = None - self._text_client: EmbeddingsClient | None = None - self._image_client: ImageEmbeddingsClient | None = None + self._text_client = text_client + self._image_client = image_client self._inference_client_kwargs: dict[str, Any] | None = None + self._endpoint = _service_url_for_inference_clients( + text_client=text_client, + image_client=image_client, + explicit_endpoint=endpoint, + resolved_endpoint=resolved_models_endpoint, + ) self.default_headers = ( {key: value for key, value in default_headers.items() if key != USER_AGENT_KEY} if default_headers else None ) if use_project_client: - if project_client is None: - if not resolved_project_endpoint: - raise ValueError( - "Foundry project endpoint is required. Set via 'project_endpoint' parameter " - "or 'FOUNDRY_PROJECT_ENDPOINT' environment variable." - ) - if credential is None: - raise ValueError( - "Azure credential is required when using project_endpoint without a project_client." - ) - if isinstance(credential, AzureKeyCredential): - raise ValueError("A token credential is required when using a Foundry project endpoint.") - - from azure.ai.projects.aio import AIProjectClient - - project_client_kwargs: dict[str, Any] = { - "endpoint": resolved_project_endpoint, - "credential": credential, - "per_retry_policies": [create_feature_usage_policy()], - } - if IS_TELEMETRY_ENABLED: - project_client_kwargs["user_agent"] = get_user_agent() - if allow_preview is not None: - project_client_kwargs["allow_preview"] = allow_preview - project_client = AIProjectClient(**project_client_kwargs) - self._owns_project_client = True - - openai_kwargs: dict[str, Any] = {} - if default_headers: - openai_kwargs["default_headers"] = default_headers - if self._owns_project_client: - openai_kwargs["http_client"] = create_foundry_feature_usage_http_client() - - self.project_client = project_client - self._openai_client = project_client.get_openai_client(**openai_kwargs) - self._endpoint = _get_openai_model_base_url(str(self._openai_client.base_url)) - self._openai_client.base_url = self._endpoint + # Image-only operations must not initialize or validate the unrelated text transport. + if not has_injected_client: + self._endpoint = str(self._ensure_openai_client().base_url) else: # An injected SDK client is a complete transport. Connection settings are # required only for a text or image operation that still needs a client created. @@ -338,8 +314,6 @@ def __init__( if IS_TELEMETRY_ENABLED: client_kwargs["user_agent"] = get_user_agent() - self._text_client = text_client - self._image_client = image_client self._inference_client_kwargs = client_kwargs # Create missing clients immediately only when the caller supplied no transport. # An injected client must not be replaced, and an unrelated or invalid ambient @@ -348,12 +322,6 @@ def __init__( if client_kwargs is not None and text_client is None and image_client is None: self._ensure_text_client() self._ensure_image_client() - self._endpoint = _service_url_for_inference_clients( - text_client=text_client, - image_client=image_client, - explicit_endpoint=endpoint, - resolved_endpoint=resolved_models_endpoint, - ) super().__init__(additional_properties=additional_properties) @@ -390,6 +358,48 @@ def service_url(self) -> str: """ return self._endpoint or "" + def _ensure_openai_client(self) -> AsyncOpenAI: + """Return the project text transport, initializing it from retained settings when needed.""" + if self._openai_client is not None: + return self._openai_client + project_client = self.project_client + if project_client is None: + if not self._project_endpoint: + raise ValueError( + "Foundry project endpoint is required. Set via 'project_endpoint' parameter " + "or 'FOUNDRY_PROJECT_ENDPOINT' environment variable." + ) + if self._project_credential is None: + raise ValueError("Azure credential is required when using project_endpoint without a project_client.") + if isinstance(self._project_credential, AzureKeyCredential): + raise ValueError("A token credential is required when using a Foundry project endpoint.") + + from azure.ai.projects.aio import AIProjectClient + + project_client_kwargs: dict[str, Any] = { + "endpoint": self._project_endpoint, + "credential": self._project_credential, + "per_retry_policies": [create_feature_usage_policy()], + } + if IS_TELEMETRY_ENABLED: + project_client_kwargs["user_agent"] = get_user_agent() + if self._project_allow_preview is not None: + project_client_kwargs["allow_preview"] = self._project_allow_preview + project_client = AIProjectClient(**project_client_kwargs) + self.project_client = project_client + self._owns_project_client = True + + openai_kwargs: dict[str, Any] = {} + if self._project_default_headers: + openai_kwargs["default_headers"] = self._project_default_headers + if self._owns_project_client: + openai_kwargs["http_client"] = create_foundry_feature_usage_http_client() + + openai_client = project_client.get_openai_client(**openai_kwargs) + openai_client.base_url = _get_openai_model_base_url(str(openai_client.base_url)) + self._openai_client = openai_client + return openai_client + def _ensure_text_client(self) -> EmbeddingsClient: """Return the text embeddings transport, creating it from stored settings when needed.""" if self._text_client is not None: @@ -508,7 +518,10 @@ async def get_embeddings( if not (text_model := opts.get("model") or self.model): raise ValueError("A model is required, either in the client or options, for text inputs.") text_inputs = [t for _, t in text_items] - if self._openai_client is not None: + openai_client = self._openai_client + if openai_client is None and (self.project_client is not None or self._project_endpoint is not None): + openai_client = self._ensure_openai_client() + if openai_client is not None: openai_kwargs: dict[str, Any] = { "input": text_inputs, "model": text_model, @@ -524,7 +537,7 @@ async def get_embeddings( if extra_body: openai_kwargs["extra_body"] = extra_body - openai_response = await self._openai_client.embeddings.create(**openai_kwargs) + openai_response = await openai_client.embeddings.create(**openai_kwargs) encoding = openai_kwargs.get("encoding_format", "float") for item in sorted(openai_response.data, key=lambda value: value.index): original_idx = text_items[item.index][0] diff --git a/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py b/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py index 80a8fec72fb..2fddc92a813 100644 --- a/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py +++ b/python/packages/foundry/tests/foundry/test_foundry_embedding_client.py @@ -662,6 +662,164 @@ async def test_injected_client_takes_precedence_over_explicit_project_settings( image_factory.assert_not_called() project_factory.assert_not_called() + @pytest.mark.parametrize("client_type", [RawFoundryEmbeddingClient, FoundryEmbeddingClient]) + @pytest.mark.parametrize("project_source", ["project_client", "project_endpoint", "environment"]) + async def test_image_client_preserves_lazy_project_text_embeddings( + self, + client_type: type[RawFoundryEmbeddingClient[Any]], + project_source: str, + mock_image_client: AsyncMock, + ) -> None: + """An injected image transport leaves project-backed text available on demand.""" + project_endpoint = "https://test.services.ai.azure.com/api/projects/test" + openai_client = _make_openai_client() + project_client = MagicMock() + project_client.get_openai_client.return_value = openai_client + project_client.close = AsyncMock() + credential = MagicMock() + default_headers = {"X-Test": "value"} + kwargs: dict[str, Any] = {"default_headers": default_headers} + env: dict[str, str] = {} + if project_source == "project_client": + kwargs["project_client"] = project_client + else: + kwargs.update(credential=credential, allow_preview=True) + if project_source == "project_endpoint": + kwargs["project_endpoint"] = project_endpoint + else: + env["FOUNDRY_PROJECT_ENDPOINT"] = project_endpoint + mock_image_client._config.endpoint = "https://image.inference.ai.azure.com" + image = Content.from_data(data=b"\x89PNG", media_type="image/png") + with ( + patch.dict(os.environ, env, clear=True), + patch("azure.ai.projects.aio.AIProjectClient", return_value=project_client) as project_factory, + patch("agent_framework_foundry._embedding_client.EmbeddingsClient") as text_factory, + patch("agent_framework_foundry._embedding_client.ImageEmbeddingsClient") as image_factory, + ): + client = client_type( + model="text-model", image_model="image-model", image_client=mock_image_client, **kwargs + ) + project_factory.assert_not_called() + project_client.get_openai_client.assert_not_called() + + image_result = await client.get_embeddings([image]) + assert image_result[0].vector == [0.4, 0.5, 0.6] + mock_image_client.embed.assert_awaited_once() + project_factory.assert_not_called() + project_client.get_openai_client.assert_not_called() + + text_result = await client.get_embeddings(["hello"]) + assert text_result[0].vector == [0.1, 0.2, 0.3] + openai_client.embeddings.create.assert_awaited_once_with(input=["hello"], model="text-model") + + mixed_result = await client.get_embeddings([image, "hello"]) + assert [embedding.vector for embedding in mixed_result] == [[0.4, 0.5, 0.6], [0.1, 0.2, 0.3]] + assert openai_client.embeddings.create.await_count == 2 + assert mock_image_client.embed.await_count == 2 + assert mock_image_client.embed.call_args.kwargs["model"] == "image-model" + assert client.service_url() == "https://image.inference.ai.azure.com" + assert str(openai_client.base_url) == "https://test.openai.azure.com/openai/v1/" + await client.close() + + text_factory.assert_not_called() + image_factory.assert_not_called() + openai_client.close.assert_awaited_once() + if project_source == "project_client": + project_factory.assert_not_called() + project_client.get_openai_client.assert_called_once_with(default_headers=default_headers) + project_client.close.assert_not_called() + else: + project_factory.assert_called_once_with( + endpoint=project_endpoint, + credential=credential, + allow_preview=True, + user_agent=get_user_agent(), + per_retry_policies=[ANY], + ) + project_client.get_openai_client.assert_called_once_with(default_headers=default_headers, http_client=ANY) + project_client.close.assert_awaited_once() + + @pytest.mark.parametrize( + ("credential", "message"), + [(None, "Azure credential is required"), (AzureKeyCredential("test-key"), "A token credential is required")], + ) + async def test_image_client_defers_project_credential_validation_until_text_is_requested( + self, credential: AzureKeyCredential | None, message: str, mock_image_client: AsyncMock + ) -> None: + """Incomplete project settings do not prevent requests through the injected image client.""" + image = Content.from_data(data=b"\x89PNG", media_type="image/png") + with ( + patch.dict(os.environ, {}, clear=True), + patch("azure.ai.projects.aio.AIProjectClient") as project_factory, + ): + client = RawFoundryEmbeddingClient( + model="embedding-model", + project_endpoint="https://test.services.ai.azure.com/api/projects/test", + credential=credential, + image_client=mock_image_client, + ) + result = await client.get_embeddings([image]) + assert result[0].vector == [0.4, 0.5, 0.6] + with pytest.raises(ValueError, match=message): + await client.get_embeddings(["hello", image]) + + mock_image_client.embed.assert_awaited_once() + project_factory.assert_not_called() + + async def test_injected_text_and_image_clients_keep_precedence_over_project( + self, mock_text_client: AsyncMock, mock_image_client: AsyncMock + ) -> None: + """A project client is unused when both modalities have injected transports.""" + project_client = MagicMock() + image = Content.from_data(data=b"\x89PNG", media_type="image/png") + with patch.dict(os.environ, {}, clear=True): + client = RawFoundryEmbeddingClient( + model="embedding-model", + project_client=project_client, + text_client=mock_text_client, + image_client=mock_image_client, + ) + result = await client.get_embeddings(["hello", image]) + + assert [embedding.vector for embedding in result] == [[0.1, 0.2, 0.3], [0.4, 0.5, 0.6]] + mock_text_client.embed.assert_awaited_once() + mock_image_client.embed.assert_awaited_once() + project_client.get_openai_client.assert_not_called() + + async def test_image_client_retains_models_precedence_when_both_endpoints_come_from_env( + self, mock_image_client: AsyncMock + ) -> None: + """An injected image client does not change the legacy text source selection from env.""" + with ( + patch.dict( + os.environ, + { + "FOUNDRY_PROJECT_ENDPOINT": "https://test.services.ai.azure.com/api/projects/test", + "FOUNDRY_MODELS_ENDPOINT": "https://test.inference.ai.azure.com", + "FOUNDRY_MODELS_API_KEY": "test-key", + }, + clear=True, + ), + patch("azure.ai.projects.aio.AIProjectClient") as project_factory, + patch("agent_framework_foundry._embedding_client.EmbeddingsClient") as text_factory, + patch("agent_framework_foundry._embedding_client.ImageEmbeddingsClient") as image_factory, + ): + text_factory.return_value.embed = AsyncMock(return_value=_make_embed_response([[0.7]])) + client = RawFoundryEmbeddingClient(model="embedding-model", image_client=mock_image_client) + text_factory.assert_not_called() + result = await client.get_embeddings(["hello"]) + + assert result[0].vector == [0.7] + text_factory.assert_called_once_with( + endpoint="https://test.inference.ai.azure.com", + credential=ANY, + user_agent=get_user_agent(), + per_retry_policies=[ANY], + ) + project_factory.assert_not_called() + image_factory.assert_not_called() + mock_image_client.embed.assert_not_awaited() + def test_minimal_reproduction_constructs(self) -> None: """The reported MagicMock reproduction no longer fails during construction.""" with patch.dict(os.environ, {}, clear=True):