From 10ff10b2fb57e1e91e5c080f8994db2cb3289b5d Mon Sep 17 00:00:00 2001 From: Kevin Kent Date: Fri, 19 Jun 2026 19:37:55 -0400 Subject: [PATCH] Log modelProvider in chat response telemetry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit response.success/cancelled/error currently log the raw `model` id. For BYOK models that id is a user-supplied string (local Ollama refs, custom Azure deployment names, etc.) that is scrubbed to "" during telemetry ingestion. When that happens the provider is unrecoverable, because `source` is frequently `api.github.copilot-chat` rather than `byok.`. Log `IChatEndpoint.modelProvider` (the model vendor — a fixed provider constant such as Ollama / Azure / Anthropic / Gemini / OpenAI / OpenRouter / xAI / CustomOAI) so BYOK requests can be attributed by provider even when the model id is redacted. The value is a controlled, non-free-form label, so no user content / PII is added to telemetry. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../copilot/src/extension/prompt/node/chatMLFetcher.ts | 2 ++ .../src/extension/prompt/node/chatMLFetcherTelemetry.ts | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts index 97a41d0380bdaa..4b2a2ebfd57e6d 100644 --- a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts +++ b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts @@ -536,6 +536,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { source: telemetryProperties.messageSource ?? 'unknown', requestId: ourRequestId, model: chatEndpoint.model, + modelProvider: chatEndpoint.modelProvider, apiType: chatEndpoint.apiType, transport, interactionType, @@ -692,6 +693,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { source: telemetryProperties.messageSource ?? 'unknown', requestId: ourRequestId, model: chatEndpoint.model, + modelProvider: chatEndpoint.modelProvider, apiType: chatEndpoint.apiType, transport, interactionType, diff --git a/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts b/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts index cabb725591e6e8..ae74da5bc322c0 100644 --- a/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts +++ b/extensions/copilot/src/extension/prompt/node/chatMLFetcherTelemetry.ts @@ -38,6 +38,7 @@ export interface IChatMLFetcherCancellationProperties { source: string; requestId: string; model: string; + modelProvider: string | undefined; apiType: string | undefined; transport: string; interactionType: string; @@ -148,6 +149,7 @@ export class ChatMLFetcherTelemetrySender { "requestKind": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Resolved X-Interaction-Type for the request: 'conversation-agent', 'conversation-subagent', 'conversation-background', 'conversation-panel', 'conversation-inline', 'conversation-edits', 'conversation-other', 'conversation-notebook', or 'conversation-terminal'" }, "model": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Model selection for the response" }, "modelInvoked": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Actual model invoked for the response" }, + "modelProvider": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Provider/vendor backing the model (e.g. for BYOK: Ollama, Azure, Anthropic, Gemini, OpenAI, OpenRouter, xAI, CustomOAI). Lets BYOK requests be attributed by provider even when the raw model id is redacted." }, "apiType": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "API type for the response- chat completions or responses" }, "conversationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Id for the current chat conversation." }, "requestId": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Id of the current turn request" }, @@ -218,6 +220,7 @@ export class ChatMLFetcherTelemetrySender { conversationId: baseTelemetry?.properties.conversationId, model: chatEndpointInfo?.model, modelInvoked: chatCompletion.model, + modelProvider: chatEndpointInfo?.modelProvider, apiType: chatEndpointInfo?.apiType, requestId: chatCompletion.requestId.headerRequestId, gitHubRequestId: chatCompletion.requestId.gitHubRequestId, @@ -270,6 +273,7 @@ export class ChatMLFetcherTelemetrySender { source, requestId, model, + modelProvider, apiType, transport, interactionType, @@ -305,6 +309,7 @@ export class ChatMLFetcherTelemetrySender { "owner": "digitarald", "comment": "Report canceled service responses for quality.", "model": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Model selection for the response" }, + "modelProvider": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Provider/vendor backing the model (e.g. for BYOK: Ollama, Azure, Anthropic, Gemini, OpenAI, OpenRouter, xAI, CustomOAI). Lets BYOK requests be attributed by provider even when the raw model id is redacted." }, "apiType": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "API type for the response- chat completions or responses" }, "source": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Source for why the request was made" }, "requestKind": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Resolved X-Interaction-Type for the request: 'conversation-agent', 'conversation-subagent', 'conversation-background', 'conversation-panel', 'conversation-inline', 'conversation-edits', 'conversation-other', 'conversation-notebook', or 'conversation-terminal'" }, @@ -357,6 +362,7 @@ export class ChatMLFetcherTelemetrySender { source, requestId, model, + modelProvider, requestKind: interactionType, conversationId, associatedRequestId, @@ -416,6 +422,7 @@ export class ChatMLFetcherTelemetrySender { "type": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Type of issue" }, "reason": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Reason of issue" }, "model": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Model selection for the response" }, + "modelProvider": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Provider/vendor backing the model (e.g. for BYOK: Ollama, Azure, Anthropic, Gemini, OpenAI, OpenRouter, xAI, CustomOAI). Lets BYOK requests be attributed by provider even when the raw model id is redacted." }, "apiType": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "API type for the response- chat completions or responses" }, "source": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Source for why the request was made" }, "requestKind": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Resolved X-Interaction-Type for the request: 'conversation-agent', 'conversation-subagent', 'conversation-background', 'conversation-panel', 'conversation-inline', 'conversation-edits', 'conversation-other', 'conversation-notebook', or 'conversation-terminal'" }, @@ -474,6 +481,7 @@ export class ChatMLFetcherTelemetrySender { requestId: processed.requestId, gitHubRequestId: processed.serverRequestId, model: chatEndpointInfo.model, + modelProvider: chatEndpointInfo.modelProvider, apiType: chatEndpointInfo.apiType, conversationId: telemetryProperties?.conversationId, reasoningEffort: requestBody.reasoning?.effort ?? requestBody.output_config?.effort,