Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions build/lib/policies/policyData.jsonc
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,24 @@
"github.copilot.chat.otel.captureContent"
]
},
{
"key": "chat.agentHost.otel.captureIdentity",
"name": "CopilotOtelCaptureIdentity",
"category": "InteractiveSession",
"minimumVersion": "1.139",
"localization": {
"description": {
"key": "chat.agentHost.otel.captureIdentity.policy",
"value": "Controls whether Copilot OpenTelemetry captures the authenticated account name, operating system username, and machine hostname. Independent of content capture."
}
},
"type": "boolean",
"default": false,
"included": false,
"referencedSettings": [
"github.copilot.chat.otel.captureIdentity"
]
},
{
"key": "chat.agentHost.otel.enabled",
"name": "CopilotOtelEnabled",
Expand Down
97 changes: 82 additions & 15 deletions extensions/copilot/docs/monitoring/agent_monitoring.md

Large diffs are not rendered by default.

15 changes: 15 additions & 0 deletions extensions/copilot/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -5373,6 +5373,21 @@
"advanced"
]
},
"github.copilot.chat.otel.captureIdentity": {
"type": [
"boolean",
"null"
],
"default": null,
"scope": "application",
"policyReference": {
"name": "CopilotOtelCaptureIdentity"
},
"markdownDescription": "Capture the authenticated Copilot account name, operating system username, and machine hostname in OpenTelemetry. **Contains personally identifying data.** Independent of content capture. `null` means no personal preference (off by default). `COPILOT_OTEL_CAPTURE_IDENTITY` overrides personal settings; explicit enterprise policy overrides both. Enabling requires window reload; an effective denial applies immediately to subsequent exports.",
"tags": [
"advanced"
]
},
"github.copilot.chat.otel.serviceName": {
"type": "string",
"default": "",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,8 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio
else if (level === 'warn') { console.warn(msg); }
else { console.info(msg); }
};
builder.define(IOTelService, new NodeOTelService(otelConfig, logFn, otelConfig.dbSpanExporter ? otelSqliteStore : undefined));
builder.define(IOTelService, new NodeOTelService(otelConfig, logFn, otelConfig.dbSpanExporter ? otelSqliteStore : undefined,
() => otelConfigResolver.resolve().config.captureIdentity));
Comment on lines +311 to +312
} else {
builder.define(IOTelService, new InMemoryOTelService(otelConfig));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import { ILogService } from '../../../platform/log/common/logService';
import { IChatEndpoint, IMakeChatRequestOptions } from '../../../platform/networking/common/networking';
import { CopilotChatAttr, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiProviderName, GitHubCopilotAttr, normalizeResponseModel, StdAttr, stringifyToolDefinitionsForOTel, truncateForOTel } from '../../../platform/otel/common/index';
import { IOTelService, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService';
import { agentIdentityAttributes } from '../../../platform/otel/common/otelIdentity';
import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService';
import { ChatResponseStreamImpl } from '../../../util/common/chatResponseStreamImpl';
import { toErrorMessage } from '../../../util/common/errorMessage';
Expand Down Expand Up @@ -204,6 +205,7 @@ class InlineChatToolCalling {
@IConfigurationService private readonly _configurationService: IConfigurationService,
@IExperimentationService private readonly _experimentationService: IExperimentationService,
@IOTelService private readonly _otelService: IOTelService,
@IAuthenticationService private readonly _authenticationService: IAuthenticationService,
) { }

async run(endpoint: IChatEndpoint, conversation: Conversation, request: vscode.ChatRequest, stream: vscode.ChatResponseStream, token: CancellationToken, documentContext: IDocumentContext, chatTelemetry: ChatTelemetryBuilder): Promise<IInlineChatEditResult> {
Expand All @@ -216,6 +218,7 @@ class InlineChatToolCalling {
kind: SpanKind.INTERNAL,
attributes: {
[GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT,
...agentIdentityAttributes(this._otelService.config, this._authenticationService),
[GenAiAttr.PROVIDER_NAME]: GenAiProviderName.GITHUB,
[GenAiAttr.AGENT_NAME]: 'Inline Chat',
[GenAiAttr.CONVERSATION_ID]: conversation.sessionId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import { IChatEndpoint, IMakeChatRequestOptions } from '../../../platform/networ
import { nanoAiuToCredits, OpenAIContextManagementResponse } from '../../../platform/networking/common/openai';
import { CopilotChatAttr, emitAgentTurnEvent, emitSessionStartEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiProviderName, GitHubCopilotAttr, normalizeResponseModel, resolveWorkspaceOTelMetadata, StdAttr, stringifyToolDefinitionsForOTel, truncateForOTel, workspaceMetadataToOTelAttributes } from '../../../platform/otel/common/index';
import { IOTelService, ISpanHandle, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService';
import { agentIdentityAttributes } from '../../../platform/otel/common/otelIdentity';
import { IAuthenticationService } from '../../../platform/authentication/common/authentication';
import { IRequestLogger } from '../../../platform/requestLogger/common/requestLogger';
import { getCurrentCapturingToken } from '../../../platform/requestLogger/node/requestLogger';
import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService';
Expand Down Expand Up @@ -416,6 +418,7 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
@IFileSystemService private readonly _fileSystemService: IFileSystemService,
@IOTelService protected readonly _otelService: IOTelService,
@IGitService private readonly _gitService: IGitService,
@IAuthenticationService private readonly _authenticationService: IAuthenticationService,
) {
super();
}
Expand Down Expand Up @@ -1271,6 +1274,7 @@ export abstract class ToolCallingLoop<TOptions extends IToolCallingLoopOptions =
kind: SpanKind.INTERNAL,
attributes: {
[GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT,
...agentIdentityAttributes(this._otelService.config, this._authenticationService),
[GenAiAttr.PROVIDER_NAME]: GenAiProviderName.GITHUB,
[GenAiAttr.AGENT_NAME]: agentName,
[GenAiAttr.CONVERSATION_ID]: this.options.conversation.sessionId,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/

import { afterEach, describe, expect, it } from 'vitest';
import type { ChatRequest, LanguageModelChat, LanguageModelToolInformation } from 'vscode';
import { IAuthenticationService } from '../../../../platform/authentication/common/authentication';
import { StaticGitHubAuthenticationService } from '../../../../platform/authentication/common/staticGitHubAuthenticationService';
import { ChatFetchResponseType, ChatResponse } from '../../../../platform/chat/common/commonTypes';
import { IEndpointProvider } from '../../../../platform/endpoint/common/endpointProvider';
import { MockEndpoint } from '../../../../platform/endpoint/test/node/mockEndpoint';
import type { IChatEndpoint, IEmbeddingsEndpoint } from '../../../../platform/networking/common/networking';
import { GenAiAttr, StdAttr } from '../../../../platform/otel/common/genAiAttributes';
import { resolveOTelConfig } from '../../../../platform/otel/common/otelConfig';
import { ICompletedSpanData, IOTelService } from '../../../../platform/otel/common/otelService';
import { InMemoryOTelService } from '../../../../platform/otel/node/inMemoryOTelService';
import { CancellationToken } from '../../../../util/vs/base/common/cancellation';
import { Event } from '../../../../util/vs/base/common/event';
import { DisposableStore } from '../../../../util/vs/base/common/lifecycle';
import { generateUuid } from '../../../../util/vs/base/common/uuid';
import { SyncDescriptor } from '../../../../util/vs/platform/instantiation/common/descriptors';
import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation';
import { Conversation, Turn } from '../../../prompt/common/conversation';
import { ToolCallRound } from '../../../prompt/common/toolCallRound';
import { IBuildPromptResult, nullRenderPromptResult } from '../../../prompt/node/intents';
import { createExtensionUnitTestingServices } from '../../../test/node/services';
import { IToolCallingLoopOptions, IToolCallSingleResult, ToolCallingLoop } from '../../node/toolCallingLoop';

class IdentityTestLoop extends ToolCallingLoop<IToolCallingLoopOptions> {
protected override async buildPrompt(): Promise<IBuildPromptResult> { return nullRenderPromptResult(); }
protected override async getAvailableTools(): Promise<LanguageModelToolInformation[]> { return []; }
protected override async fetch(): Promise<ChatResponse> { throw new Error('Not used by invocation tests'); }
override async runOne(): Promise<IToolCallSingleResult> {
return {
response: { type: ChatFetchResponseType.Success, value: 'answer', requestId: 'test', serverRequestId: undefined, usage: undefined, resolvedModel: 'test' },
round: new ToolCallRound('answer'),
hadIgnoredFiles: false,
lastRequestMessages: [],
availableTools: [],
};
}
}

/** Avoid model metadata caches and network requests: only the invocation boundary is exercised. */
class IdentityTestEndpointProvider implements IEndpointProvider {
declare readonly _serviceBrand: undefined;
readonly onDidModelsRefresh = Event.None;
constructor(@IInstantiationService private readonly _instantiationService: IInstantiationService) { }
async getAllCompletionModels() { return []; }
async getAllChatEndpoints(): Promise<IChatEndpoint[]> { return [await this.getChatEndpoint()]; }
async getChatEndpoint(): Promise<IChatEndpoint> { return this._instantiationService.createInstance(MockEndpoint, 'test'); }
async getEmbeddingsEndpoint(): Promise<IEmbeddingsEndpoint> { throw new Error('Not used by identity tests'); }
}

class IdentityTestAuthentication extends StaticGitHubAuthenticationService {
setAccount(name: string | undefined): void {
this._anyGitHubSession = name === undefined ? undefined : {
id: name, accessToken: 'test-token', scopes: [], account: { id: name, label: name },
};
}
}

function request(subagent: boolean): ChatRequest {
return {
prompt: 'hello', command: undefined, references: [], location: 1, location2: undefined,
attempt: 0, enableCommandDetection: false, isParticipantDetected: false, toolReferences: [],
toolInvocationToken: {} as ChatRequest['toolInvocationToken'], model: { family: 'test' } as LanguageModelChat,
tools: new Map(), id: generateUuid(), sessionId: generateUuid(),
sessionResource: {} as ChatRequest['sessionResource'], hasHooksEnabled: false,
...(subagent ? { subAgentInvocationId: 'child', subAgentName: 'search' } : {}),
};
}

describe('ToolCallingLoop identity attribution', () => {
const disposables = new DisposableStore();
let otel: InMemoryOTelService;
afterEach(async () => {
disposables.clear();
await otel?.shutdown();
});

it.each([false, true])('reads the current account for top-level and subagent spans (capture=%s)', async captureIdentity => {
const services = disposables.add(createExtensionUnitTestingServices());
services.define(IEndpointProvider, new SyncDescriptor(IdentityTestEndpointProvider));
services.define(IAuthenticationService, new SyncDescriptor(IdentityTestAuthentication, [undefined]));
otel = new InMemoryOTelService(resolveOTelConfig({
env: {}, settingEnabled: true, settingCaptureIdentity: captureIdentity, settingCaptureContent: false,
extensionVersion: 'test', sessionId: 'test',
}));
services.define(IOTelService, otel);
const accessor = disposables.add(services.createTestingAccessor());
const instantiation = accessor.get(IInstantiationService);
const auth = accessor.get(IAuthenticationService);
expect(auth).toBeInstanceOf(IdentityTestAuthentication);
if (!(auth instanceof IdentityTestAuthentication)) {
throw new Error('Identity test authentication was not registered');
}
const completed: ICompletedSpanData[] = [];
disposables.add(otel.onDidCompleteSpan(span => completed.push(span)));
for (const [account, subagent] of [['first', false], ['first', true], ['second', true], [undefined, false]] as const) {
auth.setAccount(account);
const chatRequest = request(subagent);
const loop = disposables.add(instantiation.createInstance(IdentityTestLoop, {
request: chatRequest, toolCallLimit: 1,
conversation: new Conversation(generateUuid(), [new Turn(generateUuid(), { type: 'user', message: 'hello' })]),
}));
await loop.run(undefined, CancellationToken.None);
}
expect(completed.filter(span => span.attributes[GenAiAttr.OPERATION_NAME] === 'invoke_agent')
.map(span => span.attributes[StdAttr.USER_NAME])).toEqual(captureIdentity ? ['first', 'first', 'second', undefined] : [undefined, undefined, undefined, undefined]);
});
});
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import type { CancellationToken, LanguageModelToolInformation, Progress } from 'vscode';
import { IAuthenticationChatUpgradeService } from '../../../platform/authentication/common/authenticationUpgrade';
import { IAuthenticationService } from '../../../platform/authentication/common/authentication';
import { IChatHookService } from '../../../platform/chat/common/chatHookService';
import { ChatLocation, ChatResponse } from '../../../platform/chat/common/commonTypes';
import { ISessionTranscriptService } from '../../../platform/chat/common/sessionTranscriptService';
Expand Down Expand Up @@ -48,8 +49,9 @@ export class McpToolCallingLoop extends ToolCallingLoop<IMcpToolCallingLoopOptio
@IFileSystemService fileSystemService: IFileSystemService,
@IOTelService otelService: IOTelService,
@IGitService gitService: IGitService,
@IAuthenticationService authenticationService: IAuthenticationService,
) {
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService);
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService, authenticationService);
}

private async getEndpoint() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,18 @@ describe('OTelStaleConfigMonitor', () => {
log = new RecordingLogService();
});

it('prompts on identity revocation without restarting or mutating the running config', async () => {
settings.policy = { ...managedPolicy, captureIdentity: true };
const resolver = new TestResolver(settings);
const monitor = new OTelStaleConfigMonitor(resolver, host, log);
settings.policy.captureIdentity = false;
expect(await monitor.check()).toBe(OTelConfigDrift.Policy);
expect({ prompts: host.prompts, restarts: host.restarts, active: resolver.activeResolution.config.captureIdentity }).toEqual({
prompts: 1, restarts: 0, active: true,
});
expect(resolver.resolve().config.captureIdentity).toBe(false);
});

it('restarts for policy that lands before the contribution can register its watcher', async () => {
const resolver = new TestResolver(settings);
expect(resolver.activeResolution.config.enabled).toBe(false);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import { randomUUID } from 'crypto';
import type { CancellationToken, ChatRequest, LanguageModelToolInformation, Progress } from 'vscode';
import { IAuthenticationChatUpgradeService } from '../../../platform/authentication/common/authenticationUpgrade';
import { IAuthenticationService } from '../../../platform/authentication/common/authentication';
import { IChatHookService } from '../../../platform/chat/common/chatHookService';
import { ChatLocation, ChatResponse } from '../../../platform/chat/common/commonTypes';
import { ISessionTranscriptService } from '../../../platform/chat/common/sessionTranscriptService';
Expand Down Expand Up @@ -52,8 +53,9 @@ export class CodebaseToolCallingLoop extends ToolCallingLoop<ICodebaseToolCallin
@IFileSystemService fileSystemService: IFileSystemService,
@IOTelService otelService: IOTelService,
@IGitService gitService: IGitService,
@IAuthenticationService authenticationService: IAuthenticationService,
) {
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService);
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService, authenticationService);
}

private async getEndpoint(request: ChatRequest) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -632,8 +632,9 @@ class DefaultToolCallingLoop extends ToolCallingLoop<IDefaultToolLoopOptions> {
@IFileSystemService fileSystemService: IFileSystemService,
@IOTelService otelService: IOTelService,
@IGitService gitService: IGitService,
@IAuthenticationService authenticationService: IAuthenticationService,
) {
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService);
super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, fileSystemService, otelService, gitService, authenticationService);

this._register(this.onDidBuildPrompt(({ result, tools, promptTokenLength, toolTokenCount }) => {
if (result.metadata.get(SummarizedConversationHistoryMetadata)) {
Expand Down
Loading
Loading