diff --git a/lambdas/functions/webhook/src/lambda.ts b/lambdas/functions/webhook/src/lambda.ts index 1f7cb0c830..5a8c1e8f25 100644 --- a/lambdas/functions/webhook/src/lambda.ts +++ b/lambdas/functions/webhook/src/lambda.ts @@ -9,15 +9,22 @@ import { EventWrapper } from './types'; import { WorkflowJobEvent } from '@octokit/webhooks-types'; import { ConfigDispatcher, ConfigWebhook, ConfigWebhookEventBridge } from './ConfigLoader'; import { dispatch } from './runners/dispatch'; +import { githubEventTracingMiddleware } from './tracing/githubEventTracingMiddleware'; export interface Response { statusCode: number; body: string; } -middy(directWebhook).use(captureLambdaHandler(tracer)); +export const directWebhook = middy(directWebhookHandler) + .use(captureLambdaHandler(tracer)) + .use(githubEventTracingMiddleware()); -export async function directWebhook(event: APIGatewayEvent, context: Context): Promise { +export const eventBridgeWebhook = middy(eventBridgeWebhookHandler) + .use(captureLambdaHandler(tracer)) + .use(githubEventTracingMiddleware()); + +async function directWebhookHandler(event: APIGatewayEvent, context: Context): Promise { setContext(context, 'lambda.ts'); logger.logEventIfEnabled(event); @@ -42,7 +49,7 @@ export async function directWebhook(event: APIGatewayEvent, context: Context): P return result; } -export async function eventBridgeWebhook(event: APIGatewayEvent, context: Context): Promise { +async function eventBridgeWebhookHandler(event: APIGatewayEvent, context: Context): Promise { setContext(context, 'lambda.ts'); logger.logEventIfEnabled(event); diff --git a/lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.test.ts b/lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.test.ts new file mode 100644 index 0000000000..b7886eb309 --- /dev/null +++ b/lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.test.ts @@ -0,0 +1,86 @@ +import { tracer } from '@aws-github-runner/aws-powertools-util'; +import { APIGatewayEvent, Context } from 'aws-lambda'; +import { describe, it, expect, vi, beforeEach } from 'vitest'; + +import { githubEventTracingMiddleware } from './githubEventTracingMiddleware'; + +describe('githubEventTracingMiddleware', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + function buildEvent(overrides: Partial = {}): APIGatewayEvent { + return { + headers: { + 'X-GitHub-Event': 'workflow_job', + 'X-GitHub-Delivery': 'delivery-id-1', + }, + body: JSON.stringify({ workflow_job: { created_at: new Date(Date.now() - 5000).toISOString() } }), + requestContext: { requestTimeEpoch: Date.now() - 20 }, + ...overrides, + } as unknown as APIGatewayEvent; + } + + it('does nothing when tracing is not enabled (no active segment)', async () => { + vi.spyOn(tracer, 'getSegment').mockReturnValue(undefined); + const putAnnotation = vi.spyOn(tracer, 'putAnnotation'); + + const { before, after } = githubEventTracingMiddleware(); + await before?.({ event: buildEvent(), context: {} as Context } as never); + await after?.({ event: buildEvent(), context: {} as Context } as never); + + expect(putAnnotation).not.toHaveBeenCalled(); + }); + + it('annotates event type, delivery id, ingress lag and workflow_job age when tracing is active', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any); + const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined); + + const { before } = githubEventTracingMiddleware(); + await before?.({ event: buildEvent(), context: {} as Context } as never); + + expect(putAnnotation).toHaveBeenCalledWith('github_event_type', 'workflow_job'); + expect(putAnnotation).toHaveBeenCalledWith('github_delivery_id', 'delivery-id-1'); + expect(putAnnotation).toHaveBeenCalledWith('api_gateway_ingress_to_lambda_ms', expect.any(Number)); + expect(putAnnotation).toHaveBeenCalledWith('workflow_job_age_ms', expect.any(Number)); + }); + + it('skips workflow_job_age_ms for non workflow_job event types', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any); + const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined); + + const { before } = githubEventTracingMiddleware(); + await before?.({ + event: buildEvent({ headers: { 'X-GitHub-Event': 'push', 'X-GitHub-Delivery': 'delivery-id-2' } }), + context: {} as Context, + } as never); + + expect(putAnnotation).toHaveBeenCalledWith('github_event_type', 'push'); + expect(putAnnotation).not.toHaveBeenCalledWith('workflow_job_age_ms', expect.any(Number)); + }); + + it('does not throw on a malformed body and skips the age annotation', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any); + const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined); + + const { before } = githubEventTracingMiddleware(); + await before?.({ event: buildEvent({ body: 'not-json' }), context: {} as Context } as never); + + expect(putAnnotation).not.toHaveBeenCalledWith('workflow_job_age_ms', expect.any(Number)); + }); + + it('adds lambda_processing_ms annotation on after and onError', async () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + vi.spyOn(tracer, 'getSegment').mockReturnValue({} as any); + const putAnnotation = vi.spyOn(tracer, 'putAnnotation').mockImplementation(() => undefined); + + const middleware = githubEventTracingMiddleware(); + await middleware.before?.({ event: buildEvent(), context: {} as Context } as never); + await middleware.after?.({ event: buildEvent(), context: {} as Context } as never); + + expect(putAnnotation).toHaveBeenCalledWith('lambda_processing_ms', expect.any(Number)); + }); +}); diff --git a/lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.ts b/lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.ts new file mode 100644 index 0000000000..929312ac4e --- /dev/null +++ b/lambdas/functions/webhook/src/tracing/githubEventTracingMiddleware.ts @@ -0,0 +1,62 @@ +import { MiddlewareObj } from '@middy/core'; +import { APIGatewayEvent, Context } from 'aws-lambda'; +import { tracer } from '@aws-github-runner/aws-powertools-util'; + +// Reusable X-Ray instrumentation for the webhook Lambda's GitHub-originated event types. +// Adds annotations (queryable/alertable) rather than synthetic subsegments, so it never +// alters the shape of the actual X-Ray trace for this invocation. +export function githubEventTracingMiddleware(): MiddlewareObj { + let lambdaStartedAt: number; + + const before = (request: { event: APIGatewayEvent; context: Context }): void => { + lambdaStartedAt = Date.now(); + + if (!tracer.getSegment()) return; + + const headers = lowerCaseKeys(request.event.headers as Record); + const eventType = headers['x-github-event']; + const deliveryId = headers['x-github-delivery']; + + if (eventType !== undefined) tracer.putAnnotation('github_event_type', eventType); + if (deliveryId !== undefined) tracer.putAnnotation('github_delivery_id', deliveryId); + + const apiGatewayIngressAtMs = request.event.requestContext?.requestTimeEpoch; + if (apiGatewayIngressAtMs !== undefined) { + tracer.putAnnotation('api_gateway_ingress_to_lambda_ms', lambdaStartedAt - apiGatewayIngressAtMs); + } + + if (eventType === 'workflow_job') { + const workflowJobAgeMs = tryGetWorkflowJobAgeMs(request.event.body, lambdaStartedAt); + if (workflowJobAgeMs !== undefined) { + tracer.putAnnotation('workflow_job_age_ms', workflowJobAgeMs); + } + } + }; + + const after = (): void => { + if (!tracer.getSegment()) return; + tracer.putAnnotation('lambda_processing_ms', Date.now() - lambdaStartedAt); + }; + + return { before, after, onError: after }; +} + +function lowerCaseKeys(headers: Record): Record { + const result: Record = {}; + for (const key in headers) { + result[key.toLowerCase()] = headers[key]; + } + return result; +} + +// Best-effort: the body isn't verified/parsed yet at this point in the request lifecycle, +// so a malformed or not-yet-signature-verified payload must not fail the request. +function tryGetWorkflowJobAgeMs(body: string | null, nowMs: number): number | undefined { + if (!body) return undefined; + try { + const createdAt = (JSON.parse(body) as { workflow_job?: { created_at?: string } }).workflow_job?.created_at; + return createdAt ? nowMs - new Date(createdAt).getTime() : undefined; + } catch { + return undefined; + } +}