diff --git a/apps/ai/src/worker/http.test.ts b/apps/ai/src/worker/http.test.ts index 859e79ec82..8c2c8440db 100644 --- a/apps/ai/src/worker/http.test.ts +++ b/apps/ai/src/worker/http.test.ts @@ -22,6 +22,7 @@ import { HttpServerResponse } from "effect/http" import { TestClock } from "effect/testing" import { AuditEventsQueueProducer, MapleDbConnection } from "@maple/backend/platform/bindings" import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" +import { AuditTrailService } from "@maple/backend/services/audit-trail/AuditTrailService" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" import { OrgId } from "@maple/primitives" @@ -216,9 +217,13 @@ describe("the maple-ai Worker through alchemy's bridge", () => { const handedOver = yield* Ref.make>>([]) const sent = yield* Ref.make(0) const ingested = yield* Ref.make(0) - const warehouse = Layer.succeed( - WarehouseQueryService, - makeWarehouseServiceStub({ ingest: () => Ref.update(ingested, (n) => n + 1) }), + const warehouse = Layer.effect(AuditTrailService, AuditTrailService.make).pipe( + Layer.provideMerge( + Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ ingest: () => Ref.update(ingested, (n) => n + 1) }), + ), + ), ) const producer = Layer.succeed(AuditEventsQueueProducer, { sendBatch: (messages) => Ref.update(sent, (n) => n + messages.length), diff --git a/apps/api/src/audit-events-runtime.test.ts b/apps/api/src/audit-events-runtime.test.ts index 2e7a011f47..41310df621 100644 --- a/apps/api/src/audit-events-runtime.test.ts +++ b/apps/api/src/audit-events-runtime.test.ts @@ -1,26 +1,32 @@ import { describe, expect, it } from "@effect/vitest" import { OrgId } from "@maple/primitives" -import type { AuditLogRow } from "@maple/domain/tinybird" import { WarehouseUpstreamError } from "@maple/domain/http" import { Effect, Layer, Schema, Array as Arr } from "effect" import { processAuditEventsBatch } from "./audit-events-runtime" import { makeWarehouseServiceStub } from "@/routes/v2/v2-test-support" import { AuditLogEvent, encodeAuditLogEventSync } from "@maple/backend/services/audit/audit-event" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { AuditTrailService } from "@maple/backend/services/audit-trail/AuditTrailService" const asOrgId = Schema.decodeUnknownSync(OrgId) const ORG = asOrgId("org_audit_consumer_test") const OTHER_ORG = asOrgId("org_audit_consumer_other") -const event = (id: string, orgId: OrgId = ORG) => +const event = ( + id: string, + orgId: OrgId = ORG, + what: { readonly action: string; readonly outcome: "allowed" | "denied" } = { + action: "dashboard.created", + outcome: "allowed", + }, +) => encodeAuditLogEventSync( new AuditLogEvent({ orgId, id: Schema.decodeSync(AuditLogEvent.fields.id)(id), actorType: "user", source: "dashboard", - action: "dashboard.created", - outcome: "allowed", + ...what, occurredAtMs: 1_700_000_000_000, }), ) @@ -45,42 +51,68 @@ const message = (body: unknown, attempts: number) => { const batchOf = (...messages: ReadonlyArray<{ readonly message: unknown }>) => ({ messages: Arr.map(messages, (entry) => entry.message) }) as never +/** The consumer's services over one warehouse stub: the trail writes through the same `ingest`. */ +const withTrail = (stub: ReturnType) => + Layer.effect(AuditTrailService, AuditTrailService.make).pipe( + Layer.provideMerge(Layer.succeed(WarehouseQueryService, stub)), + ) + /** A warehouse whose `ingest` is interrupted — a deploy tearing the isolate down. */ -const interruptedWarehouse = Layer.succeed( - WarehouseQueryService, - makeWarehouseServiceStub({ ingest: () => Effect.interrupt }), -) +const interruptedWarehouse = withTrail(makeWarehouseServiceStub({ ingest: () => Effect.interrupt })) /** A warehouse whose `ingest` dies rather than failing — an unexpected defect. */ -const dyingWarehouse = Layer.succeed( - WarehouseQueryService, +const dyingWarehouse = withTrail( makeWarehouseServiceStub({ ingest: () => Effect.die(new Error("ingest exploded")), }), ) -/** A warehouse whose `ingest` records each call, or fails every call. */ -const warehouse = (fail = false) => { - const written: Array<{ orgId: string; rows: ReadonlyArray }> = [] - const layer = Layer.succeed( - WarehouseQueryService, +const AUDIT_LOG = "audit_log" +const AUDIT_TRAIL = "audit_trail_events" + +/** + * A warehouse whose `ingest` records each call, failing the ones to the `failing` datasources. + * `trailCopy` makes the write to the audit trail die or be interrupted instead. + */ +const warehouse = (failing: ReadonlyArray = [], trailCopy?: "dies" | "interrupted") => { + const written: Array<{ + orgId: string + datasource: string + rows: ReadonlyArray + bestEffort: boolean + }> = [] + const layer = withTrail( makeWarehouseServiceStub({ - ingest: (tenant, _datasource, rows) => - fail - ? Effect.fail( - new WarehouseUpstreamError({ - message: "tinybird down", - pipeName: "audit_log", - cause: new Error("down"), - }), - ) - : Effect.sync(() => { - // SAFETY: this stub only ever receives the audit datasource's rows. - written.push({ orgId: tenant.orgId, rows: rows as ReadonlyArray }) - }), + ingest: (tenant, datasource, rows, options) => + datasource === AUDIT_TRAIL && trailCopy === "dies" + ? Effect.die(new Error("trail exploded")) + : datasource === AUDIT_TRAIL && trailCopy === "interrupted" + ? Effect.interrupt + : failing.includes(datasource) + ? Effect.fail( + new WarehouseUpstreamError({ + message: "tinybird down", + pipeName: datasource, + cause: new Error("down"), + }), + ) + : Effect.sync(() => { + written.push({ + orgId: tenant.orgId, + datasource, + rows, + bestEffort: options?.bestEffort === true, + }) + }), }), ) - return { written, layer } + /** `[orgId, row count]` per write to one datasource, in a stable order. */ + const writesTo = (datasource: string) => + Arr.map( + Arr.filter(written, (write) => write.datasource === datasource), + (write) => [write.orgId, write.rows.length] as const, + ).sort() + return { written, writesTo, layer } } describe("processAuditEventsBatch", () => { @@ -97,16 +129,107 @@ describe("processAuditEventsBatch", () => { expect(first.calls).toEqual(["ack"]) expect(second.calls).toEqual(["ack"]) expect(other.calls).toEqual(["ack"]) - expect(Arr.map(store.written, (write) => [write.orgId, write.rows.length]).sort()).toEqual([ + expect(store.writesTo(AUDIT_LOG)).toEqual([ [OTHER_ORG, 1], [ORG, 2], ]) - expect( - Arr.every( - Arr.flatMap(store.written, (write) => write.rows), - (row) => row.Action === "dashboard.created", - ), - ).toBe(true) + }), + ) + + it.effect("copies changes and refusals to the audit trail, and leaves allowed reads out", () => + Effect.gen(function* () { + const store = warehouse() + const change = message(event("11111111-1111-4111-8111-111111111111"), 1) + const read = message( + event("22222222-2222-4222-8222-222222222222", ORG, { + action: "telemetry.read", + outcome: "allowed", + }), + 1, + ) + const refusedRead = message( + event("33333333-3333-4333-8333-333333333333", ORG, { + action: "telemetry.read", + outcome: "denied", + }), + 1, + ) + yield* processAuditEventsBatch(batchOf(change, read, refusedRead), MAX_RETRIES).pipe( + Effect.provide(store.layer), + ) + + expect(store.writesTo(AUDIT_LOG)).toEqual([[ORG, 3]]) + expect(store.writesTo(AUDIT_TRAIL)).toEqual([[ORG, 2]]) + }), + ) + + it.effect("writes nothing to the audit trail for a batch of allowed reads", () => + Effect.gen(function* () { + const store = warehouse() + const read = message( + event("11111111-1111-4111-8111-111111111111", ORG, { + action: "telemetry.read", + outcome: "allowed", + }), + 1, + ) + yield* processAuditEventsBatch(batchOf(read), MAX_RETRIES).pipe(Effect.provide(store.layer)) + + expect(read.calls).toEqual(["ack"]) + expect(store.writesTo(AUDIT_TRAIL)).toEqual([]) + }), + ) + + // `audit_log` is the record. A message retried because only its copy failed would be + // rewritten there on every attempt and end in the DLQ for an entry that was stored. + it.effect("acks an entry whose audit trail copy died", () => + Effect.gen(function* () { + const store = warehouse([], "dies") + const change = message(event("11111111-1111-4111-8111-111111111111"), 1) + yield* processAuditEventsBatch(batchOf(change), MAX_RETRIES).pipe(Effect.provide(store.layer)) + + expect(change.calls).toEqual(["ack"]) + expect(store.writesTo(AUDIT_LOG)).toEqual([[ORG, 1]]) + }), + ) + + // A deploy landing between the two writes: unacked, so the platform redelivers the entry. + it.effect("leaves an entry unacked when its audit trail copy is interrupted", () => + Effect.gen(function* () { + const store = warehouse([], "interrupted") + const change = message(event("11111111-1111-4111-8111-111111111111"), 1) + const exit = yield* processAuditEventsBatch(batchOf(change), MAX_RETRIES).pipe( + Effect.provide(store.layer), + Effect.exit, + ) + + expect(exit._tag).toBe("Failure") + expect(change.calls).toEqual([]) + }), + ) + + it.effect("acks an entry whose audit trail copy failed", () => + Effect.gen(function* () { + const store = warehouse([AUDIT_TRAIL]) + const change = message(event("11111111-1111-4111-8111-111111111111"), 1) + yield* processAuditEventsBatch(batchOf(change), MAX_RETRIES).pipe(Effect.provide(store.layer)) + + expect(change.calls).toEqual(["ack"]) + expect(store.writesTo(AUDIT_LOG)).toEqual([[ORG, 1]]) + }), + ) + + // The copy is dropped on failure, so its failed insert must not read as a lost write. + it.effect("writes the audit trail copy as best effort and the entry itself as not", () => + Effect.gen(function* () { + const store = warehouse() + const change = message(event("11111111-1111-4111-8111-111111111111"), 1) + yield* processAuditEventsBatch(batchOf(change), MAX_RETRIES).pipe(Effect.provide(store.layer)) + + expect(Arr.map(store.written, (write) => [write.datasource, write.bestEffort])).toEqual([ + [AUDIT_LOG, false], + [AUDIT_TRAIL, true], + ]) }), ) @@ -117,7 +240,7 @@ describe("processAuditEventsBatch", () => { Effect.gen(function* () { const exhausted = message(event("44444444-4444-4444-8444-444444444444"), MAX_RETRIES + 1) yield* processAuditEventsBatch(batchOf(exhausted), MAX_RETRIES).pipe( - Effect.provide(warehouse(true).layer), + Effect.provide(warehouse([AUDIT_LOG]).layer), ) expect(exhausted.calls).toEqual(["retry"]) }), @@ -128,7 +251,7 @@ describe("processAuditEventsBatch", () => { const a = message(event("55555555-5555-4555-8555-555555555555"), 2) const b = message(event("66666666-6666-4666-8666-666666666666"), 2) yield* processAuditEventsBatch(batchOf(a, b), MAX_RETRIES).pipe( - Effect.provide(warehouse(true).layer), + Effect.provide(warehouse([AUDIT_LOG]).layer), ) expect(a.calls).toEqual(["retry"]) expect(b.calls).toEqual(["retry"]) @@ -175,7 +298,7 @@ describe("processAuditEventsBatch", () => { expect(malformed.calls).toEqual(["ack"]) expect(fine.calls).toEqual(["ack"]) - expect(store.written).toHaveLength(1) + expect(store.writesTo(AUDIT_LOG)).toEqual([[ORG, 1]]) }), ) }) diff --git a/apps/api/src/audit-events-runtime.ts b/apps/api/src/audit-events-runtime.ts index 276b3a3dff..cf40342e94 100644 --- a/apps/api/src/audit-events-runtime.ts +++ b/apps/api/src/audit-events-runtime.ts @@ -8,6 +8,11 @@ import type { QueueBatch } from "@maple/backend/platform/queue-batch" import { ALERTING_SYSTEM_USER_ID, systemTenant } from "@maple/backend/services/auth/system-tenant" import { AUDIT_LOG_DATASOURCE } from "@maple/backend/services/audit/AuditLogService" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { + AuditTrailService, + type AuditTrailServiceApi, +} from "@maple/backend/services/audit-trail/AuditTrailService" +import { mapleAuditTrailEvents } from "@maple/backend/services/audit-trail/providers/maple" import { type AuditLogEvent, auditEventToRow, @@ -21,7 +26,7 @@ import { * layer requires them, not because a write ever consults them. Its spans are * `maple-api`'s, through the telemetry the bridge builds into the event. */ -export const AuditEventsLive = WarehouseQueryService.layer.pipe( +export const AuditEventsLive = Layer.mergeAll(WarehouseQueryService.layer, AuditTrailService.layer).pipe( Layer.provide(Layer.mergeAll(EventBaseLive, EdgeCacheServiceLive)), ) @@ -72,8 +77,24 @@ const retryOrExhaust = (message: Message, cause: unknown, maxRetries: number) => } /** - * Audit events queue consumer: lowers each event to its `audit_log` row and - * writes one batch per org through the managed ingest pipeline. The table is a + * Changes and refusals also go to the unified audit trail. `audit_log` is the record, so a failed + * copy is logged and dropped: retrying the message for it would send stored entries to the DLQ. + */ +const copyToTrail = (trail: AuditTrailServiceApi, orgId: OrgId, events: ReadonlyArray) => + Effect.suspend(() => trail.write(orgId, mapleAuditTrailEvents(events), { bestEffort: true })).pipe( + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.logWarning("Audit trail copy failed; audit_log kept the entries").pipe( + Effect.annotateLogs({ orgId, entries: events.length, error: summarizeCause(cause) }), + ), + ), + ) + +/** + * Audit events queue consumer: lowers each event to its `audit_log` row (and, for a change or + * a refusal, its `audit_trail_events` row) and writes one batch per org through the managed + * ingest pipeline. The table is a * ReplacingMergeTree on the entry id, so queue redelivery collapses at merge * time; write failures retry through the queue's policy and, once exhausted, * land in `audit-events-dlq` rather than disappearing. @@ -83,6 +104,7 @@ export const processAuditEventsBatch = Effect.fn("AuditEvents.processBatch")(fun maxRetries: number, ) { const warehouse = yield* WarehouseQueryService + const trail = yield* AuditTrailService const now = yield* Clock.currentTimeMillis const decodedPerMessage = yield* Effect.forEach(batch.messages, (message) => @@ -113,14 +135,16 @@ export const processAuditEventsBatch = Effect.fn("AuditEvents.processBatch")(fun yield* Effect.forEach( byOrg, - ([orgId, group]) => - warehouse + ([orgId, group]) => { + const events = Arr.map(group, ({ event }) => event) + return warehouse .ingest( systemTenant(orgId, ALERTING_SYSTEM_USER_ID), AUDIT_LOG_DATASOURCE, - Arr.map(group, ({ event }) => auditEventToRow(event, now)), + Arr.map(events, (event) => auditEventToRow(event, now)), ) .pipe( + Effect.andThen(copyToTrail(trail, orgId, events)), Effect.flatMap(() => Effect.sync(() => { for (const { message } of group) message.ack() @@ -146,7 +170,8 @@ export const processAuditEventsBatch = Effect.fn("AuditEvents.processBatch")(fun }, ), ), - ), + ) + }, { concurrency: 3, discard: true }, ) }) diff --git a/apps/api/src/http/api-observability.test.ts b/apps/api/src/http/api-observability.test.ts index 2eaf3e83bd..e030ea3d95 100644 --- a/apps/api/src/http/api-observability.test.ts +++ b/apps/api/src/http/api-observability.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "vitest" import { Effect } from "effect" -import { HttpMiddleware, HttpServerRequest } from "effect/http" +import { HttpMiddleware, HttpServerRequest, UrlParams } from "effect/http" import { ApiObservabilityLive } from "./api-observability" const tracedFor = (url: string, method = "GET") => @@ -15,6 +15,23 @@ describe("ApiObservabilityLive", () => { expect(tracedFor("/hb/abc123secret/fail", "POST")).toBe(false) }) + // The audit log is filtered by a member's email. The span of that request is kept for weeks + // and read by people who cannot open the audit log. + it("keeps an audit actor filter out of the URL a server span records", () => { + const recorded = Effect.gen(function* () { + const names = yield* UrlParams.CurrentRedactedNames + return UrlParams.redactUrl( + "https://api.maple.dev/internal/audit-trail/events?actor=ada%40acme.test&category=membership&sig=abc", + names, + ) + }).pipe(Effect.provide(ApiObservabilityLive), Effect.runSync) + + expect(recorded).not.toContain("ada") + expect(recorded).toContain("category=membership") + // Effect's own defaults still apply. + expect(recorded).not.toContain("abc") + }) + it("still traces ordinary API routes", () => { expect(tracedFor("/api/v2/uptime_monitors")).toBe(true) expect(tracedFor("/hbx")).toBe(true) diff --git a/apps/api/src/http/api-observability.ts b/apps/api/src/http/api-observability.ts index 1d6d76ba15..5476cf50bf 100644 --- a/apps/api/src/http/api-observability.ts +++ b/apps/api/src/http/api-observability.ts @@ -1,5 +1,5 @@ import { Layer } from "effect" -import { Headers, HttpMiddleware } from "effect/http" +import { Headers, HttpMiddleware, UrlParams } from "effect/http" import { VCS_SYNC_BATCH_PATH } from "../worker/vcs-sync-dispatch" // OAuth callbacks whose query string carries a provider-issued authorization @@ -60,4 +60,11 @@ export const ApiObservabilityLive = Layer.mergeAll( // RFC 9449 proofs: short-lived, but a captured one replays within its window. "dpop", ]), + // Query values the tracer writes as `` in `url.full` and `url.query`. A custom + // list replaces Effect's, so its defaults are kept. `actor` is an audit filter: an admin + // types a member's email into it, and a span is read by more people than the audit log is. + Layer.succeed(UrlParams.CurrentRedactedNames, [ + ...UrlParams.CurrentRedactedNames.defaultValue(), + "actor", + ]), ) diff --git a/apps/api/src/routes/internal/audit-trail.http.test.ts b/apps/api/src/routes/internal/audit-trail.http.test.ts new file mode 100644 index 0000000000..14ebd32529 --- /dev/null +++ b/apps/api/src/routes/internal/audit-trail.http.test.ts @@ -0,0 +1,289 @@ +import { describe, expect, it } from "vitest" +import { + AlertMemberDirectoryUnavailableError, + ApiSchemaErrors, + ApiUnexpectedErrors, + AuditTrailApiGroup, + type AuditTrailEntry, + AuditTrailListResponse, + CurrentTenant, + OrgId, + RoleName, + UserId, +} from "@maple/domain/http" +import { Context, Effect, Layer, Schema, Array as Arr, String as Str } from "effect" +import { HttpRouter } from "effect/http" +import { HttpApi, HttpApiBuilder } from "effect/http-api" +import { ApiErrorBoundaryLive } from "@maple/backend/http/error-boundary" +import { + type AuditTrailListFilters, + type AuditTrailPage, + AuditTrailService, +} from "@maple/backend/services/audit-trail/AuditTrailService" +import { OrgMembersService } from "@maple/backend/services/org/OrgMembersService" +import { HttpAuditTrailLive } from "./audit-trail.http" + +class AuditTrailOnlyApi extends HttpApi.make("MapleInternalApi") + .add(AuditTrailApiGroup) + .middleware(ApiSchemaErrors) + .middleware(ApiUnexpectedErrors) {} + +const ORG_ID = Schema.decodeSync(OrgId)("org_trail_route") +const USER_ID = Schema.decodeSync(UserId)("user_trail_route") + +const unused = Effect.die("not used by the audit trail route") +const asRole = Schema.decodeSync(RoleName) +const asPage = Schema.decodeUnknownSync(AuditTrailListResponse) + +const event = (overrides: Partial = {}): AuditTrailEntry => ({ + id: Str.repeat(32)("a"), + provider: "github", + account: "acme", + providerEventId: "evt_1", + occurredAt: 1_700_000_000_000, + ingestedAt: 1_700_000_060_000, + actor: { type: "user", id: "octocat", name: null, email: null }, + action: "org.add_member", + category: "membership", + outcome: "success", + outcomeReason: null, + target: { type: "user", id: "hubot", name: null }, + originIp: null, + originCountry: null, + userAgent: null, + raw: null, + ...overrides, +}) + +/** The route over a trail that answers with `page` and records the filters it was asked for. */ +const makeHarness = (roles: ReadonlyArray, page: AuditTrailPage, directoryDown = false) => { + const asked: Array = [] + let directoryReads = 0 + const services = Layer.mergeAll( + Layer.succeed(AuditTrailService, { + write: () => unused, + list: (_orgId, filters) => Effect.sync(() => (asked.push(filters), page)), + }), + Layer.succeed(OrgMembersService, { + listMembers: () => + directoryDown + ? Effect.fail( + new AlertMemberDirectoryUnavailableError({ + message: "clerk down", + cause: new Error("down"), + }), + ) + : Effect.sync(() => { + directoryReads += 1 + return [{ userId: USER_ID, email: "ada@acme.test", name: "Ada", imageUrl: null }] + }), + listOrganizationsOf: () => unused, + resolveMembers: () => unused, + listMemberActivity: () => unused, + }), + Layer.succeed( + CurrentTenant.SessionAuthorization, + CurrentTenant.SessionAuthorization.of({ + bearer: (httpEffect) => + Effect.provideService( + httpEffect, + CurrentTenant.Context, + new CurrentTenant.TenantSchema({ + orgId: ORG_ID, + userId: USER_ID, + roles: Arr.map(roles, (role) => asRole(role)), + authMode: "clerk", + }), + ), + }), + ), + ) + const routes = HttpApiBuilder.layer(AuditTrailOnlyApi).pipe( + Layer.provide(HttpAuditTrailLive), + Layer.provide(ApiErrorBoundaryLive), + Layer.provide(services), + ) + const { handler, dispose } = HttpRouter.toWebHandler(routes, { disableLogger: true }) + + const list = async (search = "") => { + const response = await handler( + new Request(`http://maple.test/internal/audit-trail/events${search}`, { + headers: { authorization: "Bearer test-token" }, + }), + Context.empty() as never, + ) + return { status: response.status, body: await response.json() } + } + + return { list, asked, directoryReads: () => directoryReads, dispose } +} + +const ADMIN = ["org:admin"] + +describe("GET /internal/audit-trail/events", () => { + it("refuses an engineer: the trail sits behind the audit log permission, not the Security hub's", async () => { + const harness = makeHarness(["org:engineer"], { events: [event()], next: null }) + try { + const response = await harness.list() + + expect(response.status).toBe(403) + expect(harness.asked).toEqual([]) + } finally { + await harness.dispose() + } + }) + + it("hands an admin a page whose cursor asks for the events after it", async () => { + const last = event({ id: Str.repeat(32)("b"), occurredAt: 1_700_000_000_412 }) + const harness = makeHarness(ADMIN, { + events: [event(), last], + next: { occurredAtMs: last.occurredAt, id: last.id }, + }) + try { + const first = await harness.list("?provider=github&limit=2") + expect(first.status).toBe(200) + const page = asPage(first.body) + expect(page.events).toHaveLength(2) + + await harness.list(`?provider=github&limit=2&cursor=${page.nextCursor}`) + expect(harness.asked).toEqual([ + { provider: "github", limit: 2 }, + { provider: "github", limit: 2, before: { occurredAtMs: last.occurredAt, id: last.id } }, + ]) + } finally { + await harness.dispose() + } + }) + + it("answers 400 for a cursor it did not issue, before anything reaches the warehouse", async () => { + const harness = makeHarness(ADMIN, { events: [], next: null }) + try { + const response = await harness.list("?cursor=1700000000000:not-an-id") + + expect(response.status).toBe(400) + expect(harness.asked).toEqual([]) + } finally { + await harness.dispose() + } + }) + + it("answers 400 for a time bound no event could have", async () => { + const harness = makeHarness(ADMIN, { events: [], next: null }) + try { + const response = await harness.list("?since=9000000000000000") + + expect(response.status).toBe(400) + expect(harness.asked).toEqual([]) + } finally { + await harness.dispose() + } + }) + + const mapleUser = (overrides: Partial = {}) => + event({ + id: Str.repeat(32)("c"), + provider: "maple", + actor: { type: "user", id: USER_ID, name: null, email: null, ...overrides }, + }) + + it("names Maple's own users from the directory and leaves other platforms' actors alone", async () => { + const harness = makeHarness(ADMIN, { + events: [ + mapleUser(), + // Another platform's actor whose id happens to equal a Maple user id. + event({ actor: { type: "user", id: USER_ID, name: null, email: null } }), + // A label frozen when the event was written is what the actor was called then. + mapleUser({ name: "Device credential" }), + ], + next: null, + }) + try { + const { events } = asPage((await harness.list()).body) + + expect(Arr.map(events, (listed) => [listed.actor.name, listed.actor.email])).toEqual([ + ["Ada", "ada@acme.test"], + [null, null], + ["Device credential", null], + ]) + } finally { + await harness.dispose() + } + }) + + it("still serves the page, with ids, when the member directory is down", async () => { + const harness = makeHarness(ADMIN, { events: [mapleUser()], next: null }, true) + try { + const response = await harness.list() + + expect(response.status).toBe(200) + expect(asPage(response.body).events[0]?.actor).toMatchObject({ id: USER_ID, name: null }) + } finally { + await harness.dispose() + } + }) + + // Maple's events are stored under the user id: without the directory an email matches none + // of them, and a page that left them out would look complete. + it("answers 503 for an email filter while the member directory is down", async () => { + const harness = makeHarness(ADMIN, { events: [], next: null }, true) + try { + const response = await harness.list("?actor=ada@acme.test") + + expect(response.status).toBe(503) + expect(harness.asked).toEqual([]) + } finally { + await harness.dispose() + } + }) + + it("reads the directory once to resolve an email filter and name the page it returns", async () => { + const harness = makeHarness(ADMIN, { events: [mapleUser()], next: null }) + try { + const { events } = asPage((await harness.list("?actor=ada@acme.test")).body) + + expect(events[0]?.actor.name).toBe("Ada") + expect(harness.directoryReads()).toBe(1) + } finally { + await harness.dispose() + } + }) + + it("does not resolve an email against Maple's members when another platform is asked for", async () => { + const harness = makeHarness(ADMIN, { events: [], next: null }) + try { + await harness.list("?provider=github&actor=ada@acme.test") + + expect(harness.asked).toEqual([{ provider: "github", actor: "ada@acme.test", limit: 50 }]) + expect(harness.directoryReads()).toBe(0) + } finally { + await harness.dispose() + } + }) + + // Maple's own events store the user id, and the page shows the email next to it. + it("finds a member's Maple events when filtering by the email the page showed", async () => { + const harness = makeHarness(ADMIN, { events: [], next: null }) + try { + await harness.list("?actor=Ada@acme.test") + await harness.list("?actor=octocat") + + expect(harness.asked).toEqual([ + { actor: "Ada@acme.test", actorAlias: USER_ID, limit: 50 }, + { actor: "octocat", limit: 50 }, + ]) + } finally { + await harness.dispose() + } + }) + + it("does not read the member directory for a page with no Maple users on it", async () => { + const harness = makeHarness(ADMIN, { events: [event()], next: null }) + try { + await harness.list() + + expect(harness.directoryReads()).toBe(0) + } finally { + await harness.dispose() + } + }) +}) diff --git a/apps/api/src/routes/internal/audit-trail.http.ts b/apps/api/src/routes/internal/audit-trail.http.ts new file mode 100644 index 0000000000..a29c5acd75 --- /dev/null +++ b/apps/api/src/routes/internal/audit-trail.http.ts @@ -0,0 +1,119 @@ +import { HttpApiBuilder } from "effect/http-api" +import { + AuditTrailForbiddenError, + AuditTrailPersistenceError, + CurrentTenant, + MapleInternalApi, +} from "@maple/domain/http" +import type { AuditTrailEntry } from "@maple/domain/http" +import * as Policy from "@maple/domain/policy" +import { AuditTrailService } from "@maple/backend/services/audit-trail/AuditTrailService" +import type { OrgMember } from "@maple/backend/services/org/OrgMembersService" +import { OrgMembersService } from "@maple/backend/services/org/OrgMembersService" +import { Effect, Array as Arr, Option, String as Str } from "effect" +import { orgMemberDirectory, orgMemberDirectoryOrFail } from "@/routes/org-member-directory" + +const DEFAULT_PAGE_LIMIT = 50 + +/** `:`; the query schema has already checked the shape. */ +const parseCursor = (cursor: string) => { + const [occurredAtMs, id = ""] = Str.split(":")(cursor) + return { occurredAtMs: Number(occurredAtMs), id } +} + +/** + * The user id of a Maple dashboard event stored without a name, else null. An event that froze + * a label when it was written (a device credential) keeps it. + */ +const unnamedMapleUserId = (event: AuditTrailEntry): string | null => + event.provider === "maple" && event.actor.type === "user" && event.actor.name === null + ? event.actor.id + : null + +/** Put the member's current name and email on Maple's own user events; a departed id stays an id. */ +const nameMapleUsers = ( + events: ReadonlyArray, + directory: ReadonlyMap, +): ReadonlyArray => + Arr.map(events, (event) => { + const userId = unnamedMapleUserId(event) + const member = userId === null ? undefined : directory.get(userId) + return member === undefined + ? event + : { ...event, actor: { ...event.actor, name: member.name, email: member.email } } + }) + +/** The member an email belongs to. Maple's own events store the user id, never the email. */ +const memberByEmail = (directory: ReadonlyMap, email: string) => + Arr.findFirst( + [...directory.values()], + (member) => Str.toLowerCase(member.email) === Str.toLowerCase(email), + ) + +export const HttpAuditTrailLive = HttpApiBuilder.group(MapleInternalApi, "auditTrail", (handlers) => + Effect.gen(function* () { + const trail = yield* AuditTrailService + const members = yield* OrgMembersService + + return handlers.handle("list", ({ query }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + // Every platform's actor emails and origin IPs: the same gate as Maple's own audit log. + yield* Policy.check( + tenant.roles, + "audit_log:read", + (denial) => new AuditTrailForbiddenError({ message: denial.message }), + ) + // Only Maple's own events store a user id where the page shows an email. + const emailActor = + query.actor !== undefined && + Str.includes("@")(query.actor) && + (query.provider === undefined || query.provider === "maple") + ? query.actor + : undefined + // The filter's answer depends on this lookup, so an outage fails the request: a + // page that silently left the member's Maple events out would read as complete. + const filterDirectory = + emailActor === undefined + ? undefined + : yield* orgMemberDirectoryOrFail(members, tenant.orgId).pipe( + Effect.mapError( + (error) => + new AuditTrailPersistenceError({ + message: + "The member directory is unavailable, so an email cannot be matched to a member", + cause: error._tag, + }), + ), + ) + const actorAlias = + emailActor === undefined || filterDirectory === undefined + ? Option.none() + : memberByEmail(filterDirectory, emailActor) + const page = yield* trail.list(tenant.orgId, { + ...(query.provider !== undefined ? { provider: query.provider } : undefined), + ...(query.account !== undefined ? { account: query.account } : undefined), + ...(query.category !== undefined ? { category: query.category } : undefined), + ...(query.outcome !== undefined ? { outcome: query.outcome } : undefined), + ...(query.action !== undefined ? { action: query.action } : undefined), + ...(query.actor !== undefined ? { actor: query.actor } : undefined), + ...(Option.isSome(actorAlias) ? { actorAlias: actorAlias.value.userId } : undefined), + ...(query.since !== undefined ? { sinceMs: query.since } : undefined), + ...(query.until !== undefined ? { untilMs: query.until } : undefined), + ...(query.cursor !== undefined ? { before: parseCursor(query.cursor) } : undefined), + limit: query.limit ?? DEFAULT_PAGE_LIMIT, + }) + // Names are decoration: without a directory the page still shows every id. + const directory = + filterDirectory ?? + (Arr.some(page.events, (event) => unnamedMapleUserId(event) !== null) + ? yield* orgMemberDirectory(members, tenant.orgId) + : new Map()) + return { + events: nameMapleUsers(page.events, directory), + nextCursor: page.next === null ? null : `${page.next.occurredAtMs}:${page.next.id}`, + } + }), + ) + }), +) diff --git a/apps/api/src/routes/org-member-directory.ts b/apps/api/src/routes/org-member-directory.ts new file mode 100644 index 0000000000..1a4b88fbd5 --- /dev/null +++ b/apps/api/src/routes/org-member-directory.ts @@ -0,0 +1,39 @@ +import type { OrgMember, OrgMembersServiceApi } from "@maple/backend/services/org/OrgMembersService" +import { summarizeCause } from "@maple/backend/platform/describe-cause" +import type { OrgId } from "@maple/primitives" +import { Cause, Effect, Array as Arr } from "effect" + +const byUserId = (members: ReadonlyArray): ReadonlyMap => + new Map(Arr.map(members, (member) => [member.userId, member] as const)) + +/** + * The directory for a caller whose answer depends on it: an outage fails. A deployment with no + * directory at all (self-hosted without Clerk) has no members to look up, which is not a failure. + */ +export const orgMemberDirectoryOrFail = (members: Pick, orgId: OrgId) => + members.listMembers(orgId).pipe( + Effect.map(byUserId), + Effect.catchTag("@maple/http/errors/AlertMemberDirectoryNotConfiguredError", () => + Effect.succeed>(new Map()), + ), + ) + +/** + * The org's current members by user id, for putting names on audit records. Never fails: an + * unconfigured or unavailable directory answers with nobody and the records keep their ids. + */ +export const orgMemberDirectory = ( + members: Pick, + orgId: OrgId, +): Effect.Effect> => + members.listMembers(orgId).pipe( + Effect.map(byUserId), + Effect.catchCause((cause) => + Cause.hasInterruptsOnly(cause) + ? Effect.interrupt + : Effect.logWarning("Member directory unavailable; audit records keep their ids").pipe( + Effect.annotateLogs({ error: summarizeCause(cause) }), + Effect.as(new Map()), + ), + ), + ) diff --git a/apps/api/src/routes/v2/audit-log.http.ts b/apps/api/src/routes/v2/audit-log.http.ts index e0d568278b..ba323c9bb7 100644 --- a/apps/api/src/routes/v2/audit-log.http.ts +++ b/apps/api/src/routes/v2/audit-log.http.ts @@ -13,21 +13,14 @@ import { } from "@maple/domain/http/v2" import type { V2AuditLogEntry } from "@maple/domain/http/v2" import type { AuditLogEntry } from "@maple/backend/services/audit/audit-event" -import { Cause, Effect, Option, Schema, Array as Arr } from "effect" -import { summarizeCause } from "@maple/backend/platform/describe-cause" +import { Effect, Option, Schema, Array as Arr } from "effect" +import { orgMemberDirectory } from "@/routes/org-member-directory" import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" import { OrgMembersService } from "@maple/backend/services/org/OrgMembersService" import * as Policy from "@maple/domain/policy" import type { AuditLogListFilters } from "@maple/backend/services/audit/AuditLogService" import { timestampMs } from "@maple/backend/platform/time" -/** No directory, no names — the entries still carry every id they were written with. */ -const unnamed = (cause: Cause.Cause) => - Effect.logWarning("Audit log: member directory unavailable; entries keep their ids").pipe( - Effect.annotateLogs({ error: summarizeCause(cause) }), - Effect.as(new Map()), - ) - const decodeApiKeyIdOption = Schema.decodeUnknownOption(ApiKeyId) const decodeActorIdOption = Schema.decodeUnknownOption(ActorId) const decodeUserIdOption = Schema.decodeUnknownOption(UserId) @@ -152,25 +145,17 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( const members = yield* OrgMembersService /** - * Names and avatars for the acting users, or nothing at all: a directory - * that is unconfigured (self-hosted without Clerk) or briefly unavailable - * must never turn reading the audit log into an error. Ids still render. - * - * A member with no name set falls back to their email, which is what the - * rest of the product shows and what an admin reading an audit trail - * actually recognises — an opaque `user_…` is the last resort, not the - * second one. - * - * Failures and defects both fall back; interruption is re-raised, because - * a request being torn down has no page left to label. + * Names and avatars for the acting users. A member with no name set falls back to their + * email, which is what the rest of the product shows and what an admin reading an audit + * trail recognises; an opaque `user_…` is the last resort, not the second one. */ const directory = (orgId: OrgId) => - members.listMembers(orgId).pipe( + orgMemberDirectory(members, orgId).pipe( Effect.map( - (all) => + (known) => new Map( Arr.map( - all, + [...known.values()], (member) => [ member.userId, @@ -179,9 +164,6 @@ export const HttpV2AuditLogLive = HttpApiBuilder.group(MapleApiV2, "auditLog", ( ), ), ), - Effect.catchCause((cause) => - Cause.hasInterruptsOnly(cause) ? Effect.interrupt : unnamed(cause), - ), ) return handlers.handle("list", ({ query }) => diff --git a/apps/api/src/runtime/http-graph.ts b/apps/api/src/runtime/http-graph.ts index af27cdae03..156d8cb538 100644 --- a/apps/api/src/runtime/http-graph.ts +++ b/apps/api/src/runtime/http-graph.ts @@ -27,6 +27,7 @@ import { AwsOidcRouter } from "@/routes/aws-oidc.http" import { DiscoveryRouter, instanceApiV2, NotFoundRouter } from "@/routes/discovery.http" import { HttpDigestLive } from "@/routes/internal/digest.http" import { HttpCodeReviewLive } from "@/routes/internal/code-review.http" +import { HttpAuditTrailLive } from "@/routes/internal/audit-trail.http" import { HttpDependencyScanLive } from "@/routes/internal/dependency-scan.http" import { HttpErrorsLive } from "@/routes/internal/errors.http" import { HttpIntegrationsLive } from "@/routes/internal/integrations.http" @@ -165,6 +166,7 @@ const ApiInternalRoutes = HttpApiBuilder.layer(MapleInternalApi).pipe( HttpAiTriageLive, HttpBillingLive, HttpDemoLive, + HttpAuditTrailLive, HttpDependencyScanLive, HttpDigestLive, HttpGithubCostLive, diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 5ec8a58566..c679e0d7cf 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -109,6 +109,7 @@ import { AiSessionReadService } from "@maple/backend/services/ai-sessions/AiSess import { ProductEventsService } from "@maple/backend/services/product-events/ProductEventsService" import { AuditLogService } from "@maple/backend/services/audit/AuditLogService" +import { AuditTrailService } from "@maple/backend/services/audit-trail/AuditTrailService" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" import { QueryEngineService } from "@maple/backend/services/warehouse/QueryEngineService" import { TelemetryReadService } from "@maple/backend/services/warehouse/TelemetryReadService" @@ -194,6 +195,7 @@ export const HttpServicesLive = Layer.mergeAll( VercelCostSourceService.layer, ServiceMapWorldService.layer, AuditLogService.layer, + AuditTrailService.layer, WarehouseQueryService.layer, QueryEngineService.layer, RuntimeReleaseCatalog.layer, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 7bdc12a962..90f05aabf1 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -206,6 +206,7 @@ describe("current local schema identity", () => { "ai_trace_index", "ai_trace_index_mv", "audit_log", + "audit_trail_events", "ci_jobs", "ci_jobs_mv", "ci_runs", @@ -362,6 +363,7 @@ describe("current local schema identity", () => { "ai_trace_index", "ai_trace_index_mv", "audit_log", + "audit_trail_events", "ci_jobs", "ci_jobs_mv", "ci_runs", diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index c21a50a5d5..4beb2da5de 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "266cb624c1996b27cb3d435f5feb3e57ec70c6fdf5f9ecfd1f291b6425b3a692"; +pub const PROJECT_REVISION: &str = "5059e1601b59566f096b277902cfa023b21cbd0b62e1d9e53a7bb05cc92473f7"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/packages/backend/src/services/audit-trail/AuditTrailService.test.ts b/packages/backend/src/services/audit-trail/AuditTrailService.test.ts new file mode 100644 index 0000000000..67ed89a3d4 --- /dev/null +++ b/packages/backend/src/services/audit-trail/AuditTrailService.test.ts @@ -0,0 +1,303 @@ +import { describe, expect, it } from "@effect/vitest" +import type { AuditTrailEvent } from "@maple/domain/audit-trail" +import { WarehouseUpstreamError } from "@maple/domain/http" +import { OrgId } from "@maple/primitives" +import type { CompiledQuery } from "@maple/query-engine/ch" +import { Effect, Layer, Schema, Array as Arr, String as Str } from "effect" +import { TestClock } from "effect/testing" +import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { auditTrailEventId } from "./audit-trail-row" +import { AuditTrailService } from "./AuditTrailService" + +const ORG = Schema.decodeSync(OrgId)("org_trail_service") + +const down = new WarehouseUpstreamError({ + message: "tinybird down", + pipeName: "audit_trail_events", + cause: new Error("down"), +}) + +/** + * A warehouse that answers the listing with canned rows, keeps the SQL it was handed and the + * rows `ingest` receives; `failing` makes both fail. + */ +const fakeWarehouse = (rows: ReadonlyArray> = [], failing = false) => { + const sql: Array = [] + const written: Array> = [] + const bestEffort: Array = [] + const layer = Layer.succeed( + WarehouseQueryService, + makeWarehouseServiceStub({ + ingest: (_tenant, _datasource, batch, options) => + failing + ? Effect.fail(down) + : Effect.sync(() => { + bestEffort.push(options?.bestEffort === true) + // SAFETY: every warehouse row is a record of column values. + written.push(...(batch as ReadonlyArray>)) + }), + compiledQuery: ((_tenant: unknown, compiled: unknown) => + Effect.gen(function* () { + if (failing) return yield* down + const query = Effect.isEffect(compiled) ? yield* compiled : compiled + // SAFETY: the listing hands the stub a compiled query. + const compiledQuery = query as CompiledQuery + sql.push(compiledQuery.sql) + // Decode as the real executor does, so timestamps arrive as `DateTime.Utc`. + return yield* compiledQuery.decodeRows(rows) + })) as never, + }), + ) + return { sql, written, bestEffort, ingests: () => bestEffort.length, layer } +} + +const withTrail = ( + warehouse: ReturnType, + use: (trail: AuditTrailService["Service"]) => Effect.Effect, +) => + Effect.flatMap(AuditTrailService, use).pipe( + Effect.provide( + Layer.effect(AuditTrailService, AuditTrailService.make).pipe(Layer.provide(warehouse.layer)), + ), + ) + +const stored = (id: string, overrides: Partial> = {}) => ({ + id, + occurredAt: "2026-08-29 09:12:00.412", + ingestedAt: "2026-08-29 09:13:30.000", + provider: "github", + account: "acme", + providerEventId: `evt_${id}`, + actorType: "user", + actorId: "octocat", + actorName: "", + actorEmail: "", + action: "org.add_member", + category: "membership", + outcome: "success", + outcomeReason: "", + targetType: "user", + targetId: "hubot", + targetName: "", + originIp: "", + originCountry: "", + userAgent: "", + raw: JSON.stringify({ action: "org.add_member" }), + ...overrides, +}) + +const ids = [Str.repeat(32)("c"), Str.repeat(32)("b"), Str.repeat(32)("a")] as const +const OCCURRED_AT_MS = Date.UTC(2026, 7, 29, 9, 12, 0, 412) + +describe("AuditTrailService.list", () => { + it.effect("returns a page and where the next one starts when more events exist", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse(Arr.map(ids, (id) => stored(id))) + const page = yield* withTrail(warehouse, (trail) => trail.list(ORG, { limit: 2 })) + + expect(Arr.map(page.events, (event) => event.id)).toEqual([ids[0], ids[1]]) + expect(page.next).toEqual({ occurredAtMs: OCCURRED_AT_MS, id: ids[1] }) + }), + ) + + it.effect("has no next page when the events fit", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse([stored(ids[0]), stored(ids[1])]) + const page = yield* withTrail(warehouse, (trail) => trail.list(ORG, { limit: 2 })) + + expect(page.events).toHaveLength(2) + expect(page.next).toBeNull() + }), + ) + + // Two unmerged copies differ only in when they were stored; the first write is the one the + // table will keep, so it is the one to show. + it.effect("reads an event's copies oldest write first, so the kept one is the first write", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse([ + stored(ids[0], { ingestedAt: "2026-08-29 09:13:30.000" }), + stored(ids[0], { ingestedAt: "2026-08-29 09:20:00.000" }), + ]) + const page = yield* withTrail(warehouse, (trail) => trail.list(ORG, { limit: 10 })) + + expect(warehouse.sql[0]).toContain("ORDER BY occurredAt DESC, id DESC, ingestedAt ASC") + expect(Arr.map(page.events, (event) => event.ingestedAt)).toEqual([ + Date.UTC(2026, 7, 29, 9, 13, 30), + ]) + }), + ) + + it.effect("shows a redelivered event the table has not merged yet once", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse([stored(ids[0]), stored(ids[0]), stored(ids[1])]) + const page = yield* withTrail(warehouse, (trail) => trail.list(ORG, { limit: 10 })) + + expect(Arr.map(page.events, (event) => event.id)).toEqual([ids[0], ids[1]]) + }), + ) + + // Ending the list because the row past the page is a copy would hide whatever follows it. + it.effect("still offers a next page when the row past the page is an unmerged copy", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse([stored(ids[0]), stored(ids[1]), stored(ids[1])]) + const page = yield* withTrail(warehouse, (trail) => trail.list(ORG, { limit: 2 })) + + expect(page.next).toEqual({ occurredAtMs: OCCURRED_AT_MS, id: ids[1] }) + }), + ) + + it.effect("reads '' as absent and a payload that is not a JSON object as no payload", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse([ + stored(ids[0]), + stored(ids[1], { raw: "not json" }), + stored(ids[2], { raw: "" }), + ]) + const page = yield* withTrail(warehouse, (trail) => trail.list(ORG, { limit: 10 })) + + expect(page.events[0]).toMatchObject({ + providerEventId: `evt_${ids[0]}`, + actor: { type: "user", id: "octocat", name: null, email: null }, + target: { type: "user", id: "hubot", name: null }, + outcomeReason: null, + originIp: null, + raw: { action: "org.add_member" }, + }) + expect(Arr.map(page.events, (event) => event.raw === null)).toEqual([false, true, true]) + }), + ) + + // The cursor is an equality on the sorting key: a bound that dropped the milliseconds would + // skip every event sharing the boundary second. + it.effect("continues from the cursor's exact millisecond", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse() + yield* withTrail(warehouse, (trail) => + trail.list(ORG, { before: { occurredAtMs: OCCURRED_AT_MS, id: ids[1] }, limit: 2 }), + ) + + expect(warehouse.sql[0]).toContain("OccurredAt = '2026-08-29 09:12:00.412'") + expect(warehouse.sql[0]).toContain(`Id < '${ids[1]}'`) + }), + ) + + it.effect("matches an actor by id, by email, or by the alias the caller resolved", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse() + yield* withTrail(warehouse, (trail) => + trail.list(ORG, { actor: "ada@acme.test", actorAlias: "user_ada", limit: 2 }), + ) + + expect(warehouse.sql[0]).toContain( + "((audit_trail_events.ActorId = 'ada@acme.test' OR audit_trail_events.ActorEmail = 'ada@acme.test') OR audit_trail_events.ActorId = 'user_ada')", + ) + }), + ) + + it.effect("fails with a persistence error when the warehouse is down", () => + Effect.gen(function* () { + const error = yield* withTrail(fakeWarehouse([], true), (trail) => + trail.list(ORG, { limit: 2 }), + ).pipe(Effect.flip) + + expect(error._tag).toBe("@maple/http/errors/AuditTrailPersistenceError") + }), + ) +}) + +describe("AuditTrailService.write", () => { + const event: AuditTrailEvent = { + provider: "maple", + providerEventId: "evt_1", + occurredAtMs: 1_700_000_000_000, + actor: { type: "user", id: "usr_1" }, + action: "member.added", + category: "membership", + outcome: "success", + } + + it.effect("stores absent values as '' and the payload as JSON text, under the writing org", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse() + yield* withTrail(warehouse, (trail) => + trail.write(ORG, [event, { ...event, providerEventId: "evt_2", raw: { id: "evt_2" } }]), + ) + + expect(warehouse.written).toMatchObject([ + { OrgId: ORG, ProviderEventId: "evt_1", Account: "", ActorName: "", TargetType: "", Raw: "" }, + { OrgId: ORG, ProviderEventId: "evt_2", Raw: '{"id":"evt_2"}' }, + ]) + }), + ) + + // The table keeps the highest `Version`: the first write has to win, or a redelivery would + // carry its later `IngestedAt` into a detection window the event already fired in. + it.effect("writes a redelivered event to the same key with a lower version", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse() + yield* withTrail(warehouse, (trail) => + Effect.gen(function* () { + yield* trail.write(ORG, [event]) + yield* TestClock.adjust("5 minutes") + yield* trail.write(ORG, [event]) + }), + ) + const [first, redelivered] = warehouse.written + + expect(redelivered?.["Id"]).toBe(first?.["Id"]) + expect(Number(redelivered?.["Version"])).toBeLessThan(Number(first?.["Version"])) + }), + ) + + it.effect("does not reach the warehouse for an empty batch", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse() + yield* withTrail(warehouse, (trail) => trail.write(ORG, [])) + yield* withTrail(warehouse, (trail) => trail.write(ORG, [event])) + + expect(warehouse.ingests()).toBe(1) + }), + ) + + // A caller that drops the events on failure asks for a Warn, not an Error span. + it.effect("passes best effort through to the insert", () => + Effect.gen(function* () { + const warehouse = fakeWarehouse() + yield* withTrail(warehouse, (trail) => trail.write(ORG, [event], { bestEffort: true })) + yield* withTrail(warehouse, (trail) => trail.write(ORG, [event])) + + expect(warehouse.bestEffort).toEqual([true, false]) + }), + ) + + it.effect("fails with a persistence error when the warehouse is down", () => + Effect.gen(function* () { + const error = yield* withTrail(fakeWarehouse([], true), (trail) => + trail.write(ORG, [event]), + ).pipe(Effect.flip) + + expect(error._tag).toBe("@maple/http/errors/AuditTrailPersistenceError") + }), + ) +}) + +describe("auditTrailEventId", () => { + // The ingest gateway derives the same key in Rust for the providers it writes; these + // vectors are the contract between the two. + it("is the first 32 hex characters of sha256 over provider, account and event id", () => { + expect(auditTrailEventId({ provider: "maple", account: "acme", providerEventId: "evt_1" })).toBe( + "1d19ea3d78217e97b45a7a9d28300146", + ) + expect(auditTrailEventId({ provider: "maple", providerEventId: "evt_1" })).toBe( + "851ca0eff076aad987a0192777b176cd", + ) + }) + + it("keys another account's event, and another event, to a different row", () => { + const base = { provider: "maple", account: "team_1", providerEventId: "evt_1" } as const + + expect(auditTrailEventId({ ...base, account: "team_2" })).not.toBe(auditTrailEventId(base)) + expect(auditTrailEventId({ ...base, providerEventId: "evt_2" })).not.toBe(auditTrailEventId(base)) + }) +}) diff --git a/packages/backend/src/services/audit-trail/AuditTrailService.ts b/packages/backend/src/services/audit-trail/AuditTrailService.ts new file mode 100644 index 0000000000..a27854a288 --- /dev/null +++ b/packages/backend/src/services/audit-trail/AuditTrailService.ts @@ -0,0 +1,154 @@ +import type { AuditTrailEvent } from "@maple/domain/audit-trail" +import { AuditTrailPersistenceError } from "@maple/domain/http" +import type { AuditTrailEntry } from "@maple/domain/http" +import type { OrgId } from "@maple/primitives" +import * as CH from "@maple/query-engine/ch" +import { Clock, Context, DateTime, Effect, Layer, Option, Array as Arr } from "effect" +import { ALERTING_SYSTEM_USER_ID, systemTenant } from "@maple/backend/services/auth/system-tenant" +import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { auditTrailEventToRow, storedRowToEntry } from "./audit-trail-row" + +const AUDIT_TRAIL_DATASOURCE = "audit_trail_events" + +export interface AuditTrailListFilters { + readonly provider?: string + readonly account?: string + readonly category?: string + readonly outcome?: string + readonly action?: string + /** An actor's provider id or email. */ + readonly actor?: string + /** With `actor`: a second id the same person acts under. */ + readonly actorAlias?: string + readonly sinceMs?: number + readonly untilMs?: number + /** Only events that sort after this one, newest first. */ + readonly before?: { readonly occurredAtMs: number; readonly id: string } + readonly limit: number +} + +/** One page, and where the next one starts; `next` is null on the last page. */ +export interface AuditTrailPage { + readonly events: ReadonlyArray + readonly next: { readonly occurredAtMs: number; readonly id: string } | null +} + +export interface AuditTrailServiceApi { + /** + * Store one org's events. Safe to repeat: the same event writes the same row. `bestEffort` + * is for a caller that drops the events on failure: the failed insert is a Warn, not an Error span. + */ + readonly write: ( + orgId: OrgId, + events: ReadonlyArray, + options?: { readonly bestEffort?: boolean }, + ) => Effect.Effect + /** One org's audit events, newest first, `filters.limit` to a page. */ + readonly list: ( + orgId: OrgId, + filters: AuditTrailListFilters, + ) => Effect.Effect +} + +const persistenceError = (message: string) => (error: { readonly _tag: string }) => + new AuditTrailPersistenceError({ message, cause: error._tag }) + +/** Which optional filters bind, and the parameter values behind them. */ +const listQueryInputs = (orgId: OrgId, filters: AuditTrailListFilters) => { + const opts: CH.AuditTrailEventsOpts = { + provider: filters.provider !== undefined, + account: filters.account !== undefined, + category: filters.category !== undefined, + outcome: filters.outcome !== undefined, + action: filters.action !== undefined, + actor: filters.actor !== undefined, + actorAlias: filters.actorAlias !== undefined, + since: filters.sinceMs !== undefined, + until: filters.untilMs !== undefined, + before: filters.before !== undefined, + // One row past the page says whether another page exists. + limit: filters.limit + 1, + } + const values = { + orgId, + ...(filters.provider !== undefined ? { provider: filters.provider } : undefined), + ...(filters.account !== undefined ? { account: filters.account } : undefined), + ...(filters.category !== undefined ? { category: filters.category } : undefined), + ...(filters.outcome !== undefined ? { outcome: filters.outcome } : undefined), + ...(filters.action !== undefined ? { action: filters.action } : undefined), + ...(filters.actor !== undefined ? { actor: filters.actor } : undefined), + ...(filters.actorAlias !== undefined ? { actorAlias: filters.actorAlias } : undefined), + ...(filters.sinceMs !== undefined ? { since: DateTime.makeUnsafe(filters.sinceMs) } : undefined), + ...(filters.untilMs !== undefined ? { until: DateTime.makeUnsafe(filters.untilMs) } : undefined), + ...(filters.before !== undefined + ? { + beforeOccurredAt: DateTime.makeUnsafe(filters.before.occurredAtMs), + beforeId: filters.before.id, + } + : undefined), + } + return { opts, values } +} + +/** The unified audit trail: provider adapters write their events here and the Security hub lists them. */ +export class AuditTrailService extends Context.Service()( + "@maple/backend/services/AuditTrailService", + { + make: Effect.gen(function* () { + const warehouse = yield* WarehouseQueryService + + // A transient 5xx is retried: a duplicate row collapses on the event's key. + const writeRows = Effect.fn("AuditTrailService.write")(function* ( + orgId: OrgId, + events: ReadonlyArray, + bestEffort: boolean, + ) { + const now = yield* Clock.currentTimeMillis + yield* warehouse + .ingest( + systemTenant(orgId, ALERTING_SYSTEM_USER_ID), + AUDIT_TRAIL_DATASOURCE, + Arr.map(events, (event) => auditTrailEventToRow(orgId, event, now)), + { bestEffort, retryTransient: true }, + ) + .pipe(Effect.mapError(persistenceError("The audit trail could not be written"))) + }) + + const write: AuditTrailServiceApi["write"] = (orgId, events, options) => + events.length === 0 ? Effect.void : writeRows(orgId, events, options?.bestEffort === true) + + const list: AuditTrailServiceApi["list"] = Effect.fn("AuditTrailService.list")( + function* (orgId, filters) { + const { opts, values } = listQueryInputs(orgId, filters) + const rows = yield* warehouse + .compiledQuery( + systemTenant(orgId, ALERTING_SYSTEM_USER_ID), + CH.compile(CH.auditTrailEventsQuery(opts), values), + { profile: "list", context: "auditTrailEvents" }, + ) + .pipe(Effect.mapError(persistenceError("The audit trail could not be read"))) + const entries = Arr.map(Arr.take(rows, filters.limit), storedRowToEntry) + const last = Arr.last(entries) + // A redelivered event the table has not merged yet is the same entry twice. When + // the copy lands in the lookahead slot the next page can come back empty, which + // is cheaper than ending the list early. + const seen = new Set() + return { + events: Arr.filter( + entries, + (entry) => !seen.has(entry.id) && seen.add(entry.id) !== undefined, + ), + next: + rows.length > filters.limit && Option.isSome(last) + ? { occurredAtMs: last.value.occurredAt, id: last.value.id } + : null, + } + }, + ) + + return { write, list } + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(WarehouseQueryService.layer)) +} diff --git a/packages/backend/src/services/audit-trail/audit-trail-row.ts b/packages/backend/src/services/audit-trail/audit-trail-row.ts new file mode 100644 index 0000000000..dbf06eea52 --- /dev/null +++ b/packages/backend/src/services/audit-trail/audit-trail-row.ts @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto" +import type { InferQueryOutput } from "@maple-dev/effect-orm/clickhouse" +import type { AuditTrailEvent } from "@maple/domain/audit-trail" +import type { AuditTrailEntry } from "@maple/domain/http" +import type { AuditTrailEventsRow } from "@maple/domain/tinybird" +import type { OrgId } from "@maple/primitives" +import type { auditTrailEventsQuery } from "@maple/query-engine/ch" +import { warehouseDateTime64 } from "@maple/query-engine/datetime" +import { DateTime, Option, Schema, String as Str } from "effect" + +const encodeJson = Schema.encodeSync(Schema.fromJsonString(Schema.Unknown)) +const decodeRawJson = Schema.decodeUnknownOption( + Schema.fromJsonString(Schema.Record(Schema.String, Schema.Unknown)), +) + +const sha256Hex = (value: string) => createHash("sha256").update(value, "utf8").digest("hex") + +/** + * The stored row's key, the first 32 hex characters of sha256 over `provider\naccount\neventId`. + * A writer in another language has to derive the same key, or its redeliveries stop collapsing. + */ +export const auditTrailEventId = ( + event: Pick, +): string => + Str.slice(0, 32)(sha256Hex(`${event.provider}\n${event.account ?? ""}\n${event.providerEventId}`)) + +/** Above any ingest time in ms, so `Version` stays positive and falls as time rises. */ +const VERSION_CEILING_MS = 9_999_999_999_999 + +/** Lower an adapter's event to its `audit_trail_events` row; `ingestedAtMs` is the write time. */ +export const auditTrailEventToRow = ( + orgId: OrgId, + event: AuditTrailEvent, + ingestedAtMs: number, +): AuditTrailEventsRow => ({ + OrgId: orgId, + Id: auditTrailEventId(event), + OccurredAt: warehouseDateTime64(event.occurredAtMs), + IngestedAt: warehouseDateTime64(ingestedAtMs), + Provider: event.provider, + Account: event.account ?? "", + ProviderEventId: event.providerEventId, + ActorType: event.actor.type, + ActorId: event.actor.id ?? "", + ActorName: event.actor.name ?? "", + ActorEmail: event.actor.email ?? "", + Action: event.action, + Category: event.category, + Outcome: event.outcome, + OutcomeReason: event.outcomeReason ?? "", + TargetType: event.target?.type ?? "", + TargetId: event.target?.id ?? "", + TargetName: event.target?.name ?? "", + OriginIp: event.originIp ?? "", + OriginCountry: event.originCountry ?? "", + UserAgent: event.userAgent ?? "", + Raw: event.raw === undefined ? "" : encodeJson(event.raw), + Version: VERSION_CEILING_MS - ingestedAtMs, +}) + +/** `''` in the warehouse row is "absent". */ +const absent = (value: string): string | null => (value === "" ? null : value) + +/** + * A listed row as the reader sees it. `raw` is whatever the provider sent, written by adapters + * in two languages: one that is not a JSON object reads as absent and keeps the page. + */ +export const storedRowToEntry = ( + row: InferQueryOutput>, +): AuditTrailEntry => ({ + id: row.id, + provider: row.provider, + account: absent(row.account), + providerEventId: row.providerEventId, + occurredAt: DateTime.toEpochMillis(row.occurredAt), + ingestedAt: DateTime.toEpochMillis(row.ingestedAt), + actor: { + type: row.actorType, + id: absent(row.actorId), + name: absent(row.actorName), + email: absent(row.actorEmail), + }, + action: row.action, + category: row.category, + outcome: row.outcome, + outcomeReason: absent(row.outcomeReason), + target: { type: absent(row.targetType), id: absent(row.targetId), name: absent(row.targetName) }, + originIp: absent(row.originIp), + originCountry: absent(row.originCountry), + userAgent: absent(row.userAgent), + raw: Option.getOrNull(decodeRawJson(row.raw)), +}) diff --git a/packages/backend/src/services/audit-trail/providers/maple.test.ts b/packages/backend/src/services/audit-trail/providers/maple.test.ts new file mode 100644 index 0000000000..1f39e143f3 --- /dev/null +++ b/packages/backend/src/services/audit-trail/providers/maple.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from "vitest" +import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" +import { OrgId, UserId } from "@maple/primitives" +import { Schema, Array as Arr } from "effect" +import { AuditLogEvent } from "@maple/backend/services/audit/audit-event" +import { mapleAuditTrailEvents } from "./maple" + +const ORG = Schema.decodeSync(OrgId)("org_trail_maple") +const USER = Schema.decodeSync(UserId)("user_trail_maple") +const KEY_ID = "5b8f2c1e-7a55-4c0e-9a3b-0d2c7e5b9a11" +const ENTRY_ID = "9d2c1e3a-6a1b-4f0e-9c1d-2b3a4c5d6e7f" + +const entry = (overrides: Partial[0]> = {}) => + new AuditLogEvent({ + orgId: ORG, + id: Schema.decodeSync(AuditLogEvent.fields.id)(ENTRY_ID), + actorType: "user", + userId: USER, + source: "dashboard", + action: "alert_rule.updated", + outcome: "allowed", + occurredAtMs: 1_700_000_000_000, + ...overrides, + }) + +const ALERT_RULE = { resourceType: "alert_rule", resourceId: "alrt_1" } + +/** The trail event for one audit entry, or undefined when the trail leaves it out. */ +const trailEventOf = (overrides: Parameters[0] = {}) => + mapleAuditTrailEvents([entry(overrides)])[0] + +describe("mapleAuditTrailEvents", () => { + it("carries a change with its actor, target and diff", () => { + const changes = { fields: ["name"], before: { name: "a" }, after: { name: "b" } } + const event = trailEventOf({ ...ALERT_RULE, changes, originIp: "203.0.113.7", originCountry: "DE" }) + + expect(event).toMatchObject({ + provider: "maple", + // The id the Settings audit log shows for the same entry. + providerEventId: encodePublicId(PublicIdPrefixes.auditLogEntry, ENTRY_ID), + actor: { type: "user", id: USER }, + action: "alert_rule.updated", + category: "configuration", + outcome: "success", + target: { type: "alert_rule", id: "alrt_1" }, + originIp: "203.0.113.7", + originCountry: "DE", + raw: { source: "dashboard", changes }, + }) + }) + + it("leaves an allowed read out and keeps a refused one", () => { + const read = { action: "telemetry.read", resourceType: "telemetry" } + + expect(trailEventOf({ ...read, outcome: "allowed" })).toBeUndefined() + expect(trailEventOf({ ...read, outcome: "denied", denialReason: "missing scope" })).toMatchObject({ + category: "data_access", + outcome: "denied", + outcomeReason: "missing scope", + }) + }) + + // A key that authenticated and was refused for a missing scope: a rule on refused + // permissions has to find it, and the row has to say what was asked for. + it("files a refused API request under permission, aimed at its method and path", () => { + const event = trailEventOf({ + actorType: "api_key", + action: "api.request", + resourceType: "api", + outcome: "denied", + denialReason: "The key lacks the dashboards:write scope.", + metadata: { method: "POST", path: "/v2/dashboards" }, + }) + + expect(event).toMatchObject({ + category: "permission", + outcome: "denied", + outcomeReason: "The key lacks the dashboards:write scope.", + target: { type: "api", name: "POST /v2/dashboards" }, + }) + }) + + it("names an API key by its public id and the name it had, not by the person who minted it", () => { + const event = trailEventOf({ + actorType: "api_key", + apiKeyId: Schema.decodeSync(AuditLogEvent.fields.apiKeyId)(KEY_ID), + actorLabel: "CI deploy key", + }) + + expect(event?.actor).toEqual({ + type: "service", + id: encodePublicId(PublicIdPrefixes.apiKey, KEY_ID), + name: "CI deploy key", + }) + }) + + it("points a membership change at the member it happened to", () => { + const event = trailEventOf({ + actorType: "system", + source: "system", + action: "member.removed", + resourceType: "member", + affectedUserId: USER, + }) + + expect(event).toMatchObject({ + category: "membership", + actor: { type: "system" }, + target: { type: "member", id: USER }, + }) + }) + + it("files a secret set or removed on a non-credential resource under credential", () => { + const categoryOf = (action: string) => trailEventOf({ action, resourceType: "status_page" })?.category + + expect( + Arr.map( + [ + "status_page.break_glass_rotated", + "status_page.break_glass_revoked", + "status_page.password_set", + "status_page.password_removed", + ], + categoryOf, + ), + ).toEqual(["credential", "credential", "credential", "credential"]) + expect(categoryOf("status_page.updated")).toBe("resource") + }) + + it("files linking and unlinking a chat identity under permission", () => { + const categoryOf = (action: string) => + trailEventOf({ action, resourceType: "chat_integration" })?.category + + expect( + Arr.map( + ["chat_integration.identity_link_started", "chat_integration.identity_unlinked"], + categoryOf, + ), + ).toEqual(["permission", "permission"]) + expect(categoryOf("chat_integration.settings_updated")).toBe("configuration") + }) + + it("files a resource this build does not know under other", () => { + expect(trailEventOf({ action: "hologram.created", resourceType: "hologram" })?.category).toBe("other") + }) + + it("keeps the kept entries in order and drops the rest", () => { + const events = mapleAuditTrailEvents([ + entry(ALERT_RULE), + entry({ action: "telemetry.read", resourceType: "telemetry" }), + entry({ action: "dashboard.created", resourceType: "dashboard" }), + ]) + + expect(Arr.map(events, (event) => event.action)).toEqual(["alert_rule.updated", "dashboard.created"]) + }) +}) diff --git a/packages/backend/src/services/audit-trail/providers/maple.ts b/packages/backend/src/services/audit-trail/providers/maple.ts new file mode 100644 index 0000000000..c3a8838c81 --- /dev/null +++ b/packages/backend/src/services/audit-trail/providers/maple.ts @@ -0,0 +1,184 @@ +import { isAuditAccessAction } from "@maple/domain/audit-actions" +import type { AuditTrailCategory, AuditTrailEvent } from "@maple/domain/audit-trail" +import { encodePublicId, PublicIdPrefixes } from "@maple/domain/http/v2" +import { Array as Arr, Option, Record as Rec } from "effect" +import type { AuditAction, AuditResourceType } from "@maple/domain/audit-actions" +import type { AuditLogEvent } from "@maple/backend/services/audit/audit-event" + +/** + * What each Maple resource's actions are about. Complete on purpose: a new audited resource + * does not compile until someone decides where it belongs in the trail. + */ +const CATEGORY_BY_RESOURCE = { + agent: "resource", + agent_feedback: "other", + ai_provider_key: "credential", + alert_destination: "configuration", + alert_rule: "configuration", + alert_silence: "configuration", + alert_maintenance_window: "configuration", + anomaly_incident: "resource", + anomaly_settings: "configuration", + // The only `api` action is a request refused for a scope or role the credential lacks. + api: "permission", + api_key: "credential", + attribute_mapping: "configuration", + ingest_rule: "configuration", + data_source: "configuration", + dashboard: "resource", + chat_integration: "configuration", + // A share link is a grant: it lets someone without an account read the dashboard. + dashboard_share: "permission", + device_credential: "credential", + error_issue: "resource", + escalation_chain: "configuration", + oncall_schedule: "configuration", + oncall_override: "configuration", + gcp_connector: "configuration", + aws_connection: "configuration", + source: "configuration", + ingest_key: "credential", + mcp_connection: "configuration", + mcp_tool: "data_access", + investigation: "resource", + onboarding_reward: "billing", + planetscale_integration: "configuration", + cloudflare_integration: "configuration", + cost_source: "configuration", + cost_source_key: "credential", + member: "membership", + organization: "configuration", + scrape_target: "configuration", + uptime_monitor: "configuration", + slo: "configuration", + status_page: "resource", + status_page_component: "resource", + status_page_incident: "resource", + security_audit_finding: "resource", + security_audit_run: "resource", + security_audit_settings: "configuration", + session_replay: "data_access", + support_channel: "configuration", + telemetry: "data_access", + vercel_integration: "configuration", + warehouse_settings: "configuration", + widget_credential: "credential", +} as const satisfies Record + +/** Actions about a secret or a grant on a resource whose other actions are not. */ +const CATEGORY_BY_ACTION = { + "source.rolled": "credential", + "status_page.break_glass_rotated": "credential", + "status_page.break_glass_revoked": "credential", + "status_page.password_set": "credential", + "status_page.password_removed": "credential", + "planetscale_integration.metrics_token_set": "credential", + "mcp_connection.revoked": "credential", + // The link is standing authority to act as the member from the chat platform. + "chat_integration.identity_link_started": "permission", + "chat_integration.identity_unlinked": "permission", +} as const satisfies Partial> + +const isAuditResourceType = (type: string): type is AuditResourceType => + Rec.has(CATEGORY_BY_RESOURCE, type) + +/** An entry from a build that knew a resource this one does not lands in `other`. */ +const categoryOf = (event: AuditLogEvent): AuditTrailCategory => + Rec.get(CATEGORY_BY_ACTION, event.action).pipe( + Option.getOrElse(() => + event.resourceType !== undefined && isAuditResourceType(event.resourceType) + ? CATEGORY_BY_RESOURCE[event.resourceType] + : "other", + ), + ) + +/** The actor's id in the public form the rest of the product shows. */ +const actorOf = (event: AuditLogEvent): AuditTrailEvent["actor"] => { + const name = event.actorLabel !== undefined ? { name: event.actorLabel } : undefined + switch (event.actorType) { + case "user": + return { + type: "user", + ...(event.userId !== undefined ? { id: event.userId } : undefined), + ...name, + } + case "api_key": + return { + type: "service", + ...(event.apiKeyId !== undefined + ? { id: encodePublicId(PublicIdPrefixes.apiKey, event.apiKeyId) } + : undefined), + ...name, + } + case "agent": + return { + type: "service", + ...(event.actorId !== undefined + ? { id: encodePublicId(PublicIdPrefixes.actor, event.actorId) } + : undefined), + ...name, + } + case "system": + return { type: "system", ...name } + } +} + +/** What the trail's detail view shows of a Maple entry beyond the shared columns. */ +const rawOf = (event: AuditLogEvent): Record => ({ + source: event.source, + ...(event.affectedUserId !== undefined ? { affectedUserId: event.affectedUserId } : undefined), + ...(event.changes !== undefined ? { changes: event.changes } : undefined), + ...(event.metadata !== undefined ? { metadata: event.metadata } : undefined), + ...(event.requestId !== undefined ? { requestId: event.requestId } : undefined), +}) + +/** A refused API request has no resource: what it was aimed at is its method and path. */ +const requestName = (event: AuditLogEvent): string | undefined => { + const method = event.metadata?.["method"] + const path = event.metadata?.["path"] + return event.action === "api.request" && typeof method === "string" && typeof path === "string" + ? `${method} ${path}` + : undefined +} + +/** + * Maple's own audit entry as a trail event, or null for one the trail leaves out: an allowed + * read, tool call or SQL run. A heavy agent user writes hundreds a day, and they would bury + * every other platform's events. They stay in `audit_log`. + */ +const mapleAuditTrailEvent = (event: AuditLogEvent): AuditTrailEvent | null => { + if (isAuditAccessAction(event.action) && event.outcome !== "denied") return null + // A membership change names the member it happened to, not a resource id. + const targetId = event.resourceId ?? event.affectedUserId + const targetName = requestName(event) + const target = + event.resourceType !== undefined || targetId !== undefined + ? { + ...(event.resourceType !== undefined ? { type: event.resourceType } : undefined), + ...(targetId !== undefined ? { id: targetId } : undefined), + ...(targetName !== undefined ? { name: targetName } : undefined), + } + : undefined + return { + provider: "maple", + // The entry's public id, the one the Settings audit log shows and looks up. + providerEventId: encodePublicId(PublicIdPrefixes.auditLogEntry, event.id), + occurredAtMs: event.occurredAtMs, + actor: actorOf(event), + action: event.action, + category: categoryOf(event), + outcome: event.outcome === "denied" ? "denied" : "success", + ...(event.denialReason !== undefined ? { outcomeReason: event.denialReason } : undefined), + ...(target !== undefined ? { target } : undefined), + ...(event.originIp !== undefined ? { originIp: event.originIp } : undefined), + ...(event.originCountry !== undefined ? { originCountry: event.originCountry } : undefined), + raw: rawOf(event), + } +} + +/** The trail events for a batch of Maple's own audit entries. */ +export const mapleAuditTrailEvents = (events: ReadonlyArray): ReadonlyArray => + Arr.flatMap(events, (event) => { + const trailEvent = mapleAuditTrailEvent(event) + return trailEvent === null ? [] : [trailEvent] + }) diff --git a/packages/backend/src/services/audit/AuditLogService.test.ts b/packages/backend/src/services/audit/AuditLogService.test.ts index 4ada068e9b..645c2e0290 100644 --- a/packages/backend/src/services/audit/AuditLogService.test.ts +++ b/packages/backend/src/services/audit/AuditLogService.test.ts @@ -11,6 +11,7 @@ import { TestClock } from "effect/testing" import { makeWarehouseServiceStub } from "@maple/backend/testing/warehouse-test-support" import { type AuditActorInfo, CurrentAuditActor } from "@maple/backend/services/auth/audit-actor" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { AuditTrailService } from "@maple/backend/services/audit-trail/AuditTrailService" import { AUDIT_ACCESS_ACTION_SUFFIXES, AUDIT_LOG_DATASOURCE, @@ -38,15 +39,21 @@ const recordingWarehouse = (rows: ReadonlyArray> = []) = bestEffort: boolean retryTransient: boolean }> = [] + /** Rows written to the unified audit trail, the only other table the service writes. */ + const trail: Array = [] const sql: Array = [] - const layer = Layer.succeed( + const stub = Layer.succeed( WarehouseQueryService, makeWarehouseServiceStub({ ingest: (_tenant, datasource, batch, options) => Effect.sync(() => { + if (datasource !== AUDIT_LOG_DATASOURCE) { + trail.push(...batch) + return + } ingested.push({ datasource, - // SAFETY: this stub only ever receives the audit datasource's rows. + // SAFETY: only `audit_log` writes reach this branch. rows: batch as ReadonlyArray, bestEffort: options?.bestEffort === true, retryTransient: options?.retryTransient === true, @@ -63,7 +70,9 @@ const recordingWarehouse = (rows: ReadonlyArray> = []) = })) as never, }), ) - return { ingested, sql, layer } + // The audit log writes its copy through the real trail service, over the same stub. + const layer = Layer.effect(AuditTrailService, AuditTrailService.make).pipe(Layer.provideMerge(stub)) + return { ingested, trail, sql, layer } } const storedRow = (overrides: Partial> = {}) => ({ @@ -131,6 +140,48 @@ describe("AuditLogService (warehouse-backed)", () => { }), ) + it.effect("copies a change to the audit trail under the same time, and keeps reads out of it", () => + Effect.gen(function* () { + const warehouse = recordingWarehouse() + yield* Effect.gen(function* () { + const audit = yield* AuditLogService + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "dashboard.created", + resourceId: DASHBOARD_ID, + }) + yield* audit.record({ + orgId: ORG, + actor: { type: "user", userId: USER }, + source: "dashboard", + action: "telemetry.read", + }) + }).pipe( + Effect.provide( + Layer.effect(AuditLogService, AuditLogService.make).pipe(Layer.provide(warehouse.layer)), + ), + ) + + expect(warehouse.ingested).toHaveLength(2) + expect(warehouse.trail).toEqual([ + expect.objectContaining({ + OrgId: ORG, + Provider: "maple", + Action: "dashboard.created", + Category: "resource", + Outcome: "success", + ActorType: "user", + ActorId: USER, + TargetType: "dashboard", + TargetId: encodePublicId(PublicIdPrefixes.dashboard, DASHBOARD_ID), + OccurredAt: warehouse.ingested[0].rows[0].OccurredAt, + }), + ]) + }), + ) + it.effect("publishes to the audit queue instead of writing when the binding is present", () => Effect.gen(function* () { const warehouse = recordingWarehouse() diff --git a/packages/backend/src/services/audit/AuditLogService.ts b/packages/backend/src/services/audit/AuditLogService.ts index b399726b63..d37d068509 100644 --- a/packages/backend/src/services/audit/AuditLogService.ts +++ b/packages/backend/src/services/audit/AuditLogService.ts @@ -12,6 +12,8 @@ import { ALERTING_SYSTEM_USER_ID, systemTenant } from "@maple/backend/services/a import { CurrentAuditActor } from "@maple/backend/services/auth/audit-actor" import { AUDIT_ACCESS_ACTION_SUFFIXES, isAuditAccessAction } from "@maple/domain/audit-actions" import { WarehouseQueryService } from "@maple/backend/services/warehouse/WarehouseQueryService" +import { AuditTrailService } from "@maple/backend/services/audit-trail/AuditTrailService" +import { mapleAuditTrailEvents } from "@maple/backend/services/audit-trail/providers/maple" import { type AuditAction, auditResourceFields, type AuditResourceIdOption } from "./audit-actions" import { AuditLogEvent, @@ -239,6 +241,7 @@ export class AuditLogService extends Context.Service + Effect.logWarning("Audit trail copy failed", { action: event.action, cause }), + ), + ) }) /** @@ -349,7 +361,9 @@ export class AuditLogService extends Context.Service = new Set(["audit_log"]) +const RAW_SQL_HIDDEN_DATASOURCES: ReadonlySet = new Set(["audit_log", "audit_trail_events"]) function collectDatasources() { // `Datasources` exports a mix of datasource definitions, type aliases, helper diff --git a/packages/domain/package.json b/packages/domain/package.json index 1d3b012015..1694f75bb6 100644 --- a/packages/domain/package.json +++ b/packages/domain/package.json @@ -9,6 +9,7 @@ "./ai-traffic": "./src/ai-traffic.ts", "./anticipated-errors": "./src/anticipated-errors.ts", "./audit-actions": "./src/audit-actions.ts", + "./audit-trail": "./src/audit-trail.ts", "./billing": "./src/billing.ts", "./billing-tax-ids": "./src/billing-tax-ids.ts", "./clerk-js-url": "./src/clerk-js-url.ts", diff --git a/packages/domain/src/audit-trail.ts b/packages/domain/src/audit-trail.ts new file mode 100644 index 0000000000..da87d4c27c --- /dev/null +++ b/packages/domain/src/audit-trail.ts @@ -0,0 +1,65 @@ +import { Schema } from "effect" + +/** Platforms whose audit events the trail holds. One literal per built adapter; `maple` is Maple's own audit log. */ +export const AuditTrailProvider = Schema.Literals(["maple"]) +export type AuditTrailProvider = typeof AuditTrailProvider.Type + +/** + * What an event is about, the same across platforms so one rule covers all of them. + * `configuration` is how the platform behaves; `resource` is something people make and own in it. + * `permission` is what someone may do: a grant changed, or a request refused for lacking one. + */ +export const AuditTrailCategory = Schema.Literals([ + "authentication", + "membership", + "permission", + "credential", + "configuration", + "resource", + "data_access", + "billing", + "other", +]) +export type AuditTrailCategory = typeof AuditTrailCategory.Type + +/** `service`: a token, key, app, service account or agent. `system`: the platform acting on its own. */ +export const AuditTrailActorType = Schema.Literals(["user", "service", "system", "unknown"]) +export type AuditTrailActorType = typeof AuditTrailActorType.Type + +/** `failure`: attempted and it did not work. `denied`: refused for lack of access. */ +export const AuditTrailOutcome = Schema.Literals(["success", "failure", "denied", "unknown"]) +export type AuditTrailOutcome = typeof AuditTrailOutcome.Type + +/** + * One audit event as a provider adapter hands it over. `provider`, `account` and + * `providerEventId` make the stored row's key: a redelivery collapses only when it also + * carries the same `occurredAtMs`. `raw` is the platform's payload, kept for the detail view. + */ +export interface AuditTrailEvent { + readonly provider: AuditTrailProvider + readonly providerEventId: string + /** The platform account, organization, team or project the event belongs to. */ + readonly account?: string + readonly occurredAtMs: number + readonly actor: { + readonly type: AuditTrailActorType + readonly id?: string + readonly name?: string + readonly email?: string + } + /** The platform's own action name, unchanged (`org.add_member`, `alert_rule.deleted`). */ + readonly action: string + readonly category: AuditTrailCategory + readonly outcome: AuditTrailOutcome + /** Why it failed or was refused, in the platform's words. */ + readonly outcomeReason?: string + readonly target?: { + readonly type?: string + readonly id?: string + readonly name?: string + } + readonly originIp?: string + readonly originCountry?: string + readonly userAgent?: string + readonly raw?: Record +} diff --git a/packages/domain/src/clickhouse/effect-orm-parity.test.ts b/packages/domain/src/clickhouse/effect-orm-parity.test.ts index 4761f94f56..c2a5f1f015 100644 --- a/packages/domain/src/clickhouse/effect-orm-parity.test.ts +++ b/packages/domain/src/clickhouse/effect-orm-parity.test.ts @@ -90,6 +90,7 @@ const KNOWN_DIFFERENCES = { session_replays: "ENGINE = ReplacingMergeTree(Version)", // Ingest-pinned to managed Tinybird, which keeps the version; self-managed copies stay empty. search_console_daily: "ENGINE = ReplacingMergeTree(IngestedAt)", + audit_trail_events: "ENGINE = ReplacingMergeTree(Version)", cost_daily: "ENGINE = ReplacingMergeTree(IngestedAt)", } satisfies Record diff --git a/packages/domain/src/clickhouse/migrations/0064_audit_trail_events.ts b/packages/domain/src/clickhouse/migrations/0064_audit_trail_events.ts new file mode 100644 index 0000000000..f76efc0466 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0064_audit_trail_events.ts @@ -0,0 +1,12 @@ +// Migration 0064: `audit_trail_events`, audit events from connected platforms and Maple itself. +// Written through the managed ingest pipeline (pinned like `audit_log`), so a BYO cluster only +// mirrors the empty table and it never gates ingest. Frozen v64 DDL. +export const migration_0064_audit_trail_events = { + version: 64, + description: "Create audit_trail_events, normalized audit events from connected platforms", + requiredForIngest: false, + statements: [ + "CREATE TABLE IF NOT EXISTS audit_trail_events (\n OrgId LowCardinality(String),\n Id String,\n OccurredAt DateTime64(3),\n IngestedAt DateTime64(3),\n Provider LowCardinality(String),\n Account String,\n ProviderEventId String,\n ActorType LowCardinality(String),\n ActorId String,\n ActorName String,\n ActorEmail String,\n Action LowCardinality(String),\n Category LowCardinality(String),\n Outcome LowCardinality(String),\n OutcomeReason String,\n TargetType LowCardinality(String),\n TargetId String,\n TargetName String,\n OriginIp String,\n OriginCountry LowCardinality(String),\n UserAgent String,\n Raw String,\n Version UInt64,\n INDEX idx_ingested_at IngestedAt TYPE minmax GRANULARITY 1\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toYYYYMM(OccurredAt)\nORDER BY (OrgId, OccurredAt, Id)\nTTL toDate(OccurredAt) + INTERVAL 400 DAY", + "ALTER TABLE audit_trail_events MODIFY SETTING ttl_only_drop_parts = 1", + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 8d428e5a5f..517f81f813 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -62,6 +62,7 @@ import { migration_0060_search_console_daily } from "./0060_search_console_daily import { migration_0061_error_events_blank_event_fingerprint } from "./0061_error_events_blank_event_fingerprint" import { migration_0062_scanner_autonomous_agents } from "./0062_scanner_autonomous_agents" import { migration_0063_cost_daily } from "./0063_cost_daily" +import { migration_0064_audit_trail_events } from "./0064_audit_trail_events" import { Array as Arr } from "effect" /** @@ -157,6 +158,7 @@ export const migrations: ReadonlyArray = [ migration_0061_error_events_blank_event_fingerprint, migration_0062_scanner_autonomous_agents, migration_0063_cost_daily, + migration_0064_audit_trail_events, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index 2478b94bb1..00d1288c9a 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -33,6 +33,7 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/AlertSilenceNotFoundError", "@maple/http/errors/AlertValidationError", "@maple/http/errors/ApiKeyNotFoundError", + "@maple/http/errors/AuditTrailForbiddenError", "@maple/http/errors/AwsAccountAlreadyConnectedError", "@maple/http/errors/AwsIntegrationUnavailableError", "@maple/http/errors/BillingConflictError", diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 71db8944bc..8dc9d82201 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "266cb624c1996b27cb3d435f5feb3e57ec70c6fdf5f9ecfd1f291b6425b3a692" as const +export const projectRevision = "5059e1601b59566f096b277902cfa023b21cbd0b62e1d9e53a7bb05cc92473f7" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_crawler_requests (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n ServiceName LowCardinality(String),\n Crawler LowCardinality(String),\n Host LowCardinality(String),\n Path String,\n HttpStatus UInt16\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -10,6 +10,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS attribute_keys_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, Hour, AttributeKey)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS attribute_values_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n AttributeKey LowCardinality(String),\n AttributeValue String,\n AttributeScope LowCardinality(String),\n UsageCount SimpleAggregateFunction(sum, UInt64)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue)\nTTL toDate(Hour) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS audit_log (\n OrgId LowCardinality(String),\n Id String,\n OccurredAt DateTime64(3),\n RecordedAt DateTime64(3),\n ActorType LowCardinality(String),\n UserId String,\n ApiKeyId String,\n ActorId String,\n ActorLabel String,\n AffectedUserId String,\n Source LowCardinality(String),\n Action LowCardinality(String),\n Outcome LowCardinality(String),\n DenialReason String,\n ResourceType LowCardinality(String),\n ResourceId String,\n ChangedFields Array(String),\n Changes String,\n Metadata String,\n RequestId String,\n OriginIp String,\n OriginCountry LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toYYYYMM(OccurredAt)\nORDER BY (OrgId, OccurredAt, Id)\nTTL toDate(OccurredAt) + INTERVAL 2190 DAY", + "CREATE TABLE IF NOT EXISTS audit_trail_events (\n OrgId LowCardinality(String),\n Id String,\n OccurredAt DateTime64(3),\n IngestedAt DateTime64(3),\n Provider LowCardinality(String),\n Account String,\n ProviderEventId String,\n ActorType LowCardinality(String),\n ActorId String,\n ActorName String,\n ActorEmail String,\n Action LowCardinality(String),\n Category LowCardinality(String),\n Outcome LowCardinality(String),\n OutcomeReason String,\n TargetType LowCardinality(String),\n TargetId String,\n TargetName String,\n OriginIp String,\n OriginCountry LowCardinality(String),\n UserAgent String,\n Raw String,\n Version UInt64,\n INDEX idx_ingested_at IngestedAt TYPE minmax GRANULARITY 1\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toYYYYMM(OccurredAt)\nORDER BY (OrgId, OccurredAt, Id)\nTTL toDate(OccurredAt) + INTERVAL 400 DAY", "CREATE TABLE IF NOT EXISTS ci_jobs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n Attempt UInt16,\n Job LowCardinality(String),\n Result LowCardinality(String),\n StatusCode LowCardinality(String),\n Duration UInt64,\n QueueMs UInt32,\n Url String,\n FailureReason String,\n RunnerName LowCardinality(String),\n RunnerLabels LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS ci_runs (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SpanId String,\n Owner LowCardinality(String),\n Repository LowCardinality(String),\n Pipeline LowCardinality(String),\n RunId String,\n RunNumber UInt64,\n Attempt UInt16,\n Result LowCardinality(String),\n Duration UInt64,\n Url String,\n Branch String,\n Revision String,\n TriggerEvent LowCardinality(String),\n Actor LowCardinality(String),\n PullRequest UInt32,\n WorkflowPath LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS cost_daily (\n OrgId LowCardinality(String),\n SourceId LowCardinality(String),\n Provider LowCardinality(String),\n AccountId LowCardinality(String),\n Date DateTime,\n CloudService LowCardinality(String),\n ServiceCategory LowCardinality(String),\n UsageType LowCardinality(String),\n Region LowCardinality(String),\n ResourceId String,\n ResourceType LowCardinality(String),\n ServiceName LowCardinality(String),\n ChargeCategory LowCardinality(String),\n Basis LowCardinality(String),\n Currency LowCardinality(String),\n BilledCost Float64,\n EffectiveCost Float64,\n ListCost Float64,\n ConsumedQuantity Float64,\n ConsumedUnit LowCardinality(String),\n Tags Map(LowCardinality(String), String),\n Estimated UInt8,\n IngestedAt DateTime64(3)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toYYYYMM(Date)\nORDER BY (OrgId, SourceId, Date, CloudService, UsageType, Region, ResourceId, ChargeCategory, AccountId, ServiceName, Currency)\nTTL toDate(Date) + INTERVAL 760 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 4fa2926ddd..97a6e7fcd4 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "266cb624c1996b27cb3d435f5feb3e57ec70c6fdf5f9ecfd1f291b6425b3a692" as const +export const projectRevision = "5059e1601b59566f096b277902cfa023b21cbd0b62e1d9e53a7bb05cc92473f7" as const export const datasources = [ { @@ -34,6 +34,11 @@ export const datasources = [ content: 'DESCRIPTION >\n Org-wide audit trail: allowed and denied actions plus telemetry/session-replay reads, attributed to the user, API key, or agent that performed them. Admin-only; read through GET /v2/audit_log.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Id String `json:$.Id`,\n OccurredAt DateTime64(3) `json:$.OccurredAt`,\n RecordedAt DateTime64(3) `json:$.RecordedAt`,\n ActorType LowCardinality(String) `json:$.ActorType`,\n UserId String `json:$.UserId`,\n ApiKeyId String `json:$.ApiKeyId`,\n ActorId String `json:$.ActorId`,\n ActorLabel String `json:$.ActorLabel`,\n AffectedUserId String `json:$.AffectedUserId`,\n Source LowCardinality(String) `json:$.Source`,\n Action LowCardinality(String) `json:$.Action`,\n Outcome LowCardinality(String) `json:$.Outcome`,\n DenialReason String `json:$.DenialReason`,\n ResourceType LowCardinality(String) `json:$.ResourceType`,\n ResourceId String `json:$.ResourceId`,\n ChangedFields Array(String) `json:$.ChangedFields[:]`,\n Changes String `json:$.Changes`,\n Metadata String `json:$.Metadata`,\n RequestId String `json:$.RequestId`,\n OriginIp String `json:$.OriginIp`,\n OriginCountry LowCardinality(String) `json:$.OriginCountry`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(OccurredAt)"\nENGINE_SORTING_KEY "OrgId, OccurredAt, Id"\nENGINE_TTL "toDate(OccurredAt) + INTERVAL 2190 DAY"', }, + { + name: "audit_trail_events", + content: + 'DESCRIPTION >\n Audit events from connected platforms and Maple itself, normalized to one shape: who did what to which target, with what outcome. Admin-only; read through the Security hub.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.OrgId`,\n Id String `json:$.Id`,\n OccurredAt DateTime64(3) `json:$.OccurredAt`,\n IngestedAt DateTime64(3) `json:$.IngestedAt`,\n Provider LowCardinality(String) `json:$.Provider`,\n Account String `json:$.Account`,\n ProviderEventId String `json:$.ProviderEventId`,\n ActorType LowCardinality(String) `json:$.ActorType`,\n ActorId String `json:$.ActorId`,\n ActorName String `json:$.ActorName`,\n ActorEmail String `json:$.ActorEmail`,\n Action LowCardinality(String) `json:$.Action`,\n Category LowCardinality(String) `json:$.Category`,\n Outcome LowCardinality(String) `json:$.Outcome`,\n OutcomeReason String `json:$.OutcomeReason`,\n TargetType LowCardinality(String) `json:$.TargetType`,\n TargetId String `json:$.TargetId`,\n TargetName String `json:$.TargetName`,\n OriginIp String `json:$.OriginIp`,\n OriginCountry LowCardinality(String) `json:$.OriginCountry`,\n UserAgent String `json:$.UserAgent`,\n Raw String `json:$.Raw`,\n Version UInt64 `json:$.Version`\n\nENGINE "ReplacingMergeTree"\nENGINE_PARTITION_KEY "toYYYYMM(OccurredAt)"\nENGINE_SORTING_KEY "OrgId, OccurredAt, Id"\nENGINE_TTL "toDate(OccurredAt) + INTERVAL 400 DAY"\nENGINE_VER "Version"\n\nINDEXES >\n idx_ingested_at IngestedAt TYPE minmax GRANULARITY 1', + }, { name: "ci_jobs", content: diff --git a/packages/domain/src/http/audit-trail.ts b/packages/domain/src/http/audit-trail.ts new file mode 100644 index 0000000000..c096a287ab --- /dev/null +++ b/packages/domain/src/http/audit-trail.ts @@ -0,0 +1,126 @@ +import { HttpApiEndpoint, HttpApiGroup } from "effect/http-api" +import { Schema } from "effect" +import { SessionAuthorization } from "./current-tenant" +import { HttpTaggedError } from "./error-policy" + +// The Security hub's audit log: audit events from connected platforms and Maple itself in one +// list. Dashboard only (the session tier refuses API keys) and behind `audit_log:read`. + +const PAGE_LIMIT_MAX = 100 + +/** 2100-01-01: past it a bound is a typo, and far past it the warehouse cannot parse the date. */ +const EPOCH_MS_MAX = 4_102_444_800_000 + +/** + * One stored event. `provider`, `category`, `actor.type` and `outcome` are plain strings on the + * wire: the ingest gateway writes rows too and deploys on its own schedule, so a value this + * build does not know must show as itself and not fail the page. + */ +export const AuditTrailEntry = Schema.Struct({ + id: Schema.String, + provider: Schema.String, + account: Schema.NullOr(Schema.String), + /** The platform's own id for the event. */ + providerEventId: Schema.String, + /** Epoch ms. `ingestedAt` trails `occurredAt` by the provider's delivery delay. */ + occurredAt: Schema.Finite, + ingestedAt: Schema.Finite, + actor: Schema.Struct({ + type: Schema.String, + id: Schema.NullOr(Schema.String), + name: Schema.NullOr(Schema.String), + email: Schema.NullOr(Schema.String), + }), + action: Schema.String, + category: Schema.String, + outcome: Schema.String, + /** Why it failed or was refused, in the platform's words. */ + outcomeReason: Schema.NullOr(Schema.String), + target: Schema.Struct({ + type: Schema.NullOr(Schema.String), + id: Schema.NullOr(Schema.String), + name: Schema.NullOr(Schema.String), + }), + originIp: Schema.NullOr(Schema.String), + originCountry: Schema.NullOr(Schema.String), + userAgent: Schema.NullOr(Schema.String), + /** The provider's payload, or null when none was kept. */ + raw: Schema.NullOr(Schema.Record(Schema.String, Schema.Unknown)), +}) +export type AuditTrailEntry = typeof AuditTrailEntry.Type + +const EpochMs = Schema.FiniteFromString.check( + Schema.isInt(), + Schema.isBetween({ minimum: 0, maximum: EPOCH_MS_MAX }), +) + +/** `:` of the last event on the previous page. */ +const AuditTrailCursor = Schema.String.check(Schema.isPattern(/^\d{1,13}:[0-9a-f]{32}$/)) + +const Filter = Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(256)) + +export const AuditTrailListQuery = Schema.Struct({ + provider: Schema.optional(Filter), + account: Schema.optional(Filter), + category: Schema.optional(Filter), + outcome: Schema.optional(Filter), + action: Schema.optional(Filter), + /** An actor's provider id or email. A Maple member's email finds their events too. */ + actor: Schema.optional(Filter), + since: Schema.optional(EpochMs), + until: Schema.optional(EpochMs), + cursor: Schema.optional(AuditTrailCursor), + limit: Schema.optional( + Schema.FiniteFromString.check( + Schema.isInt(), + Schema.isBetween({ minimum: 1, maximum: PAGE_LIMIT_MAX }), + ), + ), +}) +export type AuditTrailListQuery = typeof AuditTrailListQuery.Type + +/** Newest first. `nextCursor` is null on the last page. */ +export const AuditTrailListResponse = Schema.Struct({ + events: Schema.Array(AuditTrailEntry), + nextCursor: Schema.NullOr(Schema.String), +}) +export type AuditTrailListResponse = typeof AuditTrailListResponse.Type + +export class AuditTrailPersistenceError extends HttpTaggedError()( + "@maple/http/errors/AuditTrailPersistenceError", + /** `cause` names the failure behind it (its tag), never its details. */ + { message: Schema.String, cause: Schema.optionalKey(Schema.String) }, + { + status: 503, + code: "audit_trail_unavailable", + title: "The audit log is temporarily unavailable", + message: "The audit log is temporarily unavailable. Retry in a few seconds.", + retry: "backoff", + recovery: "retry", + exposure: "redacted", + }, +) {} + +export class AuditTrailForbiddenError extends HttpTaggedError()( + "@maple/http/errors/AuditTrailForbiddenError", + { message: Schema.String }, + { + status: 403, + code: "audit_trail_forbidden", + title: "Permission required", + retry: "never", + recovery: "request_access", + exposure: "public_message", + }, +) {} + +export class AuditTrailApiGroup extends HttpApiGroup.make("auditTrail") + .add( + HttpApiEndpoint.get("list", "/events", { + query: AuditTrailListQuery, + success: AuditTrailListResponse, + error: [AuditTrailPersistenceError, AuditTrailForbiddenError], + }), + ) + .prefix("/internal/audit-trail") + .middleware(SessionAuthorization) {} diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index ebbeec89e8..ed6a17b2a1 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -23,6 +23,7 @@ export * from "./anomalies" export * from "./api-keys" export * from "./device-credentials" export * from "./audit-log" +export * from "./audit-trail" export * from "./analyst-access" export * from "./alerts" export * from "./escalation-chains" diff --git a/packages/domain/src/http/internal-api.ts b/packages/domain/src/http/internal-api.ts index 7a36efb4d4..a73628c399 100644 --- a/packages/domain/src/http/internal-api.ts +++ b/packages/domain/src/http/internal-api.ts @@ -6,6 +6,7 @@ import { AiTriageApiGroup } from "./ai-triage" import { BillingApiGroup } from "./billing" import { CodeReviewApiGroup } from "./code-review" import { DemoApiGroup } from "./demo" +import { AuditTrailApiGroup } from "./audit-trail" import { DependencyScanApiGroup } from "./dependency-scan" import { DigestApiGroup } from "./digest" import { ErrorsApiGroup } from "./errors" @@ -53,6 +54,7 @@ export class MapleInternalApi extends HttpApi.make("MapleInternalApi") .add(AiModelsInternalApiGroup) .add(AiSessionsInternalApiGroup) .add(AiTriageApiGroup) + .add(AuditTrailApiGroup) .add(BillingApiGroup) .add(CodeReviewApiGroup) .add(DemoApiGroup) diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 400e39ac21..db1e6cda63 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2102,6 +2102,56 @@ export const auditLog = defineDatasource("audit_log", { export type AuditLogRow = InferRow +/** + * Audit events from every connected platform, and Maple's own changes and refusals, in one shape. + * Written only by provider adapters, never from OTLP. `''` is absent; `Raw` is the payload as JSON. + */ +export const auditTrailEvents = defineDatasource("audit_trail_events", { + tenantColumn: "OrgId", + description: + "Audit events from connected platforms and Maple itself, normalized to one shape: who did what to which target, with what outcome. Admin-only; read through the Security hub.", + schema: { + OrgId: t.string().lowCardinality().brand(OrgId), + // A hash of Provider, Account and ProviderEventId: a redelivery writes the same key. + Id: t.string(), + OccurredAt: utcDateTime64(3), + // When Maple stored the row. Detections window on it: a polled event arrives late. + IngestedAt: utcDateTime64(3), + Provider: t.string().lowCardinality(), + Account: t.string(), + ProviderEventId: t.string(), + ActorType: t.string().lowCardinality(), + ActorId: t.string(), + ActorName: t.string(), + ActorEmail: t.string(), + Action: t.string().lowCardinality(), + Category: t.string().lowCardinality(), + Outcome: t.string().lowCardinality(), + // Why it failed or was refused, in the platform's words. + OutcomeReason: t.string(), + TargetType: t.string().lowCardinality(), + TargetId: t.string(), + TargetName: t.string(), + OriginIp: t.string(), + OriginCountry: t.string().lowCardinality(), + UserAgent: t.string(), + Raw: t.string(), + // Falls as IngestedAt rises, so a merge keeps the first write: a redelivered event keeps + // its first IngestedAt and never moves into a later detection window. + Version: t.uint64(), + }, + indexes: [{ name: "idx_ingested_at", expr: "IngestedAt", type: "minmax", granularity: 1 }], + engine: engine.replacingMergeTree({ + partitionKey: "toYYYYMM(OccurredAt)", + sortingKey: ["OrgId", "OccurredAt", "Id"], + ver: "Version", + // Thirteen months: a year of history plus a month to compare against. + ttl: "toDate(OccurredAt) + INTERVAL 400 DAY", + }), +}) + +export type AuditTrailEventsRow = InferRow + /** * Minute-grain operation metrics used by the service-detail Operations panel. * The operation name is normalized once by the write-side MV, while exact and diff --git a/packages/domain/src/tinybird/retention-matrix.test.ts b/packages/domain/src/tinybird/retention-matrix.test.ts index d860f635ed..2bae908891 100644 --- a/packages/domain/src/tinybird/retention-matrix.test.ts +++ b/packages/domain/src/tinybird/retention-matrix.test.ts @@ -17,6 +17,8 @@ const RETENTION_DAYS = { ai_crawler_requests: 30, // Six years — HIPAA's documentation retention floor. Never rebuildable. audit_log: 2190, + // Thirteen months: a year of audit history plus a month to compare against. + audit_trail_events: 400, attribute_keys_hourly: 90, attribute_values_hourly: 90, error_events: 90, diff --git a/packages/local-server/src/local-schema-history.ts b/packages/local-server/src/local-schema-history.ts index 6b30d3d6d2..7dcff75604 100644 --- a/packages/local-server/src/local-schema-history.ts +++ b/packages/local-server/src/local-schema-history.ts @@ -53,6 +53,7 @@ import schemaV52Sql from "./schema/history/local-schema-v52.sql" with { type: "t import schemaV53Sql from "./schema/history/local-schema-v53.sql" with { type: "text" } import schemaV54Sql from "./schema/history/local-schema-v54.sql" with { type: "text" } import schemaV55Sql from "./schema/history/local-schema-v55.sql" with { type: "text" } +import schemaV56Sql from "./schema/history/local-schema-v56.sql" with { type: "text" } import { Array as Arr } from "effect" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LEGACY_LOCAL_SCHEMA, LEGACY_LOCAL_SCHEMA_VERSION, type LocalSchemaIdentity } from "./schema-identity" @@ -122,6 +123,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV53Sql, schemaV54Sql, schemaV55Sql, + schemaV56Sql, ] export interface LocalSchemaSnapshot { diff --git a/packages/local-server/src/local-schema-version.ts b/packages/local-server/src/local-schema-version.ts index 37a51d33a6..060bb59834 100644 --- a/packages/local-server/src/local-schema-version.ts +++ b/packages/local-server/src/local-schema-version.ts @@ -1,7 +1,7 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 55 as const +export const LOCAL_SCHEMA_VERSION = 56 as const /** SQLite eventing state has its own independent version sequence. */ export const LOCAL_CONTROL_SCHEMA_VERSION = 1 as const diff --git a/packages/local-server/src/local-store-migrations/steps.ts b/packages/local-server/src/local-store-migrations/steps.ts index 53da1ffc7e..ba2a5ea63e 100644 --- a/packages/local-server/src/local-store-migrations/steps.ts +++ b/packages/local-server/src/local-store-migrations/steps.ts @@ -1939,6 +1939,27 @@ const STEP_ROWS: ReadonlyArray = [ }, ], }, + { + // Purely additive: a new empty table, created by the bootstrap. Audit events come from the + // hosted provider adapters, so a local store has nothing to backfill. + slug: "audit-trail-events", + description: "Create audit_trail_events for audit events from connected platforms", + clonedBefore: "additive DDL runs", + plan: [ + [ + "install-audit-trail-events", + "Install the empty audit_trail_events table via the v56 bootstrap", + ], + ], + dispositions: [ + { + name: "audit_trail_events", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "Created empty; no existing table is read or rewritten.", + }, + ], + }, // local-schema:bump appends the next step above this line. ] diff --git a/packages/local-server/src/schema/history/local-schema-v56.sql b/packages/local-server/src/schema/history/local-schema-v56.sql new file mode 100644 index 0000000000..885fcad907 --- /dev/null +++ b/packages/local-server/src/schema/history/local-schema-v56.sql @@ -0,0 +1,3008 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 5059e1601b59566f096b277902cfa023b21cbd0b62e1d9e53a7bb05cc92473f7 + +CREATE TABLE IF NOT EXISTS ai_crawler_requests ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + ServiceName LowCardinality(String), + Crawler LowCardinality(String), + Host LowCardinality(String), + Path String, + HttpStatus UInt16 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + Model LowCardinality(String), + AgentName LowCardinality(String), + ToolName LowCardinality(String), + SpanId String, + ParentSpanId String, + Duration UInt64, + IsError UInt8, + IsLlmCall UInt8, + IsToolCall UInt8, + Tokens Float64, + Cost Float64, + ResponseId String, + VendorVersion LowCardinality(String), + InputTokens Float64, + CacheReadTokens Float64, + CacheWriteTokens Float64, + OutputTokens Float64, + ReasoningTokens Float64, + ErrorType LowCardinality(String), + StatusMessage String, + ToolDescription String, + FailedToolCallResult String, + ErrorFingerprint UInt64 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String), + SkipReason LowCardinality(String) DEFAULT '', + Level LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS audit_log ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + RecordedAt DateTime64(3), + ActorType LowCardinality(String), + UserId String, + ApiKeyId String, + ActorId String, + ActorLabel String, + AffectedUserId String, + Source LowCardinality(String), + Action LowCardinality(String), + Outcome LowCardinality(String), + DenialReason String, + ResourceType LowCardinality(String), + ResourceId String, + ChangedFields Array(String), + Changes String, + Metadata String, + RequestId String, + OriginIp String, + OriginCountry LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 2190 DAY; + +CREATE TABLE IF NOT EXISTS audit_trail_events ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + IngestedAt DateTime64(3), + Provider LowCardinality(String), + Account String, + ProviderEventId String, + ActorType LowCardinality(String), + ActorId String, + ActorName String, + ActorEmail String, + Action LowCardinality(String), + Category LowCardinality(String), + Outcome LowCardinality(String), + OutcomeReason String, + TargetType LowCardinality(String), + TargetId String, + TargetName String, + OriginIp String, + OriginCountry LowCardinality(String), + UserAgent String, + Raw String, + Version UInt64, + INDEX idx_ingested_at IngestedAt TYPE minmax GRANULARITY 1 +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 400 DAY; + +CREATE TABLE IF NOT EXISTS ci_jobs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + Attempt UInt16, + Job LowCardinality(String), + Result LowCardinality(String), + StatusCode LowCardinality(String), + Duration UInt64, + QueueMs UInt32, + Url String, + FailureReason String, + RunnerName LowCardinality(String), + RunnerLabels LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS ci_runs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + Owner LowCardinality(String), + Repository LowCardinality(String), + Pipeline LowCardinality(String), + RunId String, + RunNumber UInt64, + Attempt UInt16, + Result LowCardinality(String), + Duration UInt64, + Url String, + Branch String, + Revision String, + TriggerEvent LowCardinality(String), + Actor LowCardinality(String), + PullRequest UInt32, + WorkflowPath LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS cost_daily ( + OrgId LowCardinality(String), + SourceId LowCardinality(String), + Provider LowCardinality(String), + AccountId LowCardinality(String), + Date DateTime, + CloudService LowCardinality(String), + ServiceCategory LowCardinality(String), + UsageType LowCardinality(String), + Region LowCardinality(String), + ResourceId String, + ResourceType LowCardinality(String), + ServiceName LowCardinality(String), + ChargeCategory LowCardinality(String), + Basis LowCardinality(String), + Currency LowCardinality(String), + BilledCost Float64, + EffectiveCost Float64, + ListCost Float64, + ConsumedQuantity Float64, + ConsumedUnit LowCardinality(String), + Tags Map(LowCardinality(String), String), + Estimated UInt8, + IngestedAt DateTime64(3) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(Date) +ORDER BY (OrgId, SourceId, Date, CloudService, UsageType, Region, ResourceId, ChargeCategory, AccountId, ServiceName, Currency) +TTL toDate(Date) + INTERVAL 760 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String), + UserId String DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL toDate(Timestamp) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String), + UserId String DEFAULT '', + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL toDate(Timestamp) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprint_lineage ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + PreviousFingerprintHash UInt64 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, Minute, FingerprintHash, PreviousFingerprintHash) +TTL toDate(Minute) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS scanner_requests ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + ServiceName LowCardinality(String), + Signature LowCardinality(String), + Scanner LowCardinality(String), + Host LowCardinality(String), + Path String, + HttpStatus UInt16, + ClientAddress String, + UserAgent String +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS search_console_daily ( + OrgId LowCardinality(String), + SiteUrl LowCardinality(String), + Grain LowCardinality(String), + SearchType LowCardinality(String), + Date DateTime, + Page String, + Query String, + Country LowCardinality(String), + Device LowCardinality(String), + Clicks UInt64, + Impressions UInt64, + PositionSum Float64, + IngestedAt DateTime64(3) +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(Date) +ORDER BY (OrgId, SiteUrl, Grain, SearchType, Date, Page, Query, Country, Device) +TTL toDate(Date) + INTERVAL 520 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_endpoint_details_hourly ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + DeploymentEnv LowCardinality(String), + SpanName String, + HttpStatusCode LowCardinality(String), + ErrorType LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + BodySizeQuantiles AggregateFunction(quantilesTDigestIf(0.5, 0.95, 0.99), UInt64, UInt8), + BodySizeMax SimpleAggregateFunction(max, UInt64), + BodyCount SimpleAggregateFunction(sum, UInt64), + UserAgents AggregateFunction(approx_top_k(10), String), + SlowestDuration SimpleAggregateFunction(max, UInt64), + SlowestTraceId AggregateFunction(argMax, String, UInt64), + SlowestTimestamp AggregateFunction(argMax, DateTime64(9), UInt64), + SlowestBodySize AggregateFunction(argMax, Int64, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, SpanName, HttpStatusCode, ErrorType) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), TraceId, ParentSpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DbNamespace LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + FirstInsertedAt SimpleAggregateFunction(min, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, Minute, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Minute) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpStatusCount SimpleAggregateFunction(sum, UInt64), + Http4xxCount SimpleAggregateFunction(sum, UInt64), + Http5xxCount SimpleAggregateFunction(sum, UInt64), + RequestBodyBytes SimpleAggregateFunction(sum, UInt64), + RequestBodyCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + HttpStatusCount SimpleAggregateFunction(sum, UInt64), + Http4xxCount SimpleAggregateFunction(sum, UInt64), + Http5xxCount SimpleAggregateFunction(sum, UInt64), + RequestBodyBytes SimpleAggregateFunction(sum, UInt64), + RequestBodyCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64), + EstimatedLadderSpanCount SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder50Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder100Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder250Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder1000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder2500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder5000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder10000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0 +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64), + EstimatedLadderSpanCount SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder50Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder100Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder250Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder1000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder2500Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder5000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0, + EstimatedUnder10000Ms SimpleAggregateFunction(sum, Float64) DEFAULT 0 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + SpanKind LowCardinality(String) DEFAULT '', + IsRoot UInt8 DEFAULT 0, + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_runtimes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + RuntimeName LowCardinality(String), + RuntimeVersion LowCardinality(String), + ExecutableName LowCardinality(String), + SdkLanguage LowCardinality(String), + SdkName LowCardinality(String), + SdkVersion LowCardinality(String), + SpanCount UInt64 +) +ENGINE = SummingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv, RuntimeName, RuntimeVersion, ExecutableName, SdkLanguage, SdkName, SdkVersion) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS source_usage_hourly ( + OrgId LowCardinality(String), + SourceId String, + Hour DateTime, + SpanCount UInt64, + SpanSizeBytes UInt64, + LogCount UInt64, + LogSizeBytes UInt64, + MetricCount UInt64, + MetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, SourceId, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_facets_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName String, + HttpMethod LowCardinality(String), + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + HasError UInt8, + TraceCount SimpleAggregateFunction(sum, UInt64), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError) +TTL toDate(Hour) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_index ( + OrgId LowCardinality(String), + TraceId String, + ServiceName LowCardinality(String), + Bucket DateTime, + MinTimestamp SimpleAggregateFunction(min, DateTime64(9)), + MaxTimestamp SimpleAggregateFunction(max, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Bucket) +ORDER BY (OrgId, TraceId, ServiceName, Bucket) +TTL toDate(Bucket) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_name SpanName TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, toStartOfFiveMinutes(Timestamp), TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS user_activity_hourly ( + OrgId LowCardinality(String), + ServiceNamespace LowCardinality(String), + UserId String, + Hour DateTime, + ServiceName LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + ErrorSpanCount SimpleAggregateFunction(sum, UInt64), + RequestCount SimpleAggregateFunction(sum, UInt64), + FailedRequestCount SimpleAggregateFunction(sum, UInt64), + TraceIds AggregateFunction(uniqIf, String, UInt8), + SessionIds AggregateFunction(uniqIf, String, UInt8), + RequestDurationQuantiles AggregateFunction(quantilesTDigestIf(0.5, 0.95), UInt64, UInt8), + TopRoutes AggregateFunction(approx_top_k(10), String), + LogCount SimpleAggregateFunction(sum, UInt64), + WarnLogCount SimpleAggregateFunction(sum, UInt64), + ErrorLogCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ClassifiedRequestCount SimpleAggregateFunction(sum, UInt64) DEFAULT 0, + WebRequestCount SimpleAggregateFunction(sum, UInt64) DEFAULT 0, + MobileRequestCount SimpleAggregateFunction(sum, UInt64) DEFAULT 0 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, ServiceNamespace, UserId, Hour, ServiceName) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_crawler_requests_mv TO ai_crawler_requests AS +SELECT + OrgId, + Timestamp, + TraceId, + ServiceName, + arrayElement(['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'Meta-ExternalAgent', 'Meta-WebIndexer', 'Meta-ExternalFetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai'])) AS Crawler, + lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')) AS Host, + leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path, + toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus + FROM traces + WHERE SpanKind = 'Server' + AND multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'anthropic-ai', 'Claude-SearchBot', 'Claude-User', 'PerplexityBot', 'Perplexity-User', 'meta-externalagent', 'meta-webindexer', 'meta-externalfetcher', 'Bytespider', 'MistralAI-User', 'CCBot', 'Amazonbot', 'DuckAssistBot', 'cohere-training-data-crawler', 'cohere-ai']) > 0 + AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + SpanAttributes['maple_ai.model'] AS Model, + SpanAttributes['maple_ai.agent.name'] AS AgentName, + SpanAttributes['maple_ai.tool.name'] AS ToolName, + SpanId, + ParentSpanId, + Duration, + toUInt8(SpanAttributes['maple_ai.error'] = '1') AS IsError, + toUInt8(SpanAttributes['maple_ai.llm_call'] = '1') AS IsLlmCall, + toUInt8(SpanAttributes['maple_ai.tool_call'] = '1') AS IsToolCall, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS Tokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cost']) AS Cost, + SpanAttributes['maple_ai.response.id'] AS ResponseId, + SpanAttributes['maple_ai.vendor.version'] AS VendorVersion, + toFloat64OrZero(SpanAttributes['maple_ai.usage.input_tokens']) AS InputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_read_tokens']) AS CacheReadTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.cache_write_tokens']) AS CacheWriteTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.output_tokens']) AS OutputTokens, + toFloat64OrZero(SpanAttributes['maple_ai.usage.reasoning_tokens']) AS ReasoningTokens, + SpanAttributes['error.type'] AS ErrorType, + leftUTF8(StatusMessage, 400) AS StatusMessage, + SpanAttributes['maple_ai.tool.description'] AS ToolDescription, + SpanAttributes['maple_ai.tool.error_result'] AS FailedToolCallResult, + if(SpanAttributes['maple_ai.error'] = '1', cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(leftUTF8(coalesce(nullIf(SpanAttributes['maple_ai.tool.error_result'], ''), leftUTF8(StatusMessage, 400)), 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#')), 0) AS ErrorFingerprint + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_jobs_mv TO ci_jobs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.task.name'] AS Job, + SpanAttributes['cicd.pipeline.task.run.result'] AS Result, + StatusCode, + Duration, + toUInt32OrZero(SpanAttributes['maple.cicd.queue_duration_ms']) AS QueueMs, + SpanAttributes['cicd.pipeline.task.run.url.full'] AS Url, + SpanAttributes['maple.cicd.failure.reason'] AS FailureReason, + SpanAttributes['cicd.worker.name'] AS RunnerName, + SpanAttributes['maple.cicd.runner_labels'] AS RunnerLabels + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId != '' + AND SpanAttributes['cicd.pipeline.task.name'] != '' + AND SpanAttributes['maple.cicd.step.number'] = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ci_runs_mv TO ci_runs AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ResourceAttributes['vcs.owner.name'] AS Owner, + ResourceAttributes['vcs.repository.name'] AS Repository, + SpanAttributes['cicd.pipeline.name'] AS Pipeline, + SpanAttributes['cicd.pipeline.run.id'] AS RunId, + toUInt64OrZero(SpanAttributes['maple.cicd.run_number']) AS RunNumber, + toUInt16OrZero(SpanAttributes['maple.cicd.run_attempt']) AS Attempt, + SpanAttributes['cicd.pipeline.result'] AS Result, + Duration, + SpanAttributes['cicd.pipeline.run.url.full'] AS Url, + SpanAttributes['vcs.ref.head.name'] AS Branch, + SpanAttributes['vcs.ref.head.revision'] AS Revision, + SpanAttributes['maple.cicd.trigger_event'] AS TriggerEvent, + SpanAttributes['maple.cicd.actor'] AS Actor, + toUInt32OrZero(SpanAttributes['vcs.change.id']) AS PullRequest, + SpanAttributes['maple.cicd.workflow_path'] AS WorkflowPath + FROM traces + WHERE ServiceName = 'github-actions' + AND ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- A user key replaces type, frames, message and salt in the hash: the + -- exception event's value unless blank after trimming, then the span's. + -- Trimmed and capped. + substringUTF8( + replaceRegexpAll(coalesce(nullIf(replaceRegexpAll(if(_ei > 0, EventsAttributes[_ei]['maple.fingerprint'], ''), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), ''), SpanAttributes['maple.fingerprint']), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), + 1, 200 + ) AS _customFp, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', if( + kv.1 = 'type', + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '"[^" ]*/[^" ]*"', '"#"'), '[0-9a-fA-F-]{6,}|[0-9]+', '#') + )), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 1000), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), 'RouteNotFound \\(([A-Z]+) [^ \\t\\r\\n]*\\)?', 'RouteNotFound (\\1 PATH)'), '^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]* not found([\\r\\n]|$)', '\\1 PATH not found\\2'), '\\bCannot (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]*', 'Cannot \\1 PATH'), '\\bRoute (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS):/[^ \\t\\r\\n]* not found', 'Route \\1:PATH not found'), 'No route found for "([A-Z]+) [^"\\r\\n]*"', 'No route found for "\\1 PATH"'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '(^|[^§0-9A-Za-z_.])([1-4][0-9][0-9]|500) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§\\2 \\3 '), '(^|[^§0-9A-Za-z_.])5(0[1-9]|[1-9][0-9]) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§5xx \\3 '), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/[^/? \\t\\r\\n\'"`:,)]*[~@%$][^/? \\t\\r\\n\'"`:,)]*', '/#'), '\'[^\' ]*([.][.][.]|[*]{3,})[^\' ]*\'', '\'#\''), '"[^" ]*([.][.][.]|[*]{3,})[^" ]*"', '"#"'), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{20,})', '\\1⟦\\2⟧'), '⟦([^⟧0-9]*)⟧', '\\1'), '⟦([^⟧A-Z]*)⟧', '\\1'), '⟦([^⟧a-z]*)⟧', '\\1'), '⟦[^⟧]*⟧', '#'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{10,})', '\\1⟦\\2⟧'), '⟦([^⟧]*[_-][^⟧]*)⟧', '\\1'), '⟦(0[xX][0-9a-fA-F]*)⟧', '\\1'), '⟦([^⟧]*[a-z][^⟧]*[A-Z][^⟧]*)⟧', '\\1'), '⟦([^⟧]*[A-Z][^⟧]*[a-z][^⟧]*)⟧', '\\1'), '⟦[^⟧]*[0-9][A-Za-z]+[0-9]+[A-Za-z]+[0-9][^⟧]*⟧', '#'), '⟦([^⟧]*)⟧', '\\1'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@{"\\[]+@[^ \\t]*:[0-9]+|^[ \\t]*[^ \\t@{"\\[]*@([a-z][a-z0-9+.-]*:)?/[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$') + AND NOT match(line, '^[ \\t]*at ([^(]* \\()?(\\)?$|native(:|\\)|$)|node:|ext:)'), + splitByChar('\n', _exStack) + ) AS _frameLines, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(^|[ \\t("@])(file://)?([/.][^ \\t()"]*)?/(node_modules|site-packages|dist-packages|gems|pkg/mod)/', '\\1\\4/'), '([/-])v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*/', '\\1#/'), '(/python)[0-9]+\\.[0-9]+/', '\\1#/'), '(/\\$bunfs|[A-Z]:/~BUN)/root/', ''), '(\\$\\$Lambda|\\$Proxy|jdk\\.proxy|Generated[A-Za-z]*Accessor)\\$?[0-9]+', '\\1'), '(\\$(HibernateProxy|ByteBuddy|MockitoMock))\\$[A-Za-z0-9]+', '\\1'), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + arraySlice( + -- Python prints the outermost frame first: read it from the raise site. + if(match(_frameLines[1], '^[ \\t]*File "'), arrayReverse(_frameLines), _frameLines), + 1, 10 + ) + ) AS _normFrames, + -- A repeat (two anonymous closures in one bundle, or recursion) collapses + -- so it does not use up the frames the hash keeps. + arrayFilter((frame, i) -> i = 1 OR frame != _normFrames[i - 1], _normFrames, arrayEnumerate(_normFrames)) AS _distinctFrames, + -- The application's own frames: a dependency's or a runtime's frames vary with + -- its internals and version. A stack with none keeps every frame. + arrayFilter(frame -> NOT match(frame, '(^|[ \\t("@/])(node_modules|site-packages|dist-packages|gems|pkg/mod)/|/python#/|/ruby/#/|/usr/local/go/src/|/rustc/|\\.cargo/registry/|^[ \\t]*at (java|javax|jdk|sun|kotlin|kotlinx|scala)\\.'), _distinctFrames) AS _appFrames, + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) AS _maxFrames, + arraySlice(if(length(_appFrames) > 0, _appFrames, _distinctFrames), 1, _maxFrames) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- The previous version's frames, for the issue evaluator's re-key while a bump settles. + arrayStringConcat(arraySlice(_distinctFrames, 1, _maxFrames), '\n') AS _prevFpFrames, + if( + _customFp != '', + cityHash64(OrgId, ServiceName, 'maple.fingerprint', _customFp), + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp4') + ) AS _hash, + if( + _customFp != '', + _hash, + cityHash64(OrgId, ServiceName, _exType, _prevFpFrames, _msgSig, 'fp3') + ) AS _prevHash, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + -- Only string values: JSONExtractString renders a number or an object + -- as its JSON text, which titled `{"error":{"code":5}}` with the object. + multiIf( + JSONType(_msgText, 'title') = 'String' AND JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONType(_msgText, 'message') = 'String' AND JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONType(_msgText, 'error') = 'String' AND JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONType(_msgText, '_tag') = 'String' AND JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONType(_msgText, 'reason') = 'String' AND JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONType(_msgText, 'name') = 'String' AND JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONType(_msgText, 'type') = 'String' AND JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + -- UTF8 variants throughout: a byte cut split a character and + -- left U+FFFD at the end of a non-ASCII title. + leftUTF8(_msgText, multiIf( + positionUTF8(_msgText, ': ') > 3, toInt64(positionUTF8(_msgText, ': ')) - 1, + positionUTF8(_msgText, ' (') > 3, toInt64(positionUTF8(_msgText, ' (')) - 1, + positionUTF8(_msgText, '\n') > 3, toInt64(positionUTF8(_msgText, '\n')) - 1, + 150 + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + _hash AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion, + if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) AS UserId + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- A user key replaces type, frames, message and salt in the hash: the + -- exception event's value unless blank after trimming, then the span's. + -- Trimmed and capped. + substringUTF8( + replaceRegexpAll(coalesce(nullIf(replaceRegexpAll(if(_ei > 0, EventsAttributes[_ei]['maple.fingerprint'], ''), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), ''), SpanAttributes['maple.fingerprint']), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), + 1, 200 + ) AS _customFp, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', if( + kv.1 = 'type', + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '"[^" ]*/[^" ]*"', '"#"'), '[0-9a-fA-F-]{6,}|[0-9]+', '#') + )), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 1000), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), 'RouteNotFound \\(([A-Z]+) [^ \\t\\r\\n]*\\)?', 'RouteNotFound (\\1 PATH)'), '^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]* not found([\\r\\n]|$)', '\\1 PATH not found\\2'), '\\bCannot (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]*', 'Cannot \\1 PATH'), '\\bRoute (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS):/[^ \\t\\r\\n]* not found', 'Route \\1:PATH not found'), 'No route found for "([A-Z]+) [^"\\r\\n]*"', 'No route found for "\\1 PATH"'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '(^|[^§0-9A-Za-z_.])([1-4][0-9][0-9]|500) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§\\2 \\3 '), '(^|[^§0-9A-Za-z_.])5(0[1-9]|[1-9][0-9]) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§5xx \\3 '), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/[^/? \\t\\r\\n\'"`:,)]*[~@%$][^/? \\t\\r\\n\'"`:,)]*', '/#'), '\'[^\' ]*([.][.][.]|[*]{3,})[^\' ]*\'', '\'#\''), '"[^" ]*([.][.][.]|[*]{3,})[^" ]*"', '"#"'), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{20,})', '\\1⟦\\2⟧'), '⟦([^⟧0-9]*)⟧', '\\1'), '⟦([^⟧A-Z]*)⟧', '\\1'), '⟦([^⟧a-z]*)⟧', '\\1'), '⟦[^⟧]*⟧', '#'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{10,})', '\\1⟦\\2⟧'), '⟦([^⟧]*[_-][^⟧]*)⟧', '\\1'), '⟦(0[xX][0-9a-fA-F]*)⟧', '\\1'), '⟦([^⟧]*[a-z][^⟧]*[A-Z][^⟧]*)⟧', '\\1'), '⟦([^⟧]*[A-Z][^⟧]*[a-z][^⟧]*)⟧', '\\1'), '⟦[^⟧]*[0-9][A-Za-z]+[0-9]+[A-Za-z]+[0-9][^⟧]*⟧', '#'), '⟦([^⟧]*)⟧', '\\1'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@{"\\[]+@[^ \\t]*:[0-9]+|^[ \\t]*[^ \\t@{"\\[]*@([a-z][a-z0-9+.-]*:)?/[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$') + AND NOT match(line, '^[ \\t]*at ([^(]* \\()?(\\)?$|native(:|\\)|$)|node:|ext:)'), + splitByChar('\n', _exStack) + ) AS _frameLines, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(^|[ \\t("@])(file://)?([/.][^ \\t()"]*)?/(node_modules|site-packages|dist-packages|gems|pkg/mod)/', '\\1\\4/'), '([/-])v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*/', '\\1#/'), '(/python)[0-9]+\\.[0-9]+/', '\\1#/'), '(/\\$bunfs|[A-Z]:/~BUN)/root/', ''), '(\\$\\$Lambda|\\$Proxy|jdk\\.proxy|Generated[A-Za-z]*Accessor)\\$?[0-9]+', '\\1'), '(\\$(HibernateProxy|ByteBuddy|MockitoMock))\\$[A-Za-z0-9]+', '\\1'), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + arraySlice( + -- Python prints the outermost frame first: read it from the raise site. + if(match(_frameLines[1], '^[ \\t]*File "'), arrayReverse(_frameLines), _frameLines), + 1, 10 + ) + ) AS _normFrames, + -- A repeat (two anonymous closures in one bundle, or recursion) collapses + -- so it does not use up the frames the hash keeps. + arrayFilter((frame, i) -> i = 1 OR frame != _normFrames[i - 1], _normFrames, arrayEnumerate(_normFrames)) AS _distinctFrames, + -- The application's own frames: a dependency's or a runtime's frames vary with + -- its internals and version. A stack with none keeps every frame. + arrayFilter(frame -> NOT match(frame, '(^|[ \\t("@/])(node_modules|site-packages|dist-packages|gems|pkg/mod)/|/python#/|/ruby/#/|/usr/local/go/src/|/rustc/|\\.cargo/registry/|^[ \\t]*at (java|javax|jdk|sun|kotlin|kotlinx|scala)\\.'), _distinctFrames) AS _appFrames, + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) AS _maxFrames, + arraySlice(if(length(_appFrames) > 0, _appFrames, _distinctFrames), 1, _maxFrames) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- The previous version's frames, for the issue evaluator's re-key while a bump settles. + arrayStringConcat(arraySlice(_distinctFrames, 1, _maxFrames), '\n') AS _prevFpFrames, + if( + _customFp != '', + cityHash64(OrgId, ServiceName, 'maple.fingerprint', _customFp), + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp4') + ) AS _hash, + if( + _customFp != '', + _hash, + cityHash64(OrgId, ServiceName, _exType, _prevFpFrames, _msgSig, 'fp3') + ) AS _prevHash, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + -- Only string values: JSONExtractString renders a number or an object + -- as its JSON text, which titled `{"error":{"code":5}}` with the object. + multiIf( + JSONType(_msgText, 'title') = 'String' AND JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONType(_msgText, 'message') = 'String' AND JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONType(_msgText, 'error') = 'String' AND JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONType(_msgText, '_tag') = 'String' AND JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONType(_msgText, 'reason') = 'String' AND JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONType(_msgText, 'name') = 'String' AND JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONType(_msgText, 'type') = 'String' AND JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + -- UTF8 variants throughout: a byte cut split a character and + -- left U+FFFD at the end of a non-ASCII title. + leftUTF8(_msgText, multiIf( + positionUTF8(_msgText, ': ') > 3, toInt64(positionUTF8(_msgText, ': ')) - 1, + positionUTF8(_msgText, ' (') > 3, toInt64(positionUTF8(_msgText, ' (')) - 1, + positionUTF8(_msgText, '\n') > 3, toInt64(positionUTF8(_msgText, '\n')) - 1, + 150 + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + _hash AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion, + if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) AS UserId + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprint_lineage_mv TO error_fingerprint_lineage AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + -- Only fill the old Unknown Error bucket. Event values (including + -- empty fields) and spans with StatusMessage keep every hash input. + _ei = 0 AND StatusMessage = '' AS _useAttrs, + if( + _ei > 0, EventsAttributes[_ei]['exception.type'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.type'], ''), SpanAttributes['error.type']), '') + ) AS _exType, + if( + _ei > 0, EventsAttributes[_ei]['exception.message'], + if(_useAttrs, coalesce(nullIf(SpanAttributes['exception.message'], ''), SpanAttributes['error.message']), StatusMessage) + ) AS _exMsg, + if( + _ei > 0, EventsAttributes[_ei]['exception.stacktrace'], + if(_useAttrs, SpanAttributes['exception.stacktrace'], '') + ) AS _exStack, + if(_useAttrs, _exMsg, StatusMessage) AS _msgText, + -- A user key replaces type, frames, message and salt in the hash: the + -- exception event's value unless blank after trimming, then the span's. + -- Trimmed and capped. + substringUTF8( + replaceRegexpAll(coalesce(nullIf(replaceRegexpAll(if(_ei > 0, EventsAttributes[_ei]['maple.fingerprint'], ''), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), ''), SpanAttributes['maple.fingerprint']), '^[ \\t\\n\\r]+|[ \\t\\n\\r]+$', ''), + 1, 200 + ) AS _customFp, + -- JSON detection for the message signature below. + isValidJSON(_msgText) AS _isJson, + _isJson AND JSONType(_msgText) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', if( + kv.1 = 'type', + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(kv.2, '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '"[^" ]*/[^" ]*"', '"#"'), '[0-9a-fA-F-]{6,}|[0-9]+', '#') + )), + JSONExtractKeysAndValuesRaw(_msgText) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See ERROR_MSG_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(_msgText, 1, 1000), '\\x1b\\[[0-9;]*m', ''), '^((\\[Error\\]|[\\s\\S]*?DB::Exception:)[\\s\\S]*?)( [Ii]n scope | [Ww]hile processing )[\\s\\S]*', '\\1'), 'RouteNotFound \\(([A-Z]+) [^ \\t\\r\\n]*\\)?', 'RouteNotFound (\\1 PATH)'), '^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]* not found([\\r\\n]|$)', '\\1 PATH not found\\2'), '\\bCannot (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) /[^ \\t\\r\\n]*', 'Cannot \\1 PATH'), '\\bRoute (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS):/[^ \\t\\r\\n]* not found', 'Route \\1:PATH not found'), 'No route found for "([A-Z]+) [^"\\r\\n]*"', 'No route found for "\\1 PATH"'), '(^|[ \\t\\r\\n\'"(`=])~/[^ \'"`:,)]*', '\\1PATH'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '(^|[ \\t\\r\\n\'"(`=])/(tmp|var|srv|private|opt|mnt|data|Volumes|System|app|work|run|Users/[^/ \\t\\r\\n\'"`:,)]+|home/[^/ \\t\\r\\n\'"`:,)]+)([/.][^ \\t\\r\\n\'"`:,)]*)?([ \\t\\r\\n\'"`:,)]|$)', '\\1PATH\\4'), '\\b[a-z]{2,8}_[a-z]*[A-Z0-9][A-Za-z0-9]{8,}\\b', 'ID'), '\\b[0-9A-HJKMNP-TV-Z]{26}\\b', 'ID'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)([1-4][0-9][0-9]|500)\\b', '\\1\\2§\\3'), '\\b(HTTP|[Ss]tatus_?[Cc]ode|[Ss]tatus|returned|code|failed|failed with)(:? ?|=)5(0[1-9]|[1-9][0-9])\\b', '\\1\\2§5xx'), '(^|[^§0-9A-Za-z_.])([1-4][0-9][0-9]|500) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§\\2 \\3 '), '(^|[^§0-9A-Za-z_.])5(0[1-9]|[1-9][0-9]) (GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS) ', '\\1§5xx \\3 '), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/[^/? \\t\\r\\n\'"`:,)]*[~@%$][^/? \\t\\r\\n\'"`:,)]*', '/#'), '\'[^\' ]*([.][.][.]|[*]{3,})[^\' ]*\'', '\'#\''), '"[^" ]*([.][.][.]|[*]{3,})[^" ]*"', '"#"'), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{20,})', '\\1⟦\\2⟧'), '⟦([^⟧0-9]*)⟧', '\\1'), '⟦([^⟧A-Z]*)⟧', '\\1'), '⟦([^⟧a-z]*)⟧', '\\1'), '⟦[^⟧]*⟧', '#'), '(^|[^A-Za-z0-9_-])([A-Za-z0-9_-]{10,})', '\\1⟦\\2⟧'), '⟦([^⟧]*[_-][^⟧]*)⟧', '\\1'), '⟦(0[xX][0-9a-fA-F]*)⟧', '\\1'), '⟦([^⟧]*[a-z][^⟧]*[A-Z][^⟧]*)⟧', '\\1'), '⟦([^⟧]*[A-Z][^⟧]*[a-z][^⟧]*)⟧', '\\1'), '⟦[^⟧]*[0-9][A-Za-z]+[0-9]+[A-Za-z]+[0-9][^⟧]*⟧', '#'), '⟦([^⟧]*)⟧', '\\1'), '(§[1-4][0-9][0-9]|§500|§5xx)|[0-9a-fA-F-]{6,}|[0-9]+', '\\1#'), '§([1-4][0-9][0-9]|500|5xx)#', '\\1'), + 1, 120 + ) + ) AS _msgSig, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@{"\\[]+@[^ \\t]*:[0-9]+|^[ \\t]*[^ \\t@{"\\[]*@([a-z][a-z0-9+.-]*:)?/[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+') + AND position(line, '(definition)') = 0 + AND NOT match(line, '^[ \\t]*at \\[') + AND NOT match(line, '^[ \\t]*at ([^(:]|:[^0-9])*:?$') + AND NOT match(line, '^[ \\t]*at ([^(]* \\()?(\\)?$|native(:|\\)|$)|node:|ext:)'), + splitByChar('\n', _exStack) + ) AS _frameLines, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, '@v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*', '@#'), '\\(index [0-9]+\\)', '(index #)'), 'file:///bundle/', ''), 'https?://[^/ )]+', ''), '(^|[ \\t("@])(file://)?([/.][^ \\t()"]*)?/(node_modules|site-packages|dist-packages|gems|pkg/mod)/', '\\1\\4/'), '([/-])v?[0-9]+\\.[0-9]+\\.[0-9]+[^/ )]*/', '\\1#/'), '(/python)[0-9]+\\.[0-9]+/', '\\1#/'), '(/\\$bunfs|[A-Z]:/~BUN)/root/', ''), '(\\$\\$Lambda|\\$Proxy|jdk\\.proxy|Generated[A-Za-z]*Accessor)\\$?[0-9]+', '\\1'), '(\\$(HibernateProxy|ByteBuddy|MockitoMock))\\$[A-Za-z0-9]+', '\\1'), '(-[A-Za-z0-9_-]{8})+\\.js', '.js'), '(-[A-Za-z0-9_-]{8})+\\.css', '.css'), '(/_astro/[^/ ).]+)\\.[A-Za-z0-9_-]{8}\\.js', '\\1.js'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + arraySlice( + -- Python prints the outermost frame first: read it from the raise site. + if(match(_frameLines[1], '^[ \\t]*File "'), arrayReverse(_frameLines), _frameLines), + 1, 10 + ) + ) AS _normFrames, + -- A repeat (two anonymous closures in one bundle, or recursion) collapses + -- so it does not use up the frames the hash keeps. + arrayFilter((frame, i) -> i = 1 OR frame != _normFrames[i - 1], _normFrames, arrayEnumerate(_normFrames)) AS _distinctFrames, + -- The application's own frames: a dependency's or a runtime's frames vary with + -- its internals and version. A stack with none keeps every frame. + arrayFilter(frame -> NOT match(frame, '(^|[ \\t("@/])(node_modules|site-packages|dist-packages|gems|pkg/mod)/|/python#/|/ruby/#/|/usr/local/go/src/|/rustc/|\\.cargo/registry/|^[ \\t]*at (java|javax|jdk|sun|kotlin|kotlinx|scala)\\.'), _distinctFrames) AS _appFrames, + -- A namespaced type plus its message names one failure: keep only the + -- throw site. With no message the frames are all that is left; keep 3. + if(position(_exType, '/') > 0 AND _msgSig != '', 1, 3) AS _maxFrames, + arraySlice(if(length(_appFrames) > 0, _appFrames, _distinctFrames), 1, _maxFrames) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- The previous version's frames, for the issue evaluator's re-key while a bump settles. + arrayStringConcat(arraySlice(_distinctFrames, 1, _maxFrames), '\n') AS _prevFpFrames, + if( + _customFp != '', + cityHash64(OrgId, ServiceName, 'maple.fingerprint', _customFp), + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig, 'fp4') + ) AS _hash, + if( + _customFp != '', + _hash, + cityHash64(OrgId, ServiceName, _exType, _prevFpFrames, _msgSig, 'fp3') + ) AS _prevHash, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + -- Only string values: JSONExtractString renders a number or an object + -- as its JSON text, which titled `{"error":{"code":5}}` with the object. + multiIf( + JSONType(_msgText, 'title') = 'String' AND JSONExtractString(_msgText, 'title') != '', JSONExtractString(_msgText, 'title'), + JSONType(_msgText, 'message') = 'String' AND JSONExtractString(_msgText, 'message') != '', JSONExtractString(_msgText, 'message'), + JSONType(_msgText, 'error') = 'String' AND JSONExtractString(_msgText, 'error') != '', JSONExtractString(_msgText, 'error'), + JSONType(_msgText, '_tag') = 'String' AND JSONExtractString(_msgText, '_tag') != '', JSONExtractString(_msgText, '_tag'), + JSONType(_msgText, 'reason') = 'String' AND JSONExtractString(_msgText, 'reason') != '', JSONExtractString(_msgText, 'reason'), + JSONType(_msgText, 'name') = 'String' AND JSONExtractString(_msgText, 'name') != '', JSONExtractString(_msgText, 'name'), + JSONType(_msgText, 'type') = 'String' AND JSONExtractString(_msgText, 'type') != '', extract(JSONExtractString(_msgText, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + _msgText = '', 'Unknown Error', + position(_msgText, '{ readonly') = 1 OR position(_msgText, '└─') > 0, + if( + extract(_msgText, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(_msgText, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(_msgText, '[') = 1, _jsonLabel, + -- UTF8 variants throughout: a byte cut split a character and + -- left U+FFFD at the end of a non-ASCII title. + leftUTF8(_msgText, multiIf( + positionUTF8(_msgText, ': ') > 3, toInt64(positionUTF8(_msgText, ': ')) - 1, + positionUTF8(_msgText, ' (') > 3, toInt64(positionUTF8(_msgText, ' (')) - 1, + positionUTF8(_msgText, '\n') > 3, toInt64(positionUTF8(_msgText, '\n')) - 1, + 150 + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + _hash AS FingerprintHash, + _prevHash AS PreviousFingerprintHash + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all hold: 4xx, no exception event, no exception.type + -- attribute, and no error.type beyond the status code itself (HTTP + -- semconv sets error.type to the bare status on a non-2xx response, + -- which carries no exception). 5xx and anything carrying a real + -- exception still count, and SpanKind is deliberately not consulted — + -- these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND SpanAttributes['exception.type'] = '' + AND (SpanAttributes['error.type'] = '' OR SpanAttributes['error.type'] = toString(_httpStatus)) + ) + AND _hash != _prevHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS scanner_requests_mv TO scanner_requests AS +SELECT + OrgId, + Timestamp, + TraceId, + ServiceName, + if(multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', ' /actuator/env', ' /actuator/heapdump', ' /actuator/configprops', ' /actuator/gateway', ' /server-status', ' /server-info', 'phpinfo', '/info.php', ' /_profiler', ' /debug/pprof', ' /telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci']) > 0, arrayElement(['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', '/actuator/env', '/actuator/heapdump', '/actuator/configprops', '/actuator/gateway', '/server-status', '/server-info', 'phpinfo', '/info.php', '/_profiler', '/debug/pprof', '/telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci'], multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', ' /actuator/env', ' /actuator/heapdump', ' /actuator/configprops', ' /actuator/gateway', ' /server-status', ' /server-info', 'phpinfo', '/info.php', ' /_profiler', ' /debug/pprof', ' /telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci'])), arrayElement(['/wp-admin', '/wp-login.php', '/xmlrpc.php', '/wp-includes/', '/wp-content/plugins/'], multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/wp-admin', '/wp-login.php', '/xmlrpc.php', '/wp-includes/', '/wp-content/plugins/']))) AS Signature, + arrayElement(['ZGrab', 'Masscan', 'Nuclei', 'sqlmap', 'Nikto', 'Nmap', 'Gobuster', 'DirBuster', 'ffuf', 'WPScan', 'WhatWeb', 'Acunetix', 'Nessus', 'Censys', 'Palo Alto Expanse', 'LeakIX', 'LeakIX', 'Modat', 'Driftnet', 'ARTEX', 'norma agent SDK'], multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['zgrab', 'masscan', 'nuclei', 'sqlmap', 'nikto', 'nmap', 'gobuster', 'dirbuster', 'fuzz faster u fool', 'wpscan', 'whatweb', 'acunetix', 'nessus', 'censysinspect', 'palo alto networks', 'l9explore', 'l9tcpid', 'modatscanner', 'internetmeasurement', 'artex-enrich', 'norma/'])) AS Scanner, + leftUTF8(lower(replaceRegexpOne(coalesce(nullIf(SpanAttributes['server.address'], ''), nullIf(SpanAttributes['http.host'], ''), nullIf(SpanAttributes['net.host.name'], ''), nullIf(domain(SpanAttributes['url.full']), ''), nullIf(domain(SpanAttributes['http.url']), ''), ''), ':[0-9]+$', '')), 255) AS Host, + leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) AS Path, + toUInt16OrZero(coalesce(nullIf(SpanAttributes['http.response.status_code'], ''), nullIf(SpanAttributes['http.status_code'], ''), '')) AS HttpStatus, + leftUTF8(coalesce(nullIf(SpanAttributes['client.address'], ''), nullIf(SpanAttributes['http.client_ip'], ''), nullIf(SpanAttributes['net.peer.ip'], ''), ''), 64) AS ClientAddress, + leftUTF8(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), 256) AS UserAgent + FROM traces + WHERE SpanKind = 'Server' + AND (multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/.env', '/.aws/', '/.ssh/', '/id_rsa', '/.npmrc', '/.htpasswd', '/.docker/config.json', '/.vscode/sftp.json', '/.s3cfg', '/wp-config.php', '/.git/', '/.svn/', '/.hg/', '/.ds_store', ' /actuator/env', ' /actuator/heapdump', ' /actuator/configprops', ' /actuator/gateway', ' /server-status', ' /server-info', 'phpinfo', '/info.php', ' /_profiler', ' /debug/pprof', ' /telescope/requests', '../', '..%2f', '%2e%2e', '..%5c', '%252e%252e', '/etc/passwd', '/proc/self/environ', 'win.ini', '/phpmyadmin', '/pma/', '/adminer', '/vendor/phpunit', 'eval-stdin.php', '/boaform', '/hnap1', '/goform/', '/cgi-bin/luci']) > 0 OR multiSearchFirstIndexCaseInsensitive(concat(' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512), ' ', leftUTF8(coalesce(nullIf(SpanAttributes['url.query'], ''), nullIf(extract(SpanAttributes['http.target'], '\\?(.*)'), ''), ''), 1024)), ['/wp-admin', '/wp-login.php', '/xmlrpc.php', '/wp-includes/', '/wp-content/plugins/']) > 0 OR multiSearchFirstIndexCaseInsensitive(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), ''), ['zgrab', 'masscan', 'nuclei', 'sqlmap', 'nikto', 'nmap', 'gobuster', 'dirbuster', 'fuzz faster u fool', 'wpscan', 'whatweb', 'acunetix', 'nessus', 'censysinspect', 'palo alto networks', 'l9explore', 'l9tcpid', 'modatscanner', 'internetmeasurement', 'artex-enrich', 'norma/']) > 0) + AND leftUTF8(coalesce(nullIf(SpanAttributes['url.path'], ''), nullIf(replaceRegexpOne(SpanAttributes['http.target'], '[?#].*$', ''), ''), nullIf(path(SpanAttributes['url.full']), ''), nullIf(path(SpanAttributes['http.url']), ''), ''), 512) != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_endpoint_details_hourly_mv TO service_endpoint_details_hourly AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + SpanAttributes['error.type'] AS ErrorType, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + quantilesTDigestIfState(0.5, 0.95, 0.99)(ifNull(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])), 0), toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS BodySizeQuantiles, + maxIf(ifNull(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])), 0), toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS BodySizeMax, + countIf(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS BodyCount, + approx_top_kState(10)(coalesce(nullIf(SpanAttributes['user_agent.original'], ''), nullIf(SpanAttributes['http.user_agent'], ''), '')) AS UserAgents, + max(Duration) AS SlowestDuration, + argMaxState(TraceId, Duration) AS SlowestTraceId, + argMaxState(Timestamp, Duration) AS SlowestTimestamp, + argMaxState(ifNull(toInt64(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length']))), -1), Duration) AS SlowestBodySize + FROM traces + WHERE SpanKind = 'Server' + AND if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) != '' + GROUP BY OrgId, ServiceName, Hour, DeploymentEnv, SpanName, HttpStatusCode, ErrorType; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_minutely_mv TO service_map_db_edges_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles, + now() AS FirstInsertedAt + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Minute, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount, + sum(HttpClassifiedSpanCount) AS HttpClassifiedSpanCount, + sum(HttpStatusCount) AS HttpStatusCount, + sum(Http4xxCount) AS Http4xxCount, + sum(Http5xxCount) AS Http5xxCount, + sum(RequestBodyBytes) AS RequestBodyBytes, + sum(RequestBodyCount) AS RequestBodyCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount, + count() AS HttpClassifiedSpanCount, + countIf(SpanKind = 'Server' AND if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) != '') AS HttpStatusCount, + countIf(SpanKind = 'Server' AND toUInt16OrZero(if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code'])) BETWEEN 400 AND 499) AS Http4xxCount, + countIf(SpanKind = 'Server' AND toUInt16OrZero(if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code'])) BETWEEN 500 AND 599) AS Http5xxCount, + sumIf(ifNull(toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])), 0), SpanKind = 'Server' AND toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS RequestBodyBytes, + countIf(SpanKind = 'Server' AND toUInt64OrNull(if(SpanAttributes['http.request.body.size'] != '', SpanAttributes['http.request.body.size'], SpanAttributes['http.request_content_length'])) IS NOT NULL) AS RequestBodyCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount, + sum(SampleRate) AS EstimatedLadderSpanCount, + sumIf(SampleRate, Duration < 50000000) AS EstimatedUnder50Ms, + sumIf(SampleRate, Duration < 100000000) AS EstimatedUnder100Ms, + sumIf(SampleRate, Duration < 250000000) AS EstimatedUnder250Ms, + sumIf(SampleRate, Duration < 500000000) AS EstimatedUnder500Ms, + sumIf(SampleRate, Duration < 1000000000) AS EstimatedUnder1000Ms, + sumIf(SampleRate, Duration < 2500000000) AS EstimatedUnder2500Ms, + sumIf(SampleRate, Duration < 5000000000) AS EstimatedUnder5000Ms, + sumIf(SampleRate, Duration < 10000000000) AS EstimatedUnder10000Ms + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount, + sum(SampleRate) AS EstimatedLadderSpanCount, + sumIf(SampleRate, Duration < 50000000) AS EstimatedUnder50Ms, + sumIf(SampleRate, Duration < 100000000) AS EstimatedUnder100Ms, + sumIf(SampleRate, Duration < 250000000) AS EstimatedUnder250Ms, + sumIf(SampleRate, Duration < 500000000) AS EstimatedUnder500Ms, + sumIf(SampleRate, Duration < 1000000000) AS EstimatedUnder1000Ms, + sumIf(SampleRate, Duration < 2500000000) AS EstimatedUnder2500Ms, + sumIf(SampleRate, Duration < 5000000000) AS EstimatedUnder5000Ms, + sumIf(SampleRate, Duration < 10000000000) AS EstimatedUnder10000Ms + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + SpanKind, + toUInt8(ParentSpanId = '') AS IsRoot + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_runtimes_hourly_mv TO service_runtimes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + leftUTF8(ResourceAttributes['process.runtime.name'], 64) AS RuntimeName, + leftUTF8(ResourceAttributes['process.runtime.version'], 64) AS RuntimeVersion, + leftUTF8(ResourceAttributes['process.executable.name'], 64) AS ExecutableName, + leftUTF8(ResourceAttributes['telemetry.sdk.language'], 64) AS SdkLanguage, + leftUTF8(ResourceAttributes['telemetry.sdk.name'], 64) AS SdkName, + leftUTF8(ResourceAttributes['telemetry.sdk.version'], 64) AS SdkVersion, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, RuntimeName, RuntimeVersion, ExecutableName, SdkLanguage, SdkName, SdkVersion; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_logs_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(TimestampTime) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS MetricCount, + 0 AS MetricSizeBytes + FROM logs + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_exp_histogram_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 300 AS MetricSizeBytes + FROM metrics_exponential_histogram + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_gauge_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 150 AS MetricSizeBytes + FROM metrics_gauge + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_histogram_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 250 AS MetricSizeBytes + FROM metrics_histogram + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_metrics_sum_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS SpanCount, + 0 AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS MetricCount, + count() * 150 AS MetricSizeBytes + FROM metrics_sum + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS source_usage_traces_mv TO source_usage_hourly AS +SELECT + OrgId, + ResourceAttributes['maple_source_id'] AS SourceId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + count() AS SpanCount, + sum(length(SpanName) + 300) AS SpanSizeBytes, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS MetricCount, + 0 AS MetricSizeBytes + FROM traces + WHERE ResourceAttributes['maple_source_id'] != '' + GROUP BY OrgId, SourceId, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_facets_hourly_mv TO trace_facets_hourly AS +SELECT + OrgId, + toStartOfHour(Timestamp) AS Hour, + ServiceName, + SpanName, + HttpMethod, + HttpStatusCode, + DeploymentEnv, + ServiceNamespace, + HasError, + count() AS TraceCount, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles + FROM trace_list_mv + GROUP BY OrgId, Hour, ServiceName, SpanName, HttpMethod, HttpStatusCode, DeploymentEnv, ServiceNamespace, HasError; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_index_mv TO trace_index AS +SELECT + OrgId, + TraceId, + ServiceName, + toStartOfFiveMinutes(Timestamp) AS Bucket, + min(Timestamp) AS MinTimestamp, + max(Timestamp) AS MaxTimestamp + FROM traces + GROUP BY OrgId, TraceId, ServiceName, Bucket; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS user_activity_hourly_logs_mv TO user_activity_hourly AS +SELECT + OrgId, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + if(LogAttributes['user.id'] != '', LogAttributes['user.id'], LogAttributes['enduser.id']) AS UserId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + toUInt64(0) AS SpanCount, + toUInt64(0) AS ErrorSpanCount, + toUInt64(0) AS RequestCount, + toUInt64(0) AS FailedRequestCount, + uniqIfState(TraceId, toUInt8(0)) AS TraceIds, + uniqIfState(LogAttributes['session.id'], LogAttributes['session.id'] != '') AS SessionIds, + quantilesTDigestIfState(0.5, 0.95)(toUInt64(0), toUInt8(0)) AS RequestDurationQuantiles, + approx_top_kState(10)('') AS TopRoutes, + count() AS LogCount, + countIf(SeverityNumber >= 13 AND SeverityNumber < 17) AS WarnLogCount, + countIf(SeverityNumber >= 17) AS ErrorLogCount, + min(TimestampTime) AS FirstSeen, + max(TimestampTime) AS LastSeen, + toUInt64(0) AS ClassifiedRequestCount, + toUInt64(0) AS WebRequestCount, + toUInt64(0) AS MobileRequestCount + FROM logs + WHERE if(LogAttributes['user.id'] != '', LogAttributes['user.id'], LogAttributes['enduser.id']) != '' + GROUP BY OrgId, ServiceNamespace, UserId, Hour, ServiceName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS user_activity_hourly_traces_mv TO user_activity_hourly AS +SELECT + OrgId, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) AS UserId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + count() AS SpanCount, + countIf(StatusCode = 'Error') AS ErrorSpanCount, + countIf(IsEntryPoint = 1) AS RequestCount, + countIf(IsEntryPoint = 1 AND StatusCode = 'Error') AS FailedRequestCount, + uniqIfState(TraceId, TraceId != '') AS TraceIds, + uniqIfState(SpanAttributes['session.id'], SpanAttributes['session.id'] != '') AS SessionIds, + quantilesTDigestIfState(0.5, 0.95)(Duration, IsEntryPoint = 1) AS RequestDurationQuantiles, + approx_top_kState(10)(if(IsEntryPoint = 1, if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName), '')) AS TopRoutes, + toUInt64(0) AS LogCount, + toUInt64(0) AS WarnLogCount, + toUInt64(0) AS ErrorLogCount, + min(toDateTime(Timestamp)) AS FirstSeen, + max(toDateTime(Timestamp)) AS LastSeen, + countIf(IsEntryPoint = 1) AS ClassifiedRequestCount, + countIf(IsEntryPoint = 1 AND (ResourceAttributes['maple.sdk.type'] IN ('browser', 'client') OR if(SpanAttributes['user_agent.original'] != '', SpanAttributes['user_agent.original'], SpanAttributes['http.user_agent']) LIKE 'Mozilla/%')) AS WebRequestCount, + countIf(IsEntryPoint = 1 AND (NOT (ResourceAttributes['maple.sdk.type'] IN ('browser', 'client') OR if(SpanAttributes['user_agent.original'] != '', SpanAttributes['user_agent.original'], SpanAttributes['http.user_agent']) LIKE 'Mozilla/%') AND (ResourceAttributes['telemetry.sdk.language'] IN ('swift', 'kotlin') OR ResourceAttributes['os.name'] IN ('iOS', 'iPadOS', 'Android') OR match(if(SpanAttributes['user_agent.original'] != '', SpanAttributes['user_agent.original'], SpanAttributes['http.user_agent']), '^(Dalvik|okhttp)/| CFNetwork/')))) AS MobileRequestCount + FROM traces + WHERE if(SpanAttributes['user.id'] != '', SpanAttributes['user.id'], SpanAttributes['enduser.id']) != '' + GROUP BY OrgId, ServiceNamespace, UserId, Hour, ServiceName; diff --git a/packages/local-server/src/schema/local-inserts.json b/packages/local-server/src/schema/local-inserts.json index 992048ebdd..4c8e0a3879 100644 --- a/packages/local-server/src/schema/local-inserts.json +++ b/packages/local-server/src/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "266cb624c1996b27cb3d435f5feb3e57ec70c6fdf5f9ecfd1f291b6425b3a692", + "projectRevision": "5059e1601b59566f096b277902cfa023b21cbd0b62e1d9e53a7bb05cc92473f7", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/packages/local-server/src/schema/local-schema.sql b/packages/local-server/src/schema/local-schema.sql index 2003e3a4af..885fcad907 100644 --- a/packages/local-server/src/schema/local-schema.sql +++ b/packages/local-server/src/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 266cb624c1996b27cb3d435f5feb3e57ec70c6fdf5f9ecfd1f291b6425b3a692 +-- projectRevision: 5059e1601b59566f096b277902cfa023b21cbd0b62e1d9e53a7bb05cc92473f7 CREATE TABLE IF NOT EXISTS ai_crawler_requests ( OrgId LowCardinality(String), @@ -137,6 +137,37 @@ PARTITION BY toYYYYMM(OccurredAt) ORDER BY (OrgId, OccurredAt, Id) TTL toDate(OccurredAt) + INTERVAL 2190 DAY; +CREATE TABLE IF NOT EXISTS audit_trail_events ( + OrgId LowCardinality(String), + Id String, + OccurredAt DateTime64(3), + IngestedAt DateTime64(3), + Provider LowCardinality(String), + Account String, + ProviderEventId String, + ActorType LowCardinality(String), + ActorId String, + ActorName String, + ActorEmail String, + Action LowCardinality(String), + Category LowCardinality(String), + Outcome LowCardinality(String), + OutcomeReason String, + TargetType LowCardinality(String), + TargetId String, + TargetName String, + OriginIp String, + OriginCountry LowCardinality(String), + UserAgent String, + Raw String, + Version UInt64, + INDEX idx_ingested_at IngestedAt TYPE minmax GRANULARITY 1 +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(OccurredAt) +ORDER BY (OrgId, OccurredAt, Id) +TTL toDate(OccurredAt) + INTERVAL 400 DAY; + CREATE TABLE IF NOT EXISTS ci_jobs ( OrgId LowCardinality(String), Timestamp DateTime64(9), diff --git a/packages/local-server/src/scoped-delete.ts b/packages/local-server/src/scoped-delete.ts index 8c524ebbc3..7b035fcc2d 100644 --- a/packages/local-server/src/scoped-delete.ts +++ b/packages/local-server/src/scoped-delete.ts @@ -103,6 +103,7 @@ export const TABLE_DELETE_PLAN = { attribute_keys_hourly: unkeyed("Hour"), attribute_values_hourly: unkeyed("Hour"), audit_log: { strategy: "excluded", reason: NOT_TELEMETRY }, + audit_trail_events: { strategy: "excluded", reason: NOT_TELEMETRY }, ci_jobs: unkeyed("Timestamp"), ci_runs: unkeyed("Timestamp"), error_events: derived("Timestamp"), diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 7c900ed750..76f56639e3 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -373,6 +373,73 @@ SELECT OFFSET 50 FORMAT JSON +-- builder:audit-trail:auditTrailEventsQuery:default [9ab77231] +SELECT + audit_trail_events.Id AS id, + audit_trail_events.OccurredAt AS occurredAt, + audit_trail_events.IngestedAt AS ingestedAt, + audit_trail_events.Provider AS provider, + audit_trail_events.Account AS account, + audit_trail_events.ProviderEventId AS providerEventId, + audit_trail_events.ActorType AS actorType, + audit_trail_events.ActorId AS actorId, + audit_trail_events.ActorName AS actorName, + audit_trail_events.ActorEmail AS actorEmail, + audit_trail_events.Action AS action, + audit_trail_events.Category AS category, + audit_trail_events.Outcome AS outcome, + audit_trail_events.OutcomeReason AS outcomeReason, + audit_trail_events.TargetType AS targetType, + audit_trail_events.TargetId AS targetId, + audit_trail_events.TargetName AS targetName, + audit_trail_events.OriginIp AS originIp, + audit_trail_events.OriginCountry AS originCountry, + audit_trail_events.UserAgent AS userAgent, + audit_trail_events.Raw AS raw + FROM audit_trail_events + WHERE audit_trail_events.OrgId = 'org_sql_catalog' + ORDER BY occurredAt DESC, id DESC, ingestedAt ASC + LIMIT 51 + FORMAT JSON + +-- builder:audit-trail:auditTrailEventsQuery:filtered [fb4fb694] +SELECT + audit_trail_events.Id AS id, + audit_trail_events.OccurredAt AS occurredAt, + audit_trail_events.IngestedAt AS ingestedAt, + audit_trail_events.Provider AS provider, + audit_trail_events.Account AS account, + audit_trail_events.ProviderEventId AS providerEventId, + audit_trail_events.ActorType AS actorType, + audit_trail_events.ActorId AS actorId, + audit_trail_events.ActorName AS actorName, + audit_trail_events.ActorEmail AS actorEmail, + audit_trail_events.Action AS action, + audit_trail_events.Category AS category, + audit_trail_events.Outcome AS outcome, + audit_trail_events.OutcomeReason AS outcomeReason, + audit_trail_events.TargetType AS targetType, + audit_trail_events.TargetId AS targetId, + audit_trail_events.TargetName AS targetName, + audit_trail_events.OriginIp AS originIp, + audit_trail_events.OriginCountry AS originCountry, + audit_trail_events.UserAgent AS userAgent, + audit_trail_events.Raw AS raw + FROM audit_trail_events + WHERE audit_trail_events.OrgId = 'org_sql_catalog' + AND audit_trail_events.Provider = 'github' + AND audit_trail_events.Account = 'acme' + AND audit_trail_events.Category = 'membership' + AND audit_trail_events.Outcome = 'success' + AND audit_trail_events.Action = 'org.add_member' + AND ((audit_trail_events.ActorId = 'octocat@acme.test' OR audit_trail_events.ActorEmail = 'octocat@acme.test') OR audit_trail_events.ActorId = 'user_1') + AND audit_trail_events.OccurredAt >= '2026-01-01 10:30:00' + AND audit_trail_events.OccurredAt <= '2026-01-03 14:15:00' + AND (audit_trail_events.OccurredAt < '2026-01-03 14:15:00' OR (audit_trail_events.OccurredAt = '2026-01-03 14:15:00' AND audit_trail_events.Id < '0f6f2c1e7a554c0e9a3b0d2c7e5b9a11')) + ORDER BY occurredAt DESC, id DESC, ingestedAt ASC + LIMIT 51 + FORMAT JSON + -- builder:aws-ingest:awsIngestQuery:default [c5891e34] SELECT metrics_sum.Attributes['aws.namespace'] AS namespace, diff --git a/packages/query-engine/src/benchmark/builders.ts b/packages/query-engine/src/benchmark/builders.ts index 38fc5f4ab7..0122c2aee8 100644 --- a/packages/query-engine/src/benchmark/builders.ts +++ b/packages/query-engine/src/benchmark/builders.ts @@ -1459,6 +1459,49 @@ export const builderFixtures: ReadonlyArray = [ }, ), }, + // Audit trail listing (packages/backend/src/services/audit-trail/AuditTrailService.ts `list`). + { + module: "audit-trail", + name: "auditTrailEventsQuery", + label: "default", + compile: () => CH.compileUnsafe(CH.auditTrailEventsQuery({ limit: 51 }), { orgId: ORG_ID }), + }, + { + // Every optional filter bound at once, with the keyset cursor. + module: "audit-trail", + name: "auditTrailEventsQuery", + label: "filtered", + compile: () => + CH.compileUnsafe( + CH.auditTrailEventsQuery({ + provider: true, + account: true, + category: true, + outcome: true, + action: true, + actor: true, + actorAlias: true, + since: true, + until: true, + before: true, + limit: 51, + }), + { + orgId: ORG_ID, + provider: "github", + account: "acme", + category: "membership", + outcome: "success", + action: "org.add_member", + actor: "octocat@acme.test", + actorAlias: "user_1", + since: START_TIME, + until: END_TIME, + beforeOccurredAt: END_TIME, + beforeId: "0f6f2c1e7a554c0e9a3b0d2c7e5b9a11", + }, + ), + }, // Cancellation evidence (CancellationTelemetryEvidence.read). { module: "cancellation-evidence", diff --git a/packages/query-engine/src/benchmark/catalog.test.ts b/packages/query-engine/src/benchmark/catalog.test.ts index b828d8ffaa..9401f07d68 100644 --- a/packages/query-engine/src/benchmark/catalog.test.ts +++ b/packages/query-engine/src/benchmark/catalog.test.ts @@ -25,6 +25,7 @@ import * as activityQueries from "../ch/queries/activity" import * as alertCheckQueries from "../ch/queries/alert-checks" import * as ciRunsQueries from "../ch/queries/ci-runs" import * as auditLogQueries from "../ch/queries/audit-log" +import * as auditTrailQueries from "../ch/queries/audit-trail" import * as cancellationEvidenceQueries from "../ch/queries/cancellation-evidence" import * as anomalyQueries from "../ch/queries/anomaly" import * as attributeKeyQueries from "../ch/queries/attribute-keys" @@ -279,6 +280,7 @@ const QUERY_MODULES: Record> = { activity: activityQueries, "alert-checks": alertCheckQueries, "audit-log": auditLogQueries, + "audit-trail": auditTrailQueries, "cancellation-evidence": cancellationEvidenceQueries, "ci-runs": ciRunsQueries, anomaly: anomalyQueries, diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 78c4d0cc11..cfc52e8f95 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -60,6 +60,7 @@ export * from "./queries/service-api" export * from "./queries/service-endpoint-details" export * from "./queries/alert-checks" export * from "./queries/audit-log" +export * from "./queries/audit-trail" export * from "./queries/cancellation-evidence" export * from "./queries/liveness" export * from "./queries/signal-presence" diff --git a/packages/query-engine/src/ch/queries/audit-trail.ts b/packages/query-engine/src/ch/queries/audit-trail.ts new file mode 100644 index 0000000000..c3c4fb400e --- /dev/null +++ b/packages/query-engine/src/ch/queries/audit-trail.ts @@ -0,0 +1,90 @@ +// The unified audit trail: audit events from connected platforms and Maple itself, in one shape. +// Every read is pinned to the managed route, like `audit_log`: the table is written through +// `ingest` and does not exist in a BYO ClickHouse. + +import * as CH from "@maple-dev/effect-orm/expr" +import { from, param } from "@maple-dev/effect-orm/clickhouse" +import * as Datasources from "@maple/domain/tinybird/datasources" +import { orgIdParam } from "../tables" + +/** + * Which optional filters a listing applies. Every set flag binds a parameter of the same name at + * compile time; the values themselves never enter the SQL. + */ +export interface AuditTrailEventsOpts { + readonly provider?: boolean + readonly account?: boolean + readonly category?: boolean + readonly outcome?: boolean + readonly action?: boolean + /** Matches the actor's provider id or their email, whichever the caller has. */ + readonly actor?: boolean + /** With `actor`: a second id the same person acts under (a Maple member's user id). */ + readonly actorAlias?: boolean + readonly since?: boolean + readonly until?: boolean + /** Keyset cursor: only events that sort after `beforeOccurredAt` / `beforeId`, newest first. */ + readonly before?: boolean + readonly limit: number +} + +/** + * One org's audit events, newest first, keyset-paginated on the sorting key. A redelivered event + * ReplacingMergeTree has not merged yet can appear twice; the service collapses it by id. + */ +export function auditTrailEventsQuery(opts: AuditTrailEventsOpts) { + return ( + from(Datasources.auditTrailEvents) + .select(($) => ({ + id: $.Id, + occurredAt: $.OccurredAt, + ingestedAt: $.IngestedAt, + provider: $.Provider, + account: $.Account, + providerEventId: $.ProviderEventId, + actorType: $.ActorType, + actorId: $.ActorId, + actorName: $.ActorName, + actorEmail: $.ActorEmail, + action: $.Action, + category: $.Category, + outcome: $.Outcome, + outcomeReason: $.OutcomeReason, + targetType: $.TargetType, + targetId: $.TargetId, + targetName: $.TargetName, + originIp: $.OriginIp, + originCountry: $.OriginCountry, + userAgent: $.UserAgent, + raw: $.Raw, + })) + .where(($) => [ + $.OrgId.eq(orgIdParam), + CH.whenTrue(!!opts.provider, () => $.Provider.eq(param.string("provider"))), + CH.whenTrue(!!opts.account, () => $.Account.eq(param.string("account"))), + CH.whenTrue(!!opts.category, () => $.Category.eq(param.string("category"))), + CH.whenTrue(!!opts.outcome, () => $.Outcome.eq(param.string("outcome"))), + CH.whenTrue(!!opts.action, () => $.Action.eq(param.string("action"))), + CH.whenTrue(!!opts.actor, () => { + const byActor = $.ActorId.eq(param.string("actor")).or( + $.ActorEmail.eq(param.string("actor")), + ) + return opts.actorAlias ? byActor.or($.ActorId.eq(param.string("actorAlias"))) : byActor + }), + CH.whenTrue(!!opts.since, () => $.OccurredAt.gte(param.dateTime("since"))), + CH.whenTrue(!!opts.until, () => $.OccurredAt.lte(param.dateTime("until"))), + CH.whenTrue(!!opts.before, () => + $.OccurredAt.lt(param.dateTime("beforeOccurredAt")).or( + $.OccurredAt.eq(param.dateTime("beforeOccurredAt")).and( + $.Id.lt(param.string("beforeId")), + ), + ), + ), + ]) + // Unmerged copies of one event: the first write first, the one the service keeps. + .orderBy(["occurredAt", "desc"], ["id", "desc"], ["ingestedAt", "asc"]) + .limit(opts.limit) + .format("JSON") + .route("ingest") + ) +} diff --git a/packages/query-engine/src/execution/datasource-routing.ts b/packages/query-engine/src/execution/datasource-routing.ts index f98d589c1a..c7f7159ce2 100644 --- a/packages/query-engine/src/execution/datasource-routing.ts +++ b/packages/query-engine/src/execution/datasource-routing.ts @@ -12,6 +12,7 @@ import { String as Str, Array as Arr, Option } from "effect" export const INGEST_PINNED_TABLES: ReadonlyArray = [ "alert_checks", "audit_log", + "audit_trail_events", "cost_daily", "search_console_daily", ]