diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 5116a41498..7ce5ac0c4c 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -21,7 +21,6 @@ import { PlanetScaleInfraTimeseriesResponse, RailwayInfraRowsResponse, VercelAiGatewayResponse, - VercelAuditLogResponse, VercelInfraDependenciesResponse, VercelInfraFunctionsResponse, VercelInfraFunctionTimeseriesResponse, @@ -1000,18 +999,6 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) - .handle("vercelAuditLog", ({ payload }) => - Effect.gen(function* () { - const tenant = yield* CurrentTenant.Context - const rows = yield* runQuery(Queries.vercelAuditLog, tenant, payload) - return new VercelAuditLogResponse({ - events: Arr.map(rows, (row) => ({ - ...row, - timestamp: DateTime.formatIso(row.timestamp), - })), - }) - }), - ) .handle("gcpInfraPresence", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/ingest/alchemy.run.ts b/apps/ingest/alchemy.run.ts index bb92b22d32..728a867f3e 100644 --- a/apps/ingest/alchemy.run.ts +++ b/apps/ingest/alchemy.run.ts @@ -522,6 +522,9 @@ export const createMapleIngest = ({ stage, region, hostname, profile, dbLogin }: ...(yield* blockedIngestKeysEnv), // The org Maple's own telemetry is filed under. Required, no fallback. MAPLE_INTERNAL_ORG_ID: yield* requiredPlain("MAPLE_INTERNAL_ORG_ID"), + // On wherever a stage is deployed whole and never rolled back to an older gateway; prd + // turns it on in its environment once `audit_trail_events` is there. + ...(stage.kind !== "prd" && { INGEST_AUDIT_TRAIL_ENABLED: "true" }), ...(yield* gatewayTuningEnv), // `INGEST_COMMIT_SHA` (prd) is the image's own revision, so a deploy that leaves // `apps/ingest` alone does not change the task definition and roll the hosts. diff --git a/apps/ingest/alchemy/gateway-env.ts b/apps/ingest/alchemy/gateway-env.ts index fc93c3c21a..be53f58776 100644 --- a/apps/ingest/alchemy/gateway-env.ts +++ b/apps/ingest/alchemy/gateway-env.ts @@ -24,6 +24,9 @@ export const gatewayTuningEnv: Config.Config = Config.all([ // switch rule application off in an emergency. optionalPlain("INGEST_RULES_ENABLED"), optionalPlain("INGEST_RULES_CACHE_TTL_SECS"), + // Provider audit events go to the audit trail instead of the logs. The gateway defaults to off: + // set "true" once `audit_trail_events` exists and every task runs a build that knows the switch. + optionalPlain("INGEST_AUDIT_TRAIL_ENABLED"), optionalPlain("INGEST_MAX_REQUEST_BODY_BYTES"), optionalPlain("INGEST_MAX_DECODED_BODY_BYTES"), optionalPlain("INGEST_MAX_DECODE_HEAP_BYTES"), diff --git a/apps/ingest/src/audit_trail.rs b/apps/ingest/src/audit_trail.rs new file mode 100644 index 0000000000..57fdd02f35 --- /dev/null +++ b/apps/ingest/src/audit_trail.rs @@ -0,0 +1,839 @@ +//! Audit events from provider feeds as rows of `audit_trail_events`, the table +//! behind the Security hub's audit log. They are written here, never as OTLP +//! logs: a log can be sent by anything holding an ingest key and read by every +//! member, and an audit record must be neither. +//! +//! The row shape and the id derivation mirror the TypeScript writer +//! (`packages/backend/src/services/audit-trail/audit-trail-row.ts`). The two +//! must agree: an event a provider redelivers has to land on the same key. + +use crate::vercel_logs::text; +use chrono::{DateTime, Utc}; +use serde_json::{json, Map as JsonMap, Value as JsonValue}; +use sha2::{Digest, Sha256}; +use std::borrow::Cow; + +type JsonObject = JsonMap; + +pub const DATASOURCE: &str = "audit_trail_events"; + +/// Above any ingest time in ms, so `Version` stays positive and falls as time +/// rises: the table keeps the highest version, which is the first write. +const VERSION_CEILING_MS: i64 = 9_999_999_999_999; + +/// A payload larger than this is not kept; the row's columns say what happened. +const RAW_LIMIT_BYTES: usize = 32 * 1024; + +/// The table drops a row this long after it happened. +const RETENTION_MS: i64 = 400 * 24 * 60 * 60 * 1000; +/// How far ahead of this gateway's clock a sender's may run. +const MAX_AHEAD_MS: i64 = 24 * 60 * 60 * 1000; + +/// The most bytes a column keeps. The sender sets every one of them, and the +/// table is shared by every org. +const NAME_LIMIT_BYTES: usize = 256; +const TEXT_LIMIT_BYTES: usize = 1024; + +/// One audit event, normalized. Empty strings are absent values. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct AuditEvent { + pub provider: &'static str, + pub account: String, + pub provider_event_id: String, + pub occurred_at_ms: i64, + pub actor_type: &'static str, + pub actor_id: String, + pub actor_name: String, + pub actor_email: String, + pub action: String, + pub category: &'static str, + pub outcome: &'static str, + pub outcome_reason: String, + pub target_type: String, + pub target_id: String, + pub target_name: String, + pub origin_ip: String, + pub user_agent: String, + pub raw: String, +} + +/// The row's key: the first 32 hex characters of sha256 over +/// `provider\naccount\neventId`. +pub fn event_id(provider: &str, account: &str, provider_event_id: &str) -> String { + sha256_prefix(format!("{provider}\n{account}\n{provider_event_id}").as_bytes()) +} + +const HEX_DIGITS: &[u8; 16] = b"0123456789abcdef"; + +/// The first 16 bytes of a sha256, as 32 hex characters. +fn sha256_prefix(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut hex = String::with_capacity(32); + for byte in &digest[..16] { + hex.push(char::from(HEX_DIGITS[usize::from(byte >> 4)])); + hex.push(char::from(HEX_DIGITS[usize::from(byte & 0x0f)])); + } + hex +} + +/// `YYYY-MM-DD HH:MM:SS.mmm` in UTC, the form the warehouse's DateTime64(3) takes. +fn warehouse_datetime(epoch_ms: i64) -> String { + DateTime::::from_timestamp_millis(epoch_ms) + .unwrap_or_default() + .format("%Y-%m-%d %H:%M:%S%.3f") + .to_string() +} + +/// `value` cut to at most `max_bytes`, on a character boundary. +fn clip(value: &str, max_bytes: usize) -> &str { + if value.len() <= max_bytes { + return value; + } + let mut end = max_bytes; + while !value.is_char_boundary(end) { + end -= 1; + } + &value[..end] +} + +/// An id is part of the row's key, so one too long to keep is hashed, not cut: +/// two long ids sharing a prefix stay two events. +fn bounded_id(id: &str) -> Cow<'_, str> { + if id.len() <= NAME_LIMIT_BYTES { + Cow::Borrowed(id) + } else { + Cow::Owned(sha256_prefix(id.as_bytes())) + } +} + +impl AuditEvent { + /// The event as one NDJSON row for `org_id`, stored at `ingested_at_ms`. + pub fn to_row(&self, org_id: &str, ingested_at_ms: i64) -> Vec { + let name = |value: &'_ str| clip(value, NAME_LIMIT_BYTES).to_owned(); + let long = |value: &'_ str| clip(value, TEXT_LIMIT_BYTES).to_owned(); + let account = name(&self.account); + let provider_event_id = bounded_id(&self.provider_event_id); + json!({ + "OrgId": org_id, + "Id": event_id(self.provider, &account, &provider_event_id), + "OccurredAt": warehouse_datetime(self.occurred_at_ms), + "IngestedAt": warehouse_datetime(ingested_at_ms), + "Provider": self.provider, + "Account": account, + "ProviderEventId": provider_event_id, + "ActorType": self.actor_type, + "ActorId": name(&self.actor_id), + "ActorName": name(&self.actor_name), + "ActorEmail": name(&self.actor_email), + "Action": name(&self.action), + "Category": self.category, + "Outcome": self.outcome, + "OutcomeReason": long(&self.outcome_reason), + "TargetType": name(&self.target_type), + "TargetId": name(&self.target_id), + "TargetName": long(&self.target_name), + "OriginIp": name(&self.origin_ip), + "OriginCountry": "", + "UserAgent": long(&self.user_agent), + "Raw": self.raw, + "Version": VERSION_CEILING_MS - ingested_at_ms, + }) + .to_string() + .into_bytes() + } +} + +fn owned(value: Option<&str>) -> String { + value.unwrap_or_default().to_owned() +} + +fn object<'a>(parent: &'a JsonObject, key: &str) -> Option<&'a JsonObject> { + parent.get(key)?.as_object() +} + +/// The payload as JSON text, or a marker of its size when it is too large to keep. +fn raw_of(payload: &JsonObject) -> String { + let full = serde_json::to_string(payload).unwrap_or_default(); + if full.len() <= RAW_LIMIT_BYTES { + full + } else { + json!({ "truncated": true, "bytes": full.len() }).to_string() + } +} + +/// A key for an event the provider gave no id: the same payload is the same event. +fn content_id(payload: &JsonObject) -> String { + sha256_prefix( + serde_json::to_string(payload) + .unwrap_or_default() + .as_bytes(), + ) +} + +/// When the event happened, as the sender says, if that is a time a row can +/// hold: inside the table's retention and not ahead of this clock. Any other +/// time is replaced by now, so a sender cannot mint partitions on the shared +/// table, plant a row that never expires, or send one that expires on arrival. +fn occurred_at_ms(claimed: Option, now_ms: i64) -> i64 { + claimed + .filter(|ms| (now_ms - RETENTION_MS..=now_ms + MAX_AHEAD_MS).contains(ms)) + .unwrap_or(now_ms) +} + +/// The words of an action name, lowercased: split at punctuation and at a +/// camelCase hump. Matching whole words keeps `environment` from reading as `env`. +fn words(name: &str) -> Vec { + let mut words = Vec::new(); + let mut word = String::new(); + let mut after_lowercase = false; + for character in name.chars() { + let hump = character.is_ascii_uppercase() && after_lowercase; + if (hump || !character.is_ascii_alphanumeric()) && !word.is_empty() { + words.push(std::mem::take(&mut word)); + } + if character.is_ascii_alphanumeric() { + word.push(character.to_ascii_lowercase()); + } + after_lowercase = character.is_ascii_lowercase() || character.is_ascii_digit(); + } + if !word.is_empty() { + words.push(word); + } + words +} + +/// Whether any of `words` is one of `needles`, or its plural. +fn has_word(words: &[String], needles: &[&str]) -> bool { + words.iter().any(|word| { + needles + .iter() + .any(|needle| word == needle || word.strip_suffix('s') == Some(*needle)) + }) +} + +/// What a Vercel action is about, read off its name. Vercel publishes no closed +/// list, so this is by the words the names use; anything unrecognised is a +/// change to a resource. The order decides a name with two readings: a member's +/// role change is about permission, and password protection is a setting. +fn vercel_category(action: &str) -> &'static str { + let words = words(action); + let has = |needles: &[&str]| has_word(&words, needles); + if has(&[ + "login", "logout", "sso", "saml", "mfa", "2fa", "passkey", "signin", + ]) { + "authentication" + } else if has(&["protection"]) { + "configuration" + } else if has(&["role", "permission", "acl"]) || (has(&["access"]) && has(&["group"])) { + "permission" + } else if has(&[ + "token", + "secret", + "env", + "variable", + "certificate", + "cert", + "password", + "key", + ]) { + "credential" + } else if has(&["member", "invite", "invitation", "join", "leave"]) { + "membership" + } else if has(&["billing", "invoice", "payment", "plan", "subscription"]) { + "billing" + } else if has(&[ + "setting", + "drain", + "integration", + "webhook", + "firewall", + "config", + "configuration", + ]) { + "configuration" + } else { + "resource" + } +} + +/// What a Vercel action was done to: the action without its closing verb +/// (`project.env-var.created` is about a `project.env-var`). +fn vercel_target_type(action: &str) -> &str { + action + .rfind(['.', '-', '_']) + .map_or("", |verb_at| &action[..verb_at]) +} + +/// The id of what a Vercel action was done to, from the event's `payload`: the +/// id named after the action's subject (`drainId` for `drain-created`), or the +/// only id there is. +fn vercel_target_id<'a>(action: &str, payload: &'a JsonObject) -> Option<&'a str> { + let subject = words(action).into_iter().next().unwrap_or_default(); + if let Some(id) = text(payload, &format!("{subject}Id")) { + return Some(id); + } + let mut ids = payload + .iter() + .filter(|(key, _)| key.ends_with("Id")) + .filter_map(|(_, value)| value.as_str()); + let only = ids.next()?; + ids.next().is_none().then_some(only) +} + +/// One event of Vercel's Audit Log drain. `None` for an event with no action: +/// there is nothing to say happened. +pub fn vercel_event(event: &JsonObject, now_ms: i64) -> Option { + let action = text(event, "action")?; + let actor = object(event, "actor"); + let actor_text = |key: &str| owned(actor.and_then(|actor| text(actor, key))); + let actor_type = match actor.and_then(|actor| text(actor, "type")) { + Some("user") => "user", + Some("system") => "system", + Some(_) => "service", + None => "unknown", + }; + // Epoch milliseconds, whole in practice; a fractional one is cut to its millisecond. + #[expect( + clippy::cast_possible_truncation, + reason = "a finite positive epoch-ms value is far inside i64" + )] + let claimed_ms = event + .get("timestamp") + .and_then(JsonValue::as_f64) + .filter(|ms| ms.is_finite() && *ms > 0.0) + .map(|ms| ms.trunc() as i64); + let payload = object(event, "payload"); + Some(AuditEvent { + provider: "vercel", + account: owned(text(event, "teamId")), + provider_event_id: text(event, "id").map_or_else(|| content_id(event), str::to_owned), + occurred_at_ms: occurred_at_ms(claimed_ms, now_ms), + actor_type, + actor_id: actor_text("id"), + actor_name: actor_text("name"), + actor_email: actor_text("email"), + action: action.to_owned(), + category: vercel_category(action), + // The drain reports what was done; Vercel does not log refused attempts in it. + outcome: "success", + target_type: vercel_target_type(action).to_owned(), + target_id: owned(payload.and_then(|payload| vercel_target_id(action, payload))), + target_name: owned(payload.and_then(|payload| text(payload, "name"))), + origin_ip: owned(text(event, "ipAddress")), + user_agent: owned(text(event, "userAgent")), + raw: raw_of(event), + ..AuditEvent::default() + }) +} + +/// Which of Google's audit logs an entry is from, by its `logName`. +fn gcp_audit_log(log_name: &str) -> Option<&'static str> { + let (_, log) = log_name.rsplit_once("/logs/")?; + match log.replace("%2F", "/").as_str() { + "cloudaudit.googleapis.com/activity" => Some("activity"), + "cloudaudit.googleapis.com/policy" => Some("policy"), + _ => None, + } +} + +/// Kubernetes API writes reach the Admin Activity log under this service. Most +/// are a cluster's own controllers at work, thousands an hour: they stay logs. +const GCP_KUBERNETES_SERVICE: &str = "k8s.io"; + +/// Services whose every change is to how the organization itself is set up. +const GCP_CONFIGURATION_SERVICES: [&str; 6] = [ + "orgpolicy", + "logging", + "serviceusage", + "accesscontextmanager", + "essentialcontacts", + "securitycenter", +]; + +/// What a Google API call is about, read off its service and method. Only the +/// call itself is read (`SetIamPolicy`), not the package it sits in. +fn gcp_category(service: &str, method: &str) -> &'static str { + let call = words(method.rsplit('.').next().unwrap_or(method)); + let has = |needles: &[&str]| has_word(&call, needles); + let service_is = |names: &[&str]| names.iter().any(|name| service.contains(name)); + if has(&["iam"]) + || (service == "iam.googleapis.com" && has(&["role", "policy", "binding", "deny"])) + { + "permission" + } else if has(&["key"]) || service_is(&["secretmanager", "apikeys", "cloudkms"]) { + "credential" + } else if service_is(&["cloudbilling", "billingbudgets"]) { + "billing" + } else if has(&["login", "signin"]) { + "authentication" + } else if service_is(&GCP_CONFIGURATION_SERVICES) || (has(&["org"]) && has(&["policy"])) { + "configuration" + } else { + "resource" + } +} + +/// Who made a Google API call: `(type, id, email)`. A federated workload has +/// no email, only a `principalSubject`. +fn gcp_actor(payload: &JsonObject) -> (&'static str, &str, &str) { + let info = object(payload, "authenticationInfo"); + let email = info + .and_then(|info| text(info, "principalEmail")) + .unwrap_or_default(); + if !email.is_empty() { + let service = email.ends_with("gserviceaccount.com") || email.starts_with("system:"); + let address = if email.contains('@') { email } else { "" }; + return (if service { "service" } else { "user" }, email, address); + } + let subject = info + .and_then(|info| text(info, "principalSubject")) + .unwrap_or_default(); + if subject.is_empty() { + ("unknown", "", "") + } else if subject.starts_with("user:") { + ("user", subject, "") + } else { + ("service", subject, "") + } +} + +/// gRPC status codes Google uses for a refusal: PERMISSION_DENIED and UNAUTHENTICATED. +const GCP_DENIED_CODES: [i64; 2] = [7, 16]; + +/// A Cloud Logging entry from the Admin Activity or Policy Denied audit log. +/// `None` for every other entry, System Event included: those are Google's own +/// actions on a workload and stay with its logs. +pub fn gcp_entry(entry: &JsonObject, now_ms: i64) -> Option { + let log_name = text(entry, "logName")?; + let log = gcp_audit_log(log_name)?; + let payload = object(entry, "protoPayload")?; + let method = text(payload, "methodName")?; + let service = text(payload, "serviceName").unwrap_or_default(); + if service == GCP_KUBERNETES_SERVICE { + return None; + } + let (actor_type, actor_id, actor_email) = gcp_actor(payload); + + let status = object(payload, "status"); + let code = status + .and_then(|status| status.get("code")) + .and_then(JsonValue::as_i64) + .unwrap_or(0); + let outcome = if log == "policy" || GCP_DENIED_CODES.contains(&code) { + "denied" + } else if code == 0 { + "success" + } else { + "failure" + }; + + let resource = object(entry, "resource"); + let project = resource + .and_then(|resource| object(resource, "labels")) + .and_then(|labels| text(labels, "project_id")); + // `projects//logs/...`, or a folder's or organization's own log. + let account = project.map_or_else( + || { + log_name + .split("/logs/") + .next() + .unwrap_or_default() + .to_owned() + }, + str::to_owned, + ); + let request = object(payload, "requestMetadata"); + let claimed_ms = text(entry, "timestamp") + .and_then(|timestamp| DateTime::parse_from_rfc3339(timestamp).ok()) + .map(|timestamp| timestamp.timestamp_millis()); + + Some(AuditEvent { + provider: "gcp", + account, + provider_event_id: text(entry, "insertId").map_or_else(|| content_id(entry), str::to_owned), + occurred_at_ms: occurred_at_ms(claimed_ms, now_ms), + actor_type, + actor_id: actor_id.to_owned(), + actor_email: actor_email.to_owned(), + action: method.to_owned(), + category: if log == "policy" { + "permission" + } else { + gcp_category(service, method) + }, + outcome, + outcome_reason: owned(status.and_then(|status| text(status, "message"))), + target_type: owned(resource.and_then(|resource| text(resource, "type"))), + target_name: owned(text(payload, "resourceName")), + origin_ip: owned(request.and_then(|request| text(request, "callerIp"))), + user_agent: owned(request.and_then(|request| text(request, "callerSuppliedUserAgent"))), + raw: raw_of(entry), + ..AuditEvent::default() + }) +} + +/// Now, as the warehouse and this module count it. +pub fn now_ms() -> i64 { + Utc::now().timestamp_millis() +} + +#[cfg(test)] +mod tests { + use super::*; + + const NOW: i64 = 1_700_000_060_000; + + fn parse(json: &str) -> JsonObject { + serde_json::from_str(json).expect("fixture is a JSON object") + } + + fn row(event: &AuditEvent) -> JsonObject { + serde_json::from_slice(&event.to_row("org_1", NOW)).expect("a row is a JSON object") + } + + // The TypeScript writer derives the same key (`auditTrailEventId`); these two + // vectors are asserted there as well. A provider that moved between the two + // writers would otherwise stop collapsing its redeliveries. + #[test] + fn the_row_key_matches_the_typescript_writer() { + assert_eq!( + event_id("maple", "", "evt_1"), + "851ca0eff076aad987a0192777b176cd" + ); + assert_eq!( + event_id("maple", "acme", "evt_1"), + "1d19ea3d78217e97b45a7a9d28300146" + ); + } + + const VERCEL_EVENT: &str = r#"{"schema":"vercel.audit_log.v1","id":"evt_1","teamId":"team_1","action":"project.env-var.created","timestamp":1700000000123,"ipAddress":"203.0.113.7","userAgent":"Mozilla/5.0","actor":{"type":"user","id":"usr_1","name":"Ada","email":"ada@acme.test"},"payload":{"projectId":"prj_1"}}"#; + + #[test] + fn a_vercel_event_becomes_a_row_with_its_actor_action_and_origin() { + let event = vercel_event(&parse(VERCEL_EVENT), NOW).expect("an event with an action"); + let row = row(&event); + + assert_eq!(row["OrgId"], "org_1"); + assert_eq!(row["Provider"], "vercel"); + assert_eq!(row["Account"], "team_1"); + assert_eq!(row["ProviderEventId"], "evt_1"); + assert_eq!(row["Id"], event_id("vercel", "team_1", "evt_1")); + assert_eq!(row["OccurredAt"], "2023-11-14 22:13:20.123"); + assert_eq!(row["IngestedAt"], "2023-11-14 22:14:20.000"); + assert_eq!(row["ActorType"], "user"); + assert_eq!(row["ActorEmail"], "ada@acme.test"); + assert_eq!(row["Action"], "project.env-var.created"); + assert_eq!(row["Category"], "credential"); + assert_eq!(row["Outcome"], "success"); + assert_eq!(row["OriginIp"], "203.0.113.7"); + assert_eq!(row["TargetType"], "project.env-var"); + assert_eq!(row["TargetId"], "prj_1"); + assert!(row["Raw"].as_str().is_some_and(|raw| raw.contains("prj_1"))); + } + + // The table keeps the highest version: a redelivery must lose to the first write. + #[test] + fn a_later_write_of_the_same_event_has_the_same_key_and_a_lower_version() { + let event = vercel_event(&parse(VERCEL_EVENT), NOW).expect("an event with an action"); + let first: JsonObject = serde_json::from_slice(&event.to_row("org_1", NOW)).unwrap(); + let later: JsonObject = + serde_json::from_slice(&event.to_row("org_1", NOW + 60_000)).unwrap(); + + assert_eq!(first["Id"], later["Id"]); + assert!(later["Version"].as_i64() < first["Version"].as_i64()); + } + + #[test] + fn a_vercel_event_without_an_id_keys_on_its_content_and_one_without_an_action_is_skipped() { + let no_id = + parse(r#"{"teamId":"team_1","action":"drain-created","timestamp":1700000000000}"#); + let first = vercel_event(&no_id, NOW).expect("an event with an action"); + let again = vercel_event(&no_id, NOW + 5).expect("an event with an action"); + + assert!(!first.provider_event_id.is_empty()); + assert_eq!(first.provider_event_id, again.provider_event_id); + assert_eq!(first.category, "configuration"); + assert_eq!(first.actor_type, "unknown"); + assert_eq!(vercel_event(&parse(r#"{"id":"evt_2"}"#), NOW), None); + } + + const GCP_ACTIVITY: &str = r#"{"logName":"projects/acme-prod/logs/cloudaudit.googleapis.com%2Factivity","insertId":"abc123","timestamp":"2023-11-14T22:13:20.123Z","resource":{"type":"service_account","labels":{"project_id":"acme-prod"}},"protoPayload":{"serviceName":"iam.googleapis.com","methodName":"google.iam.admin.v1.CreateServiceAccountKey","resourceName":"projects/-/serviceAccounts/123","authenticationInfo":{"principalEmail":"ada@acme.test"},"requestMetadata":{"callerIp":"203.0.113.7","callerSuppliedUserAgent":"gcloud"}}}"#; + + #[test] + fn an_admin_activity_entry_becomes_a_row() { + let event = gcp_entry(&parse(GCP_ACTIVITY), NOW).expect("an Admin Activity entry"); + let row = row(&event); + + assert_eq!(row["Provider"], "gcp"); + assert_eq!(row["Account"], "acme-prod"); + assert_eq!(row["ProviderEventId"], "abc123"); + assert_eq!(row["OccurredAt"], "2023-11-14 22:13:20.123"); + assert_eq!(row["ActorType"], "user"); + assert_eq!(row["ActorEmail"], "ada@acme.test"); + assert_eq!(row["Action"], "google.iam.admin.v1.CreateServiceAccountKey"); + assert_eq!(row["Category"], "credential"); + assert_eq!(row["Outcome"], "success"); + assert_eq!(row["TargetType"], "service_account"); + assert_eq!(row["TargetName"], "projects/-/serviceAccounts/123"); + assert_eq!(row["OriginIp"], "203.0.113.7"); + assert_eq!(row["UserAgent"], "gcloud"); + } + + #[test] + fn a_refused_call_is_denied_with_its_reason_and_a_service_account_is_a_service() { + let mut entry = parse(GCP_ACTIVITY); + let payload = entry["protoPayload"].as_object_mut().unwrap(); + payload.insert( + "status".into(), + json!({ "code": 7, "message": "Permission iam.serviceAccountKeys.create denied" }), + ); + payload.insert( + "authenticationInfo".into(), + json!({ "principalEmail": "ci@acme-prod.iam.gserviceaccount.com" }), + ); + let event = gcp_entry(&entry, NOW).expect("an Admin Activity entry"); + + assert_eq!(event.outcome, "denied"); + assert_eq!( + event.outcome_reason, + "Permission iam.serviceAccountKeys.create denied" + ); + assert_eq!(event.actor_type, "service"); + } + + #[test] + fn a_failed_call_that_was_not_refused_is_a_failure() { + let mut entry = parse(GCP_ACTIVITY); + entry["protoPayload"].as_object_mut().unwrap().insert( + "status".into(), + json!({ "code": 9, "message": "Precondition failed" }), + ); + + assert_eq!( + gcp_entry(&entry, NOW).map(|event| event.outcome), + Some("failure") + ); + } + + #[test] + fn an_iam_policy_change_is_a_permission_event_and_a_policy_denial_is_denied() { + let mut policy_change = parse(GCP_ACTIVITY); + policy_change["protoPayload"] + .as_object_mut() + .unwrap() + .insert("methodName".into(), json!("SetIamPolicy")); + let mut denial = parse(GCP_ACTIVITY); + denial.insert( + "logName".into(), + json!("projects/acme-prod/logs/cloudaudit.googleapis.com%2Fpolicy"), + ); + + assert_eq!( + gcp_entry(&policy_change, NOW).map(|event| event.category), + Some("permission") + ); + let denied = gcp_entry(&denial, NOW).expect("a Policy Denied entry"); + assert_eq!((denied.category, denied.outcome), ("permission", "denied")); + } + + // System Event and Data Access entries, and a workload's own log lines, are not + // audit events of the org's people: they stay logs. + #[test] + fn only_admin_activity_and_policy_denied_entries_are_audit_events() { + for log in [ + "cloudaudit.googleapis.com%2Fsystem_event", + "cloudaudit.googleapis.com%2Fdata_access", + "run.googleapis.com%2Fstdout", + ] { + let mut entry = parse(GCP_ACTIVITY); + entry.insert( + "logName".into(), + json!(format!("projects/acme-prod/logs/{log}")), + ); + assert_eq!(gcp_entry(&entry, NOW), None, "{log}"); + } + let mut no_payload = parse(GCP_ACTIVITY); + no_payload.remove("protoPayload"); + assert_eq!(gcp_entry(&no_payload, NOW), None); + } + + #[test] + fn an_oversized_payload_is_not_kept_and_the_row_still_says_what_happened() { + let mut entry = parse(GCP_ACTIVITY); + entry["protoPayload"] + .as_object_mut() + .unwrap() + .insert("request".into(), json!("x".repeat(RAW_LIMIT_BYTES))); + let event = gcp_entry(&entry, NOW).expect("an Admin Activity entry"); + let raw: JsonObject = serde_json::from_str(&event.raw).expect("raw is JSON"); + + assert_eq!(raw["truncated"], true); + assert!(event.raw.len() < 100); + assert_eq!(event.action, "google.iam.admin.v1.CreateServiceAccountKey"); + } + + // Every column is the sender's to fill, and the table is every org's. + #[test] + fn a_row_keeps_no_more_of_a_field_than_its_column_takes() { + let long = "é".repeat(4000); + let mut entry = parse(GCP_ACTIVITY); + entry.insert("insertId".into(), json!(long)); + let payload = entry["protoPayload"].as_object_mut().unwrap(); + payload.insert("methodName".into(), json!(long)); + payload.insert("resourceName".into(), json!(long)); + payload.insert("status".into(), json!({ "code": 9, "message": long })); + let row = row(&gcp_entry(&entry, NOW).expect("an Admin Activity entry")); + let bytes = |column: &str| row[column].as_str().map_or(0, str::len); + + assert_eq!(bytes("Action"), NAME_LIMIT_BYTES); + assert_eq!(bytes("TargetName"), TEXT_LIMIT_BYTES); + assert_eq!(bytes("OutcomeReason"), TEXT_LIMIT_BYTES); + // An id is hashed rather than cut: it is part of the row's key. + assert_eq!(bytes("ProviderEventId"), 32); + assert!(bytes("Raw") <= RAW_LIMIT_BYTES); + } + + #[test] + fn two_long_ids_that_share_a_prefix_stay_two_events() { + let id = |tail: &str| format!("{}{tail}", "a".repeat(NAME_LIMIT_BYTES)); + let key = |id: String| { + let event = AuditEvent { + provider: "vercel", + provider_event_id: id, + ..AuditEvent::default() + }; + row(&event)["Id"].clone() + }; + + assert_ne!(key(id("1")), key(id("2"))); + } + + // The table is partitioned by the month an event happened and drops rows by + // age: a sender's own clock must not pick the partition or the expiry. + #[test] + fn a_time_the_table_could_not_hold_is_replaced_by_the_time_of_receipt() { + let at = |timestamp: i64| { + let mut event = parse(VERCEL_EVENT); + event.insert("timestamp".into(), json!(timestamp)); + vercel_event(&event, NOW).map(|event| event.occurred_at_ms) + }; + let day = 24 * 60 * 60 * 1000; + + assert_eq!(at(NOW - 399 * day), Some(NOW - 399 * day)); + assert_eq!(at(NOW + day / 2), Some(NOW + day / 2)); + assert_eq!(at(NOW - 401 * day), Some(NOW)); + assert_eq!(at(NOW + 2 * day), Some(NOW)); + // Seconds where milliseconds belong: 1970, which would expire on arrival. + assert_eq!(at(NOW / 1000), Some(NOW)); + } + + #[test] + fn a_vercel_action_is_filed_by_the_words_in_its_name() { + for (action, category) in [ + ("team.member.role.updated", "permission"), + ("team.member.added", "membership"), + ("access-group.created", "permission"), + ("project.env-var.created", "credential"), + ("project.environment.created", "resource"), + ("password_protection.enabled", "configuration"), + ("token.created", "credential"), + ("user.login", "authentication"), + ("drain-created", "configuration"), + ("deployment.deleted", "resource"), + ] { + assert_eq!(vercel_category(action), category, "{action}"); + } + } + + #[test] + fn a_vercel_event_points_at_what_its_payload_names() { + let event = parse( + r#"{"id":"evt_3","action":"drain-created","payload":{"drainId":"drn_1","projectId":"prj_1","name":"Maple logs"}}"#, + ); + let event = vercel_event(&event, NOW).expect("an event with an action"); + + assert_eq!( + ( + event.target_type.as_str(), + event.target_id.as_str(), + event.target_name.as_str() + ), + ("drain", "drn_1", "Maple logs") + ); + } + + #[test] + fn a_google_call_is_filed_by_its_service_and_the_call_itself() { + for (service, method, category) in [ + ( + "storage.googleapis.com", + "storage.setIamPermissions", + "permission", + ), + ( + "iam.googleapis.com", + "google.iam.admin.v1.CreateRole", + "permission", + ), + ( + "iam.googleapis.com", + "google.iam.admin.v1.CreateServiceAccount", + "resource", + ), + ( + "cloudresourcemanager.googleapis.com", + "SetOrgPolicy", + "configuration", + ), + ( + "logging.googleapis.com", + "google.logging.v2.ConfigServiceV2.DeleteSink", + "configuration", + ), + ( + "secretmanager.googleapis.com", + "google.cloud.secretmanager.v1.SecretManagerService.DeleteSecret", + "credential", + ), + ( + "compute.googleapis.com", + "v1.compute.instances.insert", + "resource", + ), + ] { + assert_eq!(gcp_category(service, method), category, "{method}"); + } + } + + // A cluster's controllers write to the Kubernetes API all day, and Google + // files every write under Admin Activity. + #[test] + fn a_kubernetes_api_write_stays_a_log() { + let mut entry = parse(GCP_ACTIVITY); + let payload = entry["protoPayload"].as_object_mut().unwrap(); + payload.insert("serviceName".into(), json!("k8s.io")); + payload.insert("methodName".into(), json!("io.k8s.core.v1.pods.create")); + + assert_eq!(gcp_entry(&entry, NOW), None); + } + + #[test] + fn a_federated_workload_is_a_service_named_by_its_subject() { + let subject = "principal://iam.googleapis.com/projects/1/locations/global/workloadIdentityPools/ci/subject/repo"; + let mut entry = parse(GCP_ACTIVITY); + entry["protoPayload"].as_object_mut().unwrap().insert( + "authenticationInfo".into(), + json!({ "principalSubject": subject }), + ); + let event = gcp_entry(&entry, NOW).expect("an Admin Activity entry"); + + assert_eq!( + ( + event.actor_type, + event.actor_id.as_str(), + event.actor_email.as_str() + ), + ("service", subject, "") + ); + } +} diff --git a/apps/ingest/src/lib.rs b/apps/ingest/src/lib.rs index a2395a69f5..0ab28003d3 100644 --- a/apps/ingest/src/lib.rs +++ b/apps/ingest/src/lib.rs @@ -1,4 +1,5 @@ pub mod ai_session; +pub mod audit_trail; pub mod aws; pub mod aws_metric_stream; pub mod clickhouse_insert_mappings; diff --git a/apps/ingest/src/main.rs b/apps/ingest/src/main.rs index 64ae6185f1..e3006dfddd 100644 --- a/apps/ingest/src/main.rs +++ b/apps/ingest/src/main.rs @@ -46,6 +46,7 @@ use flate2::write::GzEncoder; use flate2::Compression; use hmac::{Hmac, Mac}; use maple_ingest::ai_session; +use maple_ingest::audit_trail; use maple_ingest::aws::CredentialsProvider as AwsCredentialsProvider; use maple_ingest::aws_metric_stream; use maple_ingest::clickhouse_insert_mappings::SCHEMA_VERSION as CLICKHOUSE_SCHEMA_VERSION; @@ -205,6 +206,9 @@ struct AppConfig { org_routing_cache_ttl_secs: u64, /// `INGEST_RULES_ENABLED` (default on): apply customer drop/redact rules. Off reads nothing and costs nothing. ingest_rules_enabled: bool, + /// `INGEST_AUDIT_TRAIL_ENABLED` (default off): keep provider audit events in + /// the audit trail instead of as logs. See `stores_audit_trail`. + audit_trail_enabled: bool, /// `INGEST_RULES_CACHE_TTL_SECS`: how long an org's compiled rules are reused. ingest_rules_cache_ttl_secs: u64, /// Ceiling on the total decompressed rrweb payload a single replay session @@ -303,6 +307,12 @@ enum KeyStoreBackend { } impl AppConfig { + /// Whether provider audit events go to the audit trail. Off, or on a gateway + /// that only forwards and so has no such table, they stay logs. + fn stores_audit_trail(&self) -> bool { + self.audit_trail_enabled && self.write_mode.uses_tinybird() + } + #[expect( clippy::too_many_lines, reason = "one branch per environment variable, read in the order the deployment docs list \ @@ -628,6 +638,14 @@ impl AppConfig { std::env::var("INGEST_RULES_ENABLED").ok(), true, )?; + // Off by default. Turn it on once the table exists and every running task + // is a build that reads the audit trail's WAL frames: an older build that + // recovers one drops the rest of that segment. + let audit_trail_enabled = parse_bool( + "INGEST_AUDIT_TRAIL_ENABLED", + std::env::var("INGEST_AUDIT_TRAIL_ENABLED").ok(), + false, + )?; let ingest_rules_cache_ttl_secs = parse_u64( "INGEST_RULES_CACHE_TTL_SECS", std::env::var("INGEST_RULES_CACHE_TTL_SECS").ok(), @@ -789,6 +807,7 @@ impl AppConfig { ingest_key_cache_ttl_secs, org_routing_cache_ttl_secs, ingest_rules_enabled, + audit_trail_enabled, ingest_rules_cache_ttl_secs, replay_max_session_bytes, replay_blob_store, @@ -921,6 +940,8 @@ enum SourceClaim { struct VerifiedClaims { active: Vec, paused: Vec, + /// The store could not be asked: a claim in neither list may still be the org's. + unresolved: bool, } fn source_claims_cache(ttl: Duration) -> Cache<(String, String), SourceClaim> { @@ -1730,6 +1751,8 @@ enum GcpPushOutcome { SourcePaused, /// The org's ingest rules need more scanning than one entry may use; redelivery would too. RulesScanBudget, + /// An audit log entry, stored in the audit trail instead of the logs. + AuditEvent, } #[derive(Clone, Copy)] @@ -4103,9 +4126,295 @@ async fn handle_vercel_event_drain( let Some(drain) = EventDrain::from_path(&drain) else { return StatusCode::NOT_FOUND.into_response(); }; + if drain == EventDrain::AuditLog && state.config.stores_audit_trail() { + return handle_vercel_audit_trail(state, headers, body).await; + } handle_signal(state, headers, body, Ingress::VercelEventDrain(drain)).await } +/// Vercel's Audit Log drain as rows of `audit_trail_events`. +async fn handle_vercel_audit_trail( + state: Arc, + headers: HeaderMap, + body: Bytes, +) -> Response { + metrics::request_started(); + let _guard = InFlightGuard; + let span = tracing::info_span!( + "ingest_vercel_audit_trail", + otel.name = "POST /v1/vercel/audit-log", + otel.kind = "server", + otel.status_code = tracing::field::Empty, + otel.status_description = tracing::field::Empty, + "maple.ingest.reject_reason" = tracing::field::Empty, + "http.request.method" = "POST", + "http.route" = "/v1/vercel/audit-log", + "http.request.body.size" = body.len(), + "http.response.status_code" = tracing::field::Empty, + "error.type" = tracing::field::Empty, + "maple.signal" = "audit_trail", + "maple.org_id" = tracing::field::Empty, + "maple.audit_trail.events" = tracing::field::Empty, + "maple.audit_trail.skipped" = tracing::field::Empty, + ); + let span_handle = span.clone(); + let start = Instant::now(); + let result = handle_vercel_audit_trail_inner(&state, &headers, &body) + .instrument(span) + .await; + let duration = start.elapsed().as_secs_f64(); + let signal = TelemetrySignal::AuditTrail.as_str(); + match result { + Ok(count) => { + span_handle.record("http.response.status_code", 200u16); + span_handle.record("otel.status_code", "Ok"); + metrics::request_completed(signal, "ok", "none", duration); + (StatusCode::OK, axum::Json(AcceptedBody { accepted: count })).into_response() + } + Err(error) => { + let status = error.status.as_u16(); + span_handle.record("http.response.status_code", status); + span_handle.record("error.type", error.error_kind()); + // A drain Vercel keeps retrying has to move an error rate, like any other ingest route. + metrics::request_completed(signal, "error", error.error_kind(), duration); + record_rejection_reason( + &span_handle, + status, + error.error_kind(), + error.message.as_str(), + ); + error.into_response() + } + } +} + +/// Audit deliveries one org may make each second, across the fleet. Well above +/// what the people of a team or a cloud organization do; past it the sender is +/// told to retry, so a burst is delayed and not lost. +const AUDIT_TRAIL_VERCEL_ORG_RPS: u32 = 10; +const AUDIT_TRAIL_GCP_ORG_RPS: u32 = 100; + +/// The most a Vercel audit delivery may be once inflated. A batch of audit +/// events is kilobytes; the telemetry ceiling is for bodies that are metered. +const AUDIT_TRAIL_MAX_DECODED_BYTES: usize = 4 * 1024 * 1024; + +/// Refuses an org's audit delivery past `rps`. The rows are not metered, so +/// this is what bounds how many a sender can store. +async fn audit_trail_rate_gate( + state: &AppState, + org_id: &str, + provider: &'static str, + rps: u32, +) -> Result<(), ApiError> { + let key = format!("audit_trail:{provider}:{org_id}"); + if state.key_rate_limiter.try_acquire(&key, rps).await { + return Ok(()); + } + metrics::org_throttled(org_id, "audit_trail_rate"); + warn!(org_id, provider, "Audit event rate limit exceeded"); + Err(ApiError::too_many_requests( + "Audit event rate limit exceeded", + )) +} + +async fn handle_vercel_audit_trail_inner( + state: &AppState, + headers: &HeaderMap, + body: &Bytes, +) -> Result { + let Some(resolved_key) = resolve_replay_key(state, headers, &[SourceSignal::Logs]).await? + else { + return Ok(0); + }; + let org_id = resolved_key.org_id.clone(); + Span::current().record("maple.org_id", org_id.as_str()); + + // The drain Maple creates sends the org's private key and names the Vercel + // connection's source. Anything else posting here could write the org an + // audit event of its own invention: a source token, the public key (it + // ships in browser bundles), or a private key naming no integration source. + let unknown_sender = || { + ApiError::unauthorized( + "Audit events are accepted only from the drain Maple creates on the Vercel team", + ) + }; + if !matches!(resolved_key.key_type, IngestKeyType::Private) + || !resolved_key.accepts_source_claims() + { + return Err(unknown_sender()); + } + let claim = headers + .get(SOURCE_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|claim| !claim.is_empty() && claim.len() <= MAX_PUBLIC_ID_LEN) + .ok_or_else(unknown_sender)?; + let verified = state + .resolver + .verify_source_claims(&org_id, vec![claim.to_owned()]) + .await; + // Paused, the delivery is acked and dropped like the source's logs are. + if verified.paused.iter().any(|paused| paused == claim) { + metrics::source_paused_drop(&org_id, TelemetrySignal::AuditTrail.as_str()); + Span::current().record("maple.ingest.reject_reason", "source_paused"); + return Ok(0); + } + if !verified.active.iter().any(|active| active == claim) { + // The store being down says nothing about the sender: Vercel retries a 503. + if verified.unresolved { + return Err(ApiError::service_unavailable( + "Ingest authentication unavailable", + )); + } + return Err(unknown_sender()); + } + + audit_trail_rate_gate(state, &org_id, "vercel", AUDIT_TRAIL_VERCEL_ORG_RPS).await?; + let _org_inflight_permit = + state + .org_inflight_limiter + .try_acquire(&org_id) + .ok_or_else(|| { + warn!(org_id = %org_id, "Per-org in-flight ingest limit exceeded"); + ApiError::too_many_requests("Per-org ingest limit exceeded") + })?; + let pipeline = native_rows_pipeline_for( + state, + ExportDestination::Tinybird, + "Audit event storage is not configured", + )?; + + let gzip = match headers + .get(CONTENT_ENCODING) + .and_then(|value| value.to_str().ok()) + .map(|value| value.trim().to_ascii_lowercase()) + .as_deref() + { + None | Some("" | "identity") => false, + Some("gzip") => true, + Some(_) => { + return Err(ApiError::unsupported_media_type( + "Unsupported content-encoding", + )) + } + }; + let (rows, skipped) = + decode_vercel_audit_drain(state, body.clone(), gzip, org_id.clone()).await?; + Span::current().record("maple.audit_trail.events", rows.len()); + Span::current().record("maple.audit_trail.skipped", skipped); + if skipped > 0 { + warn!(org_id = %org_id, skipped, "Vercel audit drain events without an action were skipped"); + } + let count = store_audit_trail_rows(pipeline, &org_id, rows).await?; + state.resolver.note_source_use(&resolved_key, false); + Ok(count) +} + +/// A Vercel audit delivery as rows, and how many of its events were skipped. +/// Inflated and parsed on the blocking pool inside the process-wide decode +/// memory budget, like an OTLP body: a small gzip can inflate into a body that +/// parses into gigabytes. +async fn decode_vercel_audit_drain( + state: &AppState, + body: Bytes, + gzip: bool, + org_id: String, +) -> Result<(Vec>, usize), ApiError> { + let mut reservation = state.decode_memory.reservation(); + let heap_budget = state.config.max_decode_heap_bytes; + let max_decoded = state + .config + .max_decoded_body_bytes + .min(AUDIT_TRAIL_MAX_DECODED_BYTES); + tokio::task::spawn_blocking(move || { + let busy = || { + ApiError::tagged( + StatusCode::SERVICE_UNAVAILABLE, + &INGEST_DECODE_BUSY, + "Ingest gateway is busy decoding other requests; retry shortly", + ) + }; + let too_large = || ApiError::payload_too_large("Audit delivery too large"); + if body.len() > max_decoded { + return Err(too_large()); + } + if !reservation.try_add(body.len() as u64) { + return Err(busy()); + } + let payload = if gzip { + let invalid = || ApiError::bad_request("Invalid gzip body"); + let declared = decode_budget::gzip_declared_len(&body).ok_or_else(invalid)?; + let declared = usize::try_from(declared).unwrap_or(usize::MAX); + if declared > max_decoded { + return Err(too_large()); + } + if !reservation.try_add(declared as u64) { + return Err(busy()); + } + let inflated = gunzip_declared(&body, declared).map_err(|error| match error { + GunzipError::Invalid(_) => invalid(), + GunzipError::TooLarge => too_large(), + })?; + drop(body); + Bytes::from(inflated) + } else { + body + }; + let estimate = decode_budget::json_cost(&payload, heap_budget) + .map_err(|decode_budget::OverBudget| over_decode_budget(heap_budget))?; + if !reservation.try_add(estimate) { + return Err(busy()); + } + let parsed = vercel_logs::parse(&payload).map_err(|e| { + ApiError::bad_request(format!("Invalid Vercel Audit Log drain payload: {e}")) + })?; + let now_ms = audit_trail::now_ms(); + let rows: Vec> = parsed + .entries + .iter() + .filter_map(|event| audit_trail::vercel_event(event, now_ms)) + .map(|event| event.to_row(&org_id, now_ms)) + .collect(); + let skipped = parsed.entries.len() - rows.len() + parsed.malformed_lines; + Ok((rows, skipped)) + }) + .await + .unwrap_or_else(|join_error| { + error!(error = %join_error, "Audit delivery decode task failed"); + Err(ApiError::service_unavailable( + "Ingest gateway failed to decode the request", + )) + }) +} + +/// Audit rows to the managed warehouse: the table is pinned there, whatever +/// warehouse the org reads its telemetry from. Not metered: an audit event is +/// evidence, not usage. +async fn store_audit_trail_rows( + pipeline: &TelemetryPipeline, + org_id: &str, + rows: Vec>, +) -> Result { + if rows.is_empty() { + return Ok(0); + } + let count = rows.len(); + pipeline + .accept_rows_to( + org_id, + audit_trail::DATASOURCE.to_owned(), + rows, + TelemetrySignal::AuditTrail, + ExportDestination::Tinybird, + ) + .await + .map_err(|e| { + warn!(org_id = %org_id, error = %e, "audit trail enqueue rejected"); + api_error_from_pipeline(&e) + })?; + Ok(count) +} + // Session replay ingest /// Running total of decompressed rrweb bytes per in-flight replay session, used @@ -6182,6 +6491,18 @@ async fn handle_gcp_logpush( span_handle.record("maple.ingest.reject_reason", "source_paused"); metrics::request_completed("logs", "ok", "none", duration); } + GcpPushOutcome::AuditEvent => { + span_handle.record("otel.status_code", "Ok"); + span_handle.record("maple.ingest.item_count", 1); + span_handle.record("maple.signal", TelemetrySignal::AuditTrail.as_str()); + metrics::request_completed( + TelemetrySignal::AuditTrail.as_str(), + "ok", + "none", + duration, + ); + metrics::gcp_entry(); + } GcpPushOutcome::RulesScanBudget => { // Acked, but the entry is gone: an `Error`, like a decode failure. let kind = INGEST_RULES_SCAN_BUDGET.error_kind; @@ -6376,6 +6697,37 @@ async fn handle_gcp_logpush_inner( return Ok((StatusCode::OK.into_response(), GcpPushOutcome::NotALogEntry)); }; + // An Admin Activity or Policy Denied entry is an audit event, not a log: + // it goes to the audit trail, unmetered, ahead of the plan check on logs + // and of the org's ingest rules, which are about what it keeps of its logs. + if state.config.stores_audit_trail() { + let now_ms = audit_trail::now_ms(); + if let Some(event) = audit_trail::gcp_entry(&entry, now_ms) { + // Refused, Pub/Sub redelivers with backoff: delayed, not lost. + audit_trail_rate_gate(state, org_id, "gcp", AUDIT_TRAIL_GCP_ORG_RPS) + .await + .map_err(|error| (error, "throttle"))?; + let pipeline = native_rows_pipeline_for( + state, + ExportDestination::Tinybird, + "Audit event storage is not configured", + ) + .map_err(|error| (error, "forward"))?; + if let Err(error) = + store_audit_trail_rows(pipeline, org_id, vec![event.to_row(org_id, now_ms)]).await + { + state + .gcp_resolver + .record_health(connector_id, Some(gcp_logging::ENTRY_NOT_STORED)) + .await; + return Err((error, "forward")); + } + state.gcp_resolver.record_health(connector_id, None).await; + state.resolver.note_source_use(&resolved.key, false); + return Ok((StatusCode::OK.into_response(), GcpPushOutcome::AuditEvent)); + } + } + if let Some(error) = entitlement_rejection(state, org_id, Signal::Logs.path()).await { state .gcp_resolver @@ -8419,6 +8771,7 @@ impl IngestKeyResolver { org_id, error, "Source claim lookup failed; replacing the claims" ); + verified.unresolved = true; return verified; } }; @@ -12400,6 +12753,7 @@ mod tests { ingest_key_cache_ttl_secs: 60, org_routing_cache_ttl_secs: 5, ingest_rules_enabled: false, + audit_trail_enabled: false, ingest_rules_cache_ttl_secs: 30, replay_max_session_bytes: 1024 * 1024 * 1024, replay_blob_store: None, @@ -12952,6 +13306,298 @@ mod tests { let _ = std::fs::remove_dir_all(&queue_dir); } + const VERCEL_AUDIT_DELIVERY: &str = concat!( + r#"{"id":"evt_1","teamId":"team_1","action":"project.created","timestamp":1700000000000,"actor":{"type":"user","id":"usr_1"}}"#, + "\n", + r#"{"id":"evt_2","teamId":"team_1","timestamp":1700000000000}"#, + "\n", + ); + + /// The Vercel connection's source in the fixtures, as its uuid and as the + /// public id a drain names in `x-maple-source-id`. + const AUDIT_SOURCE_UUID: &str = "0f8e6c1a-2b3d-4e5f-8a9b-0c1d2e3f4a5b"; + const AUDIT_SOURCE_ID: &str = "src_aXunrkWtzvXSkw9Z2smcft"; + + /// What the fake Tinybird is sent: `(datasource, NDJSON rows)`. + type TinybirdImports = tokio::sync::mpsc::UnboundedReceiver<(String, String)>; + + async fn fake_tinybird_import( + State(tx): State>, + Query(query): Query>, + body: Bytes, + ) -> StatusCode { + let rows = gunzip_bounded(&body, usize::MAX).unwrap_or_else(|_| body.to_vec()); + drop(tx.send(( + query.get("name").cloned().unwrap_or_default(), + String::from_utf8(rows).expect("Tinybird should receive UTF-8 NDJSON"), + ))); + StatusCode::OK + } + + /// Turns the audit trail on, as the hosted gateway has it, with a pipeline + /// that writes to a fake Tinybird the test can read. + async fn store_audit_trail( + state: &mut AppState, + queue_dir: &std::path::Path, + ) -> TinybirdImports { + let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let app = Router::new() + .route("/v0/events", post(fake_tinybird_import)) + .with_state(tx); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let pipeline = TelemetryPipeline::new( + TinybirdConfig { + endpoint, + token: "token".to_owned(), + ..test_tinybird_config(queue_dir.join("audit-trail")) + }, + Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(), + ) + .await + .expect("the audit trail pipeline should start"); + state.config.write_mode = WriteMode::Tinybird; + state.config.audit_trail_enabled = true; + state.telemetry_pipeline = Some(pipeline); + rx + } + + /// An org with a private key and a Vercel connection whose source is + /// `AUDIT_SOURCE_ID`, on a gateway that stores the audit trail. + async fn audit_trail_test_state( + raw_key: &str, + org_id: &str, + queue_dir: PathBuf, + ) -> (AppState, Arc, TinybirdImports) { + let store = Arc::new(FakeKeyStore::default()); + store.insert_private( + raw_key, + KeyRow { + org_id: org_id.to_owned(), + self_managed: false, + clickhouse_ready: false, + }, + ); + store.insert_source_row(org_id, AUDIT_SOURCE_UUID, "integration"); + let mut state = test_app_state( + Arc::clone(&store), + queue_dir.clone(), + "http://127.0.0.1:1".to_owned(), + Duration::from_secs(30), + ) + .await; + let imports = store_audit_trail(&mut state, &queue_dir).await; + (state, store, imports) + } + + /// The headers of the drain Maple creates: the key and the connection's source. + fn vercel_drain_headers(raw_key: &str, source_id: &str) -> HeaderMap { + let mut headers = bearer_headers(raw_key); + headers.insert(SOURCE_ID_HEADER, source_id.parse().unwrap()); + headers + } + + async fn vercel_audit_delivery( + state: &AppState, + headers: &HeaderMap, + ) -> Result { + handle_vercel_audit_trail_inner( + state, + headers, + &Bytes::from_static(VERCEL_AUDIT_DELIVERY.as_bytes()), + ) + .await + } + + async fn nothing_reaches_the_audit_trail(imports: &mut TinybirdImports) -> bool { + tokio::time::timeout(Duration::from_millis(200), imports.recv()) + .await + .is_err() + } + + #[tokio::test] + async fn vercel_audit_drain_stores_each_event_that_names_an_action() { + let queue_dir = unique_main_test_dir("vercel-audit-trail"); + let (state, _store, mut imports) = + audit_trail_test_state("maple_sk_test_audit", "org_audit", queue_dir.clone()).await; + + let accepted = vercel_audit_delivery( + &state, + &vercel_drain_headers("maple_sk_test_audit", AUDIT_SOURCE_ID), + ) + .await + .expect("the drain's delivery should be accepted"); + + // The second event names no action: there is nothing to say happened. + assert_eq!(accepted, 1); + let (datasource, rows) = tokio::time::timeout(Duration::from_secs(2), imports.recv()) + .await + .expect("the event should reach Tinybird") + .expect("the Tinybird channel should stay open"); + assert_eq!(datasource, "audit_trail_events"); + let row: serde_json::Value = serde_json::from_str(rows.trim()).expect("one NDJSON row"); + assert_eq!( + ( + row["OrgId"].as_str(), + row["ProviderEventId"].as_str(), + row["Action"].as_str() + ), + (Some("org_audit"), Some("evt_1"), Some("project.created")) + ); + drop(std::fs::remove_dir_all(&queue_dir)); + } + + // The drain Maple creates sends the private key and names the connection's + // source. Any other sender could write the org an audit event it made up. + #[tokio::test] + async fn vercel_audit_drain_refuses_a_sender_that_is_not_the_orgs_drain() { + let queue_dir = unique_main_test_dir("vercel-audit-trail-senders"); + let (state, store, mut imports) = + audit_trail_test_state("maple_sk_test_senders", "org_senders", queue_dir.clone()).await; + // The public key ships in browser bundles. + let public_hash = hash_ingest_key("maple_pk_test_senders", "test-hmac-key").unwrap(); + store.keys.lock().unwrap().insert( + (public_hash, "public_key_hash"), + KeyRow { + org_id: "org_senders".to_owned(), + self_managed: false, + clickhouse_ready: false, + }, + ); + // A log shipper's token: allowed to send logs, and nothing else. + let token = "maple_src_audit_senders_0000000000000000000000"; + store.insert_source( + token, + source_row( + "org_senders", + source_config(Some(r#"["logs"]"#), None, "[]"), + ), + ); + // Another org's integration source, named by this org's key. + let foreign = "5b8f2c1e-7a55-4c0e-9a3b-0d2c7e5b9a11"; + store.insert_source_row("org_other", foreign, "integration"); + let foreign_id = sources::public_source_id(foreign).expect("a uuid has a public id"); + + for (sender, headers) in [ + ( + "the public key", + vercel_drain_headers("maple_pk_test_senders", AUDIT_SOURCE_ID), + ), + ( + "a source token", + vercel_drain_headers(token, AUDIT_SOURCE_ID), + ), + ( + "the private key naming no source", + bearer_headers("maple_sk_test_senders"), + ), + ( + "the private key naming another org's source", + vercel_drain_headers("maple_sk_test_senders", &foreign_id), + ), + ] { + let refused = vercel_audit_delivery(&state, &headers) + .await + .expect_err(sender); + assert_eq!(refused.status, StatusCode::UNAUTHORIZED, "{sender}"); + } + assert!(nothing_reaches_the_audit_trail(&mut imports).await); + drop(std::fs::remove_dir_all(&queue_dir)); + } + + #[tokio::test] + async fn vercel_audit_drain_acks_and_drops_while_the_integration_is_paused() { + let queue_dir = unique_main_test_dir("vercel-audit-trail-paused"); + let (state, store, mut imports) = + audit_trail_test_state("maple_sk_test_paused", "org_paused", queue_dir.clone()).await; + store.pause_source(AUDIT_SOURCE_UUID, current_time_millis() - 1000, None); + + let accepted = vercel_audit_delivery( + &state, + &vercel_drain_headers("maple_sk_test_paused", AUDIT_SOURCE_ID), + ) + .await + .expect("a paused source's delivery is acked"); + + assert_eq!(accepted, 0); + assert!(nothing_reaches_the_audit_trail(&mut imports).await); + drop(std::fs::remove_dir_all(&queue_dir)); + } + + // A store that cannot be asked has not said the sender is unknown: the drain + // must read as retryable, not as misconfigured. + #[tokio::test] + async fn vercel_audit_drain_asks_for_a_retry_while_its_source_cannot_be_checked() { + let queue_dir = unique_main_test_dir("vercel-audit-trail-store-down"); + let (state, store, mut imports) = + audit_trail_test_state("maple_sk_test_down", "org_down", queue_dir.clone()).await; + let headers = vercel_drain_headers("maple_sk_test_down", AUDIT_SOURCE_ID); + store.source_errors.store(true, Ordering::Relaxed); + + let refused = vercel_audit_delivery(&state, &headers) + .await + .expect_err("an unchecked source must not be stored"); + assert_eq!(refused.status, StatusCode::SERVICE_UNAVAILABLE); + assert!(nothing_reaches_the_audit_trail(&mut imports).await); + + // The store back, the same delivery is taken. + store.source_errors.store(false, Ordering::Relaxed); + assert_eq!(vercel_audit_delivery(&state, &headers).await.ok(), Some(1)); + drop(std::fs::remove_dir_all(&queue_dir)); + } + + // A few hundred bytes of gzip can inflate into a body that parses into + // gigabytes; an audit delivery has no reason to be large. + #[tokio::test] + async fn vercel_audit_drain_refuses_a_delivery_that_inflates_past_its_limit() { + let queue_dir = unique_main_test_dir("vercel-audit-trail-bomb"); + let (state, _store, mut imports) = + audit_trail_test_state("maple_sk_test_bomb", "org_bomb", queue_dir.clone()).await; + let mut headers = vercel_drain_headers("maple_sk_test_bomb", AUDIT_SOURCE_ID); + headers.insert(CONTENT_ENCODING, "gzip".parse().unwrap()); + let body = gzip_bytes("[],".repeat(AUDIT_TRAIL_MAX_DECODED_BYTES / 2)); + + let refused = handle_vercel_audit_trail_inner(&state, &headers, &body) + .await + .expect_err("a delivery past the limit must be refused"); + + assert_eq!(refused.status, StatusCode::PAYLOAD_TOO_LARGE); + assert!(nothing_reaches_the_audit_trail(&mut imports).await); + drop(std::fs::remove_dir_all(&queue_dir)); + } + + // Audit rows are not metered, so the rate is what bounds a sender. + #[tokio::test] + async fn an_orgs_audit_deliveries_are_refused_past_their_rate() { + let queue_dir = unique_main_test_dir("audit-trail-rate"); + let (state, _store, _imports) = + audit_trail_test_state("maple_sk_test_rate", "org_rate", queue_dir.clone()).await; + + assert!(audit_trail_rate_gate(&state, "org_rate", "gcp", 1) + .await + .is_ok()); + let refused = audit_trail_rate_gate(&state, "org_rate", "gcp", 1) + .await + .expect_err("a second delivery inside the second is past a rate of one"); + assert_eq!(refused.status, StatusCode::TOO_MANY_REQUESTS); + // Another org, and the same org's other provider, have their own allowance. + assert!(audit_trail_rate_gate(&state, "org_other", "gcp", 1) + .await + .is_ok()); + assert!(audit_trail_rate_gate(&state, "org_rate", "vercel", 1) + .await + .is_ok()); + drop(std::fs::remove_dir_all(&queue_dir)); + } + #[tokio::test] async fn custom_session_events_are_metered_as_product_events_but_gated_on_browser_sessions() { let queue_dir = unique_main_test_dir("session-events-custom-meter"); @@ -14540,6 +15186,65 @@ mod tests { drop(std::fs::remove_dir_all(queue_dir)); } + const GCP_AUDIT_ENTRY: &str = r#"{"logName":"projects/acme-prod/logs/cloudaudit.googleapis.com%2Factivity","insertId":"abc123","timestamp":"2026-10-09T10:00:00Z","resource":{"type":"service_account","labels":{"project_id":"acme-prod"}},"protoPayload":{"serviceName":"iam.googleapis.com","methodName":"google.iam.admin.v1.CreateServiceAccountKey","authenticationInfo":{"principalEmail":"ada@acme.test"}}}"#; + + // An Admin Activity entry names a person and what they changed. It goes to + // the audit trail, which only admins read, and never to the org's logs. + #[tokio::test] + async fn gcp_logpush_stores_an_audit_entry_in_the_audit_trail_and_not_as_a_log() { + let (ch_url, mut ch_rx) = spawn_fake_clickhouse().await; + let (mut state, store, queue_dir) = gcp_logpush_state("gcp-logpush-audit", &ch_url).await; + let mut imports = store_audit_trail( + Arc::get_mut(&mut state).expect("the state is not shared yet"), + &queue_dir, + ) + .await; + + let accepted = gcp_push(&state, "gcp_conn_1", Some("gcp-secret"), GCP_AUDIT_ENTRY).await; + + assert_eq!(accepted.status(), StatusCode::OK); + assert_eq!(store.connector_successes.load(Ordering::Relaxed), 1); + let (datasource, rows) = tokio::time::timeout(Duration::from_secs(2), imports.recv()) + .await + .expect("the entry should reach Tinybird") + .expect("the Tinybird channel should stay open"); + assert_eq!(datasource, "audit_trail_events"); + let row: serde_json::Value = serde_json::from_str(rows.trim()).expect("one NDJSON row"); + assert_eq!( + ( + row["OrgId"].as_str(), + row["ProviderEventId"].as_str(), + row["ActorEmail"].as_str() + ), + (Some("org_gcp"), Some("abc123"), Some("ada@acme.test")) + ); + assert!( + tokio::time::timeout(Duration::from_millis(200), ch_rx.recv()) + .await + .is_err(), + "an audit entry must not be written to the org's logs" + ); + drop(std::fs::remove_dir_all(queue_dir)); + } + + // A gateway that only forwards has no audit trail table, and one where the + // trail is not turned on does not write to it: there the entry stays a log. + #[tokio::test] + async fn gcp_logpush_keeps_an_audit_entry_as_a_log_where_the_gateway_only_forwards() { + let (ch_url, mut ch_rx) = spawn_fake_clickhouse().await; + let (state, _store, queue_dir) = gcp_logpush_state("gcp-logpush-audit-fwd", &ch_url).await; + + let accepted = gcp_push(&state, "gcp_conn_1", Some("gcp-secret"), GCP_AUDIT_ENTRY).await; + + assert_eq!(accepted.status(), StatusCode::OK); + let stored = tokio::time::timeout(Duration::from_secs(2), ch_rx.recv()) + .await + .expect("the entry should reach the org's ClickHouse") + .expect("ClickHouse channel should stay open"); + assert!(stored.query.starts_with("INSERT INTO logs")); + drop(std::fs::remove_dir_all(queue_dir)); + } + #[tokio::test] async fn gcp_logpush_acks_an_entry_past_the_rule_scan_budget() { let (ch_url, mut ch_rx) = spawn_fake_clickhouse().await; diff --git a/apps/ingest/src/telemetry.rs b/apps/ingest/src/telemetry.rs index 66bf1c50fa..81babf62db 100644 --- a/apps/ingest/src/telemetry.rs +++ b/apps/ingest/src/telemetry.rs @@ -533,6 +533,9 @@ pub enum TelemetrySignal { /// `POST /v1/events` (NDJSON, gateway-written). Lands in `product_events` /// next to the browser rows the `session_events` MV materializes there. ProductEvents, + /// Audit events from provider feeds (a Vercel audit drain, GCP audit logs), + /// gateway-written into `audit_trail_events`, never as OTLP logs. + AuditTrail, } impl TelemetrySignal { @@ -549,6 +552,7 @@ impl TelemetrySignal { Self::SessionReplays => "session_replays", Self::SessionEvents => "session_events", Self::ProductEvents => "product_events", + Self::AuditTrail => "audit_trail", } } } @@ -3068,6 +3072,7 @@ fn signal_tag(signal: TelemetrySignal) -> u8 { TelemetrySignal::SessionReplays => 4, TelemetrySignal::SessionEvents => 5, TelemetrySignal::ProductEvents => 6, + TelemetrySignal::AuditTrail => 7, } } @@ -3079,6 +3084,7 @@ fn signal_from_tag(tag: u8) -> Option { 4 => Some(TelemetrySignal::SessionReplays), 5 => Some(TelemetrySignal::SessionEvents), 6 => Some(TelemetrySignal::ProductEvents), + 7 => Some(TelemetrySignal::AuditTrail), _ => None, } } @@ -5713,6 +5719,71 @@ mod tests { drop(std::fs::remove_dir_all(queue_dir)); } + // Audit rows are built by the gateway, not derived from OTLP: they have to + // reach the warehouse under their own datasource, never as a log. + #[tokio::test] + async fn audit_trail_rows_reach_tinybird_under_their_own_datasource() { + let (tx, mut rx) = mpsc::unbounded_channel(); + let listener = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let addr = listener.local_addr().unwrap(); + let app = Router::new() + .route("/v0/events", post(fake_tinybird_import)) + .with_state(tx); + tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + + let queue_dir = unique_test_dir("fake-tinybird-audit-trail"); + let mut cfg = test_cfg(); + cfg.endpoint = format!("http://{addr}"); + cfg.queue_dir = queue_dir.clone(); + cfg.wal_shards = 1; + cfg.batch_max_wait = Duration::from_millis(1); + let pipeline = TelemetryPipeline::new( + cfg, + Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(), + ) + .await + .unwrap(); + + let event: serde_json::Map = serde_json::from_str( + r#"{"id":"evt_1","teamId":"team_1","action":"project.created","timestamp":1700000000000}"#, + ) + .unwrap(); + let row = crate::audit_trail::vercel_event(&event, 1_700_000_060_000) + .expect("an event with an action") + .to_row("org_e2e", 1_700_000_060_000); + let stats = pipeline + .accept_rows_to( + "org_e2e", + crate::audit_trail::DATASOURCE.to_owned(), + vec![row], + TelemetrySignal::AuditTrail, + ExportDestination::Tinybird, + ) + .await + .unwrap(); + assert_eq!(stats.rows, 1); + + let import = tokio::time::timeout(Duration::from_secs(2), rx.recv()) + .await + .expect("fake Tinybird should receive an import") + .expect("fake Tinybird channel should stay open"); + assert_eq!(import.datasource, "audit_trail_events"); + let stored: Value = serde_json::from_str(import.body.trim()).unwrap(); + assert_eq!(stored["OrgId"], "org_e2e"); + assert_eq!(stored["Provider"], "vercel"); + assert_eq!(stored["Action"], "project.created"); + + wait_for_export_drain(queue_dir.clone(), 0, ExportDestination::Tinybird).await; + drop(std::fs::remove_dir_all(queue_dir)); + } + #[tokio::test] #[expect( clippy::too_many_lines, @@ -6457,6 +6528,7 @@ mod tests { assert_eq!(TelemetrySignal::SessionReplays.as_str(), "session_replays"); assert_eq!(TelemetrySignal::SessionEvents.as_str(), "session_events"); assert_eq!(TelemetrySignal::ProductEvents.as_str(), "product_events"); + assert_eq!(TelemetrySignal::AuditTrail.as_str(), "audit_trail"); } #[test] @@ -6470,6 +6542,7 @@ mod tests { TelemetrySignal::SessionReplays, TelemetrySignal::SessionEvents, TelemetrySignal::ProductEvents, + TelemetrySignal::AuditTrail, ] { assert_eq!(signal_from_tag(signal_tag(signal)), Some(signal)); } diff --git a/apps/landing/src/content/docs/integrations/gcp.md b/apps/landing/src/content/docs/integrations/gcp.md index 1cfa691a86..5f9cdd3ac1 100644 --- a/apps/landing/src/content/docs/integrations/gcp.md +++ b/apps/landing/src/content/docs/integrations/gcp.md @@ -76,7 +76,7 @@ Turning something off takes effect in Maple at once. Google Cloud keeps the sink ### Log filter -Platform, request and audit logs are forwarded. The default filter leaves out what costs a lot and says little: +Platform, request and audit logs are forwarded. Admin Activity and Policy Denied audit entries do not land in logs: they go to the audit log in the Security section, which only organization admins can read. System Event audit entries stay in logs, and so do Kubernetes API writes (`k8s.io`), which are mostly a cluster's own controllers. On a self-hosted gateway that forwards telemetry to a collector, every audit entry stays a log. The default filter leaves out what costs a lot and says little: | Left out | Reason | | --------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | diff --git a/apps/landing/src/content/docs/integrations/vercel.md b/apps/landing/src/content/docs/integrations/vercel.md index 35d219d618..9149058fee 100644 --- a/apps/landing/src/content/docs/integrations/vercel.md +++ b/apps/landing/src/content/docs/integrations/vercel.md @@ -69,13 +69,20 @@ Request logs carry `http.request.method`, `url.path` (and `url.query` when the r Vercel sends deployment traces with the `vercel.projectId` and `vercel.deploymentId` resource attributes. Its platform spans come in two services: `vercel.edge-network` for routing and caching, and `vercel.serverless-runtime` for functions. Maple moves each function invocation (`Invoke Function`) and its outbound calls to a service named after the project, the same name its logs use, so every project is its own service on the service map with its own dependencies. Cold-start steps (`Start VM`, `Spawn Node.js`, `Init User Code`) and `waitUntil` stay under `vercel.serverless-runtime`. Every function span keeps `maple_vercel.platform_service = vercel.serverless-runtime` as a resource attribute. -### Speed Insights, Web Analytics and audit log +### Speed Insights and Web Analytics -These drains arrive as logs, each under its own `service.name`: `vercel-speed-insights`, `vercel-web-analytics` and `vercel-audit-log`. The resource carries `cloud.provider` set to `vercel`, the team id and, where Vercel sends them, the project id, deployment id and environment. +These drains arrive as logs, each under its own `service.name`: `vercel-speed-insights` and `vercel-web-analytics`. The resource carries `cloud.provider` set to `vercel`, the team id and, where Vercel sends them, the project id, deployment id and environment. - **Speed Insights**: one log per measurement, such as `LCP 2.41s /dashboard`. The metric is in `vercel.speed_insights.metric` (`LCP`, `INP`, `CLS`, `FCP`, `TTFB` or `FID`) and the value in `vercel.speed_insights.value`, in milliseconds (CLS is a score). Each log also carries `url.path`, `vercel.route`, the visitor's country, region and city (`geo.*`), browser and OS (`user_agent.*`) and device. - **Web Analytics**: one log per page view or custom event, such as `pageview /pricing`. It carries `vercel.web_analytics.event.type` and `.event.name`, the event data as JSON in `vercel.web_analytics.event.data`, `url.path`, `url.query`, the referrer, and the same visitor fields as Speed Insights. -- **Audit log**: one log per change, such as `drain-created by Jane Doe`. It carries `vercel.audit.action`, the actor (`vercel.audit.actor.type`, `.id`, `.name`, `.email`), `client.address`, `user_agent.original`, and the event's details as JSON in `vercel.audit.payload`. + +### Audit log + +Audit events do not arrive as logs. They go to the audit log in the Security section, which only organization admins can read: an audit event names a person, their email and their IP address, and every member can read logs. + +Each event shows who acted (name, email and id), the action as Vercel names it, such as `drain-created`, what it was done to, the time, the IP address and user agent, and the event's payload as Vercel sent it. A payload over 32 KB is not kept. Maple accepts audit events only from the drain it creates on your team; a drain you add by hand on Vercel is refused. + +A self-hosted gateway that forwards telemetry to a collector has no audit log table. There, audit events arrive as logs under the `vercel-audit-log` service. ## AI Gateway @@ -105,13 +112,14 @@ The trace drain forwards these spans to Maple. You do not need to configure an e - **Infrastructure → Vercel → Overview**: requests, error rate, cache hit rate and cold-start rate across the team, AI Gateway spend by project, recent builds and failed builds, and, when those drains are on, Web Vitals, page views and audit log events. - **Infrastructure → Vercel → Projects**: each project with its last build, requests, 5xx rate, cache hit rate, cold-start rate and latency. Click a project for its own page. - **A project's page** has a tab for each part: - - **Traffic**: requests by status class, top paths with their cache hit rate, and top errors. - - **Functions**: invocations, cold-start rate and cold-start init time per route. - - **Dependencies**: the hosts its functions call, with calls, error rate and latency, and a link to the project on the service map. - - **Builds**: recent deployments with build time, state and why a failed build stopped. **Build logs** opens the logs explorer on that build's output. - - **AI Gateway**, **Speed Insights** (p75 Web Vitals by route and device) and **Web Analytics** (page views, devices, top pages, referrers, countries and custom events). + - **Traffic**: requests by status class, top paths with their cache hit rate, and top errors. + - **Functions**: invocations, cold-start rate and cold-start init time per route. + - **Dependencies**: the hosts its functions call, with calls, error rate and latency, and a link to the project on the service map. + - **Builds**: recent deployments with build time, state and why a failed build stopped. **Build logs** opens the logs explorer on that build's output. + - **AI Gateway**, **Speed Insights** (p75 Web Vitals by route and device) and **Web Analytics** (page views, devices, top pages, referrers, countries and custom events). + + A section whose drain is off says so and links to the setting that turns it on. - A section whose drain is off says so and links to the setting that turns it on. - **Service map**: each project with its outbound calls, from its function spans, and an edge from `vercel.edge-network` to every project it invokes. A project's Infrastructure tab links to its Vercel project page. - [Logs](/docs/explore/logs): filter by `service.name` (the project name) or `vercel.project.id`. Speed Insights, Web Analytics and audit log events are under their own `service.name`. - [Traces](/docs/explore/traces): your `@vercel/otel` service, the deployment traces, and `ai-gateway`. diff --git a/apps/web/src/lab/infra-lab/update.ts b/apps/web/src/lab/infra-lab/update.ts index 299a462ea3..893170e0c3 100644 --- a/apps/web/src/lab/infra-lab/update.ts +++ b/apps/web/src/lab/infra-lab/update.ts @@ -22,7 +22,6 @@ import { VERCEL_API_AI_GATEWAY, VERCEL_API_DEPLOYMENTS, VERCEL_API_DETAIL, - VERCEL_AUDIT_LOG, VERCEL_BREAKDOWN, VERCEL_BUCKETS, VERCEL_DEPENDENCIES, @@ -106,7 +105,6 @@ const vercelList = (flags: Flags): VercelList.Model => ({ deployments: answered(VERCEL_TEAM_DEPLOYMENTS), speedInsights: answered(VERCEL_SPEED_INSIGHTS), webAnalytics: answered(VERCEL_WEB_ANALYTICS), - auditLog: answered(VERCEL_AUDIT_LOG), }) const vercelProject = (flags: Flags): VercelProject.Model => ({ diff --git a/apps/web/src/lab/infra-lab/view.ts b/apps/web/src/lab/infra-lab/view.ts index a8e63e4b79..a3152c89c1 100644 --- a/apps/web/src/lab/infra-lab/view.ts +++ b/apps/web/src/lab/infra-lab/view.ts @@ -25,7 +25,6 @@ import * as VercelProject from "../../page/infra-integrations/vercel/project/mod import { projectPage } from "../../page/infra-integrations/vercel/project/view" import { projectRail } from "../../page/infra-integrations/vercel/project/view-rail" import { - auditLogSection, buildsSection, dependenciesSection, functionsSection, @@ -36,12 +35,13 @@ import { import { pickTimeSearch } from "../../page/kit/view-tabs" import { sectionHeading } from "../../page/kit/section-heading" import { sortRowsBy } from "../../page/kit/table-sort" +import { cn } from "@maple/ui/lib/utils" import { eyebrow } from "@maple/views/ui/eyebrow" +import { pageRightSidebarWidthStyles } from "@maple/views/ui/page-layout.styles" import { AsyncData } from "foldkit" import { toLoadFailure } from "@maple/views/data/load-failure" import { VERCEL_ALL_DRAINS_STATUS, - VERCEL_AUDIT_LOG, VERCEL_DEPENDENCIES, VERCEL_EMPTY_SPEED_INSIGHTS, VERCEL_EMPTY_WEB_ANALYTICS, @@ -184,7 +184,13 @@ type VercelInputs = Readonly<{ timeZone: string }> /** `VercelListPage` over the fixtures, in the view its search names. */ /** A page's main column beside its right rail, as `dashboardPage` lays them out at lg. */ const withRail = (h: HtmlBuilder, main: Html, rail: Html): Html => - h.div([h.Class("flex items-start gap-6")], [h.div([h.Class("min-w-0 flex-1")], [main]), rail]) + h.div( + [h.Class("flex items-start gap-6")], + [ + h.div([h.Class("min-w-0 flex-1")], [main]), + h.div([h.Class(cn("shrink-0", pageRightSidebarWidthStyles))], [rail]), + ], + ) const vercelListView = Submodel.defineView( (list, inputs, h) => { @@ -271,14 +277,6 @@ const vercelSections = (h: H, model: Model): Html => { compact: false, }), ), - ...variants("Audit log", (state) => - auditLogSection(h, { - read: sectionRead(state, VERCEL_AUDIT_LOG, []), - enabled: state !== "off", - clock, - projectName: nameOf, - }), - ), ]) } diff --git a/apps/web/src/lab/security-audit-log-fixture.ts b/apps/web/src/lab/security-audit-log-fixture.ts index 71345ba1cd..c8ddb6b941 100644 --- a/apps/web/src/lab/security-audit-log-fixture.ts +++ b/apps/web/src/lab/security-audit-log-fixture.ts @@ -7,6 +7,8 @@ import { Array as Arr, String as Str } from "effect" const MINUTE = 60_000 type Seed = Readonly<{ + provider?: string + account?: string minutesAgo: number action: string category: string @@ -40,6 +42,34 @@ const deployKey: AuditTrailEntry["actor"] = { const system: AuditTrailEntry["actor"] = { type: "system", id: null, name: null, email: null } const SEEDS: ReadonlyArray = [ + { + provider: "vercel", + account: "team_acme", + minutesAgo: 2, + action: "project.env-var.created", + category: "credential", + actor: { type: "user", id: "usr_9f2c", name: "Grace Hopper", email: "grace@acme.test" }, + originIp: "203.0.113.90", + raw: { id: "evt_1", action: "project.env-var.created", payload: { projectId: "prj_web" } }, + }, + { + provider: "gcp", + account: "acme-prod", + minutesAgo: 11, + action: "google.iam.admin.v1.CreateServiceAccountKey", + category: "credential", + outcome: "denied", + outcomeReason: "Permission iam.serviceAccountKeys.create denied", + actor: { + type: "service", + id: "ci@acme-prod.iam.gserviceaccount.com", + name: null, + email: "ci@acme-prod.iam.gserviceaccount.com", + }, + target: { type: "service_account", name: "projects/-/serviceAccounts/deploy@acme-prod" }, + originIp: "198.51.100.24", + raw: { insertId: "abc123", protoPayload: { serviceName: "iam.googleapis.com" } }, + }, { minutesAgo: 4, action: "alert_rule.updated", @@ -130,8 +160,8 @@ const eventOf = (seed: Seed, index: number, nowMs: number): AuditTrailEntry => { const occurredAt = nowMs - seed.minutesAgo * MINUTE return { id: Str.padStart(32, "0")(index.toString(16)), - provider: "maple", - account: null, + provider: seed.provider ?? "maple", + account: seed.account ?? null, providerEventId: `alog_4CzLmR1pTxWvYbNhQd82K${index}`, occurredAt, ingestedAt: occurredAt + 900, diff --git a/apps/web/src/lab/vercel-infra-fixtures.ts b/apps/web/src/lab/vercel-infra-fixtures.ts index fd2c11e40b..6708727e62 100644 --- a/apps/web/src/lab/vercel-infra-fixtures.ts +++ b/apps/web/src/lab/vercel-infra-fixtures.ts @@ -1,6 +1,6 @@ // Fixture rows for the /infra/vercel lab frames: four projects (one erroring, one with no traces, // one forwarded but silent), AI Gateway usage across three models, team builds, one project's -// detail, functions and dependencies, and Speed Insights, Web Analytics and audit log readings. +// detail, functions and dependencies, and Speed Insights and Web Analytics readings. import { VercelAiGatewayResponse, @@ -14,7 +14,6 @@ import { VercelProjectStatus, VercelTeam, type VercelAiGatewayBucket, - type VercelAuditEvent, type VercelInfraDependency, type VercelInfraFunction, type VercelInfraFunctionBucket, @@ -418,7 +417,7 @@ export const VERCEL_API_DEPLOYMENTS: ReadonlyArray = [ deployment("api-v1", "READY", "main", "Products: cache the price table for 60s", 3 * 24 * HOUR), ] -/** Every drain on: the overview's Speed Insights, Web Analytics and audit log sections read. */ +/** Every drain on: the Speed Insights and Web Analytics sections read, the rail points at the audit log. */ export const VERCEL_ALL_DRAINS_STATUS = new VercelIntegrationStatus({ ...VERCEL_STATUS, speedInsightsEnabled: true, @@ -556,27 +555,3 @@ export const VERCEL_EMPTY_WEB_ANALYTICS = new VercelWebAnalyticsResponse({ countries: [], events: [], }) - -export const VERCEL_AUDIT_LOG: ReadonlyArray = [ - { - timestamp: new Date(NOW - 12 * MINUTE).toISOString(), - action: "project.env-variable.updated", - actor: "Rosa Moreau", - actorType: "user", - projectId: "prj_api", - }, - { - timestamp: new Date(NOW - 3 * HOUR).toISOString(), - action: "drain-created", - actor: "Maple", - actorType: "app", - projectId: "", - }, - { - timestamp: new Date(NOW - 26 * HOUR).toISOString(), - action: "team.member.added", - actor: "Sam Okafor", - actorType: "user", - projectId: "", - }, -] diff --git a/apps/web/src/page/infra-integrations/gcp/integration/gcp-configure-copy.ts b/apps/web/src/page/infra-integrations/gcp/integration/gcp-configure-copy.ts index b5a556f14a..ff068f26ef 100644 --- a/apps/web/src/page/infra-integrations/gcp/integration/gcp-configure-copy.ts +++ b/apps/web/src/page/infra-integrations/gcp/integration/gcp-configure-copy.ts @@ -60,7 +60,8 @@ export const CAPABILITIES = { logs: { title: "Log forwarding", flag: "logs_enabled", - description: "Request, audit and managed-service logs, through a Log Router sink and Pub/Sub.", + description: + "Request and managed-service logs, through a Log Router sink and Pub/Sub. Admin activity goes to the Security audit log.", run: { on: "Log sink and Pub/Sub topic", off: "Removes the log sink" }, effects: { "starts-after-script": "Starts once it's set up in Google Cloud", diff --git a/apps/web/src/page/infra-integrations/gcp/integration/view-flow-gcp.ts b/apps/web/src/page/infra-integrations/gcp/integration/view-flow-gcp.ts index 853243c670..c6020dffd4 100644 --- a/apps/web/src/page/infra-integrations/gcp/integration/view-flow-gcp.ts +++ b/apps/web/src/page/infra-integrations/gcp/integration/view-flow-gcp.ts @@ -188,7 +188,7 @@ const chooseRail = (h: H): Html => { icon: "UnorderedListIcon", title: "Google Cloud's logs", - detail: "Request, audit and managed-service logs", + detail: "Request and managed-service logs, and admin activity for the audit log", }, { icon: "ServerIcon", diff --git a/apps/web/src/page/infra-integrations/gcp/integration/view-gcp-configure-filter.ts b/apps/web/src/page/infra-integrations/gcp/integration/view-gcp-configure-filter.ts index 7229cca8f4..7a65b3610f 100644 --- a/apps/web/src/page/infra-integrations/gcp/integration/view-gcp-configure-filter.ts +++ b/apps/web/src/page/infra-integrations/gcp/integration/view-gcp-configure-filter.ts @@ -134,7 +134,7 @@ export const filterField = ( h, icon(h, "CheckIcon", { size: 14, className: "text-severity-info" }), "Platform logs", - "Requests, audit, deploys, crashes, scaling", + "Requests, deploys, crashes, scaling, and admin activity for the audit log", ), summaryRow( h, diff --git a/apps/web/src/page/infra-integrations/vercel/data.ts b/apps/web/src/page/infra-integrations/vercel/data.ts index 9607d94121..2f31cd3a3c 100644 --- a/apps/web/src/page/infra-integrations/vercel/data.ts +++ b/apps/web/src/page/infra-integrations/vercel/data.ts @@ -6,7 +6,6 @@ import type { } from "@maple/domain/http" import { getVercelAiGateway, - getVercelAuditLog, getVercelBreakdown, getVercelLatencyTimeseries, getVercelDependencies, @@ -199,13 +198,6 @@ export const loadVercelSpeedInsights = (input: WindowInput) => export const loadVercelWebAnalytics = (input: WindowInput) => getVercelWebAnalytics({ data: { ...input } }).pipe(Effect.mapError(toLoadFailure), Effect.result) -/** `vercelAuditLogResultAtom({ data: window })`. */ -export const loadVercelAuditLog = (window: TimeWindow) => - getVercelAuditLog({ data: { startTime: window.startTime, endTime: window.endTime } }).pipe( - Effect.mapBoth({ onSuccess: (response) => response.events, onFailure: toLoadFailure }), - Effect.result, - ) - export const DEPLOYMENTS_LIMIT = 20 /** `integrations.vercelDeployments`: the newest deployments of one project, or of the whole team. */ diff --git a/apps/web/src/page/infra-integrations/vercel/integration/vercel.scene.test.ts b/apps/web/src/page/infra-integrations/vercel/integration/vercel.scene.test.ts index 9e2fe1a837..4c41df33d1 100644 --- a/apps/web/src/page/infra-integrations/vercel/integration/vercel.scene.test.ts +++ b/apps/web/src/page/infra-integrations/vercel/integration/vercel.scene.test.ts @@ -337,6 +337,9 @@ describe("vercel card: connected", () => { expectView(text("Connected")).toExist(), expectView(within(role("row", { name: /Logs/ }), text("live"))).toExist(), expectView(within(role("row", { name: /Speed Insights/ }), text("off"))).toExist(), + expectView( + within(role("row", { name: /Audit log/ }), text(/Security audit log, org admins only/)), + ).toExist(), expectView(role("link", { name: "storefront" })).toHaveAttr("href", "/infra/vercel/prj_web"), expectView(role("link", { name: /Open Vercel dashboard/ })).toHaveAttr("href", "/infra/vercel"), ) @@ -527,16 +530,14 @@ describe("vercel card: configure", () => { ).toEqual(expect.objectContaining({ request: { aiGatewayEnabled: false } })) }) - test("turning on the audit log names its plan and saves only that switch at once", () => { + test("turning on the audit log names where its events show and its plan, and saves only that switch at once", () => { const opened = configured(connected()) scene( { update: step, view }, given(opened), mounted(), expectView( - text( - "Who changed what on the team, for the whole team whatever projects you pick. Enterprise plan only.", - ), + text(/Shown in the Security audit log to org admins\. Enterprise plan only\./), ).toExist(), ) const sent = step( diff --git a/apps/web/src/page/infra-integrations/vercel/integration/view-vercel-configure.ts b/apps/web/src/page/infra-integrations/vercel/integration/view-vercel-configure.ts index f6ee9f2597..3b353ea13a 100644 --- a/apps/web/src/page/infra-integrations/vercel/integration/view-vercel-configure.ts +++ b/apps/web/src/page/infra-integrations/vercel/integration/view-vercel-configure.ts @@ -348,7 +348,7 @@ const SIGNALS = [ title: "Audit log", drain: "Maple audit log", description: - "Who changed what on the team, for the whole team whatever projects you pick. Enterprise plan only.", + "Who changed what on the team, whatever projects you pick. Shown in the Security audit log to org admins. Enterprise plan only.", }, ] as const diff --git a/apps/web/src/page/infra-integrations/vercel/integration/view-vercel.ts b/apps/web/src/page/infra-integrations/vercel/integration/view-vercel.ts index 3ad8961deb..65a45bdaab 100644 --- a/apps/web/src/page/infra-integrations/vercel/integration/view-vercel.ts +++ b/apps/web/src/page/infra-integrations/vercel/integration/view-vercel.ts @@ -270,7 +270,7 @@ const SIGNALS = { auditLog: { title: "Audit log", carries: "Team actions: env vars, domains, members", - powers: "Team activity", + powers: "Security audit log, org admins only", }, } as const satisfies Record<(typeof VERCEL_DRAINS)[number]["key"], unknown> diff --git a/apps/web/src/page/infra-integrations/vercel/list/command.ts b/apps/web/src/page/infra-integrations/vercel/list/command.ts index d1df6f932a..050bc76f0f 100644 --- a/apps/web/src/page/infra-integrations/vercel/list/command.ts +++ b/apps/web/src/page/infra-integrations/vercel/list/command.ts @@ -4,7 +4,6 @@ import { linkedCursorStream } from "@maple/views/charts/linked-cursor" import { resolveWindow, TimeWindow } from "../../../../data/time-range" import { loadVercelAiGateway, - loadVercelAuditLog, loadVercelDeployments, loadVercelLatencySeries, loadVercelProjects, @@ -102,13 +101,6 @@ export const FetchWebAnalytics = Command.define("FetchVercelWebAnalytics", { ), }) -export const FetchAuditLog = Command.define("FetchVercelAuditLog", { - args: { key: Schema.String, window: TimeWindow }, - messages: [Message.CompletedFetchAuditLog], - execute: ({ key, window }) => - Effect.map(loadVercelAuditLog(window), (result) => Message.CompletedFetchAuditLog({ key, result })), -}) - export const CompleteReloadSpin = Command.define("CompleteVercelReloadSpin", { args: { version: Schema.Number }, messages: [Message.CompletedReloadSpin], diff --git a/apps/web/src/page/infra-integrations/vercel/list/model.ts b/apps/web/src/page/infra-integrations/vercel/list/model.ts index 959b226be8..4255065a37 100644 --- a/apps/web/src/page/infra-integrations/vercel/list/model.ts +++ b/apps/web/src/page/infra-integrations/vercel/list/model.ts @@ -1,6 +1,5 @@ import { VercelAiGatewayResponse, - VercelAuditEvent, VercelDeploymentSummary, VercelInfraLatencyBucket, VercelInfraProject, @@ -37,7 +36,6 @@ export const Model = Schema.Struct({ deployments: KeyedSchema(Schema.Array(VercelDeploymentSummary), LoadFailure), speedInsights: KeyedSchema(VercelSpeedInsightsResponse, LoadFailure), webAnalytics: KeyedSchema(VercelWebAnalyticsResponse, LoadFailure), - auditLog: KeyedSchema(Schema.Array(VercelAuditEvent), LoadFailure), charts: Schema.Record(Schema.String, Chart.Model), chartRuntime: ChartRuntime, themeColors: Schema.NullOr(ThemeColors), @@ -65,7 +63,6 @@ export const Message = defineMessageUnion({ CompletedFetchDeployments: result(Schema.Array(VercelDeploymentSummary)), CompletedFetchSpeedInsights: result(VercelSpeedInsightsResponse), CompletedFetchWebAnalytics: result(VercelWebAnalyticsResponse), - CompletedFetchAuditLog: result(Schema.Array(VercelAuditEvent)), CompletedLoadChartRuntime: {}, FailedLoadChartRuntime: { reason: Schema.String }, ResolvedThemeColors: { colors: ThemeColors }, @@ -80,7 +77,6 @@ export const Message = defineMessageUnion({ ClickedRetryDeployments: {}, ClickedRetrySpeedInsights: {}, ClickedRetryWebAnalytics: {}, - ClickedRetryAuditLog: {}, ChangedSearch: { value: Schema.String }, ClickedClearSearch: {}, /** A fleet band cell: its health filter on the projects table, or none to leave it. */ diff --git a/apps/web/src/page/infra-integrations/vercel/list/update.ts b/apps/web/src/page/infra-integrations/vercel/list/update.ts index 7a07f9792f..851d163e2e 100644 --- a/apps/web/src/page/infra-integrations/vercel/list/update.ts +++ b/apps/web/src/page/infra-integrations/vercel/list/update.ts @@ -16,7 +16,6 @@ import { DEFAULT_PRESET, DEPLOYMENTS_LIMIT, seriesInput } from "../data" import { CompleteReloadSpin, FetchAiGateway, - FetchAuditLog, FetchBuckets, FetchDeployments, FetchLatency, @@ -62,7 +61,6 @@ type Slot = | "deployments" | "speedInsights" | "webAnalytics" - | "auditLog" /** The reads the overview shows, once the status says connected: each only while its drain is on. */ const sync = ( @@ -96,7 +94,6 @@ const sync = ( const deploymentsKey = requestKey(org, "integrations.vercelDeployments", { limit: DEPLOYMENTS_LIMIT }) const speedKey = requestKey(org, "getVercelSpeedInsights", { data: window }) const webKey = requestKey(org, "getVercelWebAnalytics", { data: window }) - const auditKey = requestKey(org, "getVercelAuditLog", { data: window }) return { model: { ...model, @@ -125,9 +122,6 @@ const sync = ( webAnalytics: take("webAnalytics", status.webAnalyticsEnabled, model.webAnalytics, webKey, () => FetchWebAnalytics({ key: webKey, input: window }), ), - auditLog: take("auditLog", status.auditLogEnabled, model.auditLog, auditKey, () => - FetchAuditLog({ key: auditKey, window }), - ), }, commands, } @@ -154,7 +148,6 @@ export const init = (route: { readonly search: Search }, context: PageContext): deployments: Keyed.idle(), speedInsights: Keyed.idle(), webAnalytics: Keyed.idle(), - auditLog: Keyed.idle(), charts: {}, chartRuntime: { _tag: "Loading" }, themeColors: null, @@ -221,9 +214,6 @@ export const update = (model: Model, message: Message, context: PageContext): Re CompletedFetchWebAnalytics: ({ key, result }) => ({ model: { ...model, webAnalytics: Keyed.accept(model.webAnalytics, key, result) }, }), - CompletedFetchAuditLog: ({ key, result }) => ({ - model: { ...model, auditLog: Keyed.accept(model.auditLog, key, result) }, - }), CompletedLoadChartRuntime: () => ({ model: { ...model, chartRuntime: { _tag: "Ready" } } }), FailedLoadChartRuntime: ({ reason }) => ({ model: { ...model, chartRuntime: { _tag: "Failed", reason } }, @@ -239,7 +229,6 @@ export const update = (model: Model, message: Message, context: PageContext): Re ClickedRetryDeployments: () => sync(model, context, "Refresh", ["deployments"]), ClickedRetrySpeedInsights: () => sync(model, context, "Refresh", ["speedInsights"]), ClickedRetryWebAnalytics: () => sync(model, context, "Refresh", ["webAnalytics"]), - ClickedRetryAuditLog: () => sync(model, context, "Refresh", ["auditLog"]), ChangedSearch: ({ value }) => navigate(model, { ...model.search, q: value === "" ? undefined : value }), ClickedClearSearch: () => navigate(model, { ...model.search, q: undefined }), diff --git a/apps/web/src/page/infra-integrations/vercel/list/vercel.scene.test.ts b/apps/web/src/page/infra-integrations/vercel/list/vercel.scene.test.ts index 46afb70f54..71641b89d9 100644 --- a/apps/web/src/page/infra-integrations/vercel/list/vercel.scene.test.ts +++ b/apps/web/src/page/infra-integrations/vercel/list/vercel.scene.test.ts @@ -22,7 +22,6 @@ import { toLoadFailure } from "@maple/views/data/load-failure" import { VERCEL_AI_GATEWAY, VERCEL_ALL_DRAINS_STATUS, - VERCEL_AUDIT_LOG, VERCEL_BUCKETS, VERCEL_LATENCY_BUCKETS, VERCEL_PROJECTS, @@ -108,11 +107,6 @@ const loaded = ( key: model.webAnalytics.key ?? "", result: Result.succeed(VERCEL_WEB_ANALYTICS), }), - (model) => - Message.CompletedFetchAuditLog({ - key: model.auditLog.key ?? "", - result: Result.succeed(VERCEL_AUDIT_LOG), - }), ]) /** The commands the status answer starts, the window already resolved. */ @@ -169,17 +163,12 @@ describe("vercel overview", () => { const defaults = readsFor(VERCEL_STATUS) expect(defaults).not.toContain("FetchVercelSpeedInsights") expect(defaults).not.toContain("FetchVercelWebAnalytics") - expect(defaults).not.toContain("FetchVercelAuditLog") expect(readsFor(VERCEL_TRACES_ONLY_STATUS)).not.toContain("FetchVercelRequestSeries") expect( readsFor(new VercelIntegrationStatus({ ...VERCEL_STATUS, tracesEnabled: false })), ).not.toContain("FetchVercelLatencySeries") expect(readsFor(VERCEL_ALL_DRAINS_STATUS)).toEqual( - expect.arrayContaining([ - "FetchVercelSpeedInsights", - "FetchVercelWebAnalytics", - "FetchVercelAuditLog", - ]), + expect.arrayContaining(["FetchVercelSpeedInsights", "FetchVercelWebAnalytics"]), ) }) @@ -298,22 +287,41 @@ describe("vercel overview", () => { expectView(text(VERCEL_AI_GATEWAY.models[0]?.model ?? "")).toExist(), expectView(text(/Speed Insights is off/)).toExist(), expectView(text(/Web Analytics is off/)).toExist(), - expectView(text("Team activity")).toBeAbsent(), ) }) - test("with every drain on, the overview shows Web Vitals, page views and team activity", () => { + test("with every drain on, the overview shows Web Vitals and page views", () => { scene( { update: pageUpdate, view }, given(loaded(VERCEL_ALL_DRAINS_STATUS, VERCEL_PROJECTS)), themeResolved(), expectView(text("needs work")).toExist(), expectView(text("184.2K")).toExist(), - expectView(text("project.env-variable.updated")).toExist(), expectView(text(/Speed Insights is off/)).toBeAbsent(), ) }) + test("the audit log drain puts a link to the Security audit log in the rail, and nothing when it is off", () => { + scene( + { update: pageUpdate, view }, + given(loaded(VERCEL_ALL_DRAINS_STATUS, VERCEL_PROJECTS)), + themeResolved(), + expectView(text("Team activity")).toExist(), + expectView(text("Audit events from this team are in the Security audit log.")).toExist(), + expectView(role("link", { name: "Open the audit log" })).toHaveAttr( + "href", + "/security/audit-log?provider=vercel", + ), + ) + scene( + { update: pageUpdate, view }, + given(loaded(VERCEL_STATUS, VERCEL_PROJECTS)), + themeResolved(), + expectView(text("Team activity")).toBeAbsent(), + expectView(role("link", { name: "Open the audit log" })).toBeAbsent(), + ) + }) + test("a retired projects link lands on the one overview", () => { scene( { update: pageUpdate, view }, diff --git a/apps/web/src/page/infra-integrations/vercel/list/view-rail.ts b/apps/web/src/page/infra-integrations/vercel/list/view-rail.ts index 093b67ce19..f7103935ca 100644 --- a/apps/web/src/page/infra-integrations/vercel/list/view-rail.ts +++ b/apps/web/src/page/infra-integrations/vercel/list/view-rail.ts @@ -16,14 +16,14 @@ import { type Clock, dataOf } from "../view-parts" import type { Overview } from "./derive" import type { Message, Model } from "./model" -// `VercelOverviewRail`: the team, its drains, the deploys in the window and the team's activity. +// `VercelOverviewRail`: the team, its drains, the deploys in the window and where its audit events are. type H = HtmlBuilder const RAIL_DEPLOYS = 5 -const RAIL_EVENTS = 3 const settingsHref = () => hrefOf(AppRoute.Integrations({ search: { integration: "vercel" } })) +const auditLogHref = () => hrefOf(AppRoute.SecurityAuditLog({ search: { provider: "vercel" } })) const value = (h: H, text: string): Html => typography(h, { size: "xs", tone: "foreground", mono: true, truncate: true }, [text]) @@ -170,42 +170,18 @@ const deploysGroup = (h: H, model: Model, o: Overview, clock: Clock): Html | nul ) } -const activityGroup = (h: H, model: Model, o: Overview, clock: Clock): Html | null => { - const events = dataOf(model.auditLog.data) - if (!o.status.auditLogEnabled || events === null) return null - const newest = Arr.take( - Arr.sort( - events, - Order.mapInput(Order.flip(Order.String), (e: (typeof events)[number]) => e.timestamp), - ), - RAIL_EVENTS, - ) - return detailRailGroup( - h, - "Team activity", - newest.length === 0 - ? [typography(h, { size: "2xs", tone: "muted", as: "p" }, ["No team activity in this window"])] - : Arr.map(newest, (event) => - vstack(h, { gap: "0.5", className: "py-1" }, [ - typography(h, { size: "xs", tone: "foreground", mono: true, truncate: true }, [ - event.action, - ]), - hstack(h, { gap: "1" }, [ - typography(h, { size: "3xs", tone: "muted", truncate: true }, [ - `${event.actor} · ${event.projectId === "" ? "team" : o.projectName(event.projectId)} ·`, - ]), - relativeTime(h, { - value: event.timestamp, - ...clock, - variant: "short", - mono: true, - className: "shrink-0 text-3xs text-muted-foreground", - }), - ]), - ]), - ), - ) -} +// Audit events are admin-only, so they live in the Security audit log: the rail points there. +const activityGroup = (h: H, o: Overview): Html | null => + o.status.auditLogEnabled + ? detailRailGroup(h, "Team activity", [ + typography(h, { size: "2xs", tone: "muted", as: "p" }, [ + "Audit events from this team are in the Security audit log.", + ]), + typography(h, { size: "2xs", className: "self-start" }, [ + textLink(h, { href: auditLogHref() }, ["Open the audit log"]), + ]), + ]) + : null export const overviewRail = (h: H, model: Model, o: Overview, clock: Clock): Html => vstack( @@ -216,7 +192,7 @@ export const overviewRail = (h: H, model: Model, o: Overview, clock: Clock): Htm teamGroup(h, o, clock), drainsGroup(h, o), deploysGroup(h, model, o, clock), - activityGroup(h, model, o, clock), + activityGroup(h, o), ], (group): group is Html => group !== null, ), diff --git a/apps/web/src/page/infra-integrations/vercel/sections.ts b/apps/web/src/page/infra-integrations/vercel/sections.ts index 91d140dccb..94cab70dc5 100644 --- a/apps/web/src/page/infra-integrations/vercel/sections.ts +++ b/apps/web/src/page/infra-integrations/vercel/sections.ts @@ -1,5 +1,4 @@ import type { - VercelAuditEvent, VercelDeploymentSummary, VercelInfraDependency, VercelInfraFunction, @@ -912,93 +911,3 @@ export const webAnalyticsSection = ( ] }, }) - -// Audit log - -const AUDIT_LOG_OFF: DrainOff = { - lead: "The audit log is off.", - text: "Turn on the Audit Log drain to see who changed the team: members, domains, env vars and drains.", -} - -/** The team's newest audit log events. */ -export const auditLogSection = ( - h: HtmlBuilder, - options: Readonly<{ - read: SectionRead, M> - enabled: boolean - clock: Clock - projectName: (projectId: string) => string - }>, -): Html => - readSection(h, { - title: "Audit log", - hint: "newest first", - read: options.read, - off: options.enabled ? null : AUDIT_LOG_OFF, - failure: "Couldn't load the audit log", - loading: () => tableLoading(h, "Audit log"), - body: (events) => [ - dataTable( - h, - { ariaLabel: "Audit log" }, - [ - columnHead(h, { label: "Action", width: LEAD_COLUMN }), - columnHead(h, { label: "Actor", width: "w-[160px]" }), - columnHead(h, { label: "Project", width: "w-[140px]", hidden: "hidden md:flex" }), - columnHead(h, { label: "When", width: "w-[72px]", align: "right" }), - ], - events.length === 0 - ? [dataTableEmpty(h, ["No audit events in this window"])] - : Arr.map(events, (event, index) => - h.div( - [h.Key(`${event.timestamp}:${index}`), h.Class(ROW_CLASS)], - [ - typography( - h, - { - size: "xs", - tone: "foreground", - mono: true, - truncate: true, - className: LEAD_COLUMN, - }, - [event.action], - ), - typography( - h, - { size: "xs", tone: "soft", truncate: true, className: "w-[160px]" }, - [event.actor === "" ? event.actorType || EMPTY_VALUE : event.actor], - ), - typography( - h, - { - size: "xs", - tone: "muted", - mono: true, - truncate: true, - className: "hidden w-[140px] md:block", - }, - [ - event.projectId === "" - ? "team" - : options.projectName(event.projectId), - ], - ), - h.span( - [h.Class("w-[72px] text-right")], - [ - relativeTime(h, { - value: event.timestamp, - ...options.clock, - variant: "short", - mono: true, - className: typographyClass({ size: "2xs", tone: "muted" }), - }), - ], - ), - ], - ), - ), - ), - ], - }) diff --git a/apps/web/src/page/security/audit-log/audit-log.scene.test.ts b/apps/web/src/page/security/audit-log/audit-log.scene.test.ts index 872f89cabd..502fa30e70 100644 --- a/apps/web/src/page/security/audit-log/audit-log.scene.test.ts +++ b/apps/web/src/page/security/audit-log/audit-log.scene.test.ts @@ -1,7 +1,7 @@ import { OrgId } from "@maple/domain" import { AuditTrailForbiddenError, AuditTrailPersistenceError } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" -import { Result, Schema, Array as Arr, String as Str } from "effect" +import { Option, Result, Schema, Array as Arr, String as Str } from "effect" import { expect as expectView, given, role, scene, text, within, withViewInputs } from "foldkit/scene" import { describe, expect, test } from "vitest" import { toLoadFailure } from "@maple/views/data/load-failure" @@ -186,25 +186,55 @@ describe("security audit log", () => { ) }) - test("opening an event shows what was recorded about it, and closing hides it", () => { + /** The first loaded event from `provider`, opened. */ + const openedFrom = (provider: string) => { const model = loaded() - const first = eventsOf(model)?.[0] - const opened = pageUpdate(model, Message.ToggledEvent({ id: first?.id ?? "" })).model + const event = Arr.findFirst(eventsOf(model) ?? [], (candidate) => candidate.provider === provider) + const id = Option.match(event, { onNone: () => "", onSome: (found) => found.id }) + return { + event: Option.getOrUndefined(event), + id, + opened: pageUpdate(model, Message.ToggledEvent({ id })).model, + } + } + + test("opening an event shows what was recorded about it, and closing hides it", () => { + const { event, id, opened } = openedFrom("maple") scene( { update: pageUpdate, view }, given(opened), expectView(text("Payload")).toExist(), - expectView(text(first?.providerEventId ?? "")).toExist(), - expectView(text("203.0.113.7")).toExist(), + expectView(text(event?.providerEventId ?? "")).toExist(), + expectView(text(event?.originIp ?? "")).toExist(), ) scene( { update: pageUpdate, view }, - given(pageUpdate(opened, Message.ToggledEvent({ id: first?.id ?? "" })).model), + given(pageUpdate(opened, Message.ToggledEvent({ id })).model), expectView(text("Payload")).toBeAbsent(), ) }) + test("each event names the platform it came from, and the account on it", () => { + scene( + { update: pageUpdate, view }, + given(loaded()), + expectView(within(eventsTable, text("Vercel"))).toExist(), + expectView(within(eventsTable, text("Google Cloud"))).toExist(), + ) + scene( + { update: pageUpdate, view }, + given(openedFrom("gcp").opened), + expectView(text("acme-prod")).toExist(), + ) + }) + + test("the platform filter goes to the URL like the others", () => { + const vercel = pageUpdate(loaded(), Message.ChangedFilter({ filter: "provider", value: "vercel" })) + + expect(vercel.outMessage).toMatchObject({ url: hrefOf(routeOf({ provider: "vercel" })) }) + }) + test("a filter change goes to the URL, and choosing All drops it", () => { const denied = pageUpdate(loaded(), Message.ChangedFilter({ filter: "outcome", value: "denied" })) expect(denied.outMessage).toMatchObject({ url: hrefOf(routeOf({ outcome: "denied" })) }) @@ -313,20 +343,30 @@ describe("security audit log", () => { }) test("a Maple event's id opens the same entry in the Settings audit log", () => { - const model = loaded() - const first = eventsOf(model)?.[0] - const opened = pageUpdate(model, Message.ToggledEvent({ id: first?.id ?? "" })).model + const { event, opened } = openedFrom("maple") scene( { update: pageUpdate, view }, given(opened), - expectView(role("link", { name: first?.providerEventId ?? "" })).toHaveAttr( + expectView(role("link", { name: event?.providerEventId ?? "" })).toHaveAttr( "href", - auditEntryHref(first?.occurredAt ?? 0, first?.providerEventId ?? ""), + auditEntryHref(event?.occurredAt ?? 0, event?.providerEventId ?? ""), ), ) }) + // Only Maple's own events are entries of the Settings audit log. + test("another platform's event id is text, not a link into Maple's own audit log", () => { + const { event, opened } = openedFrom("vercel") + + scene( + { update: pageUpdate, view }, + given(opened), + expectView(text(event?.providerEventId ?? "")).toExist(), + expectView(role("link", { name: event?.providerEventId ?? "" })).toBeAbsent(), + ) + }) + test("an empty log says what will show up; an empty filter result offers to clear the filters", () => { scene( { update: pageUpdate, view }, diff --git a/apps/web/src/page/security/audit-log/command.ts b/apps/web/src/page/security/audit-log/command.ts index 1109175ffc..040819ae77 100644 --- a/apps/web/src/page/security/audit-log/command.ts +++ b/apps/web/src/page/security/audit-log/command.ts @@ -23,6 +23,7 @@ const listEvents = (search: SecurityAuditLogSearch, cursor: string | undefined) Effect.flatMap(MapleInternalApiClient, (api) => api.auditTrail.list({ query: { + ...(search.provider !== undefined ? { provider: search.provider } : undefined), ...(search.category !== undefined ? { category: search.category } : undefined), ...(search.outcome !== undefined ? { outcome: search.outcome } : undefined), ...(search.actor !== undefined ? { actor: search.actor } : undefined), diff --git a/apps/web/src/page/security/audit-log/derive.ts b/apps/web/src/page/security/audit-log/derive.ts index a98638cc9c..5235d8fbed 100644 --- a/apps/web/src/page/security/audit-log/derive.ts +++ b/apps/web/src/page/security/audit-log/derive.ts @@ -1,12 +1,18 @@ -import type { AuditTrailCategory, AuditTrailOutcome } from "@maple/domain/audit-trail" +import type { AuditTrailCategory, AuditTrailOutcome, AuditTrailProvider } from "@maple/domain/audit-trail" import type { AuditTrailEntry } from "@maple/domain/http" import type { LoadFailure } from "@maple/views/data/load-failure" import { Array as Arr, Option, Record as Rec } from "effect" import { AsyncData } from "foldkit" import type { Model, Search } from "./model" -// What the audit log shows of an event: the names of categories and outcomes, who acted and on -// what. A value this build does not know shows as itself. +// What the audit log shows of an event: the names of platforms, categories and outcomes, who acted +// and on what. A value this build does not know shows as itself. + +export const PROVIDER_LABEL = { + maple: "Maple", + vercel: "Vercel", + gcp: "Google Cloud", +} satisfies Record export const CATEGORY_LABEL = { authentication: "Authentication", @@ -27,8 +33,12 @@ export const OUTCOME_LABEL = { unknown: "Unknown", } satisfies Record -/** The outcomes a platform writes today: Maple records what went through and what it refused. */ -export const OUTCOME_FILTERS = ["success", "denied"] as const satisfies ReadonlyArray +/** The outcomes worth filtering by. `unknown` is a platform that did not say, not something to look for. */ +export const OUTCOME_FILTERS = [ + "success", + "failure", + "denied", +] as const satisfies ReadonlyArray const ACTOR_TYPE_LABEL = { user: "User", @@ -41,6 +51,7 @@ const ACTOR_TYPE_LABEL = { export const labelIn = (labels: Readonly>, value: string): string => Option.getOrElse(Rec.get(labels, value), () => value) +export const providerLabel = (provider: string): string => labelIn(PROVIDER_LABEL, provider) export const categoryLabel = (category: string): string => labelIn(CATEGORY_LABEL, category) export const outcomeLabel = (outcome: string): string => labelIn(OUTCOME_LABEL, outcome) export const actorTypeLabel = (type: string): string => labelIn(ACTOR_TYPE_LABEL, type) @@ -60,7 +71,10 @@ export const targetText = (event: AuditTrailEntry): string | null => event.targe export const isMarked = (event: AuditTrailEntry): boolean => event.outcome !== "success" export const isFiltered = (search: Search): boolean => - search.category !== undefined || search.outcome !== undefined || search.actor !== undefined + search.provider !== undefined || + search.category !== undefined || + search.outcome !== undefined || + search.actor !== undefined /** The API refused the viewer: the same thing the session says for a member without the permission. */ export const isForbidden = (failure: LoadFailure): boolean => failure.code === "audit_trail_forbidden" diff --git a/apps/web/src/page/security/audit-log/model.ts b/apps/web/src/page/security/audit-log/model.ts index 5b7ad3fba9..70e7fc450b 100644 --- a/apps/web/src/page/security/audit-log/model.ts +++ b/apps/web/src/page/security/audit-log/model.ts @@ -25,7 +25,7 @@ export const LaterPages = Schema.Struct({ export type LaterPages = typeof LaterPages.Type /** The filters a select sets; the actor is typed and applied on Enter. */ -export const SelectFilter = Schema.Literals(["category", "outcome"]) +export const SelectFilter = Schema.Literals(["provider", "category", "outcome"]) export type SelectFilter = typeof SelectFilter.Type export const Model = Schema.Struct({ diff --git a/apps/web/src/page/security/audit-log/view.ts b/apps/web/src/page/security/audit-log/view.ts index a333c936d2..7a82abf7fd 100644 --- a/apps/web/src/page/security/audit-log/view.ts +++ b/apps/web/src/page/security/audit-log/view.ts @@ -1,4 +1,4 @@ -import { AuditTrailCategory } from "@maple/domain/audit-trail" +import { AuditTrailCategory, AuditTrailProvider } from "@maple/domain/audit-trail" import type { AuditTrailEntry } from "@maple/domain/http" import * as Policy from "@maple/domain/policy" import { refreshingClass } from "@maple/ui/lib/refreshing" @@ -48,7 +48,7 @@ import { expandChevron } from "../../kit/expand-chevron" import { jsonBlock, prettyJson } from "../../kit/json-block" import { refusedBadge } from "../../kit/refused-badge" import { relativeTime } from "../../kit/relative-time" -import { securityLayout, WIDE_COLUMN } from "../kit/layout" +import { ROOMY_COLUMN, securityLayout, WIDE_COLUMN } from "../kit/layout" import { placeholderRows } from "../kit/parts" import { actorDetail, @@ -65,6 +65,8 @@ import { OUTCOME_FILTERS, OUTCOME_LABEL, outcomeLabel, + PROVIDER_LABEL, + providerLabel, targetText, } from "./derive" import { accessOf, Message, type Model, type SelectFilter } from "./model" @@ -72,11 +74,11 @@ import { accessOf, Message, type Model, type SelectFilter } from "./model" type H = HtmlBuilder type Clock = Readonly<{ nowMs: number; timeZone: string }> -const COLUMN_COUNT = 5 +const COLUMN_COUNT = 6 // On a phone the two text columns share the width left of the time, so neither scrolls away. const NARROW_ACTOR = "max-w-32 sm:max-w-56" -const NARROW_ACTION = "max-w-36 sm:max-w-72" +const NARROW_ACTION = "max-w-36 sm:max-w-64" /** One filter as a native select: "All …" clears it. The URL holds the value it shows. */ const filterSelect = ( @@ -107,6 +109,15 @@ const filterSelect = ( const toolbar = (h: H, model: Model): Html => hstack(h, { gap: "2", align: "center", wrap: true, className: "border-b px-4 py-2.5" }, [ + filterSelect( + h, + "provider", + "Platform", + model.search.provider, + "All platforms", + PROVIDER_LABEL, + AuditTrailProvider.literals, + ), filterSelect( h, "category", @@ -184,6 +195,7 @@ const eventDetail = (h: H, event: AuditTrailEntry, clock: Clock): Html => { ...detailField(h, "IP address", event.originIp, true), ...detailField(h, "Country", event.originCountry), ...detailField(h, "User agent", event.userAgent, true), + keyValue(h, { label: "Platform", wrap: true }, [providerLabel(event.provider)]), ...detailField(h, "Account", event.account, true), keyValue(h, { label: "Occurred", wrap: true }, [at(event.occurredAt)]), keyValue(h, { label: "Recorded", wrap: true }, [at(event.ingestedAt)]), @@ -232,6 +244,7 @@ const eventRows = (h: H, model: Model, event: AuditTrailEntry, clock: Clock): Re ], "p-0", ), + tableCell(h, [providerLabel(event.provider)], WIDE_COLUMN), tableCell( h, [ @@ -255,6 +268,19 @@ const eventRows = (h: H, model: Model, event: AuditTrailEntry, clock: Clock): Re className: typographyClass({ size: "xs" }), }), hstack(h, { gap: "1.5", align: "center", wrap: true }, [ + // A phone has no Platform column: another platform's name leads the line. + ...(event.provider === "maple" + ? [] + : [ + h.span( + [h.Class("sm:hidden")], + [ + typography(h, { size: "2xs", tone: "muted" }, [ + providerLabel(event.provider), + ]), + ], + ), + ]), typography(h, { size: "2xs", tone: "muted" }, [ categoryLabel(event.category), ]), @@ -288,7 +314,7 @@ const eventRows = (h: H, model: Model, event: AuditTrailEntry, clock: Clock): Re className: typographyClass({ size: "xs", tone: "muted" }), }), ], - `max-w-64 ${WIDE_COLUMN}`, + `max-w-64 ${ROOMY_COLUMN}`, ), tableCell(h, outcomeMark(h, event), WIDE_COLUMN), ], @@ -340,9 +366,11 @@ const eventsTable = (h: H, model: Model, events: ReadonlyArray, tableHeader(h, [ tableRow(h, [ tableHead(h, ["Time"], "w-24"), + tableHead(h, ["Platform"], WIDE_COLUMN), tableHead(h, ["Actor"]), tableHead(h, ["Action"]), - tableHead(h, ["Target"], WIDE_COLUMN), + // Beside the rail a laptop has no room for it; the detail and the wide screen do. + tableHead(h, ["Target"], ROOMY_COLUMN), tableHead(h, ["Outcome"], `w-28 ${WIDE_COLUMN}`), ]), ]), @@ -372,7 +400,7 @@ const noEvents = (h: H): Html => emptyMedia(h, { variant: "icon" }, [icon(h, "HistoryIcon")]), emptyTitle(h, ["No audit events yet"]), emptyDescription(h, [ - "Changes made in Maple, and requests it refused, appear here as they happen.", + "Changes made in Maple, and requests it refused, appear here as they happen. So do audit events from Vercel and Google Cloud once their audit feeds are on.", ]), ]), ], @@ -397,7 +425,7 @@ const listBody = (h: H, model: Model, clock: Clock): Html => if (events.length === 0 && isFiltered(model.search)) return filteredEmpty(h, { noun: "events", - description: "Try a wider category or outcome, or another actor.", + description: "Try another platform, category or outcome, or another actor.", onClear: Message.ClickedClearFilters(), className: "py-8", }) diff --git a/apps/web/src/page/security/kit/layout.ts b/apps/web/src/page/security/kit/layout.ts index aba940519a..84a801686c 100644 --- a/apps/web/src/page/security/kit/layout.ts +++ b/apps/web/src/page/security/kit/layout.ts @@ -146,7 +146,8 @@ const DESCRIPTION = { "The language runtime each service reports, checked for known vulnerabilities and end-of-life dates.", dependencies: "Known vulnerable packages in your connected repositories' lockfiles, rescanned on every push.", - auditLog: "Who changed what in Maple, and what it refused. Visible to org admins.", + auditLog: + "Who changed what, and what was refused, in Maple and your connected platforms. Visible to org admins.", } satisfies Record /** The page's blocks at their own height: the scroll area scrolls them, never squeezes them. */ diff --git a/apps/web/src/route/search.ts b/apps/web/src/route/search.ts index f41d12cf1a..4600fec5d0 100644 --- a/apps/web/src/route/search.ts +++ b/apps/web/src/route/search.ts @@ -1087,6 +1087,8 @@ export const SecurityDependenciesSearch = Schema.Struct({ /** The Security hub's audit log filters; each absent one shows everything. */ export const SecurityAuditLogSearch = Schema.Struct({ + /** The platform the events came from (`maple`, `vercel`, `gcp`). */ + provider: StringParam, category: StringParam, outcome: StringParam, /** An actor's id or email on the platform the event came from. */ diff --git a/packages/backend/src/queries/registry.ts b/packages/backend/src/queries/registry.ts index 499bc856e6..4d4c16fc16 100644 --- a/packages/backend/src/queries/registry.ts +++ b/packages/backend/src/queries/registry.ts @@ -558,14 +558,6 @@ const vercelWebAnalyticsReferrers = vercelWebAnalyticsTop("vercelWebAnalyticsRef const vercelWebAnalyticsCountries = vercelWebAnalyticsTop("vercelWebAnalyticsCountries", "country") const vercelWebAnalyticsEvents = vercelWebAnalyticsTop("vercelWebAnalyticsEvents", "event") -const vercelAuditLog = defineQuery({ - id: "vercelAuditLog", - profile: "aggregation", - cache: 15, - compile: (payload: VercelInfraProjectsRequest, orgId: OrgId) => - CH.compile(Integrations.vercelAuditLogSQL(), vercelWindow(payload, orgId)), -}) - // /infra/supabase: gauges and counter rates are separate reads (metrics_gauge, metrics_sum), // merged per project or bucket by the handler. const supabaseInfraProjectGauges = defineQuery({ @@ -1317,7 +1309,6 @@ export const Queries = { vercelWebAnalyticsReferrers, vercelWebAnalyticsCountries, vercelWebAnalyticsEvents, - vercelAuditLog, supabaseInfraProjectGauges, supabaseInfraProjectCpu, supabaseInfraProjectGaugeSeries, diff --git a/packages/domain/src/audit-trail.ts b/packages/domain/src/audit-trail.ts index da87d4c27c..cdcd36db4d 100644 --- a/packages/domain/src/audit-trail.ts +++ b/packages/domain/src/audit-trail.ts @@ -1,7 +1,7 @@ import { Schema } from "effect" /** Platforms whose audit events the trail holds. One literal per built adapter; `maple` is Maple's own audit log. */ -export const AuditTrailProvider = Schema.Literals(["maple"]) +export const AuditTrailProvider = Schema.Literals(["maple", "vercel", "gcp"]) export type AuditTrailProvider = typeof AuditTrailProvider.Type /** diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 466cdb243b..2a656bb7b0 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -1107,21 +1107,6 @@ export class VercelWebAnalyticsResponse extends Schema.Class("VercelAuditLogResponse")({ - events: Schema.Array(VercelAuditEvent), -}) {} - // Google Cloud infrastructure page (/infra/gcp): `gcp.*` metrics from the Google Cloud poller. const GcpInfraService = Schema.Literals(GCP_INFRA_SERVICE_IDS) @@ -4426,13 +4411,6 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) - .add( - HttpApiEndpoint.post("vercelAuditLog", "/vercel-audit-log", { - payload: VercelInfraProjectsRequest, - success: VercelAuditLogResponse, - error: queryEngineEndpointErrors, - }), - ) .add( HttpApiEndpoint.post("gcpInfraPresence", "/gcp-infra-presence", { payload: GcpInfraPresenceRequest, diff --git a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql index c16dc043e2..c1b39afafe 100644 --- a/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql +++ b/packages/query-engine-integrations/src/__sql_baseline__/integrations.sql @@ -5495,24 +5495,6 @@ SELECT LIMIT 2000 FORMAT JSON --- builder:vercel-event-drains:vercelAuditLogSQL:default -SELECT - logs.Timestamp AS timestamp, - logs.LogAttributes['vercel.audit.action'] AS action, - if(logs.LogAttributes['vercel.audit.actor.name'] != '', logs.LogAttributes['vercel.audit.actor.name'], logs.LogAttributes['vercel.audit.actor.id']) AS actor, - logs.LogAttributes['vercel.audit.actor.type'] AS actorType, - logs.ResourceAttributes['vercel.project.id'] AS projectId - FROM logs - WHERE logs.OrgId = 'org_sql_catalog' - AND logs.ServiceName = 'vercel-audit-log' - AND logs.TimestampTime >= '2026-01-01 10:30:00' - AND logs.TimestampTime <= '2026-01-03 14:15:00' - AND logs.Timestamp >= '2026-01-01 10:30:00' - AND logs.Timestamp <= '2026-01-03 14:15:00' - ORDER BY timestamp DESC - LIMIT 25 - FORMAT JSON - -- builder:vercel-event-drains:vercelSpeedInsightsSQL:device SELECT logs.LogAttributes['vercel.device.type'] AS key, diff --git a/packages/query-engine-integrations/src/benchmark/index.ts b/packages/query-engine-integrations/src/benchmark/index.ts index a7455814a2..eb360a09a1 100644 --- a/packages/query-engine-integrations/src/benchmark/index.ts +++ b/packages/query-engine-integrations/src/benchmark/index.ts @@ -960,12 +960,6 @@ export const integrationFixtures: ReadonlyArray = [ compile: () => compileUnsafe(CH.vercelWebAnalyticsTopSQL("event", { projectId: VERCEL_PROJECT_ID }), window), }, - { - module: "vercel-event-drains", - name: "vercelAuditLogSQL", - label: "default", - compile: () => compileUnsafe(CH.vercelAuditLogSQL(), window), - }, { module: "vercel-ai-gateway", name: "vercelAiGatewaySummarySQL", diff --git a/packages/query-engine-integrations/src/vercel/index.ts b/packages/query-engine-integrations/src/vercel/index.ts index 52293b51ba..ea70f1f249 100644 --- a/packages/query-engine-integrations/src/vercel/index.ts +++ b/packages/query-engine-integrations/src/vercel/index.ts @@ -32,7 +32,6 @@ export { vercelProjectFunctionsSQL, } from "./vercel-functions" export { - vercelAuditLogSQL, vercelSpeedInsightsSQL, vercelWebAnalyticsSummarySQL, vercelWebAnalyticsTopSQL, diff --git a/packages/query-engine-integrations/src/vercel/vercel-event-drains.ts b/packages/query-engine-integrations/src/vercel/vercel-event-drains.ts index b3999aa325..cd3686b5a4 100644 --- a/packages/query-engine-integrations/src/vercel/vercel-event-drains.ts +++ b/packages/query-engine-integrations/src/vercel/vercel-event-drains.ts @@ -1,9 +1,10 @@ -// Vercel Speed Insights, Web Analytics and audit log (/infra/vercel) +// Vercel Speed Insights and Web Analytics (/infra/vercel) // // Each drain lands in `logs` under its own service (ingest maps them in // `apps/ingest/src/vercel_event_drains.rs`), one record per event, the project on -// resource `vercel.project.id` (empty for team events). Filtering on the service -// prunes on the logs sort key. +// resource `vercel.project.id`. Filtering on the service prunes on the logs sort key. +// Audit log events go to the admin-only `audit_trail_events` where the gateway stores the audit +// trail; a gateway that does not keeps them in `logs` under `vercel-audit-log`. import * as CH from "@maple-dev/effect-orm/expr" import { from, param, type ColumnAccessor } from "@maple-dev/effect-orm/clickhouse" @@ -16,7 +17,6 @@ type LogColumns = ColumnAccessor const SPEED_INSIGHTS_SERVICE = "vercel-speed-insights" const WEB_ANALYTICS_SERVICE = "vercel-web-analytics" -const AUDIT_LOG_SERVICE = "vercel-audit-log" const drainWhere = ($: LogColumns, service: string, scope: VercelProjectScope) => [ $.OrgId.eq(orgIdParam), @@ -152,23 +152,3 @@ export function vercelWebAnalyticsTopSQL( .limit(10) .format("JSON") } - -/** The team's newest audit events. */ -export function vercelAuditLogSQL() { - return from(Datasources.logs) - .select(($) => ({ - timestamp: $.Timestamp, - action: $.LogAttributes.get("vercel.audit.action"), - actor: CH.if_( - $.LogAttributes.get("vercel.audit.actor.name").neq(""), - $.LogAttributes.get("vercel.audit.actor.name"), - $.LogAttributes.get("vercel.audit.actor.id"), - ), - actorType: $.LogAttributes.get("vercel.audit.actor.type"), - projectId: $.ResourceAttributes.get("vercel.project.id"), - })) - .where(($) => drainWhere($, AUDIT_LOG_SERVICE, {})) - .orderBy(["timestamp", "desc"]) - .limit(25) - .format("JSON") -} diff --git a/packages/query-engine-integrations/src/vercel/vercel.test.ts b/packages/query-engine-integrations/src/vercel/vercel.test.ts index f7ca9274ae..c5849b901f 100644 --- a/packages/query-engine-integrations/src/vercel/vercel.test.ts +++ b/packages/query-engine-integrations/src/vercel/vercel.test.ts @@ -22,7 +22,6 @@ import { vercelProjectFunctionsSQL, } from "./vercel-functions" import { - vercelAuditLogSQL, vercelSpeedInsightsSQL, vercelWebAnalyticsSummarySQL, vercelWebAnalyticsTopSQL, @@ -281,18 +280,6 @@ describe("Web Analytics", () => { }) }) -describe("vercelAuditLogSQL", () => { - it("lists the newest audit events, the actor by name or else id", () => { - const { sql } = compileUnsafe(vercelAuditLogSQL(), window) - expect(sql).toContain("logs.ServiceName = 'vercel-audit-log'") - expect(sql).toContain( - "if(logs.LogAttributes['vercel.audit.actor.name'] != '', logs.LogAttributes['vercel.audit.actor.name'], logs.LogAttributes['vercel.audit.actor.id']) AS actor", - ) - expect(sql).toContain("ORDER BY timestamp DESC") - expect(sql).toContain("LIMIT 25") - }) -}) - describe("AI Gateway", () => { it("counts gateway server spans, parented or not, and sums the USD cost", () => { const { sql } = compileUnsafe(vercelAiGatewaySummarySQL(), window) diff --git a/packages/web-data/src/warehouse/vercel-infra.ts b/packages/web-data/src/warehouse/vercel-infra.ts index fd899ba465..b4fa1905a2 100644 --- a/packages/web-data/src/warehouse/vercel-infra.ts +++ b/packages/web-data/src/warehouse/vercel-infra.ts @@ -287,24 +287,6 @@ export const getVercelWebAnalytics = Effect.fn("QueryEngine.getVercelWebAnalytic ) }) -export const getVercelAuditLog = Effect.fn("QueryEngine.getVercelAuditLog")(function* ({ - data, -}: { - data: VercelTimeRangeInput -}) { - const input = yield* decodeInput(TimeRangeInputSchema, data, "getVercelAuditLog") - const nowMillis = yield* Clock.currentTimeMillis - const range = resolveRange(input, nowMillis) - return yield* runWarehouseQuery("vercelAuditLog", () => - Effect.gen(function* () { - const client = yield* MapleInternalHttp - return yield* client.queryEngine.vercelAuditLog({ - payload: new VercelInfraProjectsRequest(range), - }) - }), - ) -}) - /** One project's functions per bucket, for every route or one. */ export const getVercelFunctionTimeseries = Effect.fn("QueryEngine.getVercelFunctionTimeseries")(function* ({ data,