Skip to content

Commit e424499

Browse files
committed
feat(webapp): add additional-key rollout flag and auth observability
Add a global, org-locked kill switch (additionalApiKeyLookupEnabled, default off) gating the additional environment API-key lookup, and bounded OTel auth telemetry (api_auth.attempts counter, api_auth.duration_ms histogram, api_auth.rollout_mode gauge) to watch the rollout. Attributes are closed enums only (resolver, credential_kind, result, lookup_path); no credentials, hashes, or tenant identifiers are recorded.
1 parent 3895239 commit e424499

15 files changed

Lines changed: 471 additions & 60 deletions

apps/webapp/app/models/runtimeEnvironment.server.ts

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import { hashApiKey } from "~/utils/apiKeys";
99
import { isAdditionalApiKey } from "@trigger.dev/core/v3/apiKeys";
1010
import { isDefaultDevBranch, sanitizeBranchName } from "@trigger.dev/core/v3/utils/gitBranch";
1111
import { scopesGrantFullAccess } from "@trigger.dev/rbac";
12+
import { authFeatureControls } from "~/services/authFeatureControls.server";
1213

1314
export type { RuntimeEnvironment };
1415

@@ -98,7 +99,7 @@ export function toAuthenticated(
9899

99100
export type ApiKeyEnvironmentResolution =
100101
| { ok: true; environment: AuthenticatedEnvironment }
101-
| { ok: false; reason: "not-found" | "restricted" };
102+
| { ok: false; reason: "not-found" | "restricted" | "disabled" };
102103

103104
/**
104105
* Resolve an environment from a raw API key for legacy routes that do not
@@ -109,7 +110,8 @@ export type ApiKeyEnvironmentResolution =
109110
async function resolveEnvironmentByApiKey(
110111
apiKey: string,
111112
branchName: string | undefined,
112-
tx: PrismaClientOrTransaction
113+
tx: PrismaClientOrTransaction,
114+
additionalApiKeyLookupEnabled: () => boolean
113115
): Promise<ApiKeyEnvironmentResolution> {
114116
const branch = sanitizeBranchName(branchName) ?? undefined;
115117

@@ -127,6 +129,10 @@ async function resolveEnvironmentByApiKey(
127129

128130
const now = new Date();
129131
const routesToAdditionalKey = isAdditionalApiKey(apiKey);
132+
if (routesToAdditionalKey && !additionalApiKeyLookupEnabled()) {
133+
return { ok: false, reason: "disabled" };
134+
}
135+
130136
let rootEnvironment = routesToAdditionalKey
131137
? null
132138
: await tx.runtimeEnvironment.findFirst({
@@ -269,9 +275,15 @@ async function resolveEnvironmentByApiKey(
269275
export async function findEnvironmentByApiKey(
270276
apiKey: string,
271277
branchName: string | undefined,
272-
tx: PrismaClientOrTransaction = $replica
278+
tx: PrismaClientOrTransaction = $replica,
279+
additionalApiKeyLookupEnabled = authFeatureControls.additionalApiKeyLookupEnabled
273280
): Promise<AuthenticatedEnvironment | null> {
274-
const resolution = await resolveEnvironmentByApiKey(apiKey, branchName, tx);
281+
const resolution = await resolveEnvironmentByApiKey(
282+
apiKey,
283+
branchName,
284+
tx,
285+
additionalApiKeyLookupEnabled
286+
);
275287
return resolution.ok ? resolution.environment : null;
276288
}
277289

@@ -283,9 +295,10 @@ export async function findEnvironmentByApiKey(
283295
export async function findEnvironmentByApiKeyWithResolution(
284296
apiKey: string,
285297
branchName: string | undefined,
286-
tx: PrismaClientOrTransaction = $replica
298+
tx: PrismaClientOrTransaction = $replica,
299+
additionalApiKeyLookupEnabled = authFeatureControls.additionalApiKeyLookupEnabled
287300
): Promise<ApiKeyEnvironmentResolution> {
288-
return resolveEnvironmentByApiKey(apiKey, branchName, tx);
301+
return resolveEnvironmentByApiKey(apiKey, branchName, tx, additionalApiKeyLookupEnabled);
289302
}
290303

291304
/**

apps/webapp/app/services/apiAuth.server.ts

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ import {
3030
} from "./organizationAccessToken.server";
3131
import { isPublicJWT, validatePublicJwtKey } from "./realtime/jwtAuth.server";
3232
import { isDefaultDevBranch, sanitizeBranchName } from "@trigger.dev/core/v3/utils/gitBranch";
33-
import { rbac } from "./rbac.server";
33+
import {
34+
authenticateBearerWithTelemetry,
35+
observeLegacyBearerAuthentication,
36+
} from "~/services/authTelemetry.server";
3437

3538
const ClaimsSchema = z.object({
3639
scopes: z.array(z.string()).optional(),
@@ -92,9 +95,9 @@ export async function authenticateApiRequest(
9295
return;
9396
}
9497

95-
const authentication = await authenticateApiKey(apiKey, { ...options, branchName });
96-
97-
return authentication;
98+
return observeLegacyBearerAuthentication(request, () =>
99+
authenticateApiKey(apiKey, { ...options, branchName })
100+
);
98101
}
99102

100103
/**
@@ -114,9 +117,9 @@ export async function authenticateApiRequestWithFailure(
114117
};
115118
}
116119

117-
const authentication = await authenticateApiKeyWithFailure(apiKey, { ...options, branchName });
118-
119-
return authentication;
120+
return observeLegacyBearerAuthentication(request, () =>
121+
authenticateApiKeyWithFailure(apiKey, { ...options, branchName })
122+
);
120123
}
121124

122125
/**
@@ -306,13 +309,10 @@ export async function authenticateApiKeyWithScope(
306309
return { ok: false, status: 401, error: "Invalid or Missing API key" };
307310
}
308311

309-
const result = await rbac.authenticateAuthorizeBearer(
310-
request,
311-
{ action, resource },
312-
{ allowJWT }
313-
);
314-
if (!result.ok) {
315-
return result;
312+
const result = await authenticateBearerWithTelemetry(request, { allowJWT });
313+
if (!result.ok) return result;
314+
if (!result.ability.can(action, resource)) {
315+
return { ok: false, status: 403, error: "Unauthorized" };
316316
}
317317

318318
return {
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { resolveAuthFeatureControls } from "~/services/authFeatureControls";
2+
import { globalFlagsRegistry } from "~/v3/globalFlagsRegistry.server";
3+
4+
export { resolveAuthFeatureControls } from "~/services/authFeatureControls";
5+
export type { AuthFeatureControls } from "~/services/authFeatureControls";
6+
7+
function currentControls() {
8+
return resolveAuthFeatureControls(globalFlagsRegistry.current());
9+
}
10+
11+
export const authFeatureControls = {
12+
additionalApiKeyLookupEnabled: () => currentControls().additionalApiKeyLookupEnabled,
13+
};
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
import { FEATURE_FLAG, type FeatureFlagCatalog } from "~/v3/featureFlags";
2+
3+
export type AuthFeatureControls = {
4+
additionalApiKeyLookupEnabled: boolean;
5+
};
6+
7+
export function resolveAuthFeatureControls(
8+
flags: Partial<FeatureFlagCatalog> | Record<string, unknown> | undefined
9+
): AuthFeatureControls {
10+
return {
11+
additionalApiKeyLookupEnabled: flags?.[FEATURE_FLAG.additionalApiKeyLookupEnabled] === true,
12+
};
13+
}
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import { getMeter } from "@internal/tracing";
2+
import { isAdditionalApiKey } from "@trigger.dev/core/v3/apiKeys";
3+
import { isPublicJWT } from "@trigger.dev/core/v3/jwt";
4+
import type {
5+
BearerCredentialKind,
6+
BearerLookupPath,
7+
HostBearerAuthResult,
8+
} from "@trigger.dev/rbac";
9+
import { authFeatureControls } from "~/services/authFeatureControls.server";
10+
import { rbac } from "~/services/rbac.server";
11+
import { singleton } from "~/utils/singleton";
12+
13+
export type ApiAuthResult = "success" | "invalid" | "forbidden" | "disabled" | "error";
14+
15+
const telemetry = singleton("apiAuthTelemetry", () => {
16+
const meter = getMeter("api-auth");
17+
const attempts = meter.createCounter("api_auth.attempts", {
18+
description: "Completed environment bearer authentication attempts",
19+
});
20+
const duration = meter.createHistogram("api_auth.duration_ms", {
21+
description: "Environment bearer authentication duration",
22+
unit: "ms",
23+
});
24+
25+
meter
26+
.createObservableGauge("api_auth.rollout_mode", {
27+
description: "Active API authentication rollout modes",
28+
})
29+
.addCallback((result) => {
30+
result.observe(1, {
31+
control: "additional_key_lookup",
32+
mode: authFeatureControls.additionalApiKeyLookupEnabled() ? "enabled" : "disabled",
33+
});
34+
});
35+
36+
return { attempts, duration };
37+
});
38+
39+
export async function authenticateBearerWithTelemetry(
40+
request: Request,
41+
options: { allowJWT: boolean }
42+
): Promise<HostBearerAuthResult> {
43+
const startedAt = performance.now();
44+
const classified = classifyCredential(request, options.allowJWT);
45+
let final = { ...classified, result: "error" as ApiAuthResult };
46+
47+
try {
48+
const result = await rbac.authenticateBearer(request, options);
49+
final = {
50+
credentialKind: result.resolution.credentialKind,
51+
lookupPath: result.resolution.lookupPath,
52+
result: result.ok
53+
? "success"
54+
: result.resolution.lookupPath === "additional_skipped"
55+
? "disabled"
56+
: result.status === 403
57+
? "forbidden"
58+
: "invalid",
59+
};
60+
recordAuthAttempt("rbac", final.credentialKind, final.lookupPath, final.result);
61+
return result;
62+
} catch (error) {
63+
recordAuthAttempt("rbac", final.credentialKind, final.lookupPath, final.result);
64+
throw error;
65+
} finally {
66+
telemetry.duration.record(performance.now() - startedAt, {
67+
resolver: "rbac",
68+
credential_kind: final.credentialKind,
69+
result: final.result,
70+
lookup_path: final.lookupPath,
71+
});
72+
}
73+
}
74+
75+
export async function observeLegacyBearerAuthentication<T extends { ok: boolean } | undefined>(
76+
request: Request,
77+
operation: () => Promise<T>
78+
): Promise<T> {
79+
const startedAt = performance.now();
80+
const classified = classifyCredential(request, true);
81+
const lookupPath: BearerLookupPath =
82+
classified.credentialKind === "additional_api_key" &&
83+
!authFeatureControls.additionalApiKeyLookupEnabled()
84+
? "additional_skipped"
85+
: classified.lookupPath;
86+
let result: ApiAuthResult = "error";
87+
88+
try {
89+
const value = await operation();
90+
result = value?.ok ? "success" : lookupPath === "additional_skipped" ? "disabled" : "invalid";
91+
recordAuthAttempt("legacy", classified.credentialKind, lookupPath, result);
92+
return value;
93+
} catch (error) {
94+
recordAuthAttempt("legacy", classified.credentialKind, lookupPath, result);
95+
throw error;
96+
} finally {
97+
telemetry.duration.record(performance.now() - startedAt, {
98+
resolver: "legacy",
99+
credential_kind: classified.credentialKind,
100+
result,
101+
lookup_path: lookupPath,
102+
});
103+
}
104+
}
105+
106+
function recordAuthAttempt(
107+
resolver: "rbac" | "legacy",
108+
credentialKind: BearerCredentialKind,
109+
lookupPath: BearerLookupPath,
110+
result: ApiAuthResult
111+
) {
112+
telemetry.attempts.add(1, {
113+
resolver,
114+
credential_kind: credentialKind,
115+
result,
116+
lookup_path: lookupPath,
117+
});
118+
}
119+
120+
// Best-effort pre-classification from the raw token format. This is only used
121+
// for the metric attributes when the resolver throws before returning a
122+
// resolution; the resolver's own resolution is authoritative on success/failure.
123+
// Never records the credential itself — only its bounded format class.
124+
function classifyCredential(
125+
request: Request,
126+
allowJWT: boolean
127+
): { credentialKind: BearerCredentialKind; lookupPath: BearerLookupPath } {
128+
const token = request.headers
129+
.get("Authorization")
130+
?.replace(/^Bearer /, "")
131+
.trim();
132+
if (!token) return { credentialKind: "unknown", lookupPath: "not_found" };
133+
if (token.startsWith("pk_")) {
134+
return { credentialKind: "legacy_public_key", lookupPath: "legacy_public" };
135+
}
136+
if (allowJWT && isPublicJWT(token)) {
137+
return { credentialKind: "public_jwt", lookupPath: "jwt_current" };
138+
}
139+
if (isAdditionalApiKey(token)) {
140+
return { credentialKind: "additional_api_key", lookupPath: "additional" };
141+
}
142+
return token.startsWith("tr_")
143+
? { credentialKind: "root_api_key", lookupPath: "root_current" }
144+
: { credentialKind: "unknown", lookupPath: "not_found" };
145+
}

apps/webapp/app/services/rbac.server.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { $replica, prisma } from "~/db.server";
22
import type { PrismaClient } from "@trigger.dev/database";
33
import plugin from "@trigger.dev/rbac";
44
import { env } from "~/env.server";
5+
import { authFeatureControls } from "~/services/authFeatureControls.server";
56

67
// plugin.create() is synchronous — returns a lazy controller that resolves
78
// any installed RBAC plugin on first call. Top-level await is not used
@@ -30,6 +31,7 @@ export const rbac = plugin.create(
3031
{
3132
forceFallback: env.RBAC_FORCE_FALLBACK,
3233
userActorSecret: env.SESSION_SECRET,
34+
additionalApiKeyLookupEnabled: authFeatureControls.additionalApiKeyLookupEnabled,
3335
// A plugin that owns its own database client gets the same
3436
// writer/replica topology the webapp's Prisma clients use (see
3537
// getClient/getReplicaClient in db.server.ts): control-plane URLs win,

apps/webapp/app/services/routeBuilders/apiBuilder.server.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { fromZodError } from "zod-validation-error";
66
import { apiCors } from "~/utils/apiCors";
77
import { logger } from "../logger.server";
88
import { rbac } from "../rbac.server";
9+
import { authenticateBearerWithTelemetry } from "~/services/authTelemetry.server";
910
import type { RbacAbility, RbacResource } from "@trigger.dev/rbac";
1011
import { isUserActorToken } from "@trigger.dev/rbac";
1112
import type { PersonalAccessTokenAuthenticationResult } from "../personalAccessToken.server";
@@ -60,7 +61,7 @@ async function authenticateRequestForApiBuilder(
6061
restrictedApiKey: boolean;
6162
}
6263
> {
63-
const result = await rbac.authenticateBearer(request, { allowJWT });
64+
const result = await authenticateBearerWithTelemetry(request, { allowJWT });
6465
if (!result.ok) {
6566
// Plugin auth distinguishes 401 (who are you?) from 403 (you're not
6667
// allowed) — e.g. a suspended account or IP block returns 403.

apps/webapp/app/v3/featureFlags.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,9 @@ export const FEATURE_FLAG = {
2222
// Grace-linger stamp carried alongside runOpsMintKind on flip. See mintFlipGrace.ts.
2323
runOpsMintKindPrev: "runOpsMintKindPrev",
2424
runOpsMintKindFlippedAt: "runOpsMintKindFlippedAt",
25+
// System-wide kill switch for additional (scoped) environment API-key lookup.
26+
// Defaults off; enable during rollout once the new lookup path is trusted.
27+
additionalApiKeyLookupEnabled: "additionalApiKeyLookupEnabled",
2528
} as const;
2629

2730
export const FeatureFlagCatalog = {
@@ -61,6 +64,10 @@ export const FeatureFlagCatalog = {
6164
// by stampMintKindFlip on a genuine flip. Display-only (see ORG_LOCKED_FLAGS).
6265
[FEATURE_FLAG.runOpsMintKindPrev]: z.enum(["cuid", "runOpsId"]),
6366
[FEATURE_FLAG.runOpsMintKindFlippedAt]: z.string().datetime(),
67+
// Strict z.boolean() (not z.coerce.boolean()): coercion turns the string
68+
// "false" into true, which would silently enable this kill switch the wrong
69+
// way if written as a string. Cold/absent resolves to the safe `false`.
70+
[FEATURE_FLAG.additionalApiKeyLookupEnabled]: z.boolean(),
6471
};
6572

6673
export type FeatureFlagKey = keyof typeof FeatureFlagCatalog;
@@ -79,6 +86,8 @@ export const ORG_LOCKED_FLAGS: FeatureFlagKey[] = [
7986
FEATURE_FLAG.taskEventRepository,
8087
FEATURE_FLAG.runOpsMintKindPrev,
8188
FEATURE_FLAG.runOpsMintKindFlippedAt,
89+
// System-wide only — an org must not be able to override the rollout switch.
90+
FEATURE_FLAG.additionalApiKeyLookupEnabled,
8291
];
8392

8493
// Create a Zod schema from the existing catalog
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { describe, expect, it } from "vitest";
2+
import { resolveAuthFeatureControls } from "~/services/authFeatureControls";
3+
import { FEATURE_FLAG, FeatureFlagCatalog, ORG_LOCKED_FLAGS } from "~/v3/featureFlags";
4+
5+
describe("auth feature controls", () => {
6+
it("uses safe defaults for a cold or missing snapshot", () => {
7+
expect(resolveAuthFeatureControls(undefined)).toEqual({
8+
additionalApiKeyLookupEnabled: false,
9+
});
10+
});
11+
12+
it("accepts only strict booleans and locks org overrides", () => {
13+
const flag = FEATURE_FLAG.additionalApiKeyLookupEnabled;
14+
expect(FeatureFlagCatalog[flag].safeParse(true).success).toBe(true);
15+
// Strict z.boolean(): the stringified "false" must not coerce to true.
16+
expect(FeatureFlagCatalog[flag].safeParse("false").success).toBe(false);
17+
expect(ORG_LOCKED_FLAGS).toContain(flag);
18+
});
19+
});

0 commit comments

Comments
 (0)