fix(webapp): stop an unreadable eval override reading as consent to judge

flag() ignores a per-org override that fails the schema and falls through to the global
default, which is on. For an entitlement that is right; for this flag it means an org that
tried to turn judging off with a stringified boolean kept being judged. The eval policy now
refuses on an override it cannot parse, matching what the flag's own comment already claims.

Author: kathiekiwi

apps/webapp/app/services/dashboardAgentEvalPolicy.server.ts

@@ -5,7 +5,7 @@
 
 import { prisma } from "~/db.server";
 import { logger } from "~/services/logger.server";
-import { FEATURE_FLAG } from "~/v3/featureFlags";
+import { FEATURE_FLAG, hasUnreadableTurnEvalsOverride } from "~/v3/featureFlags";
 import { makeFlag } from "~/v3/featureFlags.server";
 
 /** Judging is on unless an org turns it off. */
@@ -31,12 +31,17 @@ export async function orgAllowsDashboardAgentTurnEvals(params: {
     });
     if (!org) return false;
 
+    const overrides = (org.featureFlags as Record<string, unknown>) ?? {};
+    // `flag()` ignores an override the schema rejects and falls through to the global default,
+    // which is on — so an org that tried to turn judging off would keep being judged.
+    if (hasUnreadableTurnEvalsOverride(overrides)) return false;
+
     const flag = makeFlag();
     return Boolean(
       await flag({
         key: FEATURE_FLAG.dashboardAgentTurnEvalsEnabled,
         defaultValue: DEFAULT_TURN_EVALS_ENABLED,
-        overrides: (org.featureFlags as Record<string, unknown>) ?? {},
+        overrides,
       })
     );
   } catch (error) {

apps/webapp/app/v3/featureFlags.ts

@@ -173,6 +173,26 @@ export function resolveInternalApiOriginEnabled({
   return globalDefault;
 }
 
+/**
+ * Whether the org set `dashboardAgentTurnEvalsEnabled` to something the schema rejects.
+ * That flag is a consent switch, not an entitlement, so an unreadable override must not fall
+ * through to the global default the way `resolveInternalApiOriginEnabled` does: the org that
+ * wrote it was trying to say something, and the only safe reading of an unknown answer is no.
+ */
+export function hasUnreadableTurnEvalsOverride(orgFeatureFlags: unknown): boolean {
+  if (!orgFeatureFlags || typeof orgFeatureFlags !== "object" || Array.isArray(orgFeatureFlags)) {
+    return false;
+  }
+
+  const override = (orgFeatureFlags as Record<string, unknown>)[
+    FEATURE_FLAG.dashboardAgentTurnEvalsEnabled
+  ];
+  if (override === undefined) return false;
+
+  return !FeatureFlagCatalog[FEATURE_FLAG.dashboardAgentTurnEvalsEnabled].safeParse(override)
+    .success;
+}
+
 export type FlagControlType =
   | { type: "boolean" }
   | { type: "enum"; options: string[] }

apps/webapp/test/featureFlags.test.ts

@@ -4,9 +4,9 @@
 // call counted by a delegating wrapper around the real client.
 import type { PrismaClient } from "@trigger.dev/database";
 import { postgresTest } from "@internal/testcontainers";
-import { describe, expect, vi } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import type { PrismaClientOrTransaction } from "~/db.server";
-import { FEATURE_FLAG } from "~/v3/featureFlags";
+import { FEATURE_FLAG, hasUnreadableTurnEvalsOverride } from "~/v3/featureFlags";
 import { makeFlag, makeSetFlag } from "~/v3/featureFlags.server";
 
 vi.setConfig({ testTimeout: 60_000 });
@@ -93,3 +93,27 @@ describe("flag() override resolution", () => {
     expect(calls.findFirst).toBe(1);
   });
 });
+
+// The fall-through above is right for an entitlement and wrong for consent: judging sends the
+// turn to a third-party model, so the eval flag refuses on an override it cannot read.
+describe("hasUnreadableTurnEvalsOverride", () => {
+  it("is false when the org has no opinion", () => {
+    expect(hasUnreadableTurnEvalsOverride({})).toBe(false);
+    expect(hasUnreadableTurnEvalsOverride(null)).toBe(false);
+    expect(hasUnreadableTurnEvalsOverride([])).toBe(false);
+    expect(hasUnreadableTurnEvalsOverride({ somethingElse: "nonsense" })).toBe(false);
+  });
+
+  it("is false for a real boolean, either way", () => {
+    expect(hasUnreadableTurnEvalsOverride({ [KEY]: false })).toBe(false);
+    expect(hasUnreadableTurnEvalsOverride({ [KEY]: true })).toBe(false);
+  });
+
+  // The dangerous one: `flag()` would drop "false" and hand back the global default, which is on.
+  it("is true for a stringified boolean or any other garbage", () => {
+    expect(hasUnreadableTurnEvalsOverride({ [KEY]: "false" })).toBe(true);
+    expect(hasUnreadableTurnEvalsOverride({ [KEY]: "true" })).toBe(true);
+    expect(hasUnreadableTurnEvalsOverride({ [KEY]: 0 })).toBe(true);
+    expect(hasUnreadableTurnEvalsOverride({ [KEY]: null })).toBe(true);
+  });
+});