feat(mfa): MFA Leg 2 (tenant config: CLI)

apps/auth0-evals/mocks/fixtures/auth0/guardian_factors.json

@@ -0,0 +1,10 @@
+[
+  { "name": "sms", "enabled": false, "trial_expired": false },
+  { "name": "push-notification", "enabled": false, "trial_expired": false },
+  { "name": "otp", "enabled": false, "trial_expired": false },
+  { "name": "email", "enabled": false, "trial_expired": false },
+  { "name": "duo", "enabled": false, "trial_expired": false },
+  { "name": "webauthn-roaming", "enabled": false, "trial_expired": false },
+  { "name": "webauthn-platform", "enabled": false, "trial_expired": false },
+  { "name": "recovery-code", "enabled": false, "trial_expired": false }
+]

apps/auth0-evals/src/evals/contexts/cli-platform/AGENTS.md

@@ -0,0 +1,11 @@
+# Platform context
+
+You are a **platform engineer** at Barkbook. Auth0 tenant configuration is managed operationally through the **Auth0 CLI** — not infrastructure-as-code and not the dashboard.
+
+When the task requires changing tenant configuration, use the Auth0 CLI's Management API passthrough. It takes an HTTP method and a Management API path:
+
+```bash
+auth0 api <METHOD> <path> --data '{ ... }'
+```
+
+The Auth0 CLI is already installed and authenticated against this tenant — run tenant commands directly; do not run `auth0 login` or look for credentials, client secrets, or environment variables.

apps/auth0-evals/src/evals/mfa/angular/PROMPT.md

@@ -4,7 +4,7 @@ name: Angular MFA Step-Up
 scaffold: src/evals/scaffolds/angular/auth0
 skills: auth0-mfa
 setup_command: npm install
-compile_command: npm run build
+compile_command: ng build
 ---
 
 ## Task

apps/auth0-evals/src/evals/mfa/angular/graders.ts

@@ -48,6 +48,9 @@ export function defineGraders() {
     ),
 
     // ── Holistic judge (no level — always runs) ───────────────────────────
+    // Tenant-side MFA enforcement (Guardian factors + policies) is measured
+    // separately by the standalone mfa_tenant_cli eval;
+    // this eval scores only the Angular client step-up flow.
     judge(
       'Does the solution correctly implement MFA step-up authentication in an Angular app — ' +
         'checking the amr claim via idTokenClaims$, requesting step-up via acr_values when ' +

apps/auth0-evals/src/evals/mfa/react/graders.ts

@@ -47,6 +47,9 @@ export function defineGraders() {
     ),
 
     // ── Holistic judge (no level — always runs) ───────────────────────────
+    // Tenant-side MFA enforcement (Guardian factors + policies) is measured
+    // separately by the standalone mfa_tenant_cli eval;
+    // this eval scores only the React client step-up flow.
     judge(
       'Does the solution correctly implement MFA step-up authentication in a React app — ' +
         'checking the amr claim via getIdTokenClaims, requesting step-up via acr_values when ' +

apps/auth0-evals/src/evals/mfa/tenant-cli/PROMPT.md

@@ -0,0 +1,15 @@
+---
+id: mfa_tenant_cli
+name: MFA Tenant Config (CLI)
+category: mfa
+skills: auth0-mfa
+---
+
+## Task
+
+Our Auth0 tenant is configured operationally through the Auth0 CLI. We need to require multi-factor authentication for step-up flows — a factor being available is not enough; MFA must actually be enforced.
+
+Using the Auth0 CLI, enable the required MFA factor on the tenant and then enforce MFA so it is required for users. Do not configure the tenant through the dashboard or Terraform — the change must be made via the Auth0 CLI.
+
+Domain: dev-barkbook.us.auth0.com
+Client ID: barkbook_client_abc123xyz

apps/auth0-evals/src/evals/mfa/tenant-cli/graders.ts

@@ -0,0 +1,36 @@
+import { ranCommand, ranCommandOneOf, ranCommandsInOrder, judge, GraderLevel } from '@a0/eval-graders';
+
+export function defineGraders() {
+  return [
+    // ── L4: Enable an MFA factor via the Auth0 CLI (event-based) ──────────
+    // Reads the tool-call trace, not file contents — the whole task is CLI
+    // invocations, so there is no artifact to inspect with contains/matches.
+    ranCommandOneOf(
+      ['guardian/factors/otp', 'guardian/factors/push', 'guardian/factors/sms'],
+      'Enabled MFA factor via Auth0 CLI',
+      GraderLevel.L4,
+    ),
+    // ── L4: Enforce MFA — the step agents skip, leaving an enabled-but- ────
+    // unenforced tenant. Arg-precise so a wrong payload (e.g. an empty policy
+    // list) fails.
+    ranCommand('guardian/policies', ['all-applications'], 'Enforced MFA via guardian/policies', GraderLevel.L4),
+    // ── L4: Sequence — factor must be enabled BEFORE the enforcement policy ─
+    // that relies on it.
+    ranCommandsInOrder(
+      [['guardian/factors/otp', 'guardian/factors/push', 'guardian/factors/sms'], 'guardian/policies'],
+      'Enabled factor before setting enforcement policy',
+      GraderLevel.L4,
+    ),
+
+    // ── Holistic judge (no level — always runs) ───────────────────────────
+    // includeCommandTrace: this eval writes no files — the artifact is the CLI
+    // trace, so the judge must see the commands the agent ran to evaluate it.
+    judge(
+      'Based on the command trace, does the solution enable an MFA factor on the Auth0 tenant via ' +
+        'the Auth0 CLI (auth0 api put guardian/factors/...) AND enforce MFA via the guardian/policies ' +
+        'endpoint with the all-applications policy — WITHOUT configuring MFA through the dashboard or Terraform?',
+      undefined,
+      { includeCommandTrace: true },
+    ),
+  ];
+}

apps/auth0-evals/src/evals/mfa/tenant-cli/routes/fixtures/guardian/policies_set.json

@@ -0,0 +1 @@
+["all-applications"]

apps/auth0-evals/src/evals/mfa/tenant-cli/routes/guardian.routes.json

@@ -0,0 +1,18 @@
+{
+  "surface": "guardian",
+  "consumedBy": ["mfa_tenant_cli"],
+  "routes": [
+    { "match": "GET guardian/factors", "verb": "handler", "handler": "listFactors" },
+    { "match": "PUT guardian/factors/*", "verb": "handler", "handler": "setFactor" },
+    { "match": "PATCH guardian/factors/*", "verb": "handler", "handler": "setFactor" },
+    { "match": "PUT guardian/policies", "verb": "create", "state": "mfa.policy", "body": "policies_set.json" },
+    { "match": "PATCH guardian/policies", "verb": "create", "state": "mfa.policy", "body": "policies_set.json" },
+    {
+      "match": "GET guardian/policies",
+      "verb": "reflect",
+      "state": "mfa.policy",
+      "present": "policies_set.json",
+      "absent": []
+    }
+  ]
+}

apps/auth0-evals/src/evals/mfa/tenant-cli/routes/handlers.js

@@ -0,0 +1,23 @@
+const FACTORS = [
+  'sms',
+  'push-notification',
+  'otp',
+  'email',
+  'duo',
+  'webauthn-roaming',
+  'webauthn-platform',
+  'recovery-code',
+];
+
+export default {
+  setFactor(ctx) {
+    const factor = ctx.path.split('/').pop();
+    const disabled = /"enabled"\s*:\s*false/.test(ctx.payload);
+    if (disabled) ctx.state.clear(`mfa.factor.${factor}`);
+    else ctx.state.set(`mfa.factor.${factor}`);
+    return { enabled: true };
+  },
+  listFactors(ctx) {
+    return FACTORS.map((name) => ({ name, enabled: ctx.state.has(`mfa.factor.${name}`), trial_expired: false }));
+  },
+};

apps/auth0-evals/src/evals/mfa/vue/PROMPT.md

@@ -9,7 +9,7 @@ compile_command: npm run build
 
 ## Task
 
-My Vue app has Auth0 login set up. I want to add a Transfer Funds feature where users must complete MFA before the transfer runs. If they haven't done MFA yet, prompt them for it.
+My Vue 3 app has Auth0 login set up. I want to add a Transfer Funds feature where users must complete MFA before the transfer runs. If they haven't done MFA yet, prompt them for it.
 
 Domain: dev-barkbook.us.auth0.com
 Client ID: barkbook_client_abc123xyz

apps/auth0-evals/src/evals/mfa/vue/graders.ts

@@ -49,6 +49,9 @@ export function defineGraders() {
     ),
 
     // ── Holistic judge (no level — always runs) ───────────────────────────
+    // Tenant-side MFA enforcement (Guardian factors + policies) is measured
+    // separately by the standalone mfa_tenant_cli eval;
+    // this eval scores only the Vue client step-up flow.
     judge(
       'Does the solution correctly implement MFA step-up authentication in a Vue 3 app — ' +
         'checking the amr claim via the idTokenClaims reactive ref, requesting step-up via acr_values when ' +

packages/eval-core/tests/guardian-route.test.ts

@@ -0,0 +1,71 @@
+/**
+ * Tests for the Guardian mock surface (`mfa/tenant-cli/routes/guardian.routes.json`).
+ *
+ * The guardian manifest is co-located with the mfa_tenant_cli eval, so this test
+ * points the dispatcher at that eval's routes/ dir via EVAL_MOCK_ROUTES_DIRS —
+ * the same wiring run.ts uses per-eval. Exercises the surface end-to-end through
+ * the dispatcher: enabling factors, setting the enforcement policy, read-after-write.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { execFileSync } from 'node:child_process';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const MOCK = fileURLToPath(new URL('../../../apps/auth0-evals/mocks/auth0', import.meta.url));
+// Guardian routes live with the eval, not in the shared mocks/ dir.
+const ROUTES_DIR = fileURLToPath(new URL('../../../apps/auth0-evals/src/evals/mfa/tenant-cli/routes', import.meta.url));
+
+let stateDir: string;
+
+function run(...args: string[]): string {
+  return execFileSync(MOCK, args, {
+    env: { ...process.env, EVAL_MOCK_STATE_DIR: stateDir, EVAL_MOCK_ROUTES_DIRS: ROUTES_DIR },
+    encoding: 'utf8',
+  }).trim();
+}
+
+beforeEach(() => {
+  stateDir = mkdtempSync(join(tmpdir(), 'guardian-route-test-'));
+});
+
+afterEach(() => {
+  rmSync(stateDir, { recursive: true, force: true });
+});
+
+describe('guardian route — factors', () => {
+  it('lists all factors disabled in fresh state', () => {
+    const out = run('api', 'get', 'guardian/factors');
+    expect(out).toContain('"name":"otp","enabled":false');
+    expect(out).toContain('"name":"sms","enabled":false');
+  });
+
+  it('reflects an enabled factor on a later read (read-after-write)', () => {
+    run('api', 'put', 'guardian/factors/otp', '--data', '{"enabled": true}');
+    expect(run('api', 'get', 'guardian/factors')).toContain('"name":"otp","enabled":true');
+  });
+
+  it('routes the full Management API URL form', () => {
+    run('api', 'PATCH', 'https://dev-barkbook.us.auth0.com/api/v2/guardian/factors/otp', '--data', '{"enabled":true}');
+    expect(run('api', 'get', 'guardian/factors')).toContain('"name":"otp","enabled":true');
+  });
+
+  it('disables a factor when the payload sets enabled:false', () => {
+    run('api', 'put', 'guardian/factors/otp', '--data', '{"enabled":true}');
+    run('api', 'patch', 'guardian/factors/otp', '--data', '{"enabled":false}');
+    expect(run('api', 'get', 'guardian/factors')).toContain('"name":"otp","enabled":false');
+  });
+});
+
+describe('guardian route — policies', () => {
+  it('returns an empty policy list in fresh state', () => {
+    expect(run('api', 'get', 'guardian/policies')).toBe('[]');
+  });
+
+  it('reflects the enforcement policy once set', () => {
+    run('api', 'put', 'guardian/policies', '--data', '["all-applications"]');
+    expect(run('api', 'get', 'guardian/policies')).toBe('["all-applications"]');
+  });
+});