Standardize scenario setup/execution failure reporting (#248)

src/scenarios/server/client-helper.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect } from 'vitest';
+import { reportSetupFailure } from './client-helper';
+
+describe('reportSetupFailure', () => {
+  it('emits a single FAILURE check id-d "<scenario>-setup"', () => {
+    const checks = reportSetupFailure(
+      'tools-list',
+      new Error('connect ECONNREFUSED')
+    );
+
+    expect(checks).toHaveLength(1);
+    expect(checks[0]).toMatchObject({
+      id: 'tools-list-setup',
+      status: 'FAILURE',
+      errorMessage: 'Setup failed: connect ECONNREFUSED'
+    });
+  });
+
+  it('stringifies a non-Error thrown value', () => {
+    const checks = reportSetupFailure('prompts-list', 'boom');
+
+    expect(checks[0]?.errorMessage).toBe('Setup failed: boom');
+  });
+
+  it('attaches spec references when provided and omits the field otherwise', () => {
+    const withRefs = reportSetupFailure('resources-list', new Error('nope'), [
+      { id: 'MCP-Resources-List' }
+    ]);
+    expect(withRefs[0]?.specReferences).toEqual([{ id: 'MCP-Resources-List' }]);
+
+    const withoutRefs = reportSetupFailure('resources-list', new Error('nope'));
+    expect(withoutRefs[0]).not.toHaveProperty('specReferences');
+  });
+
+  it('sets a timestamp', () => {
+    const checks = reportSetupFailure('server-initialize', new Error('x'));
+    expect(typeof checks[0]?.timestamp).toBe('string');
+    expect(checks[0]?.timestamp.length).toBeGreaterThan(0);
+  });
+});

src/scenarios/server/client-helper.ts

@@ -8,12 +8,55 @@ import {
   LoggingMessageNotificationSchema,
   ProgressNotificationSchema
 } from '@modelcontextprotocol/sdk/types.js';
+import { ConformanceCheck } from '../../types';
 
 export interface MCPClientConnection {
   client: Client;
   close: () => Promise<void>;
 }
 
+/**
+ * Emit a single `<scenarioName>-setup` check as FAILURE for a scenario that
+ * could not get far enough to evaluate its real checks (connect failure,
+ * missing fixture, capability not advertised, etc.).
+ *
+ * See #248: previously each scenario hand-rolled a try/catch around connect
+ * and pinned the setup error onto whichever check ID happened to be first.
+ * That mislabels the failure — the error ends up under a check that has
+ * nothing to do with the actual problem, and any *other* checks the scenario
+ * would have emitted silently disappear. Routing setup failures through this
+ * helper gives them a dedicated, semantically honest ID and a consistent
+ * output shape across scenarios.
+ *
+ * The convention is that a scenario that cannot execute counts as a FAILURE;
+ * the escape hatches are scenario filtering (`--suite`/`--scenario`) and the
+ * expected-failures baseline, not in-scenario skipping or silent passes.
+ *
+ * @param scenarioName The scenario's `name`; the emitted check id is
+ *   `<scenarioName>-setup`.
+ * @param error The thrown setup error.
+ * @param specReferences Optional spec references to attach to the check.
+ * @returns A one-element array, so a scenario can `return reportSetupFailure(...)`.
+ */
+export function reportSetupFailure(
+  scenarioName: string,
+  error: unknown,
+  specReferences?: ConformanceCheck['specReferences']
+): ConformanceCheck[] {
+  const message = error instanceof Error ? error.message : String(error);
+  return [
+    {
+      id: `${scenarioName}-setup`,
+      name: `${scenarioName} setup`,
+      description: `Scenario "${scenarioName}" could not be set up (connect/fixture/capability)`,
+      status: 'FAILURE',
+      timestamp: new Date().toISOString(),
+      errorMessage: `Setup failed: ${message}`,
+      ...(specReferences ? { specReferences } : {})
+    }
+  ];
+}
+
 /**
  * Create and connect an MCP client to a server
  */

src/scenarios/server/lifecycle.test.ts

@@ -1,9 +1,15 @@
 import { ServerInitializeScenario } from './lifecycle';
 import { connectToServer } from './client-helper';
 
-vi.mock('./client-helper', () => ({
-  connectToServer: vi.fn()
-}));
+vi.mock(import('./client-helper'), async (importOriginal) => {
+  const actual = await importOriginal();
+  return {
+    ...actual,
+    // Only the connection factory is mocked; reportSetupFailure stays real so
+    // the scenario's setup-failure path is exercised end-to-end.
+    connectToServer: vi.fn()
+  };
+});
 
 describe('ServerInitializeScenario', () => {
   const serverUrl = 'http://localhost:3000/mcp';
@@ -91,4 +97,23 @@ describe('ServerInitializeScenario', () => {
       }
     });
   });
+
+  it('reports a single setup FAILURE when the connection cannot be established', async () => {
+    vi.mocked(connectToServer).mockRejectedValueOnce(
+      new Error('connect ECONNREFUSED 127.0.0.1:3000')
+    );
+
+    const checks = await new ServerInitializeScenario().run(serverUrl);
+
+    // A connect failure should not be mislabeled as the initialize or
+    // session-id check failing (#248): a single dedicated setup check instead.
+    expect(checks).toHaveLength(1);
+    expect(checks[0]).toMatchObject({
+      id: 'server-initialize-setup',
+      status: 'FAILURE',
+      errorMessage: 'Setup failed: connect ECONNREFUSED 127.0.0.1:3000'
+    });
+    // The session-id check is never reached, so no raw fetch is attempted.
+    expect(fetchMock).not.toHaveBeenCalled();
+  });
 });

src/scenarios/server/lifecycle.ts

@@ -3,7 +3,7 @@
  */
 
 import { ClientScenario, ConformanceCheck } from '../../types';
-import { connectToServer } from './client-helper';
+import { connectToServer, reportSetupFailure } from './client-helper';
 
 const VISIBLE_ASCII_REGEX = /^[\x21-\x7E]+$/;
 
@@ -61,22 +61,10 @@ and validates session ID format if one is assigned.`;
 
       await connection.close();
     } catch (error) {
-      checks.push({
-        id: 'server-initialize',
-        name: 'ServerInitialize',
-        description:
-          'Server responds to initialize request with valid structure',
-        status: 'FAILURE',
-        timestamp: new Date().toISOString(),
-        errorMessage: `Failed to initialize: ${error instanceof Error ? error.message : String(error)}`,
-        specReferences: [
-          {
-            id: 'MCP-Initialize',
-            url: 'https://modelcontextprotocol.io/specification/2025-06-18/basic/lifecycle#initialization'
-          }
-        ]
-      });
-      return checks;
+      // The handshake never completed, so neither the initialize check nor the
+      // session-id check below can be evaluated. Report a single setup failure
+      // rather than mislabeling it as one specific check failing (#248).
+      return reportSetupFailure(this.name, error);
     }
 
     // Check: Session ID visible ASCII validation

src/scenarios/server/prompts.ts

@@ -3,7 +3,7 @@
  */
 
 import { ClientScenario, ConformanceCheck } from '../../types';
-import { connectToServer } from './client-helper';
+import { connectToServer, reportSetupFailure } from './client-helper';
 
 export class PromptsListScenario implements ClientScenario {
   name = 'prompts-list';
@@ -24,9 +24,15 @@ export class PromptsListScenario implements ClientScenario {
   async run(serverUrl: string): Promise<ConformanceCheck[]> {
     const checks: ConformanceCheck[] = [];
 
+    let connection;
     try {
-      const connection = await connectToServer(serverUrl);
+      connection = await connectToServer(serverUrl);
+    } catch (error) {
+      // Couldn't connect, so prompts/list never ran; report as setup (#248).
+      return reportSetupFailure(this.name, error);
+    }
 
+    try {
       const result = await connection.client.listPrompts();
 
       // Validate response structure

src/scenarios/server/resources.ts

@@ -7,7 +7,7 @@ import {
   ConformanceCheck,
   DRAFT_PROTOCOL_VERSION
 } from '../../types';
-import { connectToServer } from './client-helper';
+import { connectToServer, reportSetupFailure } from './client-helper';
 import { sendStatelessRequest } from './stateless-client';
 import {
   TextResourceContents,
@@ -34,9 +34,15 @@ export class ResourcesListScenario implements ClientScenario {
   async run(serverUrl: string): Promise<ConformanceCheck[]> {
     const checks: ConformanceCheck[] = [];
 
+    let connection;
     try {
-      const connection = await connectToServer(serverUrl);
+      connection = await connectToServer(serverUrl);
+    } catch (error) {
+      // Couldn't connect, so resources/list never ran; report as setup (#248).
+      return reportSetupFailure(this.name, error);
+    }
 
+    try {
       const result = await connection.client.listResources();
 
       // Validate response structure

src/scenarios/server/tools.ts

@@ -3,7 +3,11 @@
  */
 
 import { ClientScenario, ConformanceCheck } from '../../types';
-import { connectToServer, NotificationCollector } from './client-helper';
+import {
+  connectToServer,
+  NotificationCollector,
+  reportSetupFailure
+} from './client-helper';
 import {
   CallToolResultSchema,
   CreateMessageRequestSchema,
@@ -107,9 +111,17 @@ export class ToolsListScenario implements ClientScenario {
   async run(serverUrl: string): Promise<ConformanceCheck[]> {
     const checks: ConformanceCheck[] = [];
 
+    let connection;
     try {
-      const connection = await connectToServer(serverUrl);
+      connection = await connectToServer(serverUrl);
+    } catch (error) {
+      // A connect failure isn't a `tools-list` failure; pinning it there would
+      // also drop the `tools-name-format` check entirely. Report it as setup
+      // (#248).
+      return reportSetupFailure(this.name, error);
+    }
 
+    try {
       const result = await connection.client.listTools();
 
       // Validate response structure