diff --git a/CHANGELOG.md b/CHANGELOG.md index eb3691a..b43c64f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ The format is based on Keep a Changelog and this project follows Semantic Versio ### Changed +- **Design-system page conformance** — updated deterministic browser fixtures for current onboarding, provider, navigation, Settings, Catalog, and Run behavior; added responsive route, mobile navigation, focus restoration, target-size, overflow, and template-chip contrast regression coverage. - **Design-system primitive ownership** — moved the native control, shared button, field, card, list, tab, empty-state, status, and utility patterns used by the frontend into `components.css`, removed unconsumed handoff primitives and duplicate app-level definitions, and tokenized application font sizes and border radii. - **Design-system foundation** — promoted `docs/design-system/tokens/` as the durable byte-identical token source, standardized the desktop sidebar at 220px, added input and accessible semantic-status tokens, resolved invalid aliases, and restored readable template operation chips. - **Agent workflow commands** — `AGENTS.md` now records the common validation, development, E2E, and benchmark data persistence workflows agents should use when changing the repo. diff --git a/frontend/src/styles/index.css b/frontend/src/styles/index.css index 12947dc..a0aface 100644 --- a/frontend/src/styles/index.css +++ b/frontend/src/styles/index.css @@ -4459,6 +4459,22 @@ pre.code-block { grid-template-columns: 1fr; } + .run-unified-layout, + .evaluate-queue-layout { + grid-template-columns: 1fr; + } + + .run-config-rail, + .evaluate-queue-rail { + border-right: 0; + border-bottom: 1px solid var(--paper-7); + } + + .evaluate-rubric { + border-left: 0; + border-top: 1px solid var(--paper-7); + } + .filters-row { grid-template-columns: 1fr; } diff --git a/frontend/tests/e2e/evaluate.spec.ts b/frontend/tests/e2e/evaluate.spec.ts index d2c68de..6812ca5 100644 --- a/frontend/tests/e2e/evaluate.spec.ts +++ b/frontend/tests/e2e/evaluate.spec.ts @@ -45,7 +45,7 @@ test.describe('Evaluate queue', () => { await expect(page.getByRole('heading', { name: 'Evaluate' })).toBeVisible(); await expect(page.locator('.merged-page-header').getByRole('heading', { name: 'Evaluate' })).toBeVisible(); await expect(page.locator('.evaluate-header')).toHaveCount(0); - await expect(page.getByText('Params')).toBeVisible(); + await expect(page.getByText('Params')).toHaveCount(0); await expect(page.getByRole('heading', { name: 'All caught up' })).toBeVisible(); }); diff --git a/frontend/tests/e2e/helpers.ts b/frontend/tests/e2e/helpers.ts index bb31d2d..2937b6a 100644 --- a/frontend/tests/e2e/helpers.ts +++ b/frontend/tests/e2e/helpers.ts @@ -2,7 +2,7 @@ import path from 'node:path'; import { fileURLToPath } from 'node:url'; import crypto from 'node:crypto'; -import { APIRequestContext, APIResponse, expect } from '@playwright/test'; +import { APIRequestContext, APIResponse, expect, type Page } from '@playwright/test'; import { loadEnv } from 'vite'; const dirname = path.dirname(fileURLToPath(import.meta.url)); @@ -13,6 +13,16 @@ const API_BASE_URL = env.E2E_API_BASE_URL ?? 'http://localhost:8080'; const API_TOKEN = env.INFERHARNESS_API_TOKEN ?? env.VITE_INFERHARNESS_API_TOKEN; const authHeaders = API_TOKEN ? { 'x-api-token': API_TOKEN } : undefined; +export async function dismissOnboarding(page: Page) { + await page.addInitScript(() => { + window.localStorage.setItem('inferharness.onboarding.v1', JSON.stringify({ + dismissedAt: '2026-01-01T00:00:00.000Z', + replaying: false, + ribbonsDismissed: [] + })); + }); +} + export interface InferenceServerRecord { inference_server: { server_id: string; diff --git a/frontend/tests/e2e/inference-servers-archive.spec.ts b/frontend/tests/e2e/inference-servers-archive.spec.ts index ac53f1d..9498bcf 100644 --- a/frontend/tests/e2e/inference-servers-archive.spec.ts +++ b/frontend/tests/e2e/inference-servers-archive.spec.ts @@ -1,8 +1,9 @@ import { expect, test } from '@playwright/test'; -import { archiveInferenceServer, createInferenceServer } from './helpers.js'; +import { archiveInferenceServer, createInferenceServer, dismissOnboarding } from './helpers.js'; test('archives an inference server', async ({ page, request }) => { + await dismissOnboarding(page); const created = await createInferenceServer(request); await page.goto('/'); @@ -21,5 +22,10 @@ test('archives an inference server', async ({ page, request }) => { ]); expect(archiveResponse.ok()).toBeTruthy(); + await page.reload(); + await page.getByRole('button', { name: 'Archived', exact: true }).click(); + const archivedCard = page.locator('.catalog-server-card').filter({ hasText: created.inference_server.display_name }); + await expect(archivedCard).toBeVisible(); + await archivedCard.click(); await expect(page.getByRole('button', { name: 'Unarchive', exact: true })).toBeVisible(); }); diff --git a/frontend/tests/e2e/inference-servers-create.spec.ts b/frontend/tests/e2e/inference-servers-create.spec.ts index 8537884..86bd9ce 100644 --- a/frontend/tests/e2e/inference-servers-create.spec.ts +++ b/frontend/tests/e2e/inference-servers-create.spec.ts @@ -1,12 +1,16 @@ import { expect, test } from '@playwright/test'; import crypto from 'node:crypto'; -import { archiveInferenceServer, findInferenceServerByName } from './helpers.js'; +import { archiveInferenceServer, dismissOnboarding, findInferenceServerByName } from './helpers.js'; + +test.beforeEach(async ({ page }) => { + await dismissOnboarding(page); +}); test('populates safe cloud provider defaults without filling the bearer token', async ({ page }) => { await page.goto('/catalog?tab=servers'); - await page.getByRole('button', { name: '+ Add server' }).click(); + await page.getByRole('button', { name: '+ Add server', exact: true }).click(); const createDrawer = page .getByRole('dialog') @@ -20,6 +24,8 @@ test('populates safe cloud provider defaults without filling the bearer token', 'Local / custom inference server', 'OpenAI', 'Mistral', + 'Anthropic', + 'Google Gemini', 'Groq', 'Together AI', 'Fireworks AI', @@ -68,7 +74,7 @@ test('creates a new inference server from the dashboard', async ({ page, request await page.goto('/catalog?tab=servers'); - await page.getByRole('button', { name: '+ Add server' }).click(); + await page.getByRole('button', { name: '+ Add server', exact: true }).click(); const createDrawer = page .getByRole('dialog') diff --git a/frontend/tests/e2e/inference-servers-edit.spec.ts b/frontend/tests/e2e/inference-servers-edit.spec.ts index e079d0f..7bf9026 100644 --- a/frontend/tests/e2e/inference-servers-edit.spec.ts +++ b/frontend/tests/e2e/inference-servers-edit.spec.ts @@ -1,8 +1,9 @@ import { expect, test } from '@playwright/test'; -import { archiveInferenceServer, createInferenceServer, findInferenceServerByName } from './helpers.js'; +import { archiveInferenceServer, createInferenceServer, dismissOnboarding, findInferenceServerByName } from './helpers.js'; test('edits an inference server', async ({ page, request }) => { + await dismissOnboarding(page); // Create server via API with hardware pre-populated to verify edit drawer pre-fills const created = await createInferenceServer(request, { hardware: { diff --git a/frontend/tests/e2e/inference-servers-inspect.spec.ts b/frontend/tests/e2e/inference-servers-inspect.spec.ts index 8256b07..55ff867 100644 --- a/frontend/tests/e2e/inference-servers-inspect.spec.ts +++ b/frontend/tests/e2e/inference-servers-inspect.spec.ts @@ -1,8 +1,9 @@ import { expect, test } from '@playwright/test'; -import { archiveInferenceServer, createInferenceServer } from './helpers.js'; +import { archiveInferenceServer, createInferenceServer, dismissOnboarding } from './helpers.js'; test('inspects an inference server', async ({ page, request }) => { + await dismissOnboarding(page); const created = await createInferenceServer(request); await page.goto('/'); diff --git a/frontend/tests/e2e/models.spec.ts b/frontend/tests/e2e/models.spec.ts index 2f9552b..9d78fb7 100644 --- a/frontend/tests/e2e/models.spec.ts +++ b/frontend/tests/e2e/models.spec.ts @@ -97,7 +97,9 @@ test('model filter rail narrows visible model cards', async ({ page, request }) await page.goto(`/catalog?tab=models&servers=${encodeURIComponent(serverId)}`); await page.waitForLoadState('networkidle'); - await page.getByLabel('MLX').check(); + const mlxFilter = page.getByLabel('MLX'); + await mlxFilter.click(); + await expect(mlxFilter).toBeChecked(); await expect(page.locator('.catalog-model-card').filter({ hasText: 'Qwen3 Coder' })).toBeVisible(); await expect(page.locator('.catalog-model-card').filter({ hasText: 'Devstral' })).toHaveCount(0); @@ -121,7 +123,9 @@ test('model filter rail uses persisted metadata instead of inferring from model await page.goto(`/catalog?tab=models&servers=${encodeURIComponent(serverId)}`); await page.waitForLoadState('networkidle'); - await page.getByLabel('MLX').check(); + const mlxFilter = page.getByLabel('MLX'); + await mlxFilter.click(); + await expect(mlxFilter).toBeChecked(); await expect(page.locator('.catalog-model-card').filter({ hasText: 'Persisted MLX' })).toBeVisible(); await expect(page.locator('.catalog-model-card').filter({ hasText: 'Raw MLX Suffix' })).toHaveCount(0); diff --git a/frontend/tests/e2e/navigation-shell.spec.ts b/frontend/tests/e2e/navigation-shell.spec.ts index 5662f15..f328268 100644 --- a/frontend/tests/e2e/navigation-shell.spec.ts +++ b/frontend/tests/e2e/navigation-shell.spec.ts @@ -1,5 +1,7 @@ import { expect, test, type Page } from '@playwright/test'; +import { dismissOnboarding } from './helpers.js'; + function catalogServer(serverId: string, name: string, modelId: string) { return { inference_server: { @@ -169,14 +171,15 @@ async function mockSettingsRoutes(page: Page) { }); } -test('sidebar exposes five top-level destinations and follows active routes', async ({ page }) => { +test('sidebar exposes six top-level destinations and follows active routes', async ({ page }) => { await page.goto('/catalog?tab=servers'); const nav = page.getByRole('navigation', { name: 'Primary navigation' }); - await expect(nav.getByRole('link')).toHaveCount(5); + await expect(nav.getByRole('link')).toHaveCount(6); await expect(nav.locator('.sidebar-item__main span:first-child')).toHaveText([ 'Catalog', 'Templates', + 'Datasets', 'Run', 'Results', 'Evaluate' @@ -185,6 +188,7 @@ test('sidebar exposes five top-level destinations and follows active routes', as for (const [href, label] of [ ['/catalog?tab=servers', 'Catalog'], ['/templates', 'Templates'], + ['/datasets', 'Datasets'], ['/run', 'Run'], ['/results?tab=dashboard', 'Results'], ['/evaluate', 'Evaluate'] @@ -207,6 +211,121 @@ test('merged page sub-tabs preserve route state', async ({ page }) => { await expect(page).toHaveURL(/\/results\?tab=history/); }); +test('responsive shell preserves desktop width and mobile navigation behavior', async ({ page }) => { + await dismissOnboarding(page); + await mockSettingsRoutes(page); + + await page.setViewportSize({ width: 1440, height: 900 }); + await page.goto('/catalog?tab=servers'); + + const desktopSidebar = page.locator('.sidebar--desktop'); + await expect(desktopSidebar).toBeVisible(); + const desktopSidebarBox = await desktopSidebar.boundingBox(); + expect(desktopSidebarBox?.width).toBe(220); + + await page.setViewportSize({ width: 390, height: 844 }); + await expect(desktopSidebar).toBeHidden(); + + const menuButton = page.getByRole('button', { name: 'Open navigation' }); + await expect(menuButton).toBeVisible(); + const menuButtonBox = await menuButton.boundingBox(); + expect(menuButtonBox?.width).toBeGreaterThanOrEqual(44); + expect(menuButtonBox?.height).toBeGreaterThanOrEqual(44); + + await menuButton.click(); + const navigationDialog = page.getByRole('dialog', { name: 'Primary navigation' }); + await expect(navigationDialog).toBeVisible(); + await expect(navigationDialog.getByRole('link')).toHaveCount(6); + await expect(navigationDialog.getByRole('button', { name: /Settings/ })).toBeVisible(); + + const closeButton = navigationDialog.getByRole('button', { name: 'Close navigation' }); + const closeButtonBox = await closeButton.boundingBox(); + expect(closeButtonBox?.width).toBeGreaterThanOrEqual(44); + expect(closeButtonBox?.height).toBeGreaterThanOrEqual(44); + await page.keyboard.press('Tab'); + expect(await page.evaluate(() => { + const dialog = document.querySelector('.mobile-nav-dialog'); + return Boolean(dialog?.contains(document.activeElement)); + })).toBe(true); + + await page.keyboard.press('Escape'); + await expect(navigationDialog).toBeHidden(); + await expect(menuButton).toBeFocused(); + + await menuButton.click(); + await navigationDialog.getByRole('link', { name: /^Templates/ }).click(); + await expect(page).toHaveURL(/\/templates$/); + await expect(navigationDialog).toBeHidden(); + await expect(menuButton).toBeFocused(); + + await menuButton.click(); + await navigationDialog.getByRole('button', { name: /Settings/ }).click(); + await expect(navigationDialog).toBeHidden(); + await expect(page.getByRole('dialog', { name: 'Settings' })).toBeVisible(); +}); + +test('primary routes avoid page-level horizontal overflow at desktop and mobile sizes', async ({ page }) => { + await dismissOnboarding(page); + const servers = [catalogServer('srv-a', 'Inferencer', 'mistral:latest')]; + const models = [catalogModel('srv-a', 'mistral:latest', 'mistral', 'MLX')]; + await mockCatalogRoutes(page, servers, models); + + const routes = [ + '/catalog?tab=servers', + '/templates', + '/datasets', + '/run', + '/results?tab=dashboard', + '/evaluate', + '/catalog?tab=models&serverId=srv-a&modelId=mistral%3Alatest' + ]; + + for (const viewport of [ + { width: 1440, height: 900 }, + { width: 390, height: 844 } + ]) { + await page.setViewportSize(viewport); + for (const route of routes) { + await page.goto(route); + await expect(page.locator('.app-main')).toBeVisible(); + await expect.poll(() => page.evaluate(() => ( + document.documentElement.scrollWidth - document.documentElement.clientWidth + ))).toBeLessThanOrEqual(1); + } + } +}); + +test('template operation chips meet accessible text contrast', async ({ page }) => { + await dismissOnboarding(page); + await page.goto('/templates'); + + const chips = page.locator('.template-card__chip'); + await expect.poll(() => chips.count()).toBeGreaterThan(0); + const chipCount = await chips.count(); + expect(chipCount).toBeGreaterThan(0); + const contrast = await chips.first().evaluate((chip) => { + const parse = (value: string) => { + const channels = value.match(/[\d.]+/g)?.slice(0, 3).map(Number) ?? [0, 0, 0]; + return channels.map((channel) => { + const normalized = channel / 255; + return normalized <= 0.03928 + ? normalized / 12.92 + : ((normalized + 0.055) / 1.055) ** 2.4; + }); + }; + const luminance = (channels: number[]) => ( + 0.2126 * channels[0] + 0.7152 * channels[1] + 0.0722 * channels[2] + ); + const style = getComputedStyle(chip); + const foreground = luminance(parse(style.color)); + const background = luminance(parse(style.backgroundColor)); + return (Math.max(foreground, background) + 0.05) + / (Math.min(foreground, background) + 0.05); + }); + + expect(contrast).toBeGreaterThanOrEqual(4.5); +}); + test('catalog servers header owns actions and conditional filter rail', async ({ page }) => { const activeServer = catalogServer('srv-a', 'Inferencer', 'mistral:latest'); const archivedServer = catalogServer('srv-archived', 'LegacyProd', 'legacy:latest'); @@ -314,6 +433,7 @@ test('settings opens from the sidebar footer', async ({ page }) => { await expect(dialog.getByRole('button', { name: /Connectivity/ })).toBeVisible(); await expect(dialog.getByRole('button', { name: /Frontend/ })).toBeVisible(); await expect(dialog.getByRole('button', { name: /Advanced/ })).toBeVisible(); + await dialog.getByRole('button', { name: /Runtime/ }).click(); await expect(dialog.locator('.label').filter({ hasText: /^Runtime$/ })).toBeVisible(); }); @@ -324,6 +444,7 @@ test('settings supports categorized env panes, folding, secrets, saves, add, rem await page.getByRole('button', { name: /Settings/ }).click(); const dialog = page.getByRole('dialog'); + await dialog.getByRole('button', { name: /Runtime/ }).click(); await expect(dialog.locator('.label').filter({ hasText: /^Runtime$/ })).toBeVisible(); await expect(dialog.getByText('2 keys')).toBeVisible(); await expect(dialog.getByText('Execution runtime')).toBeVisible(); @@ -332,7 +453,7 @@ test('settings supports categorized env panes, folding, secrets, saves, add, rem await expect(dialog.locator('.label').filter({ hasText: /^Model Selection$/ })).toBeVisible(); const modelSelect = dialog.getByLabel('Model'); await expect(modelSelect).toBeVisible(); - await expect(modelSelect).toHaveValue('srv-a::mistral:latest'); + await expect(modelSelect).toHaveValue(''); await dialog.getByLabel('Model').selectOption('srv-b::qwen:latest'); const modelSummary = dialog.locator('.settings-model-summary'); await expect(modelSummary.getByText('srv-b')).toBeVisible(); @@ -449,16 +570,21 @@ test('catalog models funnel aligns staged rail controls', async ({ page }) => { await page.goto('/catalog?tab=models'); const catalogPage = page.locator('.catalog-models'); - await expect(catalogPage.locator('.catalog-stage-number')).toHaveText(['1']); + await expect(catalogPage.locator('.catalog-stage-number')).toHaveText(['1', '2']); + await expect(page.locator('.catalog-server-stage .server-filter-row').filter({ hasText: 'srv-a.local' }).getByRole('checkbox')).toBeChecked(); - await page.locator('.catalog-server-stage .server-filter-row').filter({ hasText: 'srv-a.local' }).getByRole('checkbox').check(); + const secondServer = page.locator('.catalog-server-stage .server-filter-row').filter({ hasText: 'srv-b.local' }).getByRole('checkbox'); + await secondServer.click(); + await expect(secondServer).toBeChecked(); await expect(catalogPage.locator('.catalog-stage-number')).toHaveText(['1', '2']); const modelFilterRail = page.locator('.catalog-model-filter-stage'); await expect(modelFilterRail.getByText('Models')).toBeVisible(); await expect(modelFilterRail.getByText('0 selected')).toBeVisible(); await expect(modelFilterRail.getByRole('button', { name: 'Collapse' })).toBeVisible(); - await page.getByLabel('Mistral').check(); + const mistralFilter = page.getByLabel('Mistral'); + await mistralFilter.click(); + await expect(mistralFilter).toBeChecked(); await expect(modelFilterRail.getByText('1 selected')).toBeVisible(); await expect(modelFilterRail.getByRole('button', { name: 'Clear' })).toBeVisible(); await modelFilterRail.getByRole('button', { name: 'Collapse' }).click(); @@ -467,6 +593,7 @@ test('catalog models funnel aligns staged rail controls', async ({ page }) => { await expect(page.getByLabel('Mistral')).toBeChecked(); await modelFilterRail.getByRole('button', { name: 'Clear' }).click(); await expect(page.locator('.catalog-server-stage .server-filter-row').filter({ hasText: 'srv-a.local' }).getByRole('checkbox')).toBeChecked(); + await expect(page.locator('.catalog-server-stage .server-filter-row').filter({ hasText: 'srv-b.local' }).getByRole('checkbox')).toBeChecked(); await expect(page.getByLabel('Mistral')).not.toBeChecked(); }); @@ -502,10 +629,15 @@ test('catalog model cards use model-level provider and capability metadata', asy for (const label of ['text', 'json schema output', 'tools', 'embeddings', 'vision', 'audio', 'reasoning', 'explicit tokens', 'thinking', 'coding', 'instruct', 'mixture of experts']) { await expect(modelFilterRail.getByLabel(label)).toBeVisible(); } - await modelFilterRail.getByLabel('vision').check(); + const visionFilter = modelFilterRail.getByLabel('vision'); + await visionFilter.click(); + await expect(visionFilter).toBeChecked(); await expect(card).toBeVisible(); - await modelFilterRail.getByLabel('vision').uncheck(); - await modelFilterRail.getByLabel('audio').check(); + await visionFilter.click(); + await expect(visionFilter).not.toBeChecked(); + const audioFilter = modelFilterRail.getByLabel('audio'); + await audioFilter.click(); + await expect(audioFilter).toBeChecked(); await expect(page.locator('.catalog-model-card')).toHaveCount(0); }); diff --git a/frontend/tests/e2e/results-dashboard.spec.ts b/frontend/tests/e2e/results-dashboard.spec.ts index 0495180..12e3a57 100644 --- a/frontend/tests/e2e/results-dashboard.spec.ts +++ b/frontend/tests/e2e/results-dashboard.spec.ts @@ -219,7 +219,9 @@ test('merged Results dashboard filter and render flow', async ({ page }) => { await expect(page.getByLabel('Local Server')).toBeVisible(); await expect(page.getByLabel('mistral:latest')).toHaveCount(0); await expect(page.getByLabel('latency-benchmark')).toHaveCount(0); - await page.getByLabel('Local Server').check(); + const localServerFilter = page.getByLabel('Local Server'); + await localServerFilter.click(); + await expect(localServerFilter).toBeChecked(); await expect(resultsRail.locator('.results-funnel-stage')).toHaveCount(2); await expect(resultsRail.locator('.catalog-stage-number')).toHaveText(['1', '2']); await expect(resultsRail.getByText('Models')).toBeVisible(); @@ -235,7 +237,9 @@ test('merged Results dashboard filter and render flow', async ({ page }) => { await page.getByRole('button', { name: 'Expand Servers filters' }).click(); await expect(page.getByRole('button', { name: 'Collapse Servers filters' })).toBeVisible(); - await page.getByLabel('mistral:latest').check(); + const mistralFilter = page.getByLabel('mistral:latest'); + await mistralFilter.click(); + await expect(mistralFilter).toBeChecked(); await expect(resultsRail.locator('.results-funnel-stage')).toHaveCount(3); await expect(resultsRail.locator('.catalog-stage-number')).toHaveText(['1', '2', '3']); await expect(resultsRail.getByText('Tests & range')).toBeVisible(); @@ -257,7 +261,9 @@ test('merged Results dashboard filter and render flow', async ({ page }) => { expect(page.url()).toBe(selectedUrl); await expect(page.getByLabel('latency-benchmark')).toBeVisible(); - await page.getByLabel('latency-benchmark').check(); + const latencyFilter = page.getByLabel('latency-benchmark'); + await latencyFilter.click(); + await expect(latencyFilter).toBeChecked(); const testsStage = resultsRail.locator('.results-funnel-stage').filter({ hasText: 'Tests & range' }); await expect(testsStage.getByText('1 selected')).toBeVisible(); await testsStage.getByRole('button', { name: 'Clear' }).click(); @@ -265,12 +271,15 @@ test('merged Results dashboard filter and render flow', async ({ page }) => { await expect(page.getByLabel('mistral:latest')).toBeChecked(); await expect(page.getByLabel('latency-benchmark')).not.toBeChecked(); await expect(testsStage.getByText('0 selected')).toBeVisible(); - await page.getByLabel('Local Server').uncheck(); + await localServerFilter.click(); + await expect(localServerFilter).not.toBeChecked(); await expect(resultsRail.locator('.results-funnel-stage')).toHaveCount(1); await expect(resultsRail.getByText('Models')).toHaveCount(0); await expect(resultsRail.getByText('Tests & range')).toHaveCount(0); - await page.getByLabel('Local Server').check(); - await page.getByLabel('mistral:latest').check(); + await localServerFilter.click(); + await expect(localServerFilter).toBeChecked(); + await mistralFilter.click(); + await expect(mistralFilter).toBeChecked(); await expect(page.getByText('Total runs')).toBeVisible(); await expect(page.getByText('Pass rate')).toBeVisible(); diff --git a/frontend/tests/e2e/run-benchmark-execution.spec.ts b/frontend/tests/e2e/run-benchmark-execution.spec.ts index 6657d1e..7c06931 100644 --- a/frontend/tests/e2e/run-benchmark-execution.spec.ts +++ b/frontend/tests/e2e/run-benchmark-execution.spec.ts @@ -4,12 +4,13 @@ import http from 'node:http'; import path from 'node:path'; import { fileURLToPath } from 'node:url'; -import { expect, test } from '@playwright/test'; +import { expect, test, type Page } from '@playwright/test'; import { archiveInferenceServer, createModel, createInferenceServer, + dismissOnboarding, findInferenceServerByName } from './helpers.js'; @@ -26,6 +27,13 @@ interface MockChatServer { async function startMockOpenAiChatServer(): Promise { const requests: Array> = []; const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify({ + data: [{ id: 'gpt-4o-mini', object: 'model', owned_by: 'openai' }] + })); + return; + } if (request.method !== 'POST' || request.url !== '/v1/chat/completions') { response.writeHead(404).end(); return; @@ -69,7 +77,43 @@ async function startMockOpenAiChatServer(): Promise { }; } +async function mockHealthyServer(page: Page, serverId: string) { + await page.route('**/inference-servers/health*', async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + results: [{ + server_id: serverId, + ok: true, + status_code: 200, + response_time_ms: 1, + checked_at: '2026-01-01T00:00:00.000Z' + }] + }) + }); + }); +} + +async function mockRunCatalog(page: Page, server: unknown, model: unknown) { + await page.route(/\/inference-servers(?:\?.*)?$/, async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify([server]) + }); + }); + await page.route(/\/models(?:\?.*)?$/, async (route) => { + await route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify([model]) + }); + }); +} + test('configures a benchmark smoke run from inline prompt inputs', async ({ page, request }) => { + await dismissOnboarding(page); const mockChat = await startMockOpenAiChatServer(); const serverDisplayName = `E2E Benchmark Run ${Date.now()}-${crypto.randomBytes(4).toString('hex')}`; const server = await createInferenceServer(request, { @@ -77,10 +121,12 @@ test('configures a benchmark smoke run from inline prompt inputs', async ({ page base_url: mockChat.baseUrl, schema_family: ['openai-compatible'] }); - await createModel(request, server.inference_server.server_id, { + const model = await createModel(request, server.inference_server.server_id, { model_id: 'gpt-4o-mini', display_name: 'gpt-4o-mini' }); + await mockHealthyServer(page, server.inference_server.server_id); + await mockRunCatalog(page, server, model); try { await expect @@ -118,28 +164,30 @@ test('configures a benchmark smoke run from inline prompt inputs', async ({ page const runButton = page.getByRole('button', { name: 'Run benchmark' }); await expect(runButton).toBeEnabled(); - const createInstantiation = page.waitForResponse( + const createPlan = page.waitForResponse( (response) => response.request().method() === 'POST' && - response.url().includes('/benchmark/instantiations') && - !response.url().includes('/run') && + /\/benchmark\/plans$/.test(new URL(response.url()).pathname) && response.status() === 201 ); - const runInstantiation = page.waitForResponse( + const runPlan = page.waitForResponse( (response) => response.request().method() === 'POST' && - /\/benchmark\/instantiations\/[^/]+\/run$/.test(new URL(response.url()).pathname) && + /\/benchmark\/plans\/[^/]+\/run$/.test(new URL(response.url()).pathname) && response.status() === 201 ); await runButton.click(); - await createInstantiation; - await runInstantiation; + await createPlan; + await runPlan; await expect(page.locator('.run-message-card pre')).toHaveText('OK'); await expect(page.locator('.run-status-pill')).toHaveText('completed'); - await expect(page.locator('.run-asserts')).toContainText('completed'); - await expect(page.locator('.run-asserts')).not.toContainText('no-instantiation'); - await expect(page.locator('.run-asserts')).not.toContainText('no-result'); + const runAudit = page.locator('.run-asserts').filter({ + has: page.getByRole('heading', { name: 'Run audit' }) + }); + await expect(runAudit).toContainText('completed'); + await expect(runAudit).not.toContainText('no-instantiation'); + await expect(runAudit).not.toContainText('no-result'); await expect(page.locator('.run-metric-grid')).toContainText('5'); await expect(page.locator('.run-metric-grid')).toContainText('1'); await expect(page.locator('.run-metric-grid')).toContainText('6'); @@ -158,6 +206,7 @@ test('configures a benchmark smoke run from inline prompt inputs', async ({ page }); test('runs a server-side JSONL dataset from the Run page', async ({ page, request }) => { + await dismissOnboarding(page); const mockChat = await startMockOpenAiChatServer(); const serverDisplayName = `E2E Dataset Run ${Date.now()}-${crypto.randomBytes(4).toString('hex')}`; const datasetName = `e2e-codegen-${Date.now()}-${crypto.randomBytes(4).toString('hex')}.jsonl`; @@ -176,10 +225,12 @@ test('runs a server-side JSONL dataset from the Run page', async ({ page, reques base_url: mockChat.baseUrl, schema_family: ['openai-compatible'] }); - await createModel(request, server.inference_server.server_id, { + const model = await createModel(request, server.inference_server.server_id, { model_id: 'gpt-4o-mini', display_name: 'gpt-4o-mini' }); + await mockHealthyServer(page, server.inference_server.server_id); + await mockRunCatalog(page, server, model); try { await page.goto('/run'); @@ -199,20 +250,23 @@ test('runs a server-side JSONL dataset from the Run page', async ({ page, reques response.url().includes('/benchmark/datasets/manifest') && response.ok() ); - const runInstantiation = page.waitForResponse( + const runPlan = page.waitForResponse( (response) => response.request().method() === 'POST' && - /\/benchmark\/instantiations\/[^/]+\/run$/.test(new URL(response.url()).pathname) && + /\/benchmark\/plans\/[^/]+\/run$/.test(new URL(response.url()).pathname) && response.status() === 201 ); await runButton.click(); await prepareManifest; - await runInstantiation; + await runPlan; await expect(page.locator('.run-status-pill')).toHaveText('completed'); await expect(page.locator('.run-message-card pre')).toHaveCount(2); - await expect(page.locator('.run-asserts')).toContainText('e2e-codegen'); - await expect(page.locator('.run-asserts')).toContainText('2'); + const runAudit = page.locator('.run-asserts').filter({ + has: page.getByRole('heading', { name: 'Run audit' }) + }); + await expect(page.locator('.run-prompt-strip')).toContainText('e2e-codegen'); + await expect(runAudit).toContainText('items 2'); expect(mockChat.requests).toHaveLength(2); expect(mockChat.requests.map((entry) => entry.messages)).toEqual([ [{ role: 'user', content: 'Write a JavaScript add function.' }], diff --git a/package.json b/package.json index 966b567..2c56341 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "predev": "npm -w backend run bootstrap-env", "prestart": "npm -w backend run bootstrap-env", "e2e:serve": "concurrently --kill-others-on-fail \"npm run start:backend\" \"npm -w frontend run dev\"", - "lint": "eslint . --ext .js,.jsx,.ts,.tsx", + "lint": "eslint .", "release:check": "npm run check:design-system && npm run lint && npm test && npm run build", "start": "concurrently \"npm run start:backend\" \"npm run start:frontend\"", "start:backend": "npm -w backend run start",