test(playwright): cover model administration lifecycle

automation/playwright/README.md

@@ -80,6 +80,12 @@ The resource-administration inventory renders every management list and creation
 npm run capture -- --plan examples/inspect-resource-administration.capture-plan.json --output .work/resource-administration
 ```
 
+The local Ollama extension backs a fully offline create/update/delete journey for a model provider, model profile, and runtime profile:
+
+```powershell
+npm run capture -- --plan examples/exercise-model-administration.capture-plan.json --output .work/model-administration
+```
+
 The solution-discovery video Flow and Entry are captured with:
 
 ```powershell

automation/playwright/examples/exercise-model-administration.capture-plan.json

@@ -0,0 +1,18 @@
+{
+  "journey": "exercise-model-administration",
+  "input": {
+    "providerName": "playwright-ollama",
+    "providerDisplayName": "Playwright Ollama",
+    "profileName": "playwright-model",
+    "profileDisplayName": "Playwright model",
+    "runtimeName": "playwright-runtime",
+    "runtimeDisplayName": "Playwright runtime"
+  },
+  "checkpoints": [
+    "model-administration-provider",
+    "model-administration-runtime",
+    "model-administration-profile",
+    "model-administration-updated",
+    "model-administration-deleted"
+  ]
+}

automation/playwright/src/contracts/checkpoints.ts

@@ -74,4 +74,11 @@ export const Checkpoints = {
     profileEditors: 'resource-administration-profile-editors',
     protectedEditors: 'resource-administration-protected-editors',
   },
+  modelAdministration: {
+    provider: 'model-administration-provider',
+    runtime: 'model-administration-runtime',
+    profile: 'model-administration-profile',
+    updated: 'model-administration-updated',
+    deleted: 'model-administration-deleted',
+  },
 } as const;

automation/playwright/src/contracts/test-ids.ts

@@ -73,6 +73,32 @@ export const TestIds = {
     triggerDetails: 'resource-trigger-details',
     triggerEditor: 'resource-trigger-editor',
   },
+  modelAdministration: {
+    providerForm: 'model-provider-form',
+    providerName: 'model-provider-name',
+    providerDisplayName: 'model-provider-display-name',
+    providerExtension: 'model-provider-extension',
+    providerContribution: 'model-provider-contribution',
+    providerSave: 'model-provider-save',
+    providerTest: 'model-provider-test',
+    providerStatus: 'model-provider-status',
+    providerDelete: 'model-provider-delete',
+    providerDeleteConfirm: 'model-provider-delete-dialog-confirm',
+    profileForm: 'model-profile-form',
+    profileName: 'model-profile-name',
+    profileDisplayName: 'model-profile-display-name',
+    profileProvider: 'model-profile-provider',
+    profileModel: 'model-profile-model',
+    profileSave: 'model-profile-save',
+    profileDelete: 'model-profile-delete',
+    profileDeleteConfirm: 'model-profile-delete-dialog-confirm',
+    runtimeForm: 'runtime-profile-form',
+    runtimeName: 'runtime-profile-name',
+    runtimeDisplayName: 'runtime-profile-display-name',
+    runtimeSave: 'runtime-profile-save',
+    runtimeDelete: 'runtime-profile-delete',
+    runtimeDeleteConfirm: 'runtime-profile-delete-dialog-confirm',
+  },
   login: {
     username: 'login-username',
     password: 'login-password',

automation/playwright/src/coverage/application-surfaces.ts

@@ -108,6 +108,15 @@ function partialResourceAdministration(
   };
 }
 
+function coveredModelAdministration(id: string, source: string, routes: readonly string[], fixtureKeys: readonly string[] = []): ApplicationSurface {
+  return {
+    id, host: 'console', source, routes, fixtureKeys,
+    pageObject: 'src/pages/model-administration.page.ts',
+    journey: 'exercise-model-administration',
+    specification: 'tests/model-administration.spec.ts', coverage: 'covered',
+  };
+}
+
 export const applicationSurfaces: readonly ApplicationSurface[] = [
   {
     id: 'console-home', host: 'console', source: `${consolePages}/Home.razor`, routes: ['/'], fixtureKeys: [],
@@ -170,12 +179,12 @@ export const applicationSurfaces: readonly ApplicationSurface[] = [
   coveredFlow('task-flow-run-details', `${consolePages}/TaskFlowRunDetails.razor`, ['/tasks/{TaskId:guid}/flowruns/{RunId}'], ['taskId', 'runId'], 'inspect-flow-observability'),
 
   partialResourceAdministration('entries', `${consolePages}/Entries.razor`, ['/entries']),
-  partialResourceAdministration('model-profiles', `${consolePages}/ModelProfiles.razor`, ['/modelprofiles']),
-  partialResourceAdministration('model-profile-editor', `${consolePages}/ModelProfileEditor.razor`, ['/modelprofiles/new', '/modelprofiles/{Name}'], ['name']),
-  partialResourceAdministration('model-providers', `${consolePages}/ModelProviders.razor`, ['/modelproviders']),
-  partialResourceAdministration('model-provider-details', `${consolePages}/ModelProviderDetails.razor`, ['/modelproviders/new', '/modelproviders/{Name}'], ['name']),
-  partialResourceAdministration('runtime-profiles', `${consolePages}/RuntimeProfiles.razor`, ['/runtimeprofiles']),
-  partialResourceAdministration('runtime-profile-editor', `${consolePages}/RuntimeProfileEditor.razor`, ['/runtimeprofiles/new', '/runtimeprofiles/{Name}'], ['name']),
+  coveredModelAdministration('model-profiles', `${consolePages}/ModelProfiles.razor`, ['/modelprofiles']),
+  coveredModelAdministration('model-profile-editor', `${consolePages}/ModelProfileEditor.razor`, ['/modelprofiles/new', '/modelprofiles/{Name}'], ['name']),
+  coveredModelAdministration('model-providers', `${consolePages}/ModelProviders.razor`, ['/modelproviders']),
+  coveredModelAdministration('model-provider-details', `${consolePages}/ModelProviderDetails.razor`, ['/modelproviders/new', '/modelproviders/{Name}'], ['name']),
+  coveredModelAdministration('runtime-profiles', `${consolePages}/RuntimeProfiles.razor`, ['/runtimeprofiles']),
+  coveredModelAdministration('runtime-profile-editor', `${consolePages}/RuntimeProfileEditor.razor`, ['/runtimeprofiles/new', '/runtimeprofiles/{Name}'], ['name']),
   partialResourceAdministration('secrets', `${consolePages}/Secrets.razor`, ['/secrets']),
   partialResourceAdministration('secret-editor', `${consolePages}/SecretEditor.razor`, ['/secrets/new', '/secrets/{Name}'], ['name']),
   partialResourceAdministration('vaults', `${consolePages}/Vaults.razor`, ['/vaults']),

automation/playwright/src/journeys/exercise-model-administration.journey.ts

@@ -0,0 +1,42 @@
+import { Checkpoints } from '../contracts/checkpoints.js';
+import type { ModelAdministrationDefinition } from '../pages/model-administration.page.js';
+import type { Journey } from './journey.js';
+import { prepareConsoleJourney } from './prepare-console.journey.js';
+
+export type ExerciseModelAdministrationInput = ModelAdministrationDefinition & {
+  username?: string;
+  password?: string;
+  workspaceName?: string;
+};
+
+export const exerciseModelAdministration: Journey<ExerciseModelAdministrationInput> = async (context, input) => {
+  validate(input);
+  await prepareConsoleJourney(context, input);
+  const administration = context.pages.modelAdministration;
+
+  const provider = await administration.createProvider(context.consoleUrl, input);
+  await context.checkpoint({ name: Checkpoints.modelAdministration.provider, page: context.pages.page, target: provider });
+  await administration.updateProviderDisplayName(`${input.providerDisplayName} updated`);
+
+  const runtime = await administration.createRuntime(context.consoleUrl, input);
+  await context.checkpoint({ name: Checkpoints.modelAdministration.runtime, page: context.pages.page, target: runtime });
+  await administration.updateRuntimeDisplayName(`${input.runtimeDisplayName} updated`);
+
+  const profile = await administration.createProfile(context.consoleUrl, input);
+  await context.checkpoint({ name: Checkpoints.modelAdministration.profile, page: context.pages.page, target: profile });
+  await administration.updateProfileDisplayName(`${input.profileDisplayName} updated`);
+  await context.checkpoint({ name: Checkpoints.modelAdministration.updated, page: context.pages.page, target: profile });
+
+  await administration.deleteProfile();
+  await administration.openRuntime(context.consoleUrl, input.runtimeName);
+  await administration.deleteRuntime();
+  await administration.openProvider(context.consoleUrl, input.providerName);
+  await administration.deleteProvider();
+  await context.checkpoint({ name: Checkpoints.modelAdministration.deleted, page: context.pages.page });
+};
+
+function validate(input: ExerciseModelAdministrationInput): void {
+  for (const key of ['providerName', 'providerDisplayName', 'profileName', 'profileDisplayName', 'runtimeName', 'runtimeDisplayName'] as const) {
+    if (!input[key]?.trim()) throw new Error(`Model administration input '${key}' is required.`);
+  }
+}

automation/playwright/src/journeys/registry.ts

@@ -10,6 +10,7 @@ import { createPackProject, type CreatePackProjectInput } from './create-pack-pr
 import { inspectFlowObservability, type InspectFlowObservabilityInput } from './inspect-flow-observability.journey.js';
 import { inspectConsoleAdministration, type InspectConsoleAdministrationInput } from './inspect-console-administration.journey.js';
 import { inspectResourceAdministration, type InspectResourceAdministrationInput } from './inspect-resource-administration.journey.js';
+import { exerciseModelAdministration, type ExerciseModelAdministrationInput } from './exercise-model-administration.journey.js';
 import type { Journey } from './journey.js';
 
 export const journeys: Readonly<Record<string, Journey<Record<string, unknown>>>> = {
@@ -25,4 +26,5 @@ export const journeys: Readonly<Record<string, Journey<Record<string, unknown>>>
   'inspect-flow-observability': (context, input) => inspectFlowObservability(context, input as unknown as InspectFlowObservabilityInput),
   'inspect-console-administration': (context, input) => inspectConsoleAdministration(context, input as unknown as InspectConsoleAdministrationInput),
   'inspect-resource-administration': (context, input) => inspectResourceAdministration(context, input as unknown as InspectResourceAdministrationInput),
+  'exercise-model-administration': (context, input) => exerciseModelAdministration(context, input as unknown as ExerciseModelAdministrationInput),
 };

automation/playwright/src/pages/model-administration.page.ts

@@ -0,0 +1,150 @@
+import { type Locator, type Page } from '@playwright/test';
+import { TestIds } from '../contracts/test-ids.js';
+import { fillAndCommit } from './controls.js';
+
+export interface ModelAdministrationDefinition {
+  providerName: string;
+  providerDisplayName: string;
+  profileName: string;
+  profileDisplayName: string;
+  runtimeName: string;
+  runtimeDisplayName: string;
+}
+
+export class ModelAdministrationPage {
+  public constructor(private readonly page: Page) {}
+
+  public async createProvider(consoleUrl: string, definition: ModelAdministrationDefinition): Promise<Locator> {
+    await this.open(consoleUrl, '/modelproviders/new', TestIds.resourceAdministration.modelProviderEditor);
+    await fillAndCommit(this.page.getByTestId(TestIds.modelAdministration.providerName), definition.providerName);
+    await fillAndCommit(this.page.getByTestId(TestIds.modelAdministration.providerDisplayName), definition.providerDisplayName);
+    await selectMatchingOption(this.page.getByTestId(TestIds.modelAdministration.providerExtension), 'ollama-extension');
+    await selectFirstOption(this.page.getByTestId(TestIds.modelAdministration.providerContribution));
+    await this.page.getByTestId(TestIds.modelAdministration.providerSave).click();
+    await this.page.waitForURL(url => url.pathname === `/modelproviders/${definition.providerName}`);
+    const marker = this.page.getByTestId(TestIds.resourceAdministration.modelProviderEditor);
+    await marker.waitFor({ state: 'visible' });
+    await this.waitForInteractive(TestIds.modelAdministration.providerForm);
+    const testResponse = this.waitForApiResponse('POST', `/api/modelproviders/${encodeURIComponent(definition.providerName)}/test`);
+    await this.page.getByTestId(TestIds.modelAdministration.providerTest).click();
+    await testResponse;
+    await this.page.getByTestId(TestIds.modelAdministration.providerStatus).waitFor({ state: 'visible' });
+    return marker;
+  }
+
+  public async createRuntime(consoleUrl: string, definition: ModelAdministrationDefinition): Promise<Locator> {
+    await this.open(consoleUrl, '/runtimeprofiles/new', TestIds.resourceAdministration.runtimeProfileEditor);
+    await fillAndCommit(this.page.getByTestId(TestIds.modelAdministration.runtimeName), definition.runtimeName);
+    await fillAndCommit(this.page.getByTestId(TestIds.modelAdministration.runtimeDisplayName), definition.runtimeDisplayName);
+    await this.page.getByTestId(TestIds.modelAdministration.runtimeSave).click();
+    await this.page.waitForURL(url => url.pathname === `/runtimeprofiles/${definition.runtimeName}`);
+    await this.waitForInteractive(TestIds.modelAdministration.runtimeForm);
+    return this.page.getByTestId(TestIds.resourceAdministration.runtimeProfileEditor);
+  }
+
+  public async createProfile(consoleUrl: string, definition: ModelAdministrationDefinition): Promise<Locator> {
+    await this.open(consoleUrl, '/modelprofiles/new', TestIds.resourceAdministration.modelProfileEditor);
+    await fillAndCommit(this.page.getByTestId(TestIds.modelAdministration.profileName), definition.profileName);
+    await fillAndCommit(this.page.getByTestId(TestIds.modelAdministration.profileDisplayName), definition.profileDisplayName);
+    await selectMatchingOption(this.page.getByTestId(TestIds.modelAdministration.profileProvider), definition.providerName);
+    await selectFirstOption(this.page.getByTestId(TestIds.modelAdministration.profileModel));
+    await this.page.getByTestId(TestIds.modelAdministration.profileSave).click();
+    await this.page.waitForURL(url => url.pathname === `/modelprofiles/${definition.profileName}`);
+    await this.waitForInteractive(TestIds.modelAdministration.profileForm);
+    return this.page.getByTestId(TestIds.resourceAdministration.modelProfileEditor);
+  }
+
+  public async updateProviderDisplayName(value: string): Promise<void> {
+    await this.updateDisplayName(
+      this.page.getByTestId(TestIds.modelAdministration.providerDisplayName),
+      this.page.getByTestId(TestIds.modelAdministration.providerSave),
+      value,
+      '/api/modelproviders/',
+    );
+  }
+
+  public async updateRuntimeDisplayName(value: string): Promise<void> {
+    await this.updateDisplayName(
+      this.page.getByTestId(TestIds.modelAdministration.runtimeDisplayName),
+      this.page.getByTestId(TestIds.modelAdministration.runtimeSave),
+      value,
+      '/api/runtimeprofiles/',
+    );
+  }
+
+  public async updateProfileDisplayName(value: string): Promise<void> {
+    await this.updateDisplayName(
+      this.page.getByTestId(TestIds.modelAdministration.profileDisplayName),
+      this.page.getByTestId(TestIds.modelAdministration.profileSave),
+      value,
+      '/api/modelprofiles/',
+    );
+  }
+
+  public async deleteProfile(): Promise<void> {
+    await this.deleteCurrent(TestIds.modelAdministration.profileDelete, TestIds.modelAdministration.profileDeleteConfirm, '/modelprofiles');
+  }
+
+  public async openRuntime(consoleUrl: string, name: string): Promise<void> {
+    await this.open(consoleUrl, `/runtimeprofiles/${name}`, TestIds.resourceAdministration.runtimeProfileEditor);
+  }
+
+  public async deleteRuntime(): Promise<void> {
+    await this.deleteCurrent(TestIds.modelAdministration.runtimeDelete, TestIds.modelAdministration.runtimeDeleteConfirm, '/runtimeprofiles');
+  }
+
+  public async openProvider(consoleUrl: string, name: string): Promise<void> {
+    await this.open(consoleUrl, `/modelproviders/${name}`, TestIds.resourceAdministration.modelProviderEditor);
+  }
+
+  public async deleteProvider(): Promise<void> {
+    await this.deleteCurrent(TestIds.modelAdministration.providerDelete, TestIds.modelAdministration.providerDeleteConfirm, '/modelproviders');
+  }
+
+  private async updateDisplayName(input: Locator, save: Locator, value: string, apiPath: string): Promise<void> {
+    await fillAndCommit(input, value);
+    const response = this.waitForApiResponse('PUT', apiPath);
+    await save.click();
+    await response;
+  }
+
+  private async waitForApiResponse(method: string, apiPath: string): Promise<void> {
+    const response = await this.page.waitForResponse(candidate =>
+      candidate.request().method() === method && new URL(candidate.url()).pathname.includes(apiPath));
+    if (!response.ok()) {
+      throw new Error(`${method} ${new URL(response.url()).pathname} returned HTTP ${response.status()}.`);
+    }
+  }
+
+  private async waitForInteractive(testId: string): Promise<void> {
+    await this.page.locator(`[data-testid="${testId}"][data-interactive="true"]`).waitFor({ state: 'visible' });
+  }
+
+  private async deleteCurrent(deleteButtonId: string, confirmButtonId: string, listPath: string): Promise<void> {
+    await this.page.getByTestId(deleteButtonId).click();
+    await this.page.getByTestId(confirmButtonId).click();
+    await this.page.waitForURL(url => url.pathname === listPath);
+  }
+
+  private async open(consoleUrl: string, path: string, marker: string): Promise<void> {
+    const response = await this.page.goto(`${consoleUrl}${path}`, { waitUntil: 'domcontentloaded' });
+    if (!response?.ok()) throw new Error(`Model administration route '${path}' returned HTTP ${response?.status() ?? 'no response'}.`);
+    await this.page.getByTestId(marker).waitFor({ state: 'visible' });
+  }
+}
+
+async function selectMatchingOption(select: Locator, text: string): Promise<void> {
+  const option = select.locator('option').filter({ hasText: text }).first();
+  await option.waitFor({ state: 'attached' });
+  const value = await option.getAttribute('value');
+  if (!value) throw new Error(`No selectable option contains '${text}'.`);
+  await select.selectOption(value);
+}
+
+async function selectFirstOption(select: Locator): Promise<void> {
+  const option = select.locator('option:not([value=""]):not([disabled])').first();
+  await option.waitFor({ state: 'attached' });
+  const value = await option.getAttribute('value');
+  if (!value) throw new Error('No selectable option is available.');
+  await select.selectOption(value);
+}

automation/playwright/src/pages/product.pages.ts

@@ -12,6 +12,7 @@ import { FlowObservabilityPage } from './flow-observability.page.js';
 import { ConsoleAdministrationPage } from './console-administration.page.js';
 import { OperationsPage } from './operations.page.js';
 import { ResourceAdministrationPage } from './resource-administration.page.js';
+import { ModelAdministrationPage } from './model-administration.page.js';
 
 export class ProductPages {
   public readonly agentEditor: AgentEditorPage;
@@ -26,6 +27,7 @@ export class ProductPages {
   public readonly consoleAdministration: ConsoleAdministrationPage;
   public readonly operations: OperationsPage;
   public readonly resourceAdministration: ResourceAdministrationPage;
+  public readonly modelAdministration: ModelAdministrationPage;
 
   public constructor(public readonly page: Page) {
     this.agentEditor = new AgentEditorPage(page);
@@ -40,6 +42,7 @@ export class ProductPages {
     this.consoleAdministration = new ConsoleAdministrationPage(page);
     this.operations = new OperationsPage(page);
     this.resourceAdministration = new ResourceAdministrationPage(page);
+    this.modelAdministration = new ModelAdministrationPage(page);
   }
 
   public async ensureTheme(theme: 'light' | 'dark'): Promise<void> {

automation/playwright/tests/model-administration.spec.ts

@@ -0,0 +1,19 @@
+import { exerciseModelAdministration, type ExerciseModelAdministrationInput } from '../src/journeys/exercise-model-administration.journey.js';
+import { ignoreCheckpoints } from '../src/journeys/journey.js';
+import { ProductPages } from '../src/pages/product.pages.js';
+import { expect, test } from '../src/fixtures/test.js';
+
+const definition: ExerciseModelAdministrationInput = {
+  providerName: 'playwright-ollama',
+  providerDisplayName: 'Playwright Ollama',
+  profileName: 'playwright-model',
+  profileDisplayName: 'Playwright model',
+  runtimeName: 'playwright-runtime',
+  runtimeDisplayName: 'Playwright runtime',
+};
+
+test('model provider, model profile, and runtime profile complete their lifecycle @smoke', async ({ page, product }) => {
+  const pages = new ProductPages(page);
+  await exerciseModelAdministration({ ...product, pages, checkpoint: ignoreCheckpoints }, definition);
+  await expect(page).toHaveURL(/\/modelproviders$/);
+});