diff --git a/.gitignore b/.gitignore index ebc2ccc6..7e518170 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,6 @@ Thumbs.db # Prisma prisma/generated/ + +# Shared videos stored by the API in development (SHARE_DIR) +apps/api/data/ diff --git a/apps/api/.env.example b/apps/api/.env.example index 7aff02e8..51547649 100644 --- a/apps/api/.env.example +++ b/apps/api/.env.example @@ -21,3 +21,17 @@ MAIL_FROM=DemoFlow # http://localhost:3670/api/auth/github/callback. GITHUB_CLIENT_ID= GITHUB_CLIENT_SECRET= + +# Share links (spec #347): the directory that holds every shared video and +# poster, relative to the API's working directory unless absolute. Nothing +# else is written there; it is not backed up. +SHARE_DIR=./data/shares +# The largest render a share may hold (200 MB), and every account's total +# across its shares, posters included (2 GB). +SHARE_MAX_FILE_BYTES=209715200 +SHARE_USER_QUOTA_BYTES=2147483648 + +# Signs the cookie a share password unlock sets (spec #348). At least 32 +# characters. APP_ENV=production refuses to start without it; below production +# an empty value means a random one per boot, so unlocks end at restart. +SHARE_SIGNING_SECRET= diff --git a/apps/api/Dockerfile b/apps/api/Dockerfile index f5546aaa..5145378c 100644 --- a/apps/api/Dockerfile +++ b/apps/api/Dockerfile @@ -52,6 +52,10 @@ COPY --from=builder --chown=node:node /app/node_modules node_modules/ COPY --from=builder --chown=node:node /app/packages/shared packages/shared/ COPY --from=builder --chown=node:node /app/apps/api apps/api/ +# The shares volume mounts here; a new named volume copies this directory's +# owner, so the API can write to it as `node` (spec #347). +RUN mkdir -p /data/shares && chown node:node /data/shares + USER node WORKDIR /app/apps/api diff --git a/apps/api/prisma/migrations/20261026120000_run_kind/migration.sql b/apps/api/prisma/migrations/20261026120000_run_kind/migration.sql new file mode 100644 index 00000000..14366199 --- /dev/null +++ b/apps/api/prisma/migrations/20261026120000_run_kind/migration.sql @@ -0,0 +1,5 @@ +-- CreateEnum +CREATE TYPE "RunKind" AS ENUM ('check'); + +-- AlterTable +ALTER TABLE "Run" ADD COLUMN "kind" "RunKind"; diff --git a/apps/api/prisma/migrations/20261027090000_shares/migration.sql b/apps/api/prisma/migrations/20261027090000_shares/migration.sql new file mode 100644 index 00000000..803ac1af --- /dev/null +++ b/apps/api/prisma/migrations/20261027090000_shares/migration.sql @@ -0,0 +1,46 @@ +-- CreateEnum +CREATE TYPE "ShareStatus" AS ENUM ('uploading', 'ready', 'failed', 'lost'); + +-- CreateTable +CREATE TABLE "Share" ( + "id" TEXT NOT NULL, + "token" TEXT NOT NULL, + "userId" TEXT NOT NULL, + "runId" TEXT NOT NULL, + "format" TEXT NOT NULL, + "status" "ShareStatus" NOT NULL DEFAULT 'uploading', + "title" TEXT NOT NULL, + "description" TEXT, + "chapters" JSONB NOT NULL DEFAULT '[]', + "sizeBytes" INTEGER NOT NULL DEFAULT 0, + "posterBytes" INTEGER NOT NULL DEFAULT 0, + "width" INTEGER, + "height" INTEGER, + "durationMs" INTEGER, + "passwordHash" TEXT, + "expiresAt" TIMESTAMP(3) NOT NULL, + "views" INTEGER NOT NULL DEFAULT 0, + "failure" TEXT, + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "Share_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "Share_token_key" ON "Share"("token"); + +-- CreateIndex +CREATE INDEX "Share_userId_createdAt_idx" ON "Share"("userId", "createdAt" DESC); + +-- CreateIndex +CREATE INDEX "Share_expiresAt_idx" ON "Share"("expiresAt"); + +-- CreateIndex +CREATE UNIQUE INDEX "Share_runId_format_key" ON "Share"("runId", "format"); + +-- AddForeignKey +ALTER TABLE "Share" ADD CONSTRAINT "Share_userId_fkey" FOREIGN KEY ("userId") REFERENCES "User"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "Share" ADD CONSTRAINT "Share_runId_fkey" FOREIGN KEY ("runId") REFERENCES "Run"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/apps/api/prisma/migrations/20261027150000_project_github_url/migration.sql b/apps/api/prisma/migrations/20261027150000_project_github_url/migration.sql new file mode 100644 index 00000000..1fc7e106 --- /dev/null +++ b/apps/api/prisma/migrations/20261027150000_project_github_url/migration.sql @@ -0,0 +1,6 @@ +-- CreateEnum +CREATE TYPE "ProjectSource" AS ENUM ('reported', 'website'); + +-- AlterTable +ALTER TABLE "Project" ADD COLUMN "githubUrl" TEXT, +ADD COLUMN "source" "ProjectSource" NOT NULL DEFAULT 'reported'; diff --git a/apps/api/prisma/migrations/20261028090000_check_schedules/migration.sql b/apps/api/prisma/migrations/20261028090000_check_schedules/migration.sql new file mode 100644 index 00000000..f26447a0 --- /dev/null +++ b/apps/api/prisma/migrations/20261028090000_check_schedules/migration.sql @@ -0,0 +1,77 @@ +-- CreateEnum +CREATE TYPE "ScheduleSource" AS ENUM ('site', 'config'); + +-- CreateEnum +CREATE TYPE "ScheduleCadence" AS ENUM ('daily', 'weekdays', 'weekly'); + +-- CreateTable +CREATE TABLE "CheckSchedule" ( + "id" TEXT NOT NULL, + "projectId" TEXT NOT NULL, + "source" "ScheduleSource" NOT NULL, + "cadence" "ScheduleCadence" NOT NULL, + "weekday" TEXT, + "at" TEXT NOT NULL, + "timeZone" TEXT NOT NULL, + "exclude" TEXT[], + "pausedReason" TEXT, + "lastSlotAt" TIMESTAMP(3), + "createdAt" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP(3) NOT NULL, + + CONSTRAINT "CheckSchedule_pkey" PRIMARY KEY ("id") +); + +-- CreateTable +CREATE TABLE "ScenarioHealth" ( + "projectId" TEXT NOT NULL, + "scenarioPath" TEXT NOT NULL, + "state" "RunStatus" NOT NULL, + "since" TIMESTAMP(3) NOT NULL, + "lastRunId" TEXT NOT NULL, + + CONSTRAINT "ScenarioHealth_pkey" PRIMARY KEY ("projectId","scenarioPath") +); + +-- CreateIndex +CREATE UNIQUE INDEX "CheckSchedule_projectId_source_key" ON "CheckSchedule"("projectId", "source"); + +-- CreateIndex +CREATE INDEX "ScenarioHealth_projectId_state_idx" ON "ScenarioHealth"("projectId", "state"); + +-- AddForeignKey +ALTER TABLE "CheckSchedule" ADD CONSTRAINT "CheckSchedule_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- AddForeignKey +ALTER TABLE "ScenarioHealth" ADD CONSTRAINT "ScenarioHealth_projectId_fkey" FOREIGN KEY ("projectId") REFERENCES "Project"("id") ON DELETE CASCADE ON UPDATE CASCADE; + +-- Backfill: each scenario's state is its latest run's status, `since` the +-- start of the oldest run after the latest run with the other status. +INSERT INTO "ScenarioHealth" ("projectId", "scenarioPath", "state", "since", "lastRunId") +SELECT + latest."projectId", + latest."scenarioPath", + latest."status", + COALESCE(( + SELECT MIN(r."startedAt") + FROM "Run" r + WHERE r."projectId" = latest."projectId" + AND r."scenarioPath" = latest."scenarioPath" + AND r."startedAt" > COALESCE( + ( + SELECT MAX(o."startedAt") + FROM "Run" o + WHERE o."projectId" = latest."projectId" + AND o."scenarioPath" = latest."scenarioPath" + AND o."status" <> latest."status" + ), + '-infinity'::timestamp + ) + ), latest."startedAt"), + latest."id" +FROM ( + SELECT DISTINCT ON ("projectId", "scenarioPath") + "projectId", "scenarioPath", "status", "startedAt", "id" + FROM "Run" + ORDER BY "projectId", "scenarioPath", "startedAt" DESC, "id" DESC +) latest; diff --git a/apps/api/prisma/schema.prisma b/apps/api/prisma/schema.prisma index bfca23f9..47fdb89e 100644 --- a/apps/api/prisma/schema.prisma +++ b/apps/api/prisma/schema.prisma @@ -20,6 +20,7 @@ model User { githubIdentity GithubIdentity? runners Runner[] projects Project[] + shares Share[] } // The GitHub account a user signs in with (spec #126). At most one per user, @@ -110,19 +111,34 @@ model Project { runner Runner? @relation(fields: [runnerId], references: [id], onDelete: SetNull) // Absolute path of the project root on that machine, as the CLI sent it. rootPath String? + // https://github.com//; display only (ADR-0022 §5). + githubUrl String? + source ProjectSource @default(reported) createdAt DateTime @default(now()) updatedAt DateTime @updatedAt runs Run[] + schedules CheckSchedule[] + health ScenarioHealth[] @@unique([runnerId, rootPath]) @@index([userId]) } +// How a project was added (spec #378): by a CLI report, or from the website. +enum ProjectSource { + reported + website +} + enum RunStatus { passed failed } +enum RunKind { + check +} + // The metadata of one `demoflow run`. Never a video, a frame or a step value: // values can hold secrets, and videos stay on the machine (ADR-0008). model Run { @@ -150,11 +166,108 @@ model Run { renders Json? failureStage String? failureMessage String? + // Spec 360: `check` for a run made with `demoflow run --check`; null otherwise. + kind RunKind? createdAt DateTime @default(now()) + shares Share[] @@index([projectId, startedAt(sort: Desc), id(sort: Desc)]) } +enum ShareStatus { + uploading + ready + failed + lost +} + +// One render of a run behind a public link (spec #347, ADR-0019). The files +// live under `SHARE_DIR//`, never in the database. One per run and format. +model Share { + id String @id @default(cuid()) + // 32 random bytes, base64url (`generateToken`). Stored as is: the owner + // copies the link again (ADR-0019, Consequences). + token String @unique + userId String + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + runId String + run Run @relation(fields: [runId], references: [id], onDelete: Cascade) + format String + status ShareStatus @default(uploading) + // The scenario's name at share time, and what the daemon sent (#349). + title String + description String? + // `[{ title, atMs }]`, at most 100. + chapters Json @default("[]") + sizeBytes Int @default(0) + posterBytes Int @default(0) + width Int? + height Int? + durationMs Int? + // argon2id (`PasswordHasher`), null without a password. + passwordHash String? + expiresAt DateTime + views Int @default(0) + failure String? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([runId, format]) + @@index([userId, createdAt(sort: Desc)]) + @@index([expiresAt]) +} + +enum ScheduleSource { + site + config +} + +enum ScheduleCadence { + daily + weekdays + weekly +} + +// When a project's scenarios are checked (spec #362, ADR-0021). A `site` row +// is set on the website and run by the API; a `config` row is what the daemon +// reported from `demoflow.config.json`, stored only to be shown. +model CheckSchedule { + id String @id @default(uuid()) + projectId String + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + source ScheduleSource + cadence ScheduleCadence + // `mon`…`sun`; set only for `weekly`. + weekday String? + // `HH:MM`, 24-hour, in `timeZone`. + at String + timeZone String + exclude String[] + // `config-schedule`, `no-machine` or `cli-too-old`; null while it runs. + pausedReason String? + // The latest slot a pass was started for (site rows only). + lastSlotAt DateTime? + createdAt DateTime @default(now()) + updatedAt DateTime @updatedAt + + @@unique([projectId, source]) +} + +// The state of a project's scenario after its latest reported run (spec #362, +// ADR-0021 §6). `failed` is "stale". `since` is the start of the run that made +// the last transition; `lastRunId` is a plain column: deleting a run keeps it. +model ScenarioHealth { + projectId String + project Project @relation(fields: [projectId], references: [id], onDelete: Cascade) + scenarioPath String + state RunStatus + since DateTime + lastRunId String + + @@id([projectId, scenarioPath]) + @@index([projectId, state]) +} + // A pending `demoflow login` (RFC 8628). Only the SHA-256 hash of the polling // secret is stored. `approvedById` is a plain column: the row lives at most // ten minutes and is deleted once its token is issued. diff --git a/apps/api/src/app.module.ts b/apps/api/src/app.module.ts index 601fc36b..9f8337bf 100644 --- a/apps/api/src/app.module.ts +++ b/apps/api/src/app.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; import { AuthModule } from './auth/auth.module'; +import { ChecksModule } from './checks/checks.module'; import { ClockModule } from './common/clock.module'; import { DatabaseModule } from './database/database.module'; import { HealthController } from './health.controller'; @@ -8,6 +9,7 @@ import { LaunchesModule } from './launches/launches.module'; import { MailModule } from './mail/mail.module'; import { ProjectsModule } from './projects/projects.module'; import { RunnerModule } from './runner/runner.module'; +import { SharesModule } from './shares/shares.module'; import { VideosModule } from './videos/videos.module'; @Module({ @@ -24,6 +26,8 @@ import { VideosModule } from './videos/videos.module'; ProjectsModule, VideosModule, LaunchesModule, + SharesModule, + ChecksModule, ], controllers: [HealthController], }) diff --git a/apps/api/src/auth/auth-throttle.ts b/apps/api/src/auth/auth-throttle.ts index d8f664ac..1851de15 100644 --- a/apps/api/src/auth/auth-throttle.ts +++ b/apps/api/src/auth/auth-throttle.ts @@ -19,6 +19,13 @@ export const THROTTLE_MAIL = 'mail-email'; * minutes, which the shared `ip` budget would cut off after 20 polls. */ export const THROTTLE_DEVICE_POLL = 'device-poll'; +/** A share's password (spec #348, Decision 6): per address and share token. */ +export const THROTTLE_SHARE_UNLOCK = 'share-unlock'; +/** + * Adding a project from a folder (spec #378, Decision 8): per user, across + * inspect, init and register, so a hijacked session cannot probe a machine. + */ +export const THROTTLE_PROJECT_FOLDERS = 'project-folders'; /** Class-level: every throttler skipped until a route opts in. */ export const SKIP_ALL_THROTTLERS = { @@ -26,6 +33,8 @@ export const SKIP_ALL_THROTTLERS = { [THROTTLE_SIGN_IN]: true, [THROTTLE_MAIL]: true, [THROTTLE_DEVICE_POLL]: true, + [THROTTLE_SHARE_UNLOCK]: true, + [THROTTLE_PROJECT_FOLDERS]: true, }; export const authThrottlerOptions: ThrottlerModuleOptions = { @@ -61,6 +70,23 @@ export const authThrottlerOptions: ThrottlerModuleOptions = { ttl: 10 * MINUTE_MS, getTracker: (req) => `ip:${(req as AuthenticatedRequest).ip ?? ''}`, }, + { + name: THROTTLE_SHARE_UNLOCK, + limit: 10, + ttl: 10 * MINUTE_MS, + getTracker: (req) => { + const request = req as AuthenticatedRequest; + return `ip:${request.ip ?? ''}:share:${routeToken(request)}`; + }, + }, + { + name: THROTTLE_PROJECT_FOLDERS, + limit: 30, + ttl: MINUTE_MS, + // Behind `SessionGuard`: there is always a user. + getTracker: (req) => + `user:${(req as AuthenticatedRequest).auth?.user.id ?? ''}`, + }, ], // The default key includes the route, which would give each endpoint its // own budget. The limits are across the auth endpoints. @@ -68,6 +94,12 @@ export const authThrottlerOptions: ThrottlerModuleOptions = { `${name}:${tracker}`, }; +/** Guards run after routing: `:token` is already parsed. */ +function routeToken(req: AuthenticatedRequest): string { + const token: unknown = req.params?.token; + return typeof token === 'string' ? token : ''; +} + /** Guards run before the validation pipe, so the body is still raw here. */ function bodyEmail(req: AuthenticatedRequest): string { const body: unknown = req.body; diff --git a/apps/api/src/auth/auth.module.ts b/apps/api/src/auth/auth.module.ts index 3bb92dd2..8edac5bf 100644 --- a/apps/api/src/auth/auth.module.ts +++ b/apps/api/src/auth/auth.module.ts @@ -29,6 +29,6 @@ import { VerifiedGuard } from './verified.guard'; GithubAuthService, { provide: GithubClient, useClass: HttpGithubClient }, ], - exports: [SessionService, SessionGuard, VerifiedGuard], + exports: [SessionService, SessionGuard, VerifiedGuard, PasswordHasher], }) export class AuthModule {} diff --git a/apps/api/src/checks/checks.controller.ts b/apps/api/src/checks/checks.controller.ts new file mode 100644 index 00000000..f5d15596 --- /dev/null +++ b/apps/api/src/checks/checks.controller.ts @@ -0,0 +1,74 @@ +import { + Body, + Controller, + Delete, + Get, + HttpCode, + HttpStatus, + Param, + Put, + UseGuards, +} from '@nestjs/common'; +import type { User } from '@prisma/client'; +import { CurrentUser } from '../auth/current-user.decorator'; +import { SessionGuard } from '../auth/session.guard'; +import { + HealthService, + type HealthView, + type StaleView, +} from './health/health.service'; +import { PutScheduleDto } from './schedules/schedules.dto'; +import { + type ProjectSchedules, + SchedulesService, + type ScheduleView, +} from './schedules/schedules.service'; + +/** + * A project's check schedule and the per-scenario state (spec #362, API). + * Another user's project is `404`, as on every project route. + */ +@Controller() +@UseGuards(SessionGuard) +export class ChecksController { + constructor( + private readonly schedules: SchedulesService, + private readonly health: HealthService, + ) {} + + @Get('projects/:id/schedule') + schedule( + @CurrentUser() user: User, + @Param('id') id: string, + ): Promise { + return this.schedules.get(user.id, id); + } + + @Put('projects/:id/schedule') + put( + @CurrentUser() user: User, + @Param('id') id: string, + @Body() dto: PutScheduleDto, + ): Promise { + return this.schedules.put(user.id, id, dto); + } + + @Delete('projects/:id/schedule') + @HttpCode(HttpStatus.NO_CONTENT) + remove(@CurrentUser() user: User, @Param('id') id: string): Promise { + return this.schedules.remove(user.id, id); + } + + @Get('projects/:id/health') + projectHealth( + @CurrentUser() user: User, + @Param('id') id: string, + ): Promise { + return this.health.project(user.id, id); + } + + @Get('checks/stale') + stale(@CurrentUser() user: User): Promise { + return this.health.stale(user.id); + } +} diff --git a/apps/api/src/checks/checks.integration.spec.ts b/apps/api/src/checks/checks.integration.spec.ts new file mode 100644 index 00000000..f1718d1b --- /dev/null +++ b/apps/api/src/checks/checks.integration.spec.ts @@ -0,0 +1,629 @@ +import { readFileSync } from 'node:fs'; +import type { AddressInfo } from 'node:net'; +import { join } from 'node:path'; +import request, { type Test } from 'supertest'; +import type { App } from 'supertest/types'; +import { LaunchesService } from '../launches/launches.service'; +import { + type Paired, + pair, + type SignedIn, + signedInUser, +} from '../runner/runner.fixtures'; +import { eventually, TestSocket } from '../runner/runner-socket.fixtures'; +import { createTestApp, type TestApp } from '../test/test-app'; +import { CheckScheduler } from './schedules/check-scheduler'; + +const ROOT = '/home/alice/shop'; +const SCENARIO = 'demoflow/checkout.demo.json'; +const MINUTE = 60 * 1000; +const DAY = 24 * 60 * MINUTE; +// The fake clock starts at 2026-10-20T12:00:00Z; this slot is five minutes on. +const DAILY = { cadence: 'daily', at: '12:05', timeZone: 'UTC' }; + +describe('check schedules (integration)', () => { + let t: TestApp; + let http: App; + let port: number; + let alice: SignedIn; + let bob: SignedIn; + let machine: Paired; + let projectId: string; + let bobProjectId: string; + const sockets: TestSocket[] = []; + + beforeEach(async () => { + t = await createTestApp(); + await t.app.listen(0, '127.0.0.1'); + http = t.app.getHttpServer(); + port = (http.address() as AddressInfo).port; + alice = await signedInUser(t, 'alice'); + bob = await signedInUser(t, 'bob'); + machine = await pair(http, alice, 'Laptop'); + projectId = (await report(machine, 'passed')).projectId; + const bobMachine = await pair(http, bob, 'Desktop'); + bobProjectId = (await report(bobMachine, 'failed', '/home/bob/blog')) + .projectId; + }); + + afterEach(async () => { + for (const socket of sockets.splice(0)) { + socket.ws.terminate(); + } + await t.app.close(); + }); + + async function report( + on: Paired, + status: 'passed' | 'failed', + rootPath = ROOT, + scenarioPath = SCENARIO, + kind?: 'check', + ): Promise<{ projectId: string; runId: string }> { + const res = await request(http) + .post('/runner/runs') + .set('Authorization', `Bearer ${on.token}`) + .send({ + rootPath, + projectName: 'shop', + scenarioName: 'Checkout', + scenarioPath, + status, + startedAt: t.clock.now().toISOString(), + durationMs: 12_345, + formats: ['mp4'], + cliVersion: '0.4.0', + ...(kind && { kind }), + }) + .expect(201); + return res.body; + } + + /** A daemon on `machine`; `checks` is what its `launches.sync` says, null for none. */ + async function daemon(checks: boolean | null = true): Promise { + const socket = await TestSocket.connect( + `ws://127.0.0.1:${port}/runner`, + machine.token, + ); + sockets.push(socket); + socket.sync([], undefined, undefined, checks ?? undefined); + await socket.roundTrip(); + return socket; + } + + function put(body: unknown, cookie = alice.cookie, id = projectId): Test { + return request(http) + .put(`/projects/${id}/schedule`) + .set('Cookie', cookie) + .send(body as object); + } + + function schedule(cookie = alice.cookie, id = projectId): Test { + return request(http).get(`/projects/${id}/schedule`).set('Cookie', cookie); + } + + function tick(): Promise { + return t.app.get(CheckScheduler).tick(); + } + + /** The `check.start`s the socket got and nobody consumed. */ + async function checkStarts(socket: TestSocket): Promise { + await socket.roundTrip(); + return socket.received.filter((m) => m.type === 'check.start').length; + } + + describe('authorization', () => { + it('answers 404 for another user’s project on every route', async () => { + await put(DAILY, alice.cookie, bobProjectId).expect(404); + await schedule(alice.cookie, bobProjectId).expect(404); + await request(http) + .delete(`/projects/${bobProjectId}/schedule`) + .set('Cookie', alice.cookie) + .expect(404); + await request(http) + .get(`/projects/${bobProjectId}/health`) + .set('Cookie', alice.cookie) + .expect(404); + await request(http) + .get('/projects/not-a-project/schedule') + .set('Cookie', alice.cookie) + .expect(404); + }); + + it('needs a session', async () => { + await request(http).get(`/projects/${projectId}/schedule`).expect(401); + await request(http).get('/checks/stale').expect(401); + }); + + it('never shows another user’s stale scenarios', async () => { + await report(machine, 'failed'); + const res = await request(http) + .get('/checks/stale') + .set('Cookie', alice.cookie) + .expect(200); + expect(res.body).toEqual([ + { + projectId, + projectName: 'shop', + scenarioPath: SCENARIO, + scenarioName: 'Checkout', + since: t.clock.now().toISOString(), + lastRunId: expect.any(String), + }, + ]); + const projects = await request(http) + .get('/projects') + .set('Cookie', bob.cookie) + .expect(200); + expect( + projects.body.projects.map((p: { id: string; staleCount: number }) => [ + p.id, + p.staleCount, + ]), + ).toEqual([[bobProjectId, 1]]); + }); + }); + + describe('PUT /projects/:id/schedule', () => { + it('stores the site schedule and computes the next slot', async () => { + const res = await put({ + ...DAILY, + exclude: ['demoflow/slow.demo.json'], + }).expect(200); + expect(res.body).toEqual({ + source: 'site', + cadence: 'daily', + weekday: null, + at: '12:05', + timeZone: 'UTC', + exclude: ['demoflow/slow.demo.json'], + pausedReason: null, + nextDueAt: '2026-10-20T12:05:00.000Z', + lastPassAt: null, + }); + const read = await schedule().expect(200); + expect(read.body).toEqual({ + site: res.body, + config: null, + machine: { + id: machine.runnerId, + name: 'Laptop', + online: false, + zone: null, + }, + }); + }); + + it('keeps weekday for weekly only', async () => { + const res = await put({ + cadence: 'weekly', + weekday: 'mon', + at: '09:00', + timeZone: 'Europe/Berlin', + }).expect(200); + expect(res.body).toMatchObject({ + weekday: 'mon', + nextDueAt: '2026-10-26T08:00:00.000Z', + }); + }); + + it.each([ + ['weekly without a weekday', { ...DAILY, cadence: 'weekly' }], + ['a weekday without weekly', { ...DAILY, weekday: 'mon' }], + ['an unknown weekday', { ...DAILY, cadence: 'weekly', weekday: 'mo' }], + ['an unknown cadence', { ...DAILY, cadence: 'hourly' }], + ['at 25:00', { ...DAILY, at: '25:00' }], + ['at 9:00', { ...DAILY, at: '9:00' }], + ['an unknown zone', { ...DAILY, timeZone: 'Mars/Olympus' }], + ['no zone', { cadence: 'daily', at: '12:05' }], + [ + '101 excludes', + { + ...DAILY, + exclude: Array.from( + { length: 101 }, + (_, i) => `demoflow/s${i}.demo.json`, + ), + }, + ], + ['an exclude that is not a scenario id', { ...DAILY, exclude: ['../x'] }], + ['an unknown key', { ...DAILY, machine: 'other' }], + ])('refuses %s with 400', async (_, body) => { + await put(body).expect(400); + }); + + it('is 409 no-machine when the project has no machine', async () => { + await t.prisma.project.update({ + where: { id: projectId }, + data: { runnerId: null }, + }); + const res = await put(DAILY).expect(409); + expect(res.body.error).toBe('no-machine'); + }); + + it('is 409 config-schedule when the config has one', async () => { + const socket = await daemon(); + syncSchedule(socket, { cadence: 'daily', at: '08:00' }); + await eventually(async () => (await configRow()) !== null); + const res = await put(DAILY).expect(409); + expect(res.body.error).toBe('config-schedule'); + }); + + it('DELETE removes the site schedule', async () => { + await put(DAILY).expect(200); + await request(http) + .delete(`/projects/${projectId}/schedule`) + .set('Cookie', alice.cookie) + .expect(204); + const res = await schedule().expect(200); + expect(res.body.site).toBeNull(); + }); + }); + + function syncSchedule( + socket: TestSocket, + synced: Record | null, + zone = 'Europe/Kyiv', + ): void { + socket.send({ + type: 'schedules.sync', + zone, + projects: [ + { rootPath: ROOT, schedule: synced }, + { rootPath: '/home/alice/unknown', schedule: synced }, + ], + }); + } + + function configRow() { + return t.prisma.checkSchedule.findUnique({ + where: { projectId_source: { projectId, source: 'config' } }, + }); + } + + function siteRow() { + return t.prisma.checkSchedule.findUnique({ + where: { projectId_source: { projectId, source: 'site' } }, + }); + } + + describe('schedules.sync', () => { + it('creates the config row and pauses the site row; null lifts it', async () => { + await put(DAILY).expect(200); + const socket = await daemon(); + syncSchedule(socket, { + cadence: 'weekly', + weekday: 'fri', + at: '18:30', + exclude: [SCENARIO], + }); + await eventually(async () => (await configRow()) !== null); + const paused = await schedule().expect(200); + expect(paused.body.config).toMatchObject({ + source: 'config', + cadence: 'weekly', + weekday: 'fri', + at: '18:30', + timeZone: 'Europe/Kyiv', + exclude: [SCENARIO], + pausedReason: null, + }); + expect(paused.body.site).toMatchObject({ + pausedReason: 'config-schedule', + nextDueAt: null, + }); + expect(paused.body.machine).toMatchObject({ + online: true, + zone: 'Europe/Kyiv', + }); + // A project the server does not know is ignored. + expect(await t.prisma.checkSchedule.count({ where: { projectId } })).toBe( + 2, + ); + + // The daemon runs config schedules; the API sends nothing for them. + t.clock.advance(DAY); + await tick(); + expect(await checkStarts(socket)).toBe(0); + + syncSchedule(socket, null); + await eventually(async () => (await configRow()) === null); + const lifted = await schedule().expect(200); + expect(lifted.body.site).toMatchObject({ pausedReason: null }); + // Lifting does not catch up the slots the daemon owned. + await tick(); + expect(await checkStarts(socket)).toBe(0); + }); + + it('refuses a sync with an unknown zone', async () => { + const socket = await daemon(); + syncSchedule(socket, { cadence: 'daily', at: '08:00' }, 'Mars/Olympus'); + await socket.roundTrip(); + expect(await configRow()).toBeNull(); + }); + }); + + describe('the scheduler', () => { + it('sends one check.start for a due slot, and never a second', async () => { + await put({ ...DAILY, exclude: [SCENARIO] }).expect(200); + const socket = await daemon(); + await tick(); + expect(await checkStarts(socket)).toBe(0); + + t.clock.advance(5 * MINUTE); + await tick(); + const ask = await socket.nextLaunch('check.start'); + expect(ask).toEqual({ + type: 'check.start', + requestId: ask.requestId, + rootPath: ROOT, + exclude: [SCENARIO], + }); + expect((await siteRow())?.lastSlotAt?.toISOString()).toBe( + '2026-10-20T12:05:00.000Z', + ); + socket.send({ type: 'check.started', requestId: ask.requestId }); + await socket.roundTrip(); + + const launch = await request(http) + .get(`/projects/${projectId}/launch`) + .set('Cookie', alice.cookie) + .expect(200); + expect(launch.body.launch).toMatchObject({ + id: ask.requestId, + kind: 'check', + scenarioId: null, + state: 'running', + passed: null, + failed: null, + }); + + socket.send({ + type: 'check.finished', + requestId: ask.requestId, + passed: 3, + failed: 1, + }); + await socket.roundTrip(); + const done = await request(http) + .get(`/projects/${projectId}/launch`) + .set('Cookie', alice.cookie) + .expect(200); + expect(done.body.launch).toMatchObject({ + kind: 'check', + state: 'failed', + passed: 3, + failed: 1, + }); + + t.clock.advance(MINUTE); + await tick(); + await tick(); + expect(await checkStarts(socket)).toBe(0); + }); + + it('sends one pass after three slots missed while offline', async () => { + await put(DAILY).expect(200); + t.clock.advance(3 * DAY); + await tick(); + const socket = await daemon(); + await tick(); + await socket.nextLaunch('check.start'); + await tick(); + expect(await checkStarts(socket)).toBe(0); + expect((await siteRow())?.lastSlotAt?.toISOString()).toBe( + '2026-10-22T12:05:00.000Z', + ); + }); + + it('waits for a busy machine’s launch to end', async () => { + await put(DAILY).expect(200); + const socket = await daemon(); + const pending = request(http) + .post(`/projects/${projectId}/launch`) + .set('Cookie', alice.cookie) + .send({ scenarioId: SCENARIO }) + .then((res) => res); + const run = await socket.nextLaunch('run.start'); + socket.send({ + type: 'run.started', + requestId: run.requestId, + scenarioName: 'Checkout', + }); + await pending; + + t.clock.advance(5 * MINUTE); + await tick(); + expect(await checkStarts(socket)).toBe(0); + + socket.send({ + type: 'run.finished', + requestId: run.requestId, + outcome: 'failed', + runId: null, + }); + const launches = t.app.get(LaunchesService); + await eventually(() => !launches.isBusy(machine.runnerId)); + await tick(); + await socket.nextLaunch('check.start'); + }); + + it('owes the slot again when the machine answers busy', async () => { + await put(DAILY).expect(200); + const socket = await daemon(); + t.clock.advance(5 * MINUTE); + await tick(); + const ask = await socket.nextLaunch('check.start'); + socket.send({ + type: 'check.error', + requestId: ask.requestId, + code: 'busy', + }); + // Back to the slot PUT started from: yesterday's. + await eventually( + async () => + (await siteRow())?.lastSlotAt?.toISOString() === + '2026-10-19T12:05:00.000Z', + ); + await tick(); + await socket.nextLaunch('check.start'); + }); + + it('pauses with cli-too-old and never sends check.start without checks: true', async () => { + await put(DAILY).expect(200); + const socket = await daemon(null); + t.clock.advance(5 * MINUTE); + await tick(); + expect(await checkStarts(socket)).toBe(0); + const res = await schedule().expect(200); + expect(res.body.site).toMatchObject({ + pausedReason: 'cli-too-old', + nextDueAt: null, + }); + + // The upgraded daemon replaces the socket and gets the slot it missed. + const upgraded = await daemon(true); + await tick(); + await upgraded.nextLaunch('check.start'); + expect((await siteRow())?.pausedReason).toBeNull(); + }); + + it('pauses with no-machine when the project lost its machine', async () => { + await put(DAILY).expect(200); + await t.prisma.project.update({ + where: { id: projectId }, + data: { runnerId: null }, + }); + await tick(); + expect((await siteRow())?.pausedReason).toBe('no-machine'); + const res = await schedule().expect(200); + expect(res.body.machine).toBeNull(); + }); + }); + + describe('the stale state', () => { + function health(): Test { + return request(http) + .get(`/projects/${projectId}/health`) + .set('Cookie', alice.cookie); + } + + it('flips on a failure, keeps since on a second, flips back on a pass', async () => { + const passedAt = t.clock.now().toISOString(); + const first = await health().expect(200); + expect(first.body).toEqual([ + { + scenarioPath: SCENARIO, + scenarioName: 'Checkout', + state: 'passed', + since: passedAt, + lastRunId: expect.any(String), + }, + ]); + + t.clock.advance(MINUTE); + const failedAt = t.clock.now().toISOString(); + const failed = await report(machine, 'failed', ROOT, SCENARIO, 'check'); + expect((await health()).body[0]).toMatchObject({ + state: 'failed', + since: failedAt, + lastRunId: failed.runId, + }); + + t.clock.advance(MINUTE); + const again = await report(machine, 'failed'); + expect((await health()).body[0]).toMatchObject({ + state: 'failed', + since: failedAt, + lastRunId: again.runId, + }); + + // The check run is the schedule's last pass. + await put(DAILY).expect(200); + expect((await schedule()).body.site.lastPassAt).toBe(failedAt); + + t.clock.advance(MINUTE); + const fixedAt = t.clock.now().toISOString(); + const fixed = await report(machine, 'passed'); + expect((await health()).body[0]).toMatchObject({ + state: 'passed', + since: fixedAt, + lastRunId: fixed.runId, + }); + }); + + it('ignores a report older than the state', async () => { + t.clock.advance(-MINUTE); + await report(machine, 'failed'); + expect((await health()).body[0]).toMatchObject({ state: 'passed' }); + }); + }); + + describe('the migration backfill', () => { + /** The backfill's `SELECT`, as the migration runs it, for one project. */ + function backfill(id: string) { + const sql = readFileSync( + join( + __dirname, + '../../prisma/migrations/20261028090000_check_schedules/migration.sql', + ), + 'utf8', + ); + const select = sql + .slice( + sql.indexOf('SELECT', sql.indexOf('INSERT INTO "ScenarioHealth"')), + ) + .trim() + .replace(/;$/, ''); + return t.prisma.$queryRawUnsafe< + { scenarioPath: string; status: string; since: Date; id: string }[] + >( + `SELECT b."scenarioPath", b."status", b."since", b."id" + FROM (${select}) AS b("projectId", "scenarioPath", "status", "since", "id") + WHERE b."projectId" = $1 ORDER BY b."scenarioPath"`, + id, + ); + } + + it('gives each scenario the state of its latest run', async () => { + t.clock.advance(MINUTE); + const failedAt = t.clock.now(); + await report(machine, 'failed'); + t.clock.advance(MINUTE); + const last = await report(machine, 'failed'); + const other = await report( + machine, + 'passed', + ROOT, + 'demoflow/signup.demo.json', + ); + expect(await backfill(projectId)).toEqual([ + { + scenarioPath: SCENARIO, + status: 'failed', + since: failedAt, + id: last.runId, + }, + { + scenarioPath: 'demoflow/signup.demo.json', + status: 'passed', + since: t.clock.now(), + id: other.runId, + }, + ]); + // And the rows the reports wrote agree with it. + const rows = await t.prisma.scenarioHealth.findMany({ + where: { projectId }, + orderBy: { scenarioPath: 'asc' }, + }); + expect( + rows.map((row) => ({ + scenarioPath: row.scenarioPath, + status: row.state, + since: row.since, + id: row.lastRunId, + })), + ).toEqual(await backfill(projectId)); + }); + }); +}); diff --git a/apps/api/src/checks/checks.module.ts b/apps/api/src/checks/checks.module.ts new file mode 100644 index 00000000..5bce79e7 --- /dev/null +++ b/apps/api/src/checks/checks.module.ts @@ -0,0 +1,20 @@ +import { Module } from '@nestjs/common'; +import { AuthModule } from '../auth/auth.module'; +import { LaunchesModule } from '../launches/launches.module'; +import { RunnerModule } from '../runner/runner.module'; +import { ChecksController } from './checks.controller'; +import { HealthService } from './health/health.service'; +import { CheckScheduler } from './schedules/check-scheduler'; +import { SchedulesService } from './schedules/schedules.service'; + +/** + * Scheduled checks on the website side (spec #362, ADR-0021): the schedules, + * the scheduler that starts a check pass when one is due, and the + * per-scenario stale state. + */ +@Module({ + imports: [AuthModule, RunnerModule, LaunchesModule], + controllers: [ChecksController], + providers: [SchedulesService, CheckScheduler, HealthService], +}) +export class ChecksModule {} diff --git a/apps/api/src/checks/health/health.service.ts b/apps/api/src/checks/health/health.service.ts new file mode 100644 index 00000000..7538df55 --- /dev/null +++ b/apps/api/src/checks/health/health.service.ts @@ -0,0 +1,85 @@ +import { HttpStatus, Injectable } from '@nestjs/common'; +import type { RunStatus } from '@prisma/client'; +import { PrismaService } from '../../database/prisma.service'; +import { runError } from '../../videos/runs.service'; + +/** At most this many in `GET /checks/stale`. */ +export const STALE_LIMIT = 50; + +export interface HealthView { + scenarioPath: string; + scenarioName: string; + state: RunStatus; + since: Date; + lastRunId: string; +} + +export interface StaleView { + projectId: string; + projectName: string; + scenarioPath: string; + scenarioName: string; + since: Date; + lastRunId: string; +} + +/** + * Reads the per-scenario state (spec #362, Decision 6). "Stale" is + * `state = failed`; `recordHealth` writes it with each report. + */ +@Injectable() +export class HealthService { + constructor(private readonly prisma: PrismaService) {} + + async project(userId: string, projectId: string): Promise { + const count = await this.prisma.project.count({ + where: { id: projectId, userId }, + }); + if (count !== 1) { + throw runError(HttpStatus.NOT_FOUND, 'not-found'); + } + const rows = await this.prisma.scenarioHealth.findMany({ + where: { projectId }, + orderBy: { scenarioPath: 'asc' }, + }); + const names = await this.names(rows.map((row) => row.lastRunId)); + return rows.map((row) => ({ + scenarioPath: row.scenarioPath, + scenarioName: names.get(row.lastRunId) ?? row.scenarioPath, + state: row.state, + since: row.since, + lastRunId: row.lastRunId, + })); + } + + /** The caller's stale scenarios across projects, newest `since` first. */ + async stale(userId: string): Promise { + const rows = await this.prisma.scenarioHealth.findMany({ + where: { state: 'failed', project: { userId } }, + orderBy: [{ since: 'desc' }, { scenarioPath: 'asc' }], + take: STALE_LIMIT, + include: { project: { select: { name: true } } }, + }); + const names = await this.names(rows.map((row) => row.lastRunId)); + return rows.map((row) => ({ + projectId: row.projectId, + projectName: row.project.name, + scenarioPath: row.scenarioPath, + scenarioName: names.get(row.lastRunId) ?? row.scenarioPath, + since: row.since, + lastRunId: row.lastRunId, + })); + } + + /** + * Each run's scenario name. A deleted run leaves the state as it was + * (Decision 6), so a missing one falls back to the path. + */ + private async names(runIds: string[]): Promise> { + const runs = await this.prisma.run.findMany({ + where: { id: { in: runIds } }, + select: { id: true, scenarioName: true }, + }); + return new Map(runs.map((run) => [run.id, run.scenarioName])); + } +} diff --git a/apps/api/src/checks/health/health.ts b/apps/api/src/checks/health/health.ts new file mode 100644 index 00000000..e220ab25 --- /dev/null +++ b/apps/api/src/checks/health/health.ts @@ -0,0 +1,98 @@ +import type { Prisma, RunStatus } from '@prisma/client'; + +/** A stored run, as the stale state needs it. */ +export interface HealthRun { + id: string; + scenarioPath: string; + status: RunStatus; + startedAt: Date; +} + +/** + * Moves a scenario's state on after a stored run (spec #362, Decision 6): + * same state → only `lastRunId`; another state → `state`, `since` and + * `lastRunId`. A run that started before the one the state comes from is + * older news and changes nothing. Called inside the report's transaction. + */ +export async function recordHealth( + tx: Prisma.TransactionClient, + projectId: string, + run: HealthRun, +): Promise { + const key = { + projectId_scenarioPath: { projectId, scenarioPath: run.scenarioPath }, + }; + const created = await tx.$executeRaw` + INSERT INTO "ScenarioHealth" ("projectId", "scenarioPath", "state", "since", "lastRunId") + VALUES (${projectId}, ${run.scenarioPath}, ${run.status}::"RunStatus", ${run.startedAt}, ${run.id}) + ON CONFLICT ("projectId", "scenarioPath") DO NOTHING`; + if (created === 1) { + return; + } + // Locks the row, so two reports of one scenario apply one after the other. + const [row] = await tx.$queryRaw< + { state: RunStatus; lastRunId: string }[] + >`SELECT "state", "lastRunId" FROM "ScenarioHealth" + WHERE "projectId" = ${projectId} AND "scenarioPath" = ${run.scenarioPath} + FOR UPDATE`; + if (!row) { + return; + } + const last = await tx.run.findUnique({ + where: { id: row.lastRunId }, + select: { startedAt: true }, + }); + if (last && last.startedAt > run.startedAt) { + return; + } + await tx.scenarioHealth.update({ + where: key, + data: + row.state === run.status + ? { lastRunId: run.id } + : { state: run.status, since: run.startedAt, lastRunId: run.id }, + }); +} + +/** + * Recomputes a project's states from its runs, as the migration's backfill + * does: each scenario's latest run gives the state, and `since` is the start + * of the oldest run after the latest run with the other status. Used when a + * merge moves another project's runs here. + */ +export async function rebuildHealth( + tx: Prisma.TransactionClient, + projectId: string, +): Promise { + await tx.scenarioHealth.deleteMany({ where: { projectId } }); + await tx.$executeRaw` + INSERT INTO "ScenarioHealth" ("projectId", "scenarioPath", "state", "since", "lastRunId") + SELECT + latest."projectId", + latest."scenarioPath", + latest."status", + COALESCE(( + SELECT MIN(r."startedAt") + FROM "Run" r + WHERE r."projectId" = latest."projectId" + AND r."scenarioPath" = latest."scenarioPath" + AND r."startedAt" > COALESCE( + ( + SELECT MAX(o."startedAt") + FROM "Run" o + WHERE o."projectId" = latest."projectId" + AND o."scenarioPath" = latest."scenarioPath" + AND o."status" <> latest."status" + ), + '-infinity'::timestamp + ) + ), latest."startedAt"), + latest."id" + FROM ( + SELECT DISTINCT ON ("scenarioPath") + "projectId", "scenarioPath", "status", "startedAt", "id" + FROM "Run" + WHERE "projectId" = ${projectId} + ORDER BY "scenarioPath", "startedAt" DESC, "id" DESC + ) latest`; +} diff --git a/apps/api/src/checks/schedules/check-scheduler.ts b/apps/api/src/checks/schedules/check-scheduler.ts new file mode 100644 index 00000000..1e996f0f --- /dev/null +++ b/apps/api/src/checks/schedules/check-scheduler.ts @@ -0,0 +1,174 @@ +import { + Injectable, + Logger, + type OnApplicationBootstrap, + type OnModuleDestroy, +} from '@nestjs/common'; +import type { CheckSchedule } from '@prisma/client'; +import { Clock } from '../../common/clock'; +import { PrismaService } from '../../database/prisma.service'; +import { LaunchesService } from '../../launches/launches.service'; +import { RunnerConnections } from '../../runner/runner-connections.service'; +import { lastDueSlot } from '../slots'; +import type { PausedReason } from './schedules.service'; + +export const SCHEDULER_INTERVAL_MS = 60 * 1000; + +/** Start failures after which the slot is still owed: the pass never began. */ +const RETRIABLE = new Set(['runner-busy', 'runner-offline']); + +interface Row extends CheckSchedule { + project: { + rootPath: string | null; + runner: { id: string; revokedAt: Date | null } | null; + }; +} + +/** + * Starts a check pass when a `site` schedule is due (spec #362, Decision 4). + * A missed slot is computed from the clock and `lastSlotAt`, so a machine + * that was offline, busy or restarting gets one catch-up pass for all of + * them, and an API restart loses nothing. + * + * One API process (deploy/README.md): two would start a pass twice. + */ +@Injectable() +export class CheckScheduler implements OnApplicationBootstrap, OnModuleDestroy { + private readonly logger = new Logger(CheckScheduler.name); + private timer: NodeJS.Timeout | undefined; + private pass: Promise | undefined; + + constructor( + private readonly prisma: PrismaService, + private readonly connections: RunnerConnections, + private readonly launches: LaunchesService, + private readonly clock: Clock, + ) {} + + onApplicationBootstrap(): void { + this.timer = setInterval(() => void this.tick(), SCHEDULER_INTERVAL_MS); + this.timer.unref(); + } + + onModuleDestroy(): void { + clearInterval(this.timer); + } + + /** + * One pass over the schedules; resolves once every due `check.start` is + * sent. A tick while one is in flight joins it. + */ + tick(): Promise { + this.pass ??= this.run().finally(() => { + this.pass = undefined; + }); + return this.pass; + } + + private async run(): Promise { + const rows = await this.prisma.checkSchedule.findMany({ + // `config-schedule` lifts only with the `config` row, in `SchedulesService`. + where: { + source: 'site', + OR: [ + { pausedReason: null }, + { pausedReason: { not: 'config-schedule' } }, + ], + }, + include: { + project: { + select: { + rootPath: true, + runner: { select: { id: true, revokedAt: true } }, + }, + }, + }, + }); + const now = this.clock.now(); + for (const row of rows) { + try { + await this.consider(row, now); + } catch (error) { + this.logger.error( + `Schedule ${row.id} failed`, + error instanceof Error ? error.stack : String(error), + ); + } + } + } + + private async consider(row: Row, now: Date): Promise { + const { runner, rootPath } = row.project; + if (!runner || runner.revokedAt !== null || rootPath === null) { + await this.pause(row, 'no-machine'); + return; + } + // Null until the machine's `launches.sync`: keep what is known. + const checks = this.connections.isOnline(runner.id) + ? this.connections.checks(runner.id) + : null; + if (checks === false) { + await this.pause(row, 'cli-too-old'); + return; + } + if (checks === true || row.pausedReason === 'no-machine') { + await this.pause(row, null); + } + if (checks !== true) { + return; + } + const slot = lastDueSlot(row, now, row.timeZone); + if ( + slot === null || + (row.lastSlotAt !== null && slot <= row.lastSlotAt) || + this.launches.isBusy(runner.id) + ) { + return; + } + // Written before the request, so a crash or restart never sends it twice. + const claimed = await this.prisma.checkSchedule.updateMany({ + where: { id: row.id, lastSlotAt: row.lastSlotAt, pausedReason: null }, + data: { lastSlotAt: slot }, + }); + if (claimed.count !== 1) { + return; + } + // Not awaited: `check.start` is sent now, the answer may take seconds. + void this.launches + .startCheckPass( + { projectId: row.projectId, runnerId: runner.id, rootPath }, + row.exclude, + ) + .then(async (outcome) => { + if (outcome.ok) { + return; + } + this.logger.warn( + `Check pass of project ${row.projectId} not started: ${outcome.error}`, + ); + if (RETRIABLE.has(outcome.error)) { + await this.prisma.checkSchedule.updateMany({ + where: { id: row.id, lastSlotAt: slot }, + data: { lastSlotAt: row.lastSlotAt }, + }); + } + }) + .catch((error: unknown) => { + this.logger.error( + `Check pass of project ${row.projectId} failed`, + error instanceof Error ? error.stack : String(error), + ); + }); + } + + private async pause(row: Row, reason: PausedReason | null): Promise { + if (row.pausedReason === reason) { + return; + } + await this.prisma.checkSchedule.updateMany({ + where: { id: row.id, pausedReason: row.pausedReason }, + data: { pausedReason: reason }, + }); + row.pausedReason = reason; + } +} diff --git a/apps/api/src/checks/schedules/schedules.dto.ts b/apps/api/src/checks/schedules/schedules.dto.ts new file mode 100644 index 00000000..6dcce347 --- /dev/null +++ b/apps/api/src/checks/schedules/schedules.dto.ts @@ -0,0 +1,50 @@ +import { + ArrayMaxSize, + ArrayUnique, + IsArray, + IsIn, + Matches, + ValidateBy, + ValidateIf, +} from 'class-validator'; +import { IsScenarioId } from '../../launches/scenario-id'; +import { CADENCES, type Cadence, TIME_OF_DAY, WEEKDAYS } from '../slots'; +import { IsTimeZone } from './time-zone'; + +/** At most this many scenarios left out of a pass (spec #362, Decision 3). */ +export const MAX_EXCLUDES = 100; + +/** `PUT /projects/:id/schedule`: the website's schedule of a project. */ +export class PutScheduleDto { + @IsIn(CADENCES) + cadence!: Cadence; + + /** Required with `weekly`, refused with any other cadence. */ + @ValidateIf( + (dto: PutScheduleDto, value) => + dto.cadence === 'weekly' || value !== undefined, + ) + @IsIn(WEEKDAYS) + @ValidateBy({ + name: 'onlyWithWeekly', + validator: { + validate: (_, args) => + (args?.object as PutScheduleDto).cadence === 'weekly', + defaultMessage: () => 'weekday is only allowed with the weekly cadence', + }, + }) + weekday?: string; + + @Matches(TIME_OF_DAY, { message: 'at must be HH:MM, 24-hour' }) + at!: string; + + @IsTimeZone() + timeZone!: string; + + @ValidateIf((_, value) => value !== undefined) + @IsArray() + @ArrayMaxSize(MAX_EXCLUDES) + @ArrayUnique() + @IsScenarioId({ each: true }) + exclude?: string[]; +} diff --git a/apps/api/src/checks/schedules/schedules.service.ts b/apps/api/src/checks/schedules/schedules.service.ts new file mode 100644 index 00000000..5f9cc358 --- /dev/null +++ b/apps/api/src/checks/schedules/schedules.service.ts @@ -0,0 +1,289 @@ +import { + HttpStatus, + Injectable, + Logger, + type OnModuleInit, +} from '@nestjs/common'; +import type { CheckSchedule, Prisma } from '@prisma/client'; +import { Clock } from '../../common/clock'; +import { PrismaService } from '../../database/prisma.service'; +import { + RunnerConnections, + type SchedulesSink, +} from '../../runner/runner-connections.service'; +import type { + SchedulesMessage, + SyncedSchedule, +} from '../../runner/runner-socket'; +import { runError } from '../../videos/runs.service'; +import { lastDueSlot, nextDueSlot } from '../slots'; +import type { PutScheduleDto } from './schedules.dto'; + +/** Why a `site` schedule does not run (spec #362). */ +export type PausedReason = 'config-schedule' | 'no-machine' | 'cli-too-old'; + +/** A schedule as the website reads it (spec #362, API). */ +export interface ScheduleView { + source: 'site' | 'config'; + cadence: CheckSchedule['cadence']; + weekday: string | null; + at: string; + timeZone: string; + exclude: string[]; + pausedReason: string | null; + /** Computed, never stored; null while paused. */ + nextDueAt: Date | null; + /** The start of the project's latest reported check run. */ + lastPassAt: Date | null; +} + +export interface ScheduleMachine { + id: string; + name: string; + online: boolean; + /** The zone of the machine's latest `schedules.sync`, for the form's default. */ + zone: string | null; +} + +export interface ProjectSchedules { + site: ScheduleView | null; + config: ScheduleView | null; + machine: ScheduleMachine | null; +} + +/** + * A project's check schedules (spec #362, Decision 2): the `site` row the + * website sets, and the `config` row the daemon reports from the project's + * config with `schedules.sync`. The API runs only `site` rows + * (`CheckScheduler`); `config` rows are shown. + */ +@Injectable() +export class SchedulesService implements SchedulesSink, OnModuleInit { + private readonly logger = new Logger(SchedulesService.name); + /** The zone of each machine's latest `schedules.sync`. */ + private readonly zones = new Map(); + /** One machine's syncs apply in the order they came. */ + private readonly syncs = new Map>(); + + constructor( + private readonly prisma: PrismaService, + private readonly connections: RunnerConnections, + private readonly clock: Clock, + ) {} + + onModuleInit(): void { + this.connections.setSchedulesSink(this); + } + + async get(userId: string, projectId: string): Promise { + const project = await this.owned(userId, projectId); + const rows = await this.prisma.checkSchedule.findMany({ + where: { projectId }, + }); + const lastPassAt = await this.lastPassAt(projectId); + const view = (source: 'site' | 'config') => { + const row = rows.find((candidate) => candidate.source === source); + return row ? this.toView(row, lastPassAt) : null; + }; + const runner = project.runner; + return { + site: view('site'), + config: view('config'), + machine: + runner && runner.revokedAt === null + ? { + id: runner.id, + name: runner.name, + online: this.connections.isOnline(runner.id), + zone: this.zones.get(runner.id) ?? null, + } + : null, + }; + } + + /** + * Creates or replaces the `site` row. A new schedule never fires at once: + * `lastSlotAt` starts at the slot already due. + */ + async put( + userId: string, + projectId: string, + dto: PutScheduleDto, + ): Promise { + const project = await this.owned(userId, projectId); + const config = await this.prisma.checkSchedule.count({ + where: { projectId, source: 'config' }, + }); + if (config > 0) { + throw runError(HttpStatus.CONFLICT, 'config-schedule'); + } + if ( + !project.runner || + project.runner.revokedAt !== null || + project.rootPath === null + ) { + throw runError(HttpStatus.CONFLICT, 'no-machine'); + } + const fields = { + cadence: dto.cadence, + weekday: dto.cadence === 'weekly' ? (dto.weekday ?? null) : null, + at: dto.at, + timeZone: dto.timeZone, + exclude: dto.exclude ?? [], + }; + const data = { + ...fields, + pausedReason: null, + lastSlotAt: lastDueSlot(fields, this.clock.now(), fields.timeZone), + }; + const row = await this.prisma.checkSchedule.upsert({ + where: { projectId_source: { projectId, source: 'site' } }, + create: { projectId, source: 'site', ...data }, + update: data, + }); + return this.toView(row, await this.lastPassAt(projectId)); + } + + async remove(userId: string, projectId: string): Promise { + await this.owned(userId, projectId); + await this.prisma.checkSchedule.deleteMany({ + where: { projectId, source: 'site' }, + }); + } + + /** `schedules.sync` (spec 361, Decision 6), from `RunnerConnections`. */ + sync(runnerId: string, message: SchedulesMessage): void { + const previous = this.syncs.get(runnerId) ?? Promise.resolve(); + const next = previous + .then(() => this.apply(runnerId, message)) + .catch((error: unknown) => { + this.logger.error( + `schedules.sync from ${runnerId} failed`, + error instanceof Error ? error.stack : String(error), + ); + }) + .finally(() => { + if (this.syncs.get(runnerId) === next) { + this.syncs.delete(runnerId); + } + }); + this.syncs.set(runnerId, next); + } + + private async apply( + runnerId: string, + message: SchedulesMessage, + ): Promise { + this.zones.set(runnerId, message.zone); + const roots = message.projects.map((project) => project.rootPath); + // A project the server does not know yet is ignored (Decision 2). + const projects = await this.prisma.project.findMany({ + where: { runnerId, rootPath: { in: roots } }, + select: { id: true, rootPath: true }, + }); + for (const project of projects) { + const synced = message.projects.find( + (candidate) => candidate.rootPath === project.rootPath, + ); + if (!synced) { + continue; + } + await this.prisma.$transaction((tx) => + synced.schedule + ? this.setConfig(tx, project.id, synced.schedule, message.zone) + : this.dropConfig(tx, project.id), + ); + } + } + + private async setConfig( + tx: Prisma.TransactionClient, + projectId: string, + schedule: SyncedSchedule, + zone: string, + ): Promise { + const data = { + cadence: schedule.cadence, + weekday: + schedule.cadence === 'weekly' ? (schedule.weekday ?? null) : null, + at: schedule.at, + timeZone: zone, + exclude: schedule.exclude ?? [], + }; + await tx.checkSchedule.upsert({ + where: { projectId_source: { projectId, source: 'config' } }, + create: { projectId, source: 'config', ...data }, + update: data, + }); + await tx.checkSchedule.updateMany({ + where: { projectId, source: 'site' }, + data: { pausedReason: 'config-schedule' }, + }); + } + + /** The pause lifts without a catch-up: the slots it skipped were the daemon's. */ + private async dropConfig( + tx: Prisma.TransactionClient, + projectId: string, + ): Promise { + const { count } = await tx.checkSchedule.deleteMany({ + where: { projectId, source: 'config' }, + }); + if (count === 0) { + return; + } + const site = await tx.checkSchedule.findUnique({ + where: { projectId_source: { projectId, source: 'site' } }, + }); + if (site?.pausedReason !== 'config-schedule') { + return; + } + await tx.checkSchedule.update({ + where: { id: site.id }, + data: { + pausedReason: null, + lastSlotAt: lastDueSlot(site, this.clock.now(), site.timeZone), + }, + }); + } + + private async owned(userId: string, projectId: string) { + const project = await this.prisma.project.findFirst({ + where: { id: projectId, userId }, + select: { + rootPath: true, + runner: { select: { id: true, name: true, revokedAt: true } }, + }, + }); + if (!project) { + throw runError(HttpStatus.NOT_FOUND, 'not-found'); + } + return project; + } + + private async lastPassAt(projectId: string): Promise { + const run = await this.prisma.run.findFirst({ + where: { projectId, kind: 'check' }, + orderBy: [{ startedAt: 'desc' }, { id: 'desc' }], + select: { startedAt: true }, + }); + return run?.startedAt ?? null; + } + + private toView(row: CheckSchedule, lastPassAt: Date | null): ScheduleView { + return { + source: row.source, + cadence: row.cadence, + weekday: row.weekday, + at: row.at, + timeZone: row.timeZone, + exclude: row.exclude, + pausedReason: row.pausedReason, + nextDueAt: + row.pausedReason === null + ? nextDueSlot(row, this.clock.now(), row.timeZone) + : null, + lastPassAt, + }; + } +} diff --git a/apps/api/src/checks/schedules/time-zone.ts b/apps/api/src/checks/schedules/time-zone.ts new file mode 100644 index 00000000..7efdd5ad --- /dev/null +++ b/apps/api/src/checks/schedules/time-zone.ts @@ -0,0 +1,36 @@ +import { ValidateBy, type ValidationOptions } from 'class-validator'; + +/** Whether `zone` is an IANA time zone this runtime knows. */ +export function isTimeZone(zone: string): boolean { + if (zone.length === 0 || zone.length > 64) { + return false; + } + if (Intl.supportedValuesOf('timeZone').includes(zone)) { + return true; + } + // `supportedValuesOf` lists canonical names only: `UTC`, `Europe/Kyiv` + // and other aliases are missing from it on some runtimes. + if (!/^[A-Za-z][A-Za-z0-9_+-]*(\/[A-Za-z0-9_+-]+)*$/.test(zone)) { + return false; + } + try { + new Intl.DateTimeFormat('en-US', { timeZone: zone }); + return true; + } catch { + return false; + } +} + +/** An IANA time zone name (spec #362, Decision 3). */ +export function IsTimeZone(options?: ValidationOptions): PropertyDecorator { + return ValidateBy( + { + name: 'isTimeZone', + validator: { + validate: (value) => typeof value === 'string' && isTimeZone(value), + defaultMessage: () => '$property is not a time zone', + }, + }, + options, + ); +} diff --git a/apps/api/src/checks/slots.spec.ts b/apps/api/src/checks/slots.spec.ts new file mode 100644 index 00000000..37ae6424 --- /dev/null +++ b/apps/api/src/checks/slots.spec.ts @@ -0,0 +1,169 @@ +import { isTimeZone } from './schedules/time-zone'; +import { lastDueSlot, nextDueSlot, type SlotSchedule } from './slots'; + +function daily(at: string): SlotSchedule { + return { cadence: 'daily', weekday: null, at }; +} + +function iso(date: Date | null): string | null { + return date ? date.toISOString() : null; +} + +describe('slots', () => { + describe('lastDueSlot', () => { + it('is today when the time has passed, yesterday when it has not', () => { + expect( + iso( + lastDueSlot( + daily('09:30'), + new Date('2026-10-20T12:00:00Z'), + 'Europe/Berlin', + ), + ), + ).toBe('2026-10-20T07:30:00.000Z'); + expect( + iso( + lastDueSlot( + daily('09:30'), + new Date('2026-10-20T07:29:59Z'), + 'Europe/Berlin', + ), + ), + ).toBe('2026-10-19T07:30:00.000Z'); + }); + + it('counts a slot exactly at now as due', () => { + expect( + iso( + lastDueSlot(daily('12:00'), new Date('2026-10-20T12:00:00Z'), 'UTC'), + ), + ).toBe('2026-10-20T12:00:00.000Z'); + }); + + it('skips the weekend for weekdays', () => { + const weekdays: SlotSchedule = { + cadence: 'weekdays', + weekday: null, + at: '09:00', + }; + // Sunday 2026-10-25. + const sunday = new Date('2026-10-25T12:00:00Z'); + expect(iso(lastDueSlot(weekdays, sunday, 'UTC'))).toBe( + '2026-10-23T09:00:00.000Z', + ); + expect(iso(nextDueSlot(weekdays, sunday, 'UTC'))).toBe( + '2026-10-26T09:00:00.000Z', + ); + }); + + it('finds the named day for weekly', () => { + const monday: SlotSchedule = { + cadence: 'weekly', + weekday: 'mon', + at: '09:00', + }; + // Tuesday 2026-10-20. + const tuesday = new Date('2026-10-20T12:00:00Z'); + expect(iso(lastDueSlot(monday, tuesday, 'UTC'))).toBe( + '2026-10-19T09:00:00.000Z', + ); + expect(iso(nextDueSlot(monday, tuesday, 'UTC'))).toBe( + '2026-10-26T09:00:00.000Z', + ); + }); + + it('uses the local day of the zone, not the UTC one', () => { + // 2026-10-20 23:30 in New York is already the 21st in UTC. + expect( + iso( + lastDueSlot( + daily('23:00'), + new Date('2026-10-21T03:30:00Z'), + 'America/New_York', + ), + ), + ).toBe('2026-10-21T03:00:00.000Z'); + }); + }); + + describe('DST', () => { + it('moves a missing time to the next valid minute (Europe/Berlin, spring)', () => { + // 2026-03-29 02:00 CET jumps to 03:00 CEST. + expect( + iso( + lastDueSlot( + daily('02:30'), + new Date('2026-03-29T12:00:00Z'), + 'Europe/Berlin', + ), + ), + ).toBe('2026-03-29T01:00:00.000Z'); + }); + + it('runs a repeated time once, at its first occurrence (Europe/Berlin, autumn)', () => { + // 2026-10-25 03:00 CEST falls back to 02:00 CET. + const schedule = daily('02:30'); + expect( + iso( + lastDueSlot( + schedule, + new Date('2026-10-25T12:00:00Z'), + 'Europe/Berlin', + ), + ), + ).toBe('2026-10-25T00:30:00.000Z'); + // Between the two occurrences, the next slot is the next day. + expect( + iso( + nextDueSlot( + schedule, + new Date('2026-10-25T01:00:00Z'), + 'Europe/Berlin', + ), + ), + ).toBe('2026-10-26T01:30:00.000Z'); + }); + + it('moves a missing time to the next valid minute (America/New_York, spring)', () => { + // 2026-03-08 02:00 EST jumps to 03:00 EDT. + expect( + iso( + lastDueSlot( + daily('02:30'), + new Date('2026-03-08T12:00:00Z'), + 'America/New_York', + ), + ), + ).toBe('2026-03-08T07:00:00.000Z'); + }); + + it('runs a repeated time once, at its first occurrence (America/New_York, autumn)', () => { + // 2026-11-01 02:00 EDT falls back to 01:00 EST. + expect( + iso( + lastDueSlot( + daily('01:30'), + new Date('2026-11-01T12:00:00Z'), + 'America/New_York', + ), + ), + ).toBe('2026-11-01T05:30:00.000Z'); + }); + }); + + describe('isTimeZone', () => { + it('accepts IANA names and their aliases', () => { + expect(isTimeZone('Europe/Berlin')).toBe(true); + expect(isTimeZone('America/New_York')).toBe(true); + expect(isTimeZone('UTC')).toBe(true); + expect(isTimeZone('Europe/Kyiv')).toBe(true); + }); + + it('refuses anything else', () => { + expect(isTimeZone('Mars/Olympus')).toBe(false); + expect(isTimeZone('')).toBe(false); + expect(isTimeZone('+02:00')).toBe(false); + expect(isTimeZone('Europe/Berlin\n')).toBe(false); + }); + }); +}); diff --git a/apps/api/src/checks/slots.ts b/apps/api/src/checks/slots.ts new file mode 100644 index 00000000..1c5ddde0 --- /dev/null +++ b/apps/api/src/checks/slots.ts @@ -0,0 +1,135 @@ +/** + * When a check schedule is due (spec #362, Decision 4). A port of spec 361's + * `lastDueSlot`: the API does not depend on `@demoflow/core`, so the rule and + * its test vectors are kept in both places. + * + * DST: a local time that does not exist that day falls on the next valid + * minute; one that repeats falls on its first occurrence. + */ + +export const WEEKDAYS = ['sun', 'mon', 'tue', 'wed', 'thu', 'fri', 'sat']; +export const CADENCES = ['daily', 'weekdays', 'weekly'] as const; + +export type Cadence = (typeof CADENCES)[number]; +/** `HH:MM`, 24-hour. */ +export const TIME_OF_DAY = /^([01]\d|2[0-3]):[0-5]\d$/; + +export interface SlotSchedule { + cadence: Cadence; + /** `mon`…`sun`; only with `weekly`. */ + weekday: string | null; + /** `HH:MM`, 24-hour, local to the zone. */ + at: string; +} + +const MINUTE = 60_000; +const DAY = 24 * 60 * MINUTE; +// Enough days to reach a weekly slot from any day, either way. +const DAYS_SEARCHED = 8; +// The longest DST gap in use is one hour; some zones have moved by more. +const GAP_MINUTES = 180; + +const formatters = new Map(); + +function formatter(zone: string): Intl.DateTimeFormat { + let format = formatters.get(zone); + if (!format) { + format = new Intl.DateTimeFormat('en-US', { + timeZone: zone, + hourCycle: 'h23', + year: 'numeric', + month: 'numeric', + day: 'numeric', + hour: 'numeric', + minute: 'numeric', + }); + formatters.set(zone, format); + } + return format; +} + +/** The wall-clock time at `instant` in `zone`, as if it were UTC, in ms. */ +function wallTime(instant: number, zone: string): number { + const parts: Record = {}; + for (const part of formatter(zone).formatToParts(instant)) { + if (part.type !== 'literal') { + parts[part.type] = Number(part.value); + } + } + return Date.UTC( + parts.year, + parts.month - 1, + parts.day, + parts.hour, + parts.minute, + ); +} + +/** The first instant whose wall time in `zone` is `wall`, or null in a gap. */ +function instantOf(wall: number, zone: string): number | null { + const candidates = [wall - 14 * 60 * MINUTE, wall, wall + 14 * 60 * MINUTE] + .map((probe) => wall - (wallTime(probe, zone) - probe)) + .filter((candidate) => wallTime(candidate, zone) === wall); + return candidates.length > 0 ? Math.min(...candidates) : null; +} + +/** The slot of the local day `day` (a UTC midnight), or null when none. */ +function slotOn( + schedule: SlotSchedule, + day: number, + zone: string, +): number | null { + const weekday = new Date(day).getUTCDay(); + if (schedule.cadence === 'weekdays' && (weekday === 0 || weekday === 6)) { + return null; + } + if (schedule.cadence === 'weekly' && WEEKDAYS[weekday] !== schedule.weekday) { + return null; + } + const [hours, minutes] = schedule.at.split(':').map(Number); + const wall = day + hours * 60 * MINUTE + minutes * MINUTE; + for (let shift = 0; shift <= GAP_MINUTES; shift += 1) { + const instant = instantOf(wall + shift * MINUTE, zone); + if (instant !== null) { + return instant; + } + } + return null; +} + +function localDay(now: Date, zone: string): number { + const wall = wallTime(now.getTime(), zone); + return wall - (wall % DAY); +} + +/** The latest slot at or before `now`, or null when there is none. */ +export function lastDueSlot( + schedule: SlotSchedule, + now: Date, + zone: string, +): Date | null { + const today = localDay(now, zone); + for (let back = 0; back <= DAYS_SEARCHED; back += 1) { + const slot = slotOn(schedule, today - back * DAY, zone); + if (slot !== null && slot <= now.getTime()) { + return new Date(slot); + } + } + return null; +} + +/** The first slot after `now`, or null when there is none. */ +export function nextDueSlot( + schedule: SlotSchedule, + now: Date, + zone: string, +): Date | null { + const today = localDay(now, zone); + for (let ahead = 0; ahead <= DAYS_SEARCHED; ahead += 1) { + const slot = slotOn(schedule, today + ahead * DAY, zone); + if (slot !== null && slot > now.getTime()) { + return new Date(slot); + } + } + return null; +} diff --git a/apps/api/src/launches/launches.integration.spec.ts b/apps/api/src/launches/launches.integration.spec.ts index 274559a4..285ba280 100644 --- a/apps/api/src/launches/launches.integration.spec.ts +++ b/apps/api/src/launches/launches.integration.spec.ts @@ -252,6 +252,7 @@ describe('starting and cancelling a run from the website (integration)', () => { expect(res.status).toBe(202); const launch = { id: ask.requestId, + kind: 'run', scenarioId: SCENARIO, scenarioName: 'Checkout', state: 'running', @@ -260,6 +261,8 @@ describe('starting and cancelling a run from the website (integration)', () => { runId: null, progress: null, formats: null, + passed: null, + failed: null, }; expect(res.body).toEqual({ launch }); expect((await latest()).body).toEqual({ launch }); diff --git a/apps/api/src/launches/launches.module.ts b/apps/api/src/launches/launches.module.ts index e54419f6..ee9c1588 100644 --- a/apps/api/src/launches/launches.module.ts +++ b/apps/api/src/launches/launches.module.ts @@ -12,5 +12,6 @@ import { LaunchesService } from './launches.service'; imports: [AuthModule, RunnerModule], controllers: [LaunchesController], providers: [LaunchesService], + exports: [LaunchesService], }) export class LaunchesModule {} diff --git a/apps/api/src/launches/launches.service.ts b/apps/api/src/launches/launches.service.ts index 8d1cb280..ac624119 100644 --- a/apps/api/src/launches/launches.service.ts +++ b/apps/api/src/launches/launches.service.ts @@ -30,6 +30,8 @@ export const START_TIMEOUT_MS = 15 * 1000; export const CANCEL_TIMEOUT_MS = 30 * 1000; /** No `run.finished` this long after `run.started`: `lost`. The daemon's own cap is 15 min. */ export const RUN_TIMEOUT_MS = 20 * 60 * 1000; +/** No `check.finished` this long after `check.started`: `lost`. The daemon caps a pass at 2 h (spec 360). */ +export const CHECK_TIMEOUT_MS = 2 * 60 * 60 * 1000 + 5 * 60 * 1000; /** A finished launch is kept this long for `GET` (Decision 9). */ export const LAUNCH_RETENTION_MS = 10 * 60 * 1000; /** A detached launch whose machine has no new socket this long is `lost` (spec #284, Decision 3). */ @@ -59,10 +61,15 @@ export interface RunProgress { steps: number | null; } +/** A run of one scenario, or a check pass of the whole project (spec #362, Decision 5). */ +export type LaunchKind = 'run' | 'check'; + /** What the browser polls (Decision 1). */ export interface LaunchView { id: string; - scenarioId: string; + kind: LaunchKind; + /** Null for a check pass. */ + scenarioId: string | null; scenarioName: string | null; state: LaunchState; startedAt: string; @@ -72,6 +79,9 @@ export interface LaunchView { progress: RunProgress | null; /** The formats the browser chose; null for the project's own (spec #325, Decision 5). */ formats: RunFormat[] | null; + /** A finished check pass's counts (`check.finished`); null otherwise. */ + passed: number | null; + failed: number | null; } export interface ScenarioList { @@ -130,6 +140,10 @@ const TIMEOUT: Failure = { status: HttpStatus.GATEWAY_TIMEOUT, error: 'runner-timeout', }; +const BUSY: Failure = RUNNER_ERRORS.busy; + +/** How `startCheckPass` ended: started, or why not (`runner-busy`, `runner-offline`, …). */ +export type CheckPassStart = { ok: true } | { ok: false; error: string }; function runnerFailure(code: string): Failure { return Object.hasOwn(RUNNER_ERRORS, code) @@ -148,9 +162,10 @@ interface PendingList { interface Launch { id: string; + kind: LaunchKind; projectId: string; runnerId: string; - scenarioId: string; + scenarioId: string | null; scenarioName: string | null; /** Set at start, never changed. */ formats: RunFormat[] | null; @@ -170,6 +185,8 @@ interface Launch { detachedAt: number | null; /** The latest valid `run.progress`, kept across a detach (spec #301, Decision 6). */ progress: RunProgress | null; + passed: number | null; + failed: number | null; /** The `POST` waiting while `starting`. */ settle?: (outcome: Outcome) => void; } @@ -254,24 +271,8 @@ export class LaunchesService throw runError(HttpStatus.CONFLICT, 'runner-busy'); } const outcome = await new Promise>((settle) => { - const now = this.now(); - const launch: Launch = { - id: randomUUID(), - projectId: target.projectId, - runnerId: target.runnerId, - scenarioId, - scenarioName: null, - formats: formats ?? null, - state: 'starting', - startedAt: now, - since: now, - finishedAt: null, - runId: null, - finishing: false, - detachedAt: null, - progress: null, - settle, - }; + const launch = this.newLaunch('run', target, scenarioId, settle); + launch.formats = formats ?? null; this.launches.set(launch.id, launch); const sent = this.connections.sendTo(target.runnerId, { type: 'run.start', @@ -291,6 +292,39 @@ export class LaunchesService return outcome.value; } + /** + * Sends `check.start` for the whole project (spec #362, Decision 5) and + * settles once the machine says it started. The same one-launch-per-Runner + * rule as `start`; never throws: the scheduler decides what to do next. + */ + async startCheckPass( + target: LaunchTarget, + exclude: string[], + ): Promise { + if (this.busy(target.runnerId)) { + return { ok: false, error: BUSY.error }; + } + const outcome = await new Promise>((settle) => { + const launch = this.newLaunch('check', target, null, settle); + this.launches.set(launch.id, launch); + const sent = this.connections.sendTo(target.runnerId, { + type: 'check.start', + requestId: launch.id, + rootPath: target.rootPath, + exclude, + }); + if (!sent) { + this.failStart(launch, OFFLINE, OFFLINE.error); + } + }); + return outcome.ok ? { ok: true } : { ok: false, error: outcome.error }; + } + + /** Whether the machine has an active launch: a run or a check pass. */ + isBusy(runnerId: string): boolean { + return this.busy(runnerId); + } + /** The project's latest launch, or null. */ latest(projectId: string): LaunchView | null { const launch = this.latestOf(projectId); @@ -353,27 +387,60 @@ export class LaunchesService return true; } case 'run.started': - case 'run.error': { + case 'run.error': + case 'check.started': + case 'check.error': { const launch = this.launches.get(message.requestId); - if (launch?.runnerId !== runnerId || launch.state !== 'starting') { + if ( + launch?.runnerId !== runnerId || + launch.state !== 'starting' || + launch.kind !== kindOf(message.type) + ) { return false; } - if (message.type === 'run.error') { + if (message.type === 'run.error' || message.type === 'check.error') { this.failStart(launch, runnerFailure(message.code), message.code); return true; } launch.state = 'running'; launch.since = this.now(); - launch.scenarioName = message.scenarioName; + if (message.type === 'run.started') { + launch.scenarioName = message.scenarioName; + } const settle = launch.settle; launch.settle = undefined; settle?.({ ok: true, value: toView(launch) }); return true; } + case 'check.finished': { + const launch = this.launches.get(message.requestId); + if ( + launch?.runnerId !== runnerId || + launch.kind !== 'check' || + (launch.state !== 'running' && launch.state !== 'cancelling') || + launch.finishing + ) { + return false; + } + launch.passed = message.passed; + launch.failed = message.failed; + // A pass that verified nothing did not pass: cancelled, timed out or crashed. + this.end( + launch, + launch.state === 'cancelling' + ? 'cancelled' + : message.failed > 0 || message.passed === 0 + ? 'failed' + : 'passed', + null, + ); + return true; + } case 'run.finished': { const launch = this.launches.get(message.requestId); if ( launch?.runnerId !== runnerId || + launch.kind !== 'run' || (launch.state !== 'running' && launch.state !== 'cancelling') || launch.finishing ) { @@ -499,7 +566,9 @@ export class LaunchesService } break; case 'running': - if (age >= RUN_TIMEOUT_MS) { + if ( + age >= (launch.kind === 'check' ? CHECK_TIMEOUT_MS : RUN_TIMEOUT_MS) + ) { this.end(launch, 'lost', null); } break; @@ -567,6 +636,35 @@ export class LaunchesService ); } + private newLaunch( + kind: LaunchKind, + target: LaunchTarget, + scenarioId: string | null, + settle: (outcome: Outcome) => void, + ): Launch { + const now = this.now(); + return { + id: randomUUID(), + kind, + projectId: target.projectId, + runnerId: target.runnerId, + scenarioId, + scenarioName: null, + formats: null, + state: 'starting', + startedAt: now, + since: now, + finishedAt: null, + runId: null, + finishing: false, + detachedAt: null, + progress: null, + passed: null, + failed: null, + settle, + }; + } + /** A `run.cancel` the machine cannot get leaves the launch `lost`. */ private sendCancel(launch: Launch): void { const sent = this.connections.sendTo(launch.runnerId, { @@ -669,6 +767,7 @@ export class LaunchesService this.logger.log( JSON.stringify({ event: 'launch', + kind: launch.kind, projectId: launch.projectId, runnerId: launch.runnerId, outcome: label, @@ -687,6 +786,7 @@ export class LaunchesService function toView(launch: Launch): LaunchView { return { id: launch.id, + kind: launch.kind, scenarioId: launch.scenarioId, scenarioName: launch.scenarioName, state: launch.state, @@ -698,9 +798,15 @@ function toView(launch: Launch): LaunchView { runId: launch.runId, progress: launch.state === 'running' ? launch.progress : null, formats: launch.formats, + passed: launch.passed, + failed: launch.failed, }; } +function kindOf(type: LaunchMessage['type']): LaunchKind { + return type.startsWith('check.') ? 'check' : 'run'; +} + /** * Spec #301, Decision 3: `steps` carries both numbers with `step ≤ steps`, * any other phase carries neither. diff --git a/apps/api/src/projects/folders/folders.controller.ts b/apps/api/src/projects/folders/folders.controller.ts new file mode 100644 index 00000000..75baf5fa --- /dev/null +++ b/apps/api/src/projects/folders/folders.controller.ts @@ -0,0 +1,58 @@ +import { + Body, + Controller, + HttpCode, + HttpStatus, + Post, + UseGuards, +} from '@nestjs/common'; +import { SkipThrottle, ThrottlerGuard } from '@nestjs/throttler'; +import type { User } from '@prisma/client'; +import { + SKIP_ALL_THROTTLERS, + THROTTLE_PROJECT_FOLDERS, +} from '../../auth/auth-throttle'; +import { CurrentUser } from '../../auth/current-user.decorator'; +import { SessionGuard } from '../../auth/session.guard'; +import type { ProjectView } from '../projects.service'; +import { FolderDto, RegisterFolderDto } from './folders.dto'; +import { + FoldersService, + type InitView, + type InspectionView, +} from './folders.service'; + +/** + * Adding a project from a folder on one of the user's machines (spec #378). + * The three routes share one per-user budget (Decision 8). + */ +@Controller('projects/folders') +// One decorator: two on a class set the same keys, and the outer one wins. +@SkipThrottle({ ...SKIP_ALL_THROTTLERS, [THROTTLE_PROJECT_FOLDERS]: false }) +@UseGuards(SessionGuard, ThrottlerGuard) +export class FoldersController { + constructor(private readonly folders: FoldersService) {} + + @Post('inspect') + @HttpCode(HttpStatus.OK) + inspect( + @CurrentUser() user: User, + @Body() body: FolderDto, + ): Promise { + return this.folders.inspect(user.id, body.runnerId, body.path); + } + + @Post('init') + @HttpCode(HttpStatus.OK) + init(@CurrentUser() user: User, @Body() body: FolderDto): Promise { + return this.folders.init(user.id, body.runnerId, body.path); + } + + @Post('register') + async register( + @CurrentUser() user: User, + @Body() body: RegisterFolderDto, + ): Promise<{ project: ProjectView }> { + return { project: await this.folders.register(user.id, body) }; + } +} diff --git a/apps/api/src/projects/folders/folders.dto.ts b/apps/api/src/projects/folders/folders.dto.ts new file mode 100644 index 00000000..48bbb161 --- /dev/null +++ b/apps/api/src/projects/folders/folders.dto.ts @@ -0,0 +1,35 @@ +import { Transform } from 'class-transformer'; +import { IsOptional, IsString, Length, ValidateIf } from 'class-validator'; +import { IsGithubUrl, PATH_MAX, PROJECT_NAME_MAX } from '../projects.dto'; + +const trim = ({ value }: { value: unknown }): unknown => + typeof value === 'string' ? value.trim() : value; + +/** `POST /projects/folders/inspect` and `/init` (spec #378, Decision 1). */ +export class FolderDto { + @IsString() + @Length(1, 64) + runnerId!: string; + + /** As the person typed it: the machine resolves it, and only its answer is stored. */ + @IsString() + @Length(1, PATH_MAX) + path!: string; +} + +/** + * `POST /projects/folders/register`. Without `name`, the folder's name; without + * `githubUrl`, the inspection's; `githubUrl: null` stores none. + */ +export class RegisterFolderDto extends FolderDto { + // Not `@IsOptional()`: `null` is no second spelling of the default. + @ValidateIf((_, value) => value !== undefined) + @Transform(trim) + @IsString() + @Length(1, PROJECT_NAME_MAX) + name?: string; + + @IsOptional() + @IsGithubUrl() + githubUrl?: string | null; +} diff --git a/apps/api/src/projects/folders/folders.integration.spec.ts b/apps/api/src/projects/folders/folders.integration.spec.ts new file mode 100644 index 00000000..fbf9d4fb --- /dev/null +++ b/apps/api/src/projects/folders/folders.integration.spec.ts @@ -0,0 +1,514 @@ +import type { AddressInfo } from 'node:net'; +import request, { type Response } from 'supertest'; +import type { App } from 'supertest/types'; +import { + type Paired, + pair, + type SignedIn, + signedInUser, +} from '../../runner/runner.fixtures'; +import { + type ProjectRequest, + TestSocket, +} from '../../runner/runner-socket.fixtures'; +import { createTestApp, type TestApp } from '../../test/test-app'; +import { + FOLDER_TIMEOUT_MS, + FoldersService, + folderName, +} from './folders.service'; + +const ROOT = '/home/u/app'; +const GITHUB = 'https://github.com/acme/app'; +const PROJECT_TYPES = ['project.inspect', 'project.init', 'project.register']; + +function inspection(overrides: Record = {}) { + return { + rootPath: ROOT, + name: 'app', + hasConfig: true, + configValid: true, + scenarios: [{ id: 'demoflow/checkout.demo.json', name: 'Checkout' }], + githubUrl: GITHUB, + ...overrides, + }; +} + +describe('adding a project from a folder (integration)', () => { + let t: TestApp; + let http: App; + let port: number; + let alice: SignedIn; + let bob: SignedIn; + let machine: Paired; + const sockets: TestSocket[] = []; + + beforeEach(async () => { + t = await createTestApp(); + await t.app.listen(0, '127.0.0.1'); + http = t.app.getHttpServer(); + port = (http.address() as AddressInfo).port; + alice = await signedInUser(t, 'alice'); + bob = await signedInUser(t, 'bob'); + machine = await pair(http, alice, 'Laptop'); + }); + + afterEach(async () => { + for (const socket of sockets.splice(0)) { + socket.ws.terminate(); + } + await t.app.close(); + }); + + /** A connected daemon announcing `features` (`projects` by default). */ + async function daemon( + features: string[] = ['projects'], + on = machine, + ): Promise { + const socket = await TestSocket.connect( + `ws://127.0.0.1:${port}/runner`, + on.token, + ); + sockets.push(socket); + socket.sync([], undefined, features); + await socket.roundTrip(); + return socket; + } + + /** Starts the request, so a test can answer as the daemon meanwhile. */ + function post( + action: 'inspect' | 'init' | 'register', + body: object, + cookie = alice.cookie, + ): Promise { + return request(http) + .post(`/projects/folders/${action}`) + .set('Cookie', cookie) + .send(body) + .then((res) => res); + } + + function projects(cookie = alice.cookie): Promise { + return request(http) + .get('/projects') + .set('Cookie', cookie) + .then((res) => res); + } + + function sentProjectMessages(socket: TestSocket): unknown[] { + return socket.received.filter((m) => PROJECT_TYPES.includes(m.type)); + } + + /** Plays the daemon through a whole `register` without a `githubUrl`. */ + async function register( + socket: TestSocket, + body: object = { runnerId: machine.runnerId, path: '~/app/../app' }, + rootPath = ROOT, + ): Promise { + const pending = post('register', body); + const inspect = await socket.nextProject('project.inspect'); + socket.send({ + type: 'project.inspected', + requestId: inspect.requestId, + inspection: inspection({ rootPath }), + }); + const ask = await socket.nextProject('project.register'); + socket.send({ + type: 'project.registered', + requestId: ask.requestId, + rootPath, + }); + return pending; + } + + describe('relaying to the machine', () => { + it('inspects a folder and answers what the Runner said', async () => { + const socket = await daemon(); + const pending = post('inspect', { + runnerId: machine.runnerId, + path: '~/app', + }); + const ask = await socket.nextProject('project.inspect'); + expect(ask.path).toBe('~/app'); + socket.send({ + type: 'project.inspected', + requestId: ask.requestId, + inspection: inspection(), + }); + + const res = await pending; + expect(res.status).toBe(200); + expect(res.body).toEqual(inspection()); + }); + + it('drops a githubUrl the API would not accept', async () => { + const socket = await daemon(); + const pending = post('inspect', { + runnerId: machine.runnerId, + path: '~/app', + }); + const ask = await socket.nextProject('project.inspect'); + socket.send({ + type: 'project.inspected', + requestId: ask.requestId, + inspection: inspection({ githubUrl: 'https://gitlab.com/acme/app' }), + }); + + expect((await pending).body.githubUrl).toBeNull(); + }); + + it('initialises a folder and answers the files and the new inspection', async () => { + const socket = await daemon(); + const pending = post('init', { + runnerId: machine.runnerId, + path: '~/app', + }); + const ask = await socket.nextProject('project.init'); + socket.send({ + type: 'project.initialised', + requestId: ask.requestId, + created: ['demoflow.config.json', 'demoflow/hello.demo.json'], + kept: [], + inspection: inspection(), + }); + + const res = await pending; + expect(res.status).toBe(200); + expect(res.body).toEqual({ + created: ['demoflow.config.json', 'demoflow/hello.demo.json'], + kept: [], + inspection: inspection(), + }); + }); + + it('registers the canonical rootPath the Runner answered, not the request path', async () => { + const socket = await daemon(); + const res = await register(socket); + + expect(res.status).toBe(201); + expect(res.body.project).toEqual( + expect.objectContaining({ + name: 'app', + rootPath: ROOT, + githubUrl: GITHUB, + source: 'website', + runner: { id: machine.runnerId, name: 'Laptop' }, + runCount: 0, + lastRun: null, + }), + ); + const row = await t.prisma.project.findUniqueOrThrow({ + where: { id: res.body.project.id }, + }); + expect(row.rootPath).toBe(ROOT); + expect(row.userId).toBe(alice.user.id); + }); + + it('takes the name and githubUrl of the body without inspecting', async () => { + const socket = await daemon(); + const pending = post('register', { + runnerId: machine.runnerId, + path: '~/app', + name: ' Shop ', + githubUrl: null, + }); + const ask = await socket.nextProject('project.register'); + socket.send({ + type: 'project.registered', + requestId: ask.requestId, + rootPath: ROOT, + }); + + const res = await pending; + expect(res.status).toBe(201); + expect(res.body.project).toEqual( + expect.objectContaining({ name: 'Shop', githubUrl: null }), + ); + expect(sentProjectMessages(socket)).toEqual([]); + }); + + it('answers 409 project-exists with the first project for the same folder', async () => { + const socket = await daemon(); + const first = await register(socket); + const second = await register(socket, { + runnerId: machine.runnerId, + path: ROOT, + }); + + expect(second.status).toBe(409); + expect(second.body).toEqual({ + error: 'project-exists', + projectId: first.body.project.id, + }); + }); + + it('answers 409 runner-offline and sends nothing without a socket', async () => { + for (const action of ['inspect', 'init', 'register'] as const) { + const res = await post(action, { + runnerId: machine.runnerId, + path: '~/app', + }); + expect(res.status).toBe(409); + expect(res.body).toEqual({ error: 'runner-offline' }); + } + }); + + it('answers 409 runner-outdated to a Runner without the projects feature', async () => { + const socket = await daemon(['share']); + const res = await post('inspect', { + runnerId: machine.runnerId, + path: '~/app', + }); + + expect(res.status).toBe(409); + expect(res.body).toEqual({ error: 'runner-outdated' }); + await socket.roundTrip(); + expect(sentProjectMessages(socket)).toEqual([]); + }); + + it('lists the features of an online machine', async () => { + await daemon(); + const res = await request(http) + .get('/runners') + .set('Cookie', alice.cookie) + .expect(200); + expect(res.body.runners[0]).toEqual( + expect.objectContaining({ online: true, features: ['projects'] }), + ); + }); + + it.each([ + ['path-not-allowed', 422, 'path-not-allowed'], + ['config-missing', 422, 'config-missing'], + ['config-invalid', 422, 'config-invalid'], + ['busy', 409, 'runner-busy'], + ['internal', 502, 'runner-error'], + ['something-new', 502, 'runner-error'], + ])('maps project.error %s to %i %s', async (code, status, error) => { + const socket = await daemon(); + const pending = post('inspect', { + runnerId: machine.runnerId, + path: '/etc', + }); + const ask = await socket.nextProject('project.inspect'); + socket.send({ type: 'project.error', requestId: ask.requestId, code }); + + const res = await pending; + expect(res.status).toBe(status); + expect(res.body).toEqual({ error }); + }); + + it('ends a request with runner-offline when the socket closes', async () => { + const socket = await daemon(); + const pending = post('inspect', { + runnerId: machine.runnerId, + path: '~/app', + }); + await socket.nextProject('project.inspect'); + await socket.close(); + + const res = await pending; + expect(res.status).toBe(409); + expect(res.body).toEqual({ error: 'runner-offline' }); + }); + }); + + describe('replies', () => { + it('ignores an unknown requestId, a reply of the wrong kind and another machine', async () => { + const socket = await daemon(); + const other = await pair(http, alice, 'Desktop'); + const otherSocket = await daemon(['projects'], other); + const pending = post('inspect', { + runnerId: machine.runnerId, + path: '~/app', + }); + const ask: ProjectRequest = await socket.nextProject('project.inspect'); + + socket.send({ + type: 'project.inspected', + requestId: '6a1f0d4e-2b9c-4c3e-8f7a-1d2e3f4a5b6c', + inspection: inspection({ name: 'unknown' }), + }); + socket.send({ + type: 'project.registered', + requestId: ask.requestId, + rootPath: '/elsewhere', + }); + otherSocket.send({ + type: 'project.inspected', + requestId: ask.requestId, + inspection: inspection({ name: 'other machine' }), + }); + await socket.roundTrip(); + await otherSocket.roundTrip(); + socket.send({ + type: 'project.inspected', + requestId: ask.requestId, + inspection: inspection(), + }); + + const res = await pending; + expect(res.status).toBe(200); + expect(res.body.name).toBe('app'); + // Unanswerable frames never close the socket. + await socket.roundTrip(); + }); + + it('answers 504 runner-timeout when the Runner stays silent for 10 s', async () => { + const socket = await daemon(); + const pending = post('inspect', { + runnerId: machine.runnerId, + path: '~/app', + }); + const ask = await socket.nextProject('project.inspect'); + const folders = t.app.get(FoldersService); + + t.clock.advance(FOLDER_TIMEOUT_MS - 1); + folders.tick(); + t.clock.advance(1); + folders.tick(); + + const res = await pending; + expect(res.status).toBe(504); + expect(res.body).toEqual({ error: 'runner-timeout' }); + // A late answer finds nothing to settle. + socket.send({ + type: 'project.inspected', + requestId: ask.requestId, + inspection: inspection(), + }); + await socket.roundTrip(); + }); + }); + + describe('authorization', () => { + it("answers 404 runner-not-found to another account's machine and sends nothing", async () => { + const socket = await daemon(); + for (const action of ['inspect', 'init', 'register'] as const) { + const res = await post( + action, + { runnerId: machine.runnerId, path: '~/app' }, + bob.cookie, + ); + expect(res.status).toBe(404); + expect(res.body).toEqual({ error: 'runner-not-found' }); + } + await socket.roundTrip(); + expect(sentProjectMessages(socket)).toEqual([]); + }); + + it('answers 404 to an unknown machine', async () => { + const res = await post('inspect', { + runnerId: 'not-a-machine', + path: '~/app', + }); + expect(res.status).toBe(404); + expect(res.body).toEqual({ error: 'runner-not-found' }); + }); + + it("keeps a website-added project out of another account's reach", async () => { + const socket = await daemon(); + const { body } = await register(socket); + + await request(http) + .patch(`/projects/${body.project.id}`) + .set('Cookie', bob.cookie) + .send({ name: 'mine' }) + .expect(404); + expect((await projects(bob.cookie)).body.projects).toEqual([]); + expect((await projects()).body.projects).toEqual([ + expect.objectContaining({ id: body.project.id, name: 'app' }), + ]); + }); + + it('shares 30 requests a minute per user across the three routes', async () => { + const body = { runnerId: 'not-a-machine', path: '~/app' }; + for (let i = 0; i < 30; i++) { + const action = (['inspect', 'init', 'register'] as const)[i % 3]; + expect((await post(action, body)).status).toBe(404); + } + expect((await post('inspect', body)).status).toBe(429); + // Per user: another account still has its budget. + expect((await post('inspect', body, bob.cookie)).status).toBe(404); + }); + + it('needs a session', async () => { + await request(http) + .post('/projects/folders/inspect') + .send({ runnerId: machine.runnerId, path: '~/app' }) + .expect(401); + }); + }); + + describe('githubUrl', () => { + it.each([ + 'https://gitlab.com/acme/app', + 'https://user:secret@github.com/acme/app', + 'http://github.com/acme/app', + 'https://github.com/acme', + 'https://github.com/acme/app/tree/main', + 'https://github.com.evil.example/acme/app', + ])('rejects %s with 400 and sends nothing', async (githubUrl) => { + const socket = await daemon(); + const res = await post('register', { + runnerId: machine.runnerId, + path: '~/app', + githubUrl, + }); + expect(res.status).toBe(400); + await socket.roundTrip(); + expect(sentProjectMessages(socket)).toEqual([]); + }); + + it('is set, rejected and cleared with PATCH', async () => { + const created = await request(http) + .post('/projects') + .set('Cookie', alice.cookie) + .send({ name: 'shop' }) + .expect(201); + const id = created.body.project.id; + const patch = (body: object) => + request(http) + .patch(`/projects/${id}`) + .set('Cookie', alice.cookie) + .send(body); + + expect(created.body.project).toEqual( + expect.objectContaining({ githubUrl: null, source: 'reported' }), + ); + expect( + (await patch({ githubUrl: GITHUB }).expect(200)).body.project, + ).toEqual(expect.objectContaining({ githubUrl: GITHUB })); + await patch({ githubUrl: 'https://user:pw@github.com/acme/app' }).expect( + 400, + ); + expect( + (await patch({ githubUrl: null }).expect(200)).body.project, + ).toEqual(expect.objectContaining({ githubUrl: null })); + }); + }); + + describe('input', () => { + it.each([ + [{ path: '~/app' }], + [{ runnerId: 'r' }], + [{ runnerId: 'r', path: '' }], + [{ runnerId: 'r', path: '~/app', extra: true }], + ])('rejects %j with 400', async (body) => { + const res = await post('inspect', body); + expect(res.status).toBe(400); + }); + }); +}); + +describe('folderName', () => { + it.each([ + ['/home/u/app', 'app'], + ['/home/u/app/', 'app'], + ['C:\\Users\\u\\shop', 'shop'], + ['\\\\server\\share\\demo', 'demo'], + ['/', '/'], + ])('names %s %s', (rootPath, name) => { + expect(folderName(rootPath)).toBe(name); + }); +}); diff --git a/apps/api/src/projects/folders/folders.service.ts b/apps/api/src/projects/folders/folders.service.ts new file mode 100644 index 00000000..e2a4c3a2 --- /dev/null +++ b/apps/api/src/projects/folders/folders.service.ts @@ -0,0 +1,330 @@ +import { randomUUID } from 'node:crypto'; +import { posix, win32 } from 'node:path'; +import { + HttpStatus, + Injectable, + Logger, + type OnApplicationBootstrap, + type OnModuleDestroy, + type OnModuleInit, +} from '@nestjs/common'; +import { Clock } from '../../common/clock'; +import { PrismaService } from '../../database/prisma.service'; +import { + type ProjectSink, + RunnerConnections, +} from '../../runner/runner-connections.service'; +import type { + FolderInspection, + ProjectMessage, +} from '../../runner/runner-socket'; +import { runError } from '../../videos/runs.service'; +import { GITHUB_URL_PATTERN, PROJECT_NAME_MAX } from '../projects.dto'; +import { ProjectsService, type ProjectView } from '../projects.service'; +import type { RegisterFolderDto } from './folders.dto'; + +/** No answer this long after a `project.*` request: 504 (spec #378, Decision 4). */ +export const FOLDER_TIMEOUT_MS = 10 * 1000; +/** How often the timeout is checked. */ +const SWEEP_INTERVAL_MS = 1000; + +/** What the browser gets of a Runner's inspection (spec #377, Decision 5). */ +export interface InspectionView { + rootPath: string; + name: string; + hasConfig: boolean; + configValid: boolean; + scenarios: { id: string; name: string }[]; + githubUrl: string | null; +} + +export interface InitView { + created: string[]; + kept: string[]; + inspection: InspectionView; +} + +type FolderRequest = 'project.inspect' | 'project.init' | 'project.register'; + +/** The reply each request waits for, besides `project.error`. */ +const REPLIES = { + 'project.inspect': 'project.inspected', + 'project.init': 'project.initialised', + 'project.register': 'project.registered', +} as const satisfies Record; + +type Reply = Extract< + ProjectMessage, + { type: (typeof REPLIES)[R] } +>; + +interface Failure { + status: HttpStatus; + error: string; +} + +type Outcome = { ok: true; reply: ProjectMessage } | ({ ok: false } & Failure); + +/** `project.error` codes as HTTP (Decision 4); anything else is `runner-error`. */ +const RUNNER_ERRORS: Record = { + 'path-not-allowed': { + status: HttpStatus.UNPROCESSABLE_ENTITY, + error: 'path-not-allowed', + }, + 'config-missing': { + status: HttpStatus.UNPROCESSABLE_ENTITY, + error: 'config-missing', + }, + 'config-invalid': { + status: HttpStatus.UNPROCESSABLE_ENTITY, + error: 'config-invalid', + }, + busy: { status: HttpStatus.CONFLICT, error: 'runner-busy' }, +}; +const RUNNER_ERROR: Failure = { + status: HttpStatus.BAD_GATEWAY, + error: 'runner-error', +}; +const OFFLINE: Failure = { + status: HttpStatus.CONFLICT, + error: 'runner-offline', +}; +const TIMEOUT: Failure = { + status: HttpStatus.GATEWAY_TIMEOUT, + error: 'runner-timeout', +}; + +function runnerFailure(code: string): Failure { + return Object.hasOwn(RUNNER_ERRORS, code) + ? RUNNER_ERRORS[code] + : RUNNER_ERROR; +} + +interface Pending { + id: string; + runnerId: string; + type: FolderRequest; + /** Clock ms. */ + sentAt: number; + settle: (outcome: Outcome) => void; +} + +/** + * Adds a project from a folder on one of the user's machines (spec #378, + * ADR-0022): relays `project.inspect`, `project.init` and `project.register` + * to the machine and answers what it said. Only `register` stores anything, + * and only the path the machine answered. + * + * Works only on the instance that holds the Runner's socket, like the + * scenario list (spec #247). + */ +@Injectable() +export class FoldersService + implements ProjectSink, OnModuleInit, OnApplicationBootstrap, OnModuleDestroy +{ + private readonly logger = new Logger(FoldersService.name); + private readonly pending = new Map(); + private timer: NodeJS.Timeout | undefined; + + constructor( + private readonly connections: RunnerConnections, + private readonly prisma: PrismaService, + private readonly projects: ProjectsService, + private readonly clock: Clock, + ) {} + + onModuleInit(): void { + this.connections.setProjectSink(this); + } + + onApplicationBootstrap(): void { + this.timer = setInterval(() => this.tick(), SWEEP_INTERVAL_MS); + this.timer.unref(); + } + + onModuleDestroy(): void { + clearInterval(this.timer); + } + + async inspect( + userId: string, + runnerId: string, + path: string, + ): Promise { + await this.target(userId, runnerId); + const reply = await this.ask(runnerId, 'project.inspect', path); + return toInspectionView(reply.inspection); + } + + async init( + userId: string, + runnerId: string, + path: string, + ): Promise { + await this.target(userId, runnerId); + const reply = await this.ask(runnerId, 'project.init', path); + return { + created: [...reply.created], + kept: [...reply.kept], + inspection: toInspectionView(reply.inspection), + }; + } + + /** + * Decision 5: the project row holds the canonical `rootPath` of + * `project.registered`. Without a `githubUrl` in the body, the folder is + * inspected first for its default. + */ + async register( + userId: string, + body: RegisterFolderDto, + ): Promise { + const { runnerId, path } = body; + await this.target(userId, runnerId); + let githubUrl = body.githubUrl; + if (githubUrl === undefined) { + const inspected = await this.ask(runnerId, 'project.inspect', path); + githubUrl = toInspectionView(inspected.inspection).githubUrl; + } + const { rootPath } = await this.ask(runnerId, 'project.register', path); + return this.projects.createFromFolder(userId, { + runnerId, + rootPath, + name: body.name ?? folderName(rootPath), + githubUrl, + }); + } + + message(runnerId: string, message: ProjectMessage): boolean { + const request = this.pending.get(message.requestId); + // Only the machine a request was sent to can answer it, and only in kind. + if ( + request?.runnerId !== runnerId || + (message.type !== 'project.error' && + message.type !== REPLIES[request.type]) + ) { + return false; + } + if (message.type === 'project.error') { + this.end( + request, + { ok: false, ...runnerFailure(message.code) }, + message.code, + ); + } else { + this.end(request, { ok: true, reply: message }, 'answered'); + } + return true; + } + + /** Every request on the machine's socket ends with it. */ + closed(runnerId: string): void { + for (const request of [...this.pending.values()]) { + if (request.runnerId === runnerId) { + this.end(request, { ok: false, ...OFFLINE }, OFFLINE.error); + } + } + } + + /** The timeout, on the injected clock. Runs every `SWEEP_INTERVAL_MS`. */ + tick(): void { + const now = this.now(); + for (const request of [...this.pending.values()]) { + if (now - request.sentAt >= FOLDER_TIMEOUT_MS) { + this.end(request, { ok: false, ...TIMEOUT }, TIMEOUT.error); + } + } + } + + /** + * Decisions 2 and 3, in order, before anything is sent: the user's own + * machine (another account's answers as an unknown one), online, and new + * enough to answer `project.*`. + */ + private async target(userId: string, runnerId: string): Promise { + const count = await this.prisma.runner.count({ + where: { id: runnerId, userId, revokedAt: null }, + }); + if (count !== 1) { + throw runError(HttpStatus.NOT_FOUND, 'runner-not-found'); + } + if (!this.connections.isOnline(runnerId)) { + throw runError(OFFLINE.status, OFFLINE.error); + } + if (!this.connections.features(runnerId).includes('projects')) { + throw runError(HttpStatus.CONFLICT, 'runner-outdated'); + } + } + + private async ask( + runnerId: string, + type: R, + path: string, + ): Promise> { + const outcome = await new Promise((settle) => { + const request: Pending = { + id: randomUUID(), + runnerId, + type, + sentAt: this.now(), + settle, + }; + this.pending.set(request.id, request); + const sent = this.connections.sendTo(runnerId, { + type, + requestId: request.id, + path, + }); + if (!sent) { + this.end(request, { ok: false, ...OFFLINE }, OFFLINE.error); + } + }); + if (!outcome.ok) { + throw runError(outcome.status, outcome.error); + } + // `message` settles a request only with its own reply type. + return outcome.reply as Reply; + } + + private end(request: Pending, outcome: Outcome, label: string): void { + this.pending.delete(request.id); + // No path, no folder name: the machine's layout stays off the logs. + this.logger.log( + JSON.stringify({ + event: request.type, + runnerId: request.runnerId, + outcome: label, + durationMs: this.now() - request.sentAt, + }), + ); + request.settle(outcome); + } + + private now(): number { + return this.clock.now().getTime(); + } +} + +/** A URL the API would not accept from the browser is dropped, not passed on. */ +function toInspectionView(inspection: FolderInspection): InspectionView { + const { githubUrl } = inspection; + return { + rootPath: inspection.rootPath, + name: inspection.name, + hasConfig: inspection.hasConfig, + configValid: inspection.configValid, + scenarios: inspection.scenarios.map(({ id, name }) => ({ id, name })), + githubUrl: + githubUrl !== undefined && GITHUB_URL_PATTERN.test(githubUrl) + ? githubUrl + : null, + }; +} + +/** The canonical root's base name, as the inspection names it; Windows paths too. */ +export function folderName(rootPath: string): string { + const name = /^[A-Za-z]:[\\/]|^\\\\/.test(rootPath) + ? win32.basename(rootPath) + : posix.basename(rootPath); + return (name || rootPath).slice(0, PROJECT_NAME_MAX); +} diff --git a/apps/api/src/projects/projects.dto.ts b/apps/api/src/projects/projects.dto.ts index b4d3ffda..70094701 100644 --- a/apps/api/src/projects/projects.dto.ts +++ b/apps/api/src/projects/projects.dto.ts @@ -1,3 +1,4 @@ +import { applyDecorators } from '@nestjs/common'; import { Transform, Type } from 'class-transformer'; import { ArrayMaxSize, @@ -11,6 +12,7 @@ import { IsOptional, IsString, Length, + Matches, Max, MaxLength, Min, @@ -23,6 +25,21 @@ const trim = ({ value }: { value: unknown }): unknown => export const PROJECT_NAME_MAX = 200; export const PATH_MAX = 4096; +/** + * A repository page and nothing else (spec #378, Decision 1): no other host, + * no credentials, no path below the repository. + */ +export const GITHUB_URL_PATTERN = + /^https:\/\/github\.com\/[A-Za-z0-9._-]{1,100}\/[A-Za-z0-9._-]{1,100}$/; + +/** With `@IsOptional()`, which also lets `null` through: no URL. */ +export const IsGithubUrl = () => + applyDecorators( + IsString(), + Matches(GITHUB_URL_PATTERN, { + message: 'githubUrl must be https://github.com//', + }), + ); /** `RENDER_FORMATS` of `@demoflow/core`, which the API does not depend on. */ export const RUN_FORMATS = [ @@ -38,6 +55,9 @@ export const RUN_FORMATS = [ ] as const; export type RunFormat = (typeof RUN_FORMATS)[number]; export const RUN_STATUSES = ['passed', 'failed'] as const; +/** A run's kind (spec 360); an ordinary run sends none. */ +export const RUN_KINDS = ['check'] as const; +export type RunKind = (typeof RUN_KINDS)[number]; /** `STAGE_NAMES` of `@demoflow/core`. */ export const RUN_STAGES = [ @@ -194,6 +214,11 @@ export class RunReportDto { @ValidateNested() @Type(() => RunFailureDto) failure?: RunFailureDto; + + /** `check` for a check run (spec 360); absent for an ordinary run. */ + @IsOptional() + @IsIn(RUN_KINDS) + kind?: RunKind; } export class CreateProjectDto { @@ -203,7 +228,10 @@ export class CreateProjectDto { name!: string; } -/** Renames and binds. `runnerId` and `rootPath` come together or not at all. */ +/** + * Renames, binds, and sets or clears the GitHub URL (spec #378, Decision 7). + * `runnerId` and `rootPath` come together or not at all. + */ export class UpdateProjectDto { @IsOptional() @Transform(trim) @@ -211,6 +239,10 @@ export class UpdateProjectDto { @Length(1, PROJECT_NAME_MAX) name?: string; + @IsOptional() + @IsGithubUrl() + githubUrl?: string | null; + @ValidateIf((dto: UpdateProjectDto) => dto.rootPath !== undefined) @IsString() @Length(1, 64) diff --git a/apps/api/src/projects/projects.integration.spec.ts b/apps/api/src/projects/projects.integration.spec.ts index 9a282e20..92718772 100644 --- a/apps/api/src/projects/projects.integration.spec.ts +++ b/apps/api/src/projects/projects.integration.spec.ts @@ -88,8 +88,11 @@ describe('projects (integration)', () => { id: res.body.projectId, name: 'shop', rootPath: ROOT, + githubUrl: null, + source: 'reported', runner: { id: laptop.runnerId, name: 'Laptop' }, runCount: 1, + staleCount: 0, lastRun: { id: res.body.runId, scenarioName: 'checkout', @@ -101,6 +104,7 @@ describe('projects (integration)', () => { failedStep: null, cliVersion: '0.4.0', artifactsDeletedAt: null, + kind: null, }, createdAt: expect.any(String), updatedAt: expect.any(String), @@ -127,6 +131,7 @@ describe('projects (integration)', () => { expect.objectContaining({ name: 'shop', runCount: 2, + staleCount: 1, lastRun: expect.objectContaining({ id: second.body.runId, status: 'failed', @@ -171,6 +176,43 @@ describe('projects (integration)', () => { expect(projects[0].lastRun.formats).toEqual(formats); }); + it('stores a check run with kind check, and an ordinary run with none (#360)', async () => { + const check = await report( + laptop.token, + runReport({ kind: 'check', formats: [] }), + ).expect(201); + const ordinary = await report(laptop.token).expect(201); + + const rows = await t.prisma.run.findMany({ + where: { id: { in: [check.body.runId, ordinary.body.runId] } }, + select: { id: true, kind: true }, + }); + expect(new Map(rows.map((row) => [row.id, row.kind]))).toEqual( + new Map([ + [check.body.runId, 'check'], + [ordinary.body.runId, null], + ]), + ); + const page = (await runs(alice.cookie, check.body.projectId).expect(200)) + .body; + expect( + page.runs.map((run: { id: string; kind: string | null }) => [ + run.id, + run.kind, + ]), + ).toEqual( + expect.arrayContaining([ + [check.body.runId, 'check'], + [ordinary.body.runId, null], + ]), + ); + }); + + it('refuses any kind but check', async () => { + await report(laptop.token, runReport({ kind: 'video' })).expect(400); + await report(laptop.token, runReport({ kind: null })).expect(201); + }); + it('stores no field it was not given a column for', async () => { await report(laptop.token, { ...runReport(), @@ -275,8 +317,15 @@ describe('projects (integration)', () => { rootPath: ROOT, runner: { id: laptop.runnerId, name: 'Laptop' }, runCount: 2, + staleCount: 0, }), ); + expect( + await t.prisma.scenarioHealth.findMany({ + where: { projectId: body.project.id }, + select: { scenarioPath: true, state: true }, + }), + ).toEqual([{ scenarioPath: 'demos/checkout.yaml', state: 'passed' }]); expect( await t.prisma.project.findUnique({ where: { id: auto1.body.projectId }, diff --git a/apps/api/src/projects/projects.module.ts b/apps/api/src/projects/projects.module.ts index 978d0d4c..a1b928c8 100644 --- a/apps/api/src/projects/projects.module.ts +++ b/apps/api/src/projects/projects.module.ts @@ -1,14 +1,18 @@ import { Module } from '@nestjs/common'; import { AuthModule } from '../auth/auth.module'; import { RunnerModule } from '../runner/runner.module'; +import { ShareStorageModule } from '../shares/share-storage'; +import { FoldersController } from './folders/folders.controller'; +import { FoldersService } from './folders/folders.service'; import { ProjectsController } from './projects.controller'; import { ProjectsService } from './projects.service'; import { RunnerRunsController } from './runner-runs.controller'; /** Projects and the demo library (spec #178): metadata only, never video. */ @Module({ - imports: [AuthModule, RunnerModule], - controllers: [ProjectsController, RunnerRunsController], - providers: [ProjectsService], + imports: [AuthModule, RunnerModule, ShareStorageModule], + // `projects/folders/*` before `projects/:id/*`. + controllers: [FoldersController, ProjectsController, RunnerRunsController], + providers: [ProjectsService, FoldersService], }) export class ProjectsModule {} diff --git a/apps/api/src/projects/projects.service.ts b/apps/api/src/projects/projects.service.ts index b7d7c310..214c4a87 100644 --- a/apps/api/src/projects/projects.service.ts +++ b/apps/api/src/projects/projects.service.ts @@ -1,12 +1,22 @@ import { BadRequestException, ConflictException, + HttpException, + HttpStatus, Injectable, Logger, NotFoundException, } from '@nestjs/common'; -import { Prisma, type Run, type Runner, type RunStatus } from '@prisma/client'; +import { + Prisma, + type ProjectSource, + type Run, + type Runner, + type RunStatus, +} from '@prisma/client'; +import { rebuildHealth, recordHealth } from '../checks/health/health'; import { PrismaService } from '../database/prisma.service'; +import { ShareStorage } from '../shares/share-storage'; import { type CreateProjectDto, RUN_FORMATS, @@ -35,6 +45,8 @@ export interface RunView { cliVersion: string; /** When the machine deleted the run's directory (spec #233); null while it is there. */ artifactsDeletedAt: Date | null; + /** `check` for a check run (spec 360); null for an ordinary run. */ + kind: 'check' | null; } export interface ProjectView { @@ -43,7 +55,13 @@ export interface ProjectView { rootPath: string | null; /** Null for a project never bound, or whose machine was revoked. */ runner: { id: string; name: string } | null; + /** `https://github.com//`, display only (ADR-0022 §5). */ + githubUrl: string | null; + /** `website` when added through the folder dialog (spec #378). */ + source: ProjectSource; runCount: number; + /** Scenarios whose latest run failed (spec #362, Decision 6). */ + staleCount: number; lastRun: RunView | null; createdAt: Date; updatedAt: Date; @@ -62,7 +80,9 @@ const NEWEST_FIRST: Prisma.RunOrderByWithRelationInput[] = [ const PROJECT_INCLUDE = { runner: { select: { id: true, name: true } }, runs: { orderBy: NEWEST_FIRST, take: 1 }, - _count: { select: { runs: true } }, + _count: { + select: { runs: true, health: { where: { state: 'failed' } } }, + }, } satisfies Prisma.ProjectInclude; type ProjectWithSummary = Prisma.ProjectGetPayload<{ @@ -81,6 +101,7 @@ export function toRunView(run: Run): RunView { failedStep: run.failedStep, cliVersion: run.cliVersion, artifactsDeletedAt: run.artifactsDeletedAt, + kind: run.kind, }; } @@ -153,7 +174,7 @@ function parseStages(value: Prisma.JsonValue): RunStages | null { const RENDER_KEYS = ['format', 'width', 'height', 'sizeBytes', 'durationMs']; -function parseRenders(value: Prisma.JsonValue): RunRender[] | null { +export function parseRenders(value: Prisma.JsonValue): RunRender[] | null { if (!Array.isArray(value)) { return null; } @@ -189,7 +210,10 @@ function toProjectView(project: ProjectWithSummary): ProjectView { name: project.name, rootPath: project.rootPath, runner: project.runner, + githubUrl: project.githubUrl, + source: project.source, runCount: project._count.runs, + staleCount: project._count.health, lastRun: lastRun ? toRunView(lastRun) : null, createdAt: project.createdAt, updatedAt: project.updatedAt, @@ -209,7 +233,10 @@ function lastActivity(project: ProjectView): number { export class ProjectsService { private readonly logger = new Logger(ProjectsService.name); - constructor(private readonly prisma: PrismaService) {} + constructor( + private readonly prisma: PrismaService, + private readonly shareStorage: ShareStorage, + ) {} /** Upserts the folder's project and adds the run (Decision 3). */ async report( @@ -249,9 +276,11 @@ export class ProjectsService { : Prisma.DbNull, failureStage: report.failure?.stage ?? null, failureMessage: report.failure?.message ?? null, + kind: report.kind ?? null, }, - select: { id: true }, + select: { id: true, scenarioPath: true, status: true, startedAt: true }, }); + await recordHealth(tx, project.id, run); return { projectId: project.id, runId: run.id }; }); } @@ -278,6 +307,50 @@ export class ProjectsService { return toProjectView(project); } + /** + * The project of a folder the machine registered (spec #378, Decision 5): + * `rootPath` is the machine's canonical answer, never the browser's path. + * A folder another project already holds answers `409 project-exists` + * with that project's id. + */ + async createFromFolder( + userId: string, + folder: { + runnerId: string; + rootPath: string; + name: string; + githubUrl: string | null; + }, + ): Promise { + try { + const project = await this.prisma.project.create({ + data: { userId, ...folder, source: 'website' }, + include: PROJECT_INCLUDE, + }); + return toProjectView(project); + } catch (error) { + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + error.code === 'P2002' + ) { + const holder = await this.prisma.project.findUnique({ + where: { + runnerId_rootPath: { + runnerId: folder.runnerId, + rootPath: folder.rootPath, + }, + }, + select: { id: true }, + }); + throw new HttpException( + { error: 'project-exists', projectId: holder?.id ?? null }, + HttpStatus.CONFLICT, + ); + } + throw error; + } + } + /** * Renames and binds. Binding to a folder another project already holds * merges it into this one, in one transaction: its runs move here, it is @@ -301,6 +374,9 @@ export class ProjectsService { if (body.name !== undefined) { data.name = body.name; } + if (body.githubUrl !== undefined) { + data.githubUrl = body.githubUrl; + } if (body.runnerId !== undefined && body.rootPath !== undefined) { const runner = await tx.runner.findFirst({ where: { id: body.runnerId, userId, revokedAt: null }, @@ -327,6 +403,7 @@ export class ProjectsService { data: { projectId: id }, }); await tx.project.delete({ where: { id: holder.id } }); + await rebuildHealth(tx, id); } data.runner = { connect: { id: runner.id } }; data.rootPath = body.rootPath; @@ -346,14 +423,22 @@ export class ProjectsService { return this.get(userId, id); } - /** Deletes the project; its runs cascade. */ + /** + * Deletes the project; its runs and their shares cascade, and the shares' + * directories go once the rows are gone (spec #347, Decision 9). + */ async remove(userId: string, id: string): Promise { + const shares = await this.prisma.share.findMany({ + where: { run: { project: { id, userId } } }, + select: { id: true }, + }); const { count } = await this.prisma.project.deleteMany({ where: { id, userId }, }); if (count !== 1) { throw new NotFoundException(); } + await this.shareStorage.remove(shares.map((share) => share.id)); } /** @@ -400,7 +485,7 @@ export class ProjectsService { /** * Deletes the project's runs among `runIds` in one statement (spec #286). * Ids that are not runs of this project are skipped; only the rows go, the - * video files on the machine stay. + * video files on the machine stay. Shares of those runs go with them. */ async removeRuns( userId: string, @@ -414,12 +499,17 @@ export class ProjectsService { if (!project) { throw new NotFoundException(); } + const shares = await this.prisma.share.findMany({ + where: { run: { projectId: id, id: { in: runIds } } }, + select: { id: true }, + }); const { count } = await this.prisma.run.deleteMany({ where: { projectId: id, id: { in: runIds } }, }); this.logger.log( `runs.delete project=${id} asked=${runIds.length} deleted=${count}`, ); + await this.shareStorage.remove(shares.map((share) => share.id)); return { deleted: count }; } diff --git a/apps/api/src/runner/runner-connections.service.ts b/apps/api/src/runner/runner-connections.service.ts index 3d1d587c..8c9078ec 100644 --- a/apps/api/src/runner/runner-connections.service.ts +++ b/apps/api/src/runner/runner-connections.service.ts @@ -16,15 +16,20 @@ import { CLOSE_REPLACED, CLOSE_REVOKED, type ClientMessage, + FEATURES, + type Feature, HEARTBEAT_TICK_MS, IDLE_TIMEOUT_MS, LAST_SEEN_WRITE_INTERVAL_MS, type LaunchMessage, PING_INTERVAL_MS, + type ProjectMessage, parseClientMessage, RUN_OPTIONS, type RunOption, + type SchedulesMessage, type ServerMessage, + type ShareMessage, type VideoChunk, type VideoMessage, } from './runner-socket'; @@ -49,6 +54,14 @@ interface Connection { warnedStrayLaunch: boolean; /** The known `run.start` options this socket's `launches.sync` announced (spec #325, Decision 4). */ runOptions: RunOption[]; + /** The known features the same frame announced (spec #347, Decision 6). */ + features: Feature[]; + /** Whether the same frame said it answers `check.start` (spec 360); null before it. */ + checks: boolean | null; + /** A `share.*` answer for no pending request was logged on this socket. */ + warnedStrayShare: boolean; + /** A `project.*` answer for no pending request was logged on this socket. */ + warnedStrayProject: boolean; } /** @@ -84,6 +97,31 @@ export interface LaunchSink { closed(runnerId: string): void; } +/** + * Where share uploads (spec #347) hear from the sockets. `message` answers + * false for a request it does not hold. + */ +export interface ShareSink { + message(runnerId: string, message: ShareMessage): boolean; + /** The machine's socket is gone: closed, replaced, revoked or shut down. */ + closed(runnerId: string): void; +} + +/** Where check schedules (spec #362) hear a machine's config schedules. */ +export interface SchedulesSink { + sync(runnerId: string, message: SchedulesMessage): void; +} + +/** + * Where folder requests (spec #378) hear from the sockets. `message` answers + * false for a request it does not hold. + */ +export interface ProjectSink { + message(runnerId: string, message: ProjectMessage): boolean; + /** The machine's socket is gone: closed, replaced, revoked or shut down. */ + closed(runnerId: string): void; +} + /** * The open runner sockets of this API instance (one today, #120): online is * "has an entry here", never a column. One socket per machine — a second one @@ -101,6 +139,9 @@ export class RunnerConnections private videoSink: VideoSink | undefined; private artifactsSink: ArtifactsSink | undefined; private launchSink: LaunchSink | undefined; + private shareSink: ShareSink | undefined; + private projectSink: ProjectSink | undefined; + private schedulesSink: SchedulesSink | undefined; constructor( private readonly prisma: PrismaService, @@ -154,6 +195,19 @@ export class RunnerConnections return this.open.get(runnerId)?.runOptions ?? []; } + /** What the machine's current socket can do (spec #347); none offline or before its sync. */ + features(runnerId: string): readonly Feature[] { + return this.open.get(runnerId)?.features ?? []; + } + + /** + * Whether the machine's current socket answers `check.start` (spec #362, + * Decision 4); null offline or before its `launches.sync`. + */ + checks(runnerId: string): boolean | null { + return this.open.get(runnerId)?.checks ?? null; + } + /** To the machine's current socket; false when it is not online. */ sendTo(runnerId: string, message: ServerMessage): boolean { const connection = this.open.get(runnerId); @@ -179,6 +233,21 @@ export class RunnerConnections this.launchSink = sink; } + /** Share uploads' hook (spec #347). One sink per instance. */ + setShareSink(sink: ShareSink): void { + this.shareSink = sink; + } + + /** Folder requests' hook (spec #378). One sink per instance. */ + setProjectSink(sink: ProjectSink): void { + this.projectSink = sink; + } + + /** Check schedules' hook (spec #362). One sink per instance. */ + setSchedulesSink(sink: SchedulesSink): void { + this.schedulesSink = sink; + } + /** An authenticated socket, straight after the upgrade. */ async accept( socket: WebSocket, @@ -196,6 +265,10 @@ export class RunnerConnections warnedStrayArtifacts: false, warnedStrayLaunch: false, runOptions: [], + features: [], + checks: null, + warnedStrayShare: false, + warnedStrayProject: false, }; // Listeners first: a frame that arrives during the writes below counts. socket.on('message', (data, isBinary) => @@ -357,15 +430,35 @@ export class RunnerConnections case 'run.error': case 'run.finished': case 'run.progress': + case 'check.started': + case 'check.finished': + case 'check.error': this.relayLaunch(connection, message); return; case 'launches.sync': - // Unknown options are dropped without a log (spec #325). + // Unknown options and features are dropped without a log (specs #325, #347). connection.runOptions = RUN_OPTIONS.filter((option) => message.runOptions?.includes(option), ); + connection.features = FEATURES.filter((feature) => + message.features?.includes(feature), + ); + connection.checks = message.checks === true; this.relayLaunch(connection, message); return; + case 'schedules.sync': + this.schedulesSink?.sync(connection.runnerId, message); + return; + case 'share.prepared': + case 'share.error': + this.relayShare(connection, message); + return; + case 'project.inspected': + case 'project.initialised': + case 'project.registered': + case 'project.error': + this.relayProject(connection, message); + return; } } @@ -382,6 +475,32 @@ export class RunnerConnections } } + /** As `relayArtifacts`, with its own once-per-socket warning. */ + private relayShare(connection: Connection, message: ShareMessage): void { + if (this.shareSink?.message(connection.runnerId, message)) { + return; + } + if (!connection.warnedStrayShare) { + connection.warnedStrayShare = true; + this.logger.warn( + `Dropped ${message.type} from runner ${connection.runnerId} for a request that is not pending`, + ); + } + } + + /** As `relayArtifacts`, with its own once-per-socket warning. */ + private relayProject(connection: Connection, message: ProjectMessage): void { + if (this.projectSink?.message(connection.runnerId, message)) { + return; + } + if (!connection.warnedStrayProject) { + connection.warnedStrayProject = true; + this.logger.warn( + `Dropped ${message.type} from runner ${connection.runnerId} for a request that is not pending`, + ); + } + } + /** * An answer for no pending request — a late one after a timeout, or a * skewed daemon — is dropped, logged once per socket, and the socket stays. @@ -405,6 +524,8 @@ export class RunnerConnections this.videoSink?.closed(runnerId); this.artifactsSink?.closed(runnerId); this.launchSink?.closed(runnerId); + this.shareSink?.closed(runnerId); + this.projectSink?.closed(runnerId); } private touch(connection: Connection): void { diff --git a/apps/api/src/runner/runner-socket.fixtures.ts b/apps/api/src/runner/runner-socket.fixtures.ts index 8889fc14..f1450aca 100644 --- a/apps/api/src/runner/runner-socket.fixtures.ts +++ b/apps/api/src/runner/runner-socket.fixtures.ts @@ -25,6 +25,19 @@ export interface LaunchRequest extends Received { requestId: string; } +/** `share.prepare` (spec #347). */ +export interface SharePrepare extends Received { + requestId: string; + runId: string; + format: string; +} + +/** `project.inspect`, `project.init` or `project.register` (spec #378). */ +export interface ProjectRequest extends Received { + requestId: string; + path: string; +} + export interface Closed { code: number; reason: string; @@ -128,9 +141,9 @@ export class TestSocket { return ask as ArtifactsDelete; } - /** The next `scenarios.list`, `run.start` or `run.cancel`, with its request id. */ + /** The next `scenarios.list`, `run.start`, `run.cancel` or `check.start`, with its request id. */ async nextLaunch( - type: 'scenarios.list' | 'run.start' | 'run.cancel', + type: 'scenarios.list' | 'run.start' | 'run.cancel' | 'check.start', ): Promise { const ask = await this.next(type); if (typeof ask.requestId !== 'string') { @@ -139,16 +152,45 @@ export class TestSocket { return ask as LaunchRequest; } + /** The next `share.prepare` (spec #347), with its request id as a string. */ + async nextShare(): Promise { + const ask = await this.next('share.prepare'); + if (typeof ask.requestId !== 'string') { + throw new Error('share.prepare carries no requestId'); + } + return ask as SharePrepare; + } + + /** The next `project.*` request (spec #378), with its request id and path. */ + async nextProject( + type: 'project.inspect' | 'project.init' | 'project.register', + ): Promise { + const ask = await this.next(type); + if (typeof ask.requestId !== 'string' || typeof ask.path !== 'string') { + throw new Error(`${type} carries no requestId or path`); + } + return ask as ProjectRequest; + } + /** * `launches.sync` (spec #284): the launch ids this daemon still runs, and * the `run.start` options it accepts (spec #325) — none sent when omitted. + * `features` is its capability list, as `share` (spec #347); `checks` + * says it answers `check.start` (spec 360). */ - sync(running: string[], runOptions?: unknown): void { - this.send( - runOptions === undefined - ? { type: 'launches.sync', running } - : { type: 'launches.sync', running, runOptions }, - ); + sync( + running: string[], + runOptions?: unknown, + features?: string[], + checks?: boolean, + ): void { + this.send({ + type: 'launches.sync', + running, + ...(runOptions === undefined ? {} : { runOptions }), + ...(features === undefined ? {} : { features }), + ...(checks === undefined ? {} : { checks }), + }); } /** `run.progress` (spec #301): `step` of `steps` for `steps`, both null otherwise. */ diff --git a/apps/api/src/runner/runner-socket.spec.ts b/apps/api/src/runner/runner-socket.spec.ts index 56323aed..b7e83062 100644 --- a/apps/api/src/runner/runner-socket.spec.ts +++ b/apps/api/src/runner/runner-socket.spec.ts @@ -364,3 +364,86 @@ describe('launches.sync (spec #284)', () => { }); }); }); + +describe('project messages (spec #378)', () => { + const requestId = randomUUID(); + const inspection = { + rootPath: '/home/u/app', + name: 'app', + hasConfig: true, + configValid: true, + scenarios: [{ id: 'demoflow/checkout.demo.json', name: 'Checkout' }], + githubUrl: 'https://github.com/acme/app', + }; + + function parse(message: unknown) { + return parseClientMessage(Buffer.from(JSON.stringify(message)), false); + } + + it('reads the four replies', () => { + for (const message of [ + { type: 'project.inspected', requestId, inspection }, + { + type: 'project.inspected', + requestId, + inspection: { ...inspection, githubUrl: undefined, scenarios: [] }, + }, + { + type: 'project.initialised', + requestId, + created: ['demoflow.config.json'], + kept: [], + inspection, + }, + { type: 'project.registered', requestId, rootPath: '/home/u/app' }, + { type: 'project.error', requestId, code: 'path-not-allowed' }, + { type: 'project.error', requestId, code: 'a-code-from-a-newer-cli' }, + ]) { + expect(parse(message)).toEqual({ + ok: true, + message: expect.objectContaining(message), + }); + } + }); + + it.each([ + [ + 'an inspection without a request id', + { type: 'project.inspected', inspection }, + ], + ['a missing inspection', { type: 'project.inspected', requestId }], + [ + 'a non-boolean hasConfig', + { + type: 'project.inspected', + requestId, + inspection: { ...inspection, hasConfig: 'yes' }, + }, + ], + [ + 'a scenario without a name', + { + type: 'project.inspected', + requestId, + inspection: { ...inspection, scenarios: [{ id: 'x' }] }, + }, + ], + [ + 'an empty created file', + { + type: 'project.initialised', + requestId, + created: [''], + kept: [], + inspection, + }, + ], + [ + 'an empty rootPath', + { type: 'project.registered', requestId, rootPath: '' }, + ], + ['an error without a code', { type: 'project.error', requestId }], + ])('refuses %s', (_label, message) => { + expect(parse(message).ok).toBe(false); + }); +}); diff --git a/apps/api/src/runner/runner-socket.ts b/apps/api/src/runner/runner-socket.ts index 97406fa3..0c0ca907 100644 --- a/apps/api/src/runner/runner-socket.ts +++ b/apps/api/src/runner/runner-socket.ts @@ -11,10 +11,12 @@ import { IsBoolean, IsIn, IsInt, + IsObject, IsOptional, IsString, IsUUID, Length, + Matches, Max, MaxLength, Min, @@ -23,8 +25,10 @@ import { validateSync, } from 'class-validator'; import type { RawData } from 'ws'; +import { IsTimeZone } from '../checks/schedules/time-zone'; +import { CADENCES, type Cadence, TIME_OF_DAY, WEEKDAYS } from '../checks/slots'; import { IsScenarioId } from '../launches/scenario-id'; -import type { RunFormat } from '../projects/projects.dto'; +import { PATH_MAX, type RunFormat } from '../projects/projects.dto'; import type { RunnerContext } from './runner-token'; /** @@ -75,6 +79,8 @@ export type ServerMessage = runId: string; format: string; range: VideoRange | null; + /** The share's poster instead of the render (spec #347), only to a daemon with `share`. */ + asset?: VideoAsset; } | { type: 'video.ack'; streamId: string; bytes: number } | { type: 'video.cancel'; streamId: string } @@ -88,7 +94,23 @@ export type ServerMessage = /** Only when the browser chose formats (spec #325): the default frame is unchanged. */ options?: RunStartOptions; } - | { type: 'run.cancel'; requestId: string }; + | { type: 'run.cancel'; requestId: string } + /** A check pass of the whole project (spec 360), cancelled with `run.cancel`. */ + | { + type: 'check.start'; + requestId: string; + rootPath: string; + exclude: string[]; + } + | { type: 'share.prepare'; requestId: string; runId: string; format: string } + | { + type: 'project.inspect' | 'project.init' | 'project.register'; + requestId: string; + path: string; + }; + +/** What `video.open` may ask for besides the render itself (spec #347). */ +export type VideoAsset = 'poster'; /** What a website-started run may choose (ADR-0017): the output formats, nothing else. */ export interface RunStartOptions { @@ -102,7 +124,10 @@ export type ClientMessage = | ByeMessage | VideoMessage | ArtifactsMessage - | LaunchMessage; + | LaunchMessage + | ShareMessage + | SchedulesMessage + | ProjectMessage; export type VideoMessage = | VideoHeadMessage @@ -127,7 +152,38 @@ export type LaunchMessage = | RunErrorMessage | RunFinishedMessage | RunProgressMessage - | LaunchesSyncMessage; + | LaunchesSyncMessage + | CheckMessage; + +/** A machine's project-config schedules (spec 361, Decision 6). */ +export type SchedulesMessage = SchedulesSyncMessage; + +/** A Runner's answers to `check.start` (spec 360, Decision 6). */ +export type CheckMessage = + | CheckStartedMessage + | CheckFinishedMessage + | CheckErrorMessage; + +/** A Runner's answer to `share.prepare` (spec #347, Decision 4). */ +export type ShareMessage = SharePreparedMessage | ShareErrorMessage; + +/** A Runner's answers to `project.*` (spec #377, Decision 4; spec #378). */ +export type ProjectMessage = + | ProjectInspectedMessage + | ProjectInitialisedMessage + | ProjectRegisteredMessage + | ProjectErrorMessage; + +/** A file name `project.initialised` lists, relative to the folder. */ +const INIT_FILE_MAX = 512; +/** `demoflow init` writes two files; room for a newer template. */ +const MAX_INIT_FILES = 16; + +/** `share.prepared` limits (Decision 4). */ +export const SHARE_DESCRIPTION_MAX = 2000; +export const SHARE_CHAPTERS_MAX = 100; +export const SHARE_CHAPTER_TITLE_MAX = 200; +export const SHARE_POSTER_MAX_BYTES = 1024 * 1024; /** `scenarios.listed` carries this many entries at most. */ export const MAX_LISTED_SCENARIOS = 100; @@ -151,6 +207,22 @@ export type RunOption = (typeof RUN_OPTIONS)[number]; /** `launches.sync` announces this many options at most, each this long at most. */ export const MAX_RUN_OPTIONS = 8; export const RUN_OPTION_MAX_LENGTH = 32; +/** + * The capabilities this API knows a daemon may announce (ADR-0019 §7): `share` + * answers `share.prepare` and `video.open` with an `asset`; `projects` answers + * `project.*` (ADR-0022 §6). + */ +export const FEATURES = ['share', 'projects'] as const; +export type Feature = (typeof FEATURES)[number]; +/** `launches.sync` announces this many features at most, each this long at most. */ +export const MAX_FEATURES = 8; +export const FEATURE_MAX_LENGTH = 32; +/** `schedules.sync` carries this many projects at most (spec 361, Decision 6). */ +export const MAX_SYNCED_SCHEDULES = 500; +/** A schedule excludes this many scenarios at most (ADR-0021 §5). */ +export const MAX_SCHEDULE_EXCLUDES = 100; +/** `check.finished` counts this many scenarios at most. */ +const MAX_CHECK_COUNT = 100_000; class PingMessage { @IsIn(['ping']) @@ -379,9 +451,9 @@ class RunProgressMessage { /** * The daemon's first frame on every socket (spec #284, Decision 1): the - * launch ids whose child it still has, possibly none, and the `run.start` - * options it accepts (spec #325). Unknown options pass here and are dropped - * when the frame is routed. + * launch ids whose child it still has, possibly none, the `run.start` + * options it accepts (spec #325) and its features (spec #347). Unknown + * options and features pass here and are dropped when the frame is routed. */ class LaunchesSyncMessage { @IsIn(['launches.sync']) @@ -399,6 +471,264 @@ class LaunchesSyncMessage { @IsString({ each: true }) @MaxLength(RUN_OPTION_MAX_LENGTH, { each: true }) runOptions?: string[]; + + /** Absent from a daemon before #349: it can do none. Unknown ones are dropped. */ + @IsOptional() + @IsArray() + @ArrayMaxSize(MAX_FEATURES) + @IsString({ each: true }) + @MaxLength(FEATURE_MAX_LENGTH, { each: true }) + features?: string[]; + + /** `true` from a daemon that answers `check.start` (spec 360); absent before. */ + @IsOptional() + @IsBoolean() + checks?: boolean; +} + +/** The `demoflow run --check` child is running. `requestId` is the launch id. */ +class CheckStartedMessage { + @IsIn(['check.started']) + type!: 'check.started'; + + @IsUUID('4') + requestId!: string; +} + +/** The pass ended; both counts are 0 when it was cancelled, timed out or crashed. */ +class CheckFinishedMessage { + @IsIn(['check.finished']) + type!: 'check.finished'; + + @IsUUID('4') + requestId!: string; + + @IsInt() + @Min(0) + @Max(MAX_CHECK_COUNT) + passed!: number; + + @IsInt() + @Min(0) + @Max(MAX_CHECK_COUNT) + failed!: number; +} + +/** Sent instead of `check.started`; any code string, as for `run.error`. */ +class CheckErrorMessage { + @IsIn(['check.error']) + type!: 'check.error'; + + @IsUUID('4') + requestId!: string; + + @IsString() + @MaxLength(64) + code!: string; +} + +/** A project config's `schedule` block (spec 361, Decision 1). */ +export class SyncedSchedule { + @IsIn(CADENCES) + cadence!: Cadence; + + @ValidateIf((schedule: SyncedSchedule) => schedule.cadence === 'weekly') + @IsIn(WEEKDAYS) + weekday?: string; + + @Matches(TIME_OF_DAY) + at!: string; + + @IsOptional() + @IsArray() + @ArrayMaxSize(MAX_SCHEDULE_EXCLUDES) + @IsScenarioId({ each: true }) + exclude?: string[]; +} + +class SyncedProject { + @IsString() + @Length(1, PATH_MAX) + rootPath!: string; + + @ValidateIf((_, value) => value !== null) + @ValidateNested() + @Type(() => SyncedSchedule) + schedule!: SyncedSchedule | null; +} + +/** + * The schedules of a machine's project configs (spec 361, Decision 6), with + * the machine's time zone: sent after `launches.sync` and on every change. + */ +class SchedulesSyncMessage { + @IsIn(['schedules.sync']) + type!: 'schedules.sync'; + + @IsTimeZone() + zone!: string; + + @IsArray() + @ArrayMaxSize(MAX_SYNCED_SCHEDULES) + @ValidateNested({ each: true }) + @Type(() => SyncedProject) + projects!: SyncedProject[]; +} + +class ShareChapter { + @IsString() + @Length(1, SHARE_CHAPTER_TITLE_MAX) + title!: string; + + @IsInt() + @Min(0) + @Max(Number.MAX_SAFE_INTEGER) + atMs!: number; +} + +/** The daemon has the render and its poster; the bytes follow over `video.open`. */ +class SharePreparedMessage { + @IsIn(['share.prepared']) + type!: 'share.prepared'; + + @IsUUID('4') + requestId!: string; + + /** The scenario's description, read from its file at share time (ADR-0019 §2). */ + @IsOptional() + @IsString() + @MaxLength(SHARE_DESCRIPTION_MAX) + description?: string; + + @IsArray() + @ArrayMaxSize(SHARE_CHAPTERS_MAX) + @ValidateNested({ each: true }) + @Type(() => ShareChapter) + chapters!: ShareChapter[]; + + @IsInt() + @Min(1) + @Max(SHARE_POSTER_MAX_BYTES) + posterSize!: number; +} + +/** + * `not-found`, `no-render` or `io`. Any string, so a code a newer daemon adds + * still ends the request instead of being dropped. + */ +class ShareErrorMessage { + @IsIn(['share.error']) + type!: 'share.error'; + + @IsUUID('4') + requestId!: string; + + @IsString() + @MaxLength(64) + code!: string; +} + +/** + * What a Runner says about a folder (spec #377, Decision 5): never a file's + * contents nor a config value. `githubUrl` is any string here; the API keeps + * it only when it is a plain `https://github.com//`. + */ +export class FolderInspection { + /** Canonical: the folder's real path. */ + @IsString() + @Length(1, PATH_MAX) + rootPath!: string; + + /** The folder's base name. */ + @IsString() + @MaxLength(PATH_MAX) + name!: string; + + @IsBoolean() + hasConfig!: boolean; + + @IsBoolean() + configValid!: boolean; + + @IsArray() + @ArrayMaxSize(MAX_LISTED_SCENARIOS) + @ValidateNested({ each: true }) + @Type(() => ListedScenario) + scenarios!: ListedScenario[]; + + @IsOptional() + @IsString() + @MaxLength(PATH_MAX) + githubUrl?: string; +} + +class ProjectInspectedMessage { + @IsIn(['project.inspected']) + type!: 'project.inspected'; + + @IsUUID('4') + requestId!: string; + + @IsObject() + @ValidateNested() + @Type(() => FolderInspection) + inspection!: FolderInspection; +} + +/** `demoflow init`'s files: those written now, and those already there. */ +class ProjectInitialisedMessage { + @IsIn(['project.initialised']) + type!: 'project.initialised'; + + @IsUUID('4') + requestId!: string; + + @IsArray() + @ArrayMaxSize(MAX_INIT_FILES) + @IsString({ each: true }) + @Length(1, INIT_FILE_MAX, { each: true }) + created!: string[]; + + @IsArray() + @ArrayMaxSize(MAX_INIT_FILES) + @IsString({ each: true }) + @Length(1, INIT_FILE_MAX, { each: true }) + kept!: string[]; + + @IsObject() + @ValidateNested() + @Type(() => FolderInspection) + inspection!: FolderInspection; +} + +/** The folder is in the machine's registry; `rootPath` is its canonical path. */ +class ProjectRegisteredMessage { + @IsIn(['project.registered']) + type!: 'project.registered'; + + @IsUUID('4') + requestId!: string; + + @IsString() + @Length(1, PATH_MAX) + rootPath!: string; +} + +/** + * `path-not-allowed`, `config-missing`, `config-invalid`, `busy` or + * `internal`. Any string, so a code a newer daemon adds still ends the + * request instead of being dropped; the API maps it to `runner-error`. + */ +class ProjectErrorMessage { + @IsIn(['project.error']) + type!: 'project.error'; + + @IsUUID('4') + requestId!: string; + + @IsString() + @MaxLength(64) + code!: string; } const CLIENT_MESSAGES: Record< @@ -421,6 +751,16 @@ const CLIENT_MESSAGES: Record< 'run.finished': RunFinishedMessage, 'run.progress': RunProgressMessage, 'launches.sync': LaunchesSyncMessage, + 'check.started': CheckStartedMessage, + 'check.finished': CheckFinishedMessage, + 'check.error': CheckErrorMessage, + 'schedules.sync': SchedulesSyncMessage, + 'share.prepared': SharePreparedMessage, + 'share.error': ShareErrorMessage, + 'project.inspected': ProjectInspectedMessage, + 'project.initialised': ProjectInitialisedMessage, + 'project.registered': ProjectRegisteredMessage, + 'project.error': ProjectErrorMessage, }; /** A binary frame: part of a stream's file, in order. */ diff --git a/apps/api/src/runner/runner.module.ts b/apps/api/src/runner/runner.module.ts index 4ab1500c..a12239a0 100644 --- a/apps/api/src/runner/runner.module.ts +++ b/apps/api/src/runner/runner.module.ts @@ -1,5 +1,6 @@ import { Module } from '@nestjs/common'; import { AuthModule } from '../auth/auth.module'; +import { ShareStorageModule } from '../shares/share-storage'; import { AccountController } from '../users/account.controller'; import { AccountService } from '../users/account.service'; import { DeviceCodeService } from './device-code.service'; @@ -18,7 +19,7 @@ import { RunnersService } from './runners.service'; * would close an import cycle with `AuthModule`. */ @Module({ - imports: [AuthModule], + imports: [AuthModule, ShareStorageModule], controllers: [RunnerController, RunnersController, AccountController], providers: [ AccountService, diff --git a/apps/api/src/runner/runners.integration.spec.ts b/apps/api/src/runner/runners.integration.spec.ts index aa394a8f..34d2b0e8 100644 --- a/apps/api/src/runner/runners.integration.spec.ts +++ b/apps/api/src/runner/runners.integration.spec.ts @@ -59,6 +59,7 @@ describe('runners (integration)', () => { createdAt: expect.any(String), lastUsedAt: null, online: false, + features: [], lastSeenAt: null, lastSeenIp: null, }, diff --git a/apps/api/src/runner/runners.service.ts b/apps/api/src/runner/runners.service.ts index 751fc1f4..96102bcf 100644 --- a/apps/api/src/runner/runners.service.ts +++ b/apps/api/src/runner/runners.service.ts @@ -3,7 +3,7 @@ import type { Runner } from '@prisma/client'; import { Clock } from '../common/clock'; import { PrismaService } from '../database/prisma.service'; import { RunnerConnections } from './runner-connections.service'; -import { CLOSE_REVOKED } from './runner-socket'; +import { CLOSE_REVOKED, type Feature } from './runner-socket'; /** A machine as the API returns it: never its token hash. */ export interface RunnerView { @@ -15,11 +15,17 @@ export interface RunnerView { lastUsedAt: Date | null; /** Has an open socket on this API right now (spec #151). */ online: boolean; + /** What its current socket can do (spec #378: `projects`); none offline. */ + features: Feature[]; lastSeenAt: Date | null; lastSeenIp: string | null; } -export function toRunnerView(runner: Runner, online: boolean): RunnerView { +export function toRunnerView( + runner: Runner, + online: boolean, + features: readonly Feature[], +): RunnerView { return { id: runner.id, name: runner.name, @@ -28,6 +34,7 @@ export function toRunnerView(runner: Runner, online: boolean): RunnerView { createdAt: runner.createdAt, lastUsedAt: runner.lastUsedAt, online, + features: [...features], lastSeenAt: runner.lastSeenAt, lastSeenIp: runner.lastSeenIp, }; @@ -95,6 +102,10 @@ export class RunnersService { } private view(runner: Runner): RunnerView { - return toRunnerView(runner, this.connections.isOnline(runner.id)); + return toRunnerView( + runner, + this.connections.isOnline(runner.id), + this.connections.features(runner.id), + ); } } diff --git a/apps/api/src/shares/public-shares.controller.ts b/apps/api/src/shares/public-shares.controller.ts new file mode 100644 index 00000000..1cb2b097 --- /dev/null +++ b/apps/api/src/shares/public-shares.controller.ts @@ -0,0 +1,68 @@ +import { + Body, + Controller, + Get, + HttpCode, + HttpStatus, + Param, + Post, + Query, + Req, + Res, + UseGuards, +} from '@nestjs/common'; +import { SkipThrottle, ThrottlerGuard } from '@nestjs/throttler'; +import type { Request, Response } from 'express'; +import { + SKIP_ALL_THROTTLERS, + THROTTLE_SHARE_UNLOCK, +} from '../auth/auth-throttle'; +import { ShareFileQueryDto, UnlockShareDto } from './public-shares.dto'; +import { PublicSharesService, type SharePage } from './public-shares.service'; + +/** + * A share link, opened without an account (spec #348). No session guard: the + * token is the only key. Express answers HEAD through the GET routes. + */ +@Controller('s') +@SkipThrottle(SKIP_ALL_THROTTLERS) +export class PublicSharesController { + constructor(private readonly shares: PublicSharesService) {} + + @Get(':token') + page(@Param('token') token: string, @Req() req: Request): Promise { + return this.shares.page(token, req.cookies); + } + + @Get(':token/file') + file( + @Param('token') token: string, + @Query() { download }: ShareFileQueryDto, + @Req() req: Request, + @Res() res: Response, + ): Promise { + return this.shares.serve(req, res, token, 'file', download === '1'); + } + + @Get(':token/poster') + poster( + @Param('token') token: string, + @Req() req: Request, + @Res() res: Response, + ): Promise { + return this.shares.serve(req, res, token, 'poster', false); + } + + /** 10 attempts per 10 minutes per address and token (Decision 6). */ + @Post(':token/unlock') + @HttpCode(HttpStatus.NO_CONTENT) + @UseGuards(ThrottlerGuard) + @SkipThrottle({ [THROTTLE_SHARE_UNLOCK]: false }) + unlock( + @Param('token') token: string, + @Body() { password }: UnlockShareDto, + @Res({ passthrough: true }) res: Response, + ): Promise { + return this.shares.unlock(token, password, res); + } +} diff --git a/apps/api/src/shares/public-shares.dto.ts b/apps/api/src/shares/public-shares.dto.ts new file mode 100644 index 00000000..bb8233c8 --- /dev/null +++ b/apps/api/src/shares/public-shares.dto.ts @@ -0,0 +1,17 @@ +import { IsIn, IsOptional, IsString, Length } from 'class-validator'; +import { SHARE_PASSWORD_MAX, SHARE_PASSWORD_MIN } from './shares.dto'; + +/** `POST /s/:token/unlock` (spec #348, Decision 5). */ +export class UnlockShareDto { + /** Never trimmed: a password is what the person typed. */ + @IsString() + @Length(SHARE_PASSWORD_MIN, SHARE_PASSWORD_MAX) + password!: string; +} + +/** `?download=1` saves the file instead of playing it (Decision 4). */ +export class ShareFileQueryDto { + @IsOptional() + @IsIn(['1']) + download?: '1'; +} diff --git a/apps/api/src/shares/public-shares.integration.spec.ts b/apps/api/src/shares/public-shares.integration.spec.ts new file mode 100644 index 00000000..8a60f6b4 --- /dev/null +++ b/apps/api/src/shares/public-shares.integration.spec.ts @@ -0,0 +1,533 @@ +import { randomBytes } from 'node:crypto'; +import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Logger } from '@nestjs/common'; +import type { ShareStatus } from '@prisma/client'; +import request from 'supertest'; +import type { App } from 'supertest/types'; +import { PasswordHasher } from '../auth/password-hasher'; +import { generateToken } from '../auth/tokens'; +import type { RunFormat } from '../projects/projects.dto'; +import { type SignedIn, signedInUser } from '../runner/runner.fixtures'; +import { createTestApp, type TestApp } from '../test/test-app'; +import { SHARE_ACCESS_MS } from './share-access'; +import { ShareConfig } from './share-config'; +import { extensionOf } from './share-storage'; + +const FILE = randomBytes(5000); +const POSTER = randomBytes(300); +const DAY_MS = 24 * 60 * 60 * 1000; +const TITLE = 'Checkout flow 7f3a'; +const DESCRIPTION = 'Buys one item.'; +const PASSWORD = 'hunter22-correct-horse'; +const WEB = 'http://web.test'; + +interface Made { + id: string; + token: string; + runId: string; + projectId: string; +} + +interface MakeOptions { + status?: ShareStatus; + format?: string; + /** From now; negative is already expired. */ + expiresInMs?: number; + password?: string; + /** Write `file.` and `poster.jpg` under the share's directory. */ + files?: boolean; +} + +describe('public share links (integration)', () => { + let t: TestApp; + let http: App; + let dir: string; + let alice: SignedIn; + let hasher: PasswordHasher; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'demoflow-public-shares-')); + const config: ShareConfig = { + dir, + maxFileBytes: 10_000, + quotaBytes: 20_000, + webUrl: WEB, + }; + t = await createTestApp([], [{ provide: ShareConfig, useValue: config }]); + http = t.app.getHttpServer(); + alice = await signedInUser(t, 'alice'); + hasher = t.app.get(PasswordHasher); + }); + + afterEach(async () => { + await t.app.close(); + await rm(dir, { recursive: true, force: true }); + }); + + /** A share row as #347 leaves it, made directly, with its files on disk. */ + async function make(options: MakeOptions = {}): Promise { + const format = options.format ?? 'mp4'; + const project = await t.prisma.project.create({ + data: { userId: alice.user.id, name: 'shop' }, + }); + const run = await t.prisma.run.create({ + data: { + projectId: project.id, + scenarioName: TITLE, + scenarioPath: 'demos/checkout.demo.json', + status: 'passed', + startedAt: t.clock.now(), + durationMs: 12_345, + formats: [format], + cliVersion: '0.7.0', + }, + }); + const token = generateToken(); + const share = await t.prisma.share.create({ + data: { + token, + userId: alice.user.id, + runId: run.id, + format, + status: options.status ?? 'ready', + title: TITLE, + description: DESCRIPTION, + chapters: [ + { title: 'Open the cart', atMs: 0 }, + { title: 'Pay', atMs: 4000 }, + ], + sizeBytes: FILE.length, + posterBytes: POSTER.length, + width: 1280, + height: 720, + durationMs: 12_000, + passwordHash: + options.password === undefined + ? null + : await hasher.hash(options.password), + expiresAt: new Date( + t.clock.now().getTime() + (options.expiresInMs ?? 30 * DAY_MS), + ), + }, + }); + if (options.files ?? true) { + await mkdir(join(dir, share.id), { recursive: true }); + await writeFile( + join(dir, share.id, `file.${extensionOf(format as RunFormat)}`), + FILE, + ); + await writeFile(join(dir, share.id, 'poster.jpg'), POSTER); + } + return { id: share.id, token, runId: run.id, projectId: project.id }; + } + + function page(token: string, cookie?: string): request.Test { + const req = request(http).get(`/s/${token}`); + return cookie === undefined ? req : req.set('Cookie', cookie); + } + + function unlock(token: string, password = PASSWORD): request.Test { + return request(http).post(`/s/${token}/unlock`).send({ password }); + } + + /** The `name=value` of the unlock cookie a right password answers with. */ + async function unlocked(token: string): Promise { + const res = await unlock(token).expect(204); + const header = res.headers['set-cookie'] as unknown as string[]; + expect(header).toHaveLength(1); + return header[0].split(';')[0]; + } + + async function views(id: string): Promise { + const share = await t.prisma.share.findUniqueOrThrow({ where: { id } }); + return share.views; + } + + describe('what a share answers (Decision 2)', () => { + it('an unknown or malformed token is 404', async () => { + await page(generateToken()).expect(404, { error: 'not-found' }); + await page('nope').expect(404, { error: 'not-found' }); + await request(http).get(`/s/${generateToken()}/file`).expect(404); + await request(http).get(`/s/${generateToken()}/poster`).expect(404); + }); + + it.each([ + 'uploading', + 'failed', + ])('a share still %s is 404', async (status) => { + const share = await make({ status }); + await page(share.token).expect(404, { error: 'not-found' }); + await request(http).get(`/s/${share.token}/file`).expect(404); + await request(http).get(`/s/${share.token}/poster`).expect(404); + }); + + it('an expired share is 410 expired before the sweeper ran', async () => { + const share = await make({ expiresInMs: DAY_MS }); + await page(share.token).expect(200); + t.clock.advance(DAY_MS); + await page(share.token).expect(410, { error: 'expired' }); + await request(http) + .get(`/s/${share.token}/file`) + .expect(410, { error: 'expired' }); + await request(http).get(`/s/${share.token}/poster`).expect(410); + // The row is still there: only the sweeper deletes it. + expect(await t.prisma.share.count({ where: { id: share.id } })).toBe(1); + }); + + it('a lost share is 410 unavailable', async () => { + const share = await make({ status: 'lost' }); + await page(share.token).expect(410, { error: 'unavailable' }); + await request(http) + .get(`/s/${share.token}/file`) + .expect(410, { error: 'unavailable' }); + await request(http).get(`/s/${share.token}/poster`).expect(410); + }); + + it('a ready share whose file is gone is 410 unavailable and marked lost', async () => { + const page1 = await make({ files: false }); + await page(page1.token).expect(410, { error: 'unavailable' }); + expect( + (await t.prisma.share.findUniqueOrThrow({ where: { id: page1.id } })) + .status, + ).toBe('lost'); + + const file = await make({ files: false }); + await request(http) + .get(`/s/${file.token}/file`) + .expect(410, { error: 'unavailable' }); + expect( + (await t.prisma.share.findUniqueOrThrow({ where: { id: file.id } })) + .status, + ).toBe('lost'); + }); + + it('a share with a password and no cookie is locked, and nothing else', async () => { + const share = await make({ password: PASSWORD }); + const res = await page(share.token).expect(200); + expect(res.body).toEqual({ locked: true }); + expect(Object.keys(res.body)).toEqual(['locked']); + await request(http) + .get(`/s/${share.token}/file`) + .expect(403, { error: 'locked' }); + await request(http) + .get(`/s/${share.token}/poster`) + .expect(403, { error: 'locked' }); + }); + + it('a ready share answers its data, file and poster', async () => { + const share = await make(); + const res = await page(share.token).expect(200); + const url = `${WEB}/s/${share.token}`; + expect(res.body).toEqual({ + title: TITLE, + description: DESCRIPTION, + chapters: [ + { title: 'Open the cart', atMs: 0 }, + { title: 'Pay', atMs: 4000 }, + ], + format: 'mp4', + kind: 'video', + width: 1280, + height: 720, + durationMs: 12_000, + sizeBytes: FILE.length, + expiresAt: new Date( + t.clock.now().getTime() + 30 * DAY_MS, + ).toISOString(), + fileUrl: `${url}/file`, + posterUrl: `${url}/poster`, + downloadUrl: `${url}/file?download=1`, + }); + + const file = await request(http) + .get(`/s/${share.token}/file`) + .expect(200); + expect(file.headers['content-type']).toBe('video/mp4'); + expect(file.headers['content-length']).toBe(String(FILE.length)); + expect(file.headers['content-disposition']).toBe('inline'); + expect(Buffer.compare(file.body as Buffer, FILE)).toBe(0); + + const poster = await request(http) + .get(`/s/${share.token}/poster`) + .expect(200); + expect(poster.headers['content-type']).toBe('image/jpeg'); + expect(Buffer.compare(poster.body as Buffer, POSTER)).toBe(0); + }); + }); + + describe('the file (Decision 4)', () => { + it('answers a byte range with 206 and Content-Range', async () => { + const share = await make(); + const res = await request(http) + .get(`/s/${share.token}/file`) + .set('Range', 'bytes=0-99') + .expect(206); + expect(res.headers['content-range']).toBe(`bytes 0-99/${FILE.length}`); + expect(res.headers['content-length']).toBe('100'); + expect(res.headers['accept-ranges']).toBe('bytes'); + expect(Buffer.compare(res.body as Buffer, FILE.subarray(0, 100))).toBe(0); + }); + + it('answers a range past the end with 416', async () => { + const share = await make(); + const res = await request(http) + .get(`/s/${share.token}/file`) + .set('Range', `bytes=${FILE.length}-`) + .expect(416); + expect(res.headers['content-range']).toBe(`bytes */${FILE.length}`); + }); + + it('answers HEAD with the headers and no body', async () => { + const share = await make(); + const res = await request(http) + .head(`/s/${share.token}/file`) + .expect(200); + expect(res.headers['content-length']).toBe(String(FILE.length)); + expect(res.headers['content-type']).toBe('video/mp4'); + await request(http).head(`/s/${share.token}/poster`).expect(200); + }); + + it('saves as an attachment with ?download=1', async () => { + const share = await make(); + const res = await request(http) + .get(`/s/${share.token}/file?download=1`) + .expect(200); + expect(res.headers['content-disposition']).toBe( + 'attachment; filename="checkout-flow-7f3a-2026-10-20.mp4"', + ); + }); + + it('serves a GIF share as image/gif, kind image', async () => { + const share = await make({ format: 'gif-vertical' }); + const data = await page(share.token).expect(200); + expect(data.body).toMatchObject({ + format: 'gif-vertical', + kind: 'image', + }); + const res = await request(http).get(`/s/${share.token}/file`).expect(200); + expect(res.headers['content-type']).toBe('image/gif'); + }); + + it('without a password embeds anywhere and caches for 5 minutes', async () => { + const share = await make(); + for (const asset of ['file', 'poster']) { + const res = await request(http) + .get(`/s/${share.token}/${asset}`) + .expect(200); + expect(res.headers['cross-origin-resource-policy']).toBe( + 'cross-origin', + ); + expect(res.headers['cache-control']).toBe('public, max-age=300'); + } + }); + + it('with a password opens only with the unlock cookie, never cached', async () => { + const share = await make({ password: PASSWORD }); + await request(http).get(`/s/${share.token}/file`).expect(403); + const cookie = await unlocked(share.token); + const res = await request(http) + .get(`/s/${share.token}/file`) + .set('Cookie', cookie) + .expect(200); + expect(Buffer.compare(res.body as Buffer, FILE)).toBe(0); + expect(res.headers['cache-control']).toBe('private, no-store'); + expect(res.headers['cross-origin-resource-policy']).toBe('same-origin'); + await request(http) + .get(`/s/${share.token}/poster`) + .set('Cookie', cookie) + .expect(200); + const data = await page(share.token, cookie).expect(200); + expect(data.body).toMatchObject({ title: TITLE }); + }); + }); + + describe('unlock (Decisions 5–6)', () => { + it('sets an HttpOnly, SameSite=Lax cookie for 12 hours', async () => { + const share = await make({ password: PASSWORD }); + const res = await unlock(share.token).expect(204); + const [cookie] = res.headers['set-cookie'] as unknown as string[]; + expect(cookie).toMatch(new RegExp(`^df_share_${share.id}=\\d+\\.`)); + expect(cookie).toContain('HttpOnly'); + expect(cookie).toContain('SameSite=Lax'); + expect(cookie).toContain('Path=/'); + expect(cookie).toContain(`Max-Age=${SHARE_ACCESS_MS / 1000}`); + }); + + it('the cookie ends after 12 hours', async () => { + const share = await make({ password: PASSWORD }); + const cookie = await unlocked(share.token); + t.clock.advance(SHARE_ACCESS_MS); + const res = await page(share.token, cookie).expect(200); + expect(res.body).toEqual({ locked: true }); + }); + + it('a wrong password is 401', async () => { + const share = await make({ password: PASSWORD }); + const res = await unlock(share.token, 'wrong-password-1').expect(401, { + error: 'wrong-password', + }); + expect(res.headers['set-cookie']).toBeUndefined(); + }); + + it('a share without a password has nothing to unlock', async () => { + const share = await make(); + const res = await unlock(share.token).expect(204); + expect(res.headers['set-cookie']).toBeUndefined(); + }); + + it('a password outside 8–128 characters is 400', async () => { + const share = await make({ password: PASSWORD }); + await unlock(share.token, 'short').expect(400); + await unlock(share.token, 'x'.repeat(129)).expect(400); + }); + + it('unlocks follow expiry and the 404 rows too', async () => { + await unlock(generateToken()).expect(404); + const expired = await make({ password: PASSWORD, expiresInMs: -1 }); + await unlock(expired.token).expect(410, { error: 'expired' }); + }); + + it('the 11th attempt in 10 minutes from one address for one token is 429', async () => { + const share = await make({ password: PASSWORD }); + const other = await make({ password: PASSWORD }); + for (let attempt = 0; attempt < 10; attempt += 1) { + await unlock(share.token, 'wrong-password-1').expect(401); + } + await unlock(share.token, 'wrong-password-1').expect(429); + // Right or wrong, the budget is spent for this token only. + await unlock(share.token).expect(429); + await unlock(other.token).expect(204); + }); + + it('a cookie signed for share A does not open share B', async () => { + const a = await make({ password: PASSWORD }); + const b = await make({ password: PASSWORD }); + const cookie = await unlocked(a.token); + const value = cookie.split('=')[1]; + for (const sent of [cookie, `df_share_${b.id}=${value}`]) { + const res = await page(b.token, sent).expect(200); + expect(res.body).toEqual({ locked: true }); + await request(http) + .get(`/s/${b.token}/file`) + .set('Cookie', sent) + .expect(403); + } + }); + + it('a cross-origin unlock is refused', async () => { + const share = await make({ password: PASSWORD }); + await unlock(share.token) + .set('Origin', 'https://evil.example') + .expect(403); + }); + }); + + describe('views (Decision 7)', () => { + it('grow by one per unlocked page load, not on file, poster or locked', async () => { + const open = await make(); + await page(open.token).expect(200); + await page(open.token).expect(200); + await request(http).get(`/s/${open.token}/file`).expect(200); + await request(http).get(`/s/${open.token}/poster`).expect(200); + await request(http).head(`/s/${open.token}/file`).expect(200); + expect(await views(open.id)).toBe(2); + + const locked = await make({ password: PASSWORD }); + await page(locked.token).expect(200); + expect(await views(locked.id)).toBe(0); + const cookie = await unlocked(locked.token); + await page(locked.token, cookie).expect(200); + await request(http) + .get(`/s/${locked.token}/file`) + .set('Cookie', cookie) + .expect(200); + expect(await views(locked.id)).toBe(1); + }); + }); + + describe('authorization', () => { + it('never answers by id, and never names a user, run, project or share', async () => { + const share = await make(); + for (const key of [ + share.id, + share.runId, + share.projectId, + alice.user.id, + ]) { + await page(key).expect(404); + await request(http).get(`/s/${key}/file`).expect(404); + } + const res = await page(share.token).expect(200); + const body = JSON.stringify(res.body); + for (const key of [ + share.id, + share.runId, + share.projectId, + alice.user.id, + ]) { + expect(body).not.toContain(key); + } + for (const key of [ + 'id', + 'userId', + 'runId', + 'projectId', + 'token', + 'views', + ]) { + expect(res.body).not.toHaveProperty(key); + } + }); + + it("one share's token never reaches another's file", async () => { + const a = await make(); + const b = await make(); + await writeFile(join(dir, b.id, 'file.mp4'), randomBytes(5000)); + const res = await request(http).get(`/s/${a.token}/file`).expect(200); + expect(Buffer.compare(res.body as Buffer, FILE)).toBe(0); + }); + }); + + describe('logging', () => { + it('never logs an address or a token', async () => { + const lines: string[] = []; + const spies = (['log', 'warn', 'error', 'debug', 'verbose'] as const).map( + (level) => + jest + .spyOn(Logger.prototype, level) + .mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(' ')); + }), + ); + try { + const open = await make(); + const locked = await make({ password: PASSWORD }); + const gone = await make({ files: false }); + await page(open.token).expect(200); + await request(http).get(`/s/${open.token}/file`).expect(200); + await page(locked.token).expect(200); + await unlock(locked.token, 'wrong-password-1').expect(401); + await unlocked(locked.token); + await page(gone.token).expect(410); + + const all = lines.join('\n'); + expect(all).toContain('share.lost'); + for (const secret of [ + open.token, + locked.token, + gone.token, + '127.0.0.1', + PASSWORD, + ]) { + expect(all).not.toContain(secret); + } + } finally { + for (const spy of spies) { + spy.mockRestore(); + } + } + }); + }); +}); diff --git a/apps/api/src/shares/public-shares.service.ts b/apps/api/src/shares/public-shares.service.ts new file mode 100644 index 00000000..ca288fd6 --- /dev/null +++ b/apps/api/src/shares/public-shares.service.ts @@ -0,0 +1,325 @@ +import { type FileHandle, open } from 'node:fs/promises'; +import { pipeline } from 'node:stream/promises'; +import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'; +import type { Share } from '@prisma/client'; +import type { Request, Response } from 'express'; +import { PasswordHasher } from '../auth/password-hasher'; +import { Clock } from '../common/clock'; +import { PrismaService } from '../database/prisma.service'; +import { RUN_FORMATS, type RunFormat } from '../projects/projects.dto'; +import { parseRange, spanOf } from '../videos/byte-range'; +import { type DownloadNameRun, downloadName } from '../videos/download-name'; +import { VIDEO_CONTENT_TYPES } from '../videos/video-relay.service'; +import { ShareAccess } from './share-access'; +import { ShareConfig } from './share-config'; +import { ShareStorage } from './share-storage'; + +/** + * `generateToken()`: 32 random bytes, base64url. Caddy routes + * `/s//file` and `/s//poster` with the same pattern (Decision 1). + */ +export const SHARE_TOKEN = /^[A-Za-z0-9_-]{43}$/; + +/** `kind` is `image` for the GIF formats (Decision 3). */ +export type ShareKind = 'video' | 'image'; + +export interface ShareChapter { + title: string; + atMs: number; +} + +/** What the share page shows (Decision 3). Nothing that names an owner, a run or a share id. */ +export interface ShareData { + title: string; + description: string | null; + chapters: ShareChapter[]; + format: RunFormat; + kind: ShareKind; + width: number | null; + height: number | null; + durationMs: number | null; + sizeBytes: number; + expiresAt: Date; + fileUrl: string; + posterUrl: string; + downloadUrl: string; +} + +/** A share with a password, before its cookie: nothing about the video. */ +export interface LockedSharePage { + locked: true; +} + +export type SharePage = ShareData | LockedSharePage; + +export type ShareAsset = 'file' | 'poster'; + +type PublicShare = Share & { run: DownloadNameRun }; + +/** `{ error }` with a status, as the owner's share API answers. */ +function publicError(status: HttpStatus, error: string): HttpException { + return new HttpException({ error }, status); +} + +function notFound(): HttpException { + return publicError(HttpStatus.NOT_FOUND, 'not-found'); +} + +function unavailable(): HttpException { + return publicError(HttpStatus.GONE, 'unavailable'); +} + +/** Not `instanceof Error`: Node's fs errors can come from another realm. */ +function isMissing(error: unknown): boolean { + return ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === 'ENOENT' + ); +} + +function parseChapters(value: unknown): ShareChapter[] { + if (!Array.isArray(value)) { + return []; + } + return value.flatMap((entry: unknown) => { + if (typeof entry !== 'object' || entry === null) { + return []; + } + const { title, atMs } = entry as Record; + return typeof title === 'string' && typeof atMs === 'number' + ? [{ title, atMs }] + : []; + }); +} + +/** + * What a viewer without an account reaches (spec #348, ADR-0019 §5): the + * share page's data, its file and poster, and the password unlock. Only the + * token opens a share; expiry is checked on every request, before the + * sweeper cleans up. Nothing about the viewer is stored or logged. + */ +@Injectable() +export class PublicSharesService { + private readonly logger = new Logger(PublicSharesService.name); + + constructor( + private readonly prisma: PrismaService, + private readonly storage: ShareStorage, + private readonly config: ShareConfig, + private readonly access: ShareAccess, + private readonly hasher: PasswordHasher, + private readonly clock: Clock, + ) {} + + /** `GET /s/:token`. Counts a view only when it returns the data (Decision 7). */ + async page( + token: string, + cookies: Record | undefined, + ): Promise { + const share = await this.resolve(token); + const format = await this.formatOf(share); + if (!(await this.storage.exists(this.storage.filePath(share.id, format)))) { + await this.markLost(share.id); + throw unavailable(); + } + if (!this.opens(share, cookies)) { + return { locked: true }; + } + await this.prisma.share.updateMany({ + where: { id: share.id }, + data: { views: { increment: 1 } }, + }); + return this.toData(share, token, format); + } + + /** + * `POST /s/:token/unlock`. Right: the cookie. A share without a password + * has nothing to unlock and answers the same, without one. + */ + async unlock(token: string, password: string, res: Response): Promise { + const share = await this.resolve(token); + if (share.passwordHash === null) { + return; + } + if (!(await this.hasher.verify(share.passwordHash, password))) { + throw publicError(HttpStatus.UNAUTHORIZED, 'wrong-password'); + } + this.access.grant(res, { id: share.id, passwordHash: share.passwordHash }); + } + + /** `GET|HEAD /s/:token/file` and `/s/:token/poster`, with byte ranges (Decision 4). */ + async serve( + req: Request, + res: Response, + token: string, + asset: ShareAsset, + download: boolean, + ): Promise { + const share = await this.resolve(token); + if (!this.opens(share, req.cookies)) { + throw publicError(HttpStatus.FORBIDDEN, 'locked'); + } + const format = await this.formatOf(share); + const path = + asset === 'file' + ? this.storage.filePath(share.id, format) + : this.storage.posterPath(share.id); + let handle: FileHandle; + try { + handle = await open(path, 'r'); + } catch (error) { + if (!isMissing(error)) { + throw error; + } + // The poster alone missing leaves the video playable. + if (asset === 'file') { + await this.markLost(share.id); + } + throw unavailable(); + } + + let streaming = false; + try { + const { size } = await handle.stat(); + const range = parseRange(req.headers.range); + const span = spanOf(range, size); + const headers: Record = { + 'Accept-Ranges': 'bytes', + // Without a password the file embeds in another site; with one it + // stays behind the cookie (ADR-0019 §5). + ...(share.passwordHash === null + ? { + 'Cache-Control': 'public, max-age=300', + 'Cross-Origin-Resource-Policy': 'cross-origin', + } + : { 'Cache-Control': 'private, no-store' }), + }; + if (span === null && range !== null) { + res.writeHead(HttpStatus.REQUESTED_RANGE_NOT_SATISFIABLE, { + ...headers, + 'Content-Range': `bytes */${size}`, + }); + res.end(); + return; + } + const partial = range !== null && span !== null; + res.writeHead(partial ? HttpStatus.PARTIAL_CONTENT : HttpStatus.OK, { + ...headers, + 'Content-Type': + asset === 'file' ? VIDEO_CONTENT_TYPES[format] : 'image/jpeg', + 'Content-Length': String(span === null ? 0 : span.end - span.start + 1), + 'Content-Disposition': + asset === 'file' && download + ? `attachment; filename="${downloadName(share.run, format)}"` + : 'inline', + ...(partial + ? { 'Content-Range': `bytes ${span.start}-${span.end}/${size}` } + : {}), + }); + if (req.method === 'HEAD' || span === null) { + res.end(); + return; + } + streaming = true; + // A viewer closing the tab mid-file is no error. + await pipeline( + handle.createReadStream({ start: span.start, end: span.end }), + res, + ).catch(() => undefined); + } finally { + if (!streaming) { + await handle.close(); + } + } + } + + /** + * The Decision 2 table, up to the file: unknown, uploading and failed are + * 404; expired and lost are 410. + */ + private async resolve(token: string): Promise { + if (!SHARE_TOKEN.test(token)) { + throw notFound(); + } + const share = await this.prisma.share.findUnique({ + where: { token }, + include: { + run: { + select: { scenarioName: true, scenarioPath: true, startedAt: true }, + }, + }, + }); + if (!share || (share.status !== 'ready' && share.status !== 'lost')) { + throw notFound(); + } + if (share.expiresAt.getTime() <= this.clock.now().getTime()) { + throw publicError(HttpStatus.GONE, 'expired'); + } + if (share.status === 'lost') { + throw unavailable(); + } + return share; + } + + private opens( + share: PublicShare, + cookies: Record | undefined, + ): boolean { + return ( + share.passwordHash === null || + this.access.allows(cookies, { + id: share.id, + passwordHash: share.passwordHash, + }) + ); + } + + /** A format the API no longer knows has no file it can name: lost. */ + private async formatOf(share: PublicShare): Promise { + const format = RUN_FORMATS.find((known) => known === share.format); + if (format === undefined) { + await this.markLost(share.id); + throw unavailable(); + } + return format; + } + + /** Not backed up (ADR-0019 §4): the owner shares the run again. */ + private async markLost(id: string): Promise { + const { count } = await this.prisma.share.updateMany({ + where: { id, status: 'ready' }, + data: { status: 'lost' }, + }); + if (count === 1) { + // As the owner's API logs (spec #347, Decision 10): never the token. + this.logger.log( + JSON.stringify({ event: 'share.lost', shareId: id, outcome: 'lost' }), + ); + } + } + + private toData( + share: PublicShare, + token: string, + format: RunFormat, + ): ShareData { + const fileUrl = `${this.config.webUrl}/s/${token}/file`; + return { + title: share.title, + description: share.description, + chapters: parseChapters(share.chapters), + format, + kind: format.startsWith('gif') ? 'image' : 'video', + width: share.width, + height: share.height, + durationMs: share.durationMs, + sizeBytes: share.sizeBytes, + expiresAt: share.expiresAt, + fileUrl, + posterUrl: `${this.config.webUrl}/s/${token}/poster`, + downloadUrl: `${fileUrl}?download=1`, + }; + } +} diff --git a/apps/api/src/shares/runner-shares.controller.ts b/apps/api/src/shares/runner-shares.controller.ts new file mode 100644 index 00000000..6c2acab2 --- /dev/null +++ b/apps/api/src/shares/runner-shares.controller.ts @@ -0,0 +1,42 @@ +import { + Body, + Controller, + Get, + HttpCode, + HttpStatus, + Param, + Post, + UseGuards, +} from '@nestjs/common'; +import type { RunnerContext } from '../runner/runner-token'; +import { CurrentRunner, RunnerTokenGuard } from '../runner/runner-token.guard'; +import { CreateShareDto, ShareIdParamsDto } from './shares.dto'; +import { SharesService, type ShareView } from './shares.service'; + +/** + * `demoflow share` and the MCP tool (spec #347, Decision 7): only a run this + * machine reported. Whether an MCP client may share is checked on the + * machine, where the project's config is. + */ +@Controller('runner/shares') +@UseGuards(RunnerTokenGuard) +export class RunnerSharesController { + constructor(private readonly shares: SharesService) {} + + @Post() + @HttpCode(HttpStatus.ACCEPTED) + create( + @CurrentRunner() caller: RunnerContext, + @Body() body: CreateShareDto, + ): Promise { + return this.shares.create(caller.user.id, body, caller.runner.id); + } + + @Get(':id') + get( + @CurrentRunner() caller: RunnerContext, + @Param() { id }: ShareIdParamsDto, + ): Promise { + return this.shares.get(caller.user.id, id, caller.runner.id); + } +} diff --git a/apps/api/src/shares/share-access.spec.ts b/apps/api/src/shares/share-access.spec.ts new file mode 100644 index 00000000..48b0c5c9 --- /dev/null +++ b/apps/api/src/shares/share-access.spec.ts @@ -0,0 +1,108 @@ +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { generateToken } from '../auth/tokens'; +import { SHARE_TOKEN } from './public-shares.service'; +import { + accessAllows, + accessValue, + resolveSigningSecret, + SHARE_ACCESS_MS, +} from './share-access'; + +const SECRET = 'a'.repeat(32); +const SHARE = { id: 'cshare1', passwordHash: '$argon2id$v=19$one' }; +const NOW = Date.parse('2026-10-20T12:00:00.000Z'); + +describe('resolveSigningSecret', () => { + it('uses the secret when it is long enough', () => { + expect( + resolveSigningSecret({ + APP_ENV: 'production', + SHARE_SIGNING_SECRET: SECRET, + }), + ).toBe(SECRET); + }); + + it('refuses a short secret', () => { + expect(() => + resolveSigningSecret({ SHARE_SIGNING_SECRET: 'short' }), + ).toThrow('SHARE_SIGNING_SECRET must be at least 32 characters'); + }); + + it('refuses to start in production without one, naming the variable', () => { + expect(() => resolveSigningSecret({ APP_ENV: 'production' })).toThrow( + 'SHARE_SIGNING_SECRET is not set', + ); + }); + + it('makes a random one per boot below production', () => { + const first = resolveSigningSecret({ APP_ENV: 'development' }); + const second = resolveSigningSecret({ APP_ENV: 'development' }); + expect(first.length).toBeGreaterThanOrEqual(32); + expect(first).not.toBe(second); + }); +}); + +describe('the unlock cookie', () => { + const value = accessValue(SECRET, SHARE, NOW + SHARE_ACCESS_MS); + + it('is . and opens its share until it expires', () => { + expect(value).toMatch(/^\d+\.[A-Za-z0-9_-]{43}$/); + expect(accessAllows(SECRET, SHARE, value, NOW)).toBe(true); + expect(accessAllows(SECRET, SHARE, value, NOW + SHARE_ACCESS_MS)).toBe( + false, + ); + }); + + it('does not open another share', () => { + expect(accessAllows(SECRET, { ...SHARE, id: 'cshare2' }, value, NOW)).toBe( + false, + ); + }); + + it('is void once the password changes', () => { + expect( + accessAllows( + SECRET, + { ...SHARE, passwordHash: '$argon2id$v=19$two' }, + value, + NOW, + ), + ).toBe(false); + }); + + it('is void under another secret', () => { + expect(accessAllows('b'.repeat(32), SHARE, value, NOW)).toBe(false); + }); + + it('refuses a later expiry with the old signature, and garbage', () => { + const [, hmac] = value.split('.'); + const later = `${Math.floor(NOW / 1000) + 10 * 24 * 3600}.${hmac}`; + expect(accessAllows(SECRET, SHARE, later, NOW)).toBe(false); + expect(accessAllows(SECRET, SHARE, undefined, NOW)).toBe(false); + expect(accessAllows(SECRET, SHARE, 'nope', NOW)).toBe(false); + }); +}); + +describe('the share token and the Caddy route', () => { + const route = /path_regexp\s+(?:\S+\s+)?(\^\/s\/\S+\$)/; + + it.each([ + 'staging', + 'production', + ])('the %s Caddyfile routes /s//file and /poster for a real token', (env) => { + const caddyfile = readFileSync( + join(__dirname, '../../../../deploy', env, 'Caddyfile'), + 'utf8', + ); + const pattern = route.exec(caddyfile)?.[1]; + expect(pattern).toBeDefined(); + const regexp = new RegExp(pattern ?? '$^'); + const token = generateToken(); + expect(token).toMatch(SHARE_TOKEN); + expect(regexp.test(`/s/${token}/file`)).toBe(true); + expect(regexp.test(`/s/${token}/poster`)).toBe(true); + expect(regexp.test(`/s/${token}`)).toBe(false); + expect(regexp.test(`/s/${token}/unlock`)).toBe(false); + }); +}); diff --git a/apps/api/src/shares/share-access.ts b/apps/api/src/shares/share-access.ts new file mode 100644 index 00000000..fcf25d14 --- /dev/null +++ b/apps/api/src/shares/share-access.ts @@ -0,0 +1,150 @@ +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; +import { Injectable, Logger } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; +import type { CookieOptions, Response } from 'express'; +import { Clock } from '../common/clock'; + +/** How long an unlock lasts (spec #348, Decision 5). */ +export const SHARE_ACCESS_MS = 12 * 60 * 60 * 1000; +export const SHARE_SIGNING_SECRET_MIN = 32; + +const COOKIE_PREFIX = 'df_share_'; +/** `.`: seconds since the epoch, then base64url SHA-256. */ +const COOKIE_VALUE = /^(\d{1,12})\.([A-Za-z0-9_-]{43})$/; + +/** What an unlock is bound to: the share, and its password as it is now. */ +export interface LockedShare { + id: string; + passwordHash: string; +} + +export interface SigningEnv { + APP_ENV?: string; + SHARE_SIGNING_SECRET?: string; +} + +/** + * `SHARE_SIGNING_SECRET` (Decision 8): at least 32 characters. Production + * refuses to start without it; below production a random value per boot is + * used, so unlocks end at a restart. + */ +export function resolveSigningSecret(env: SigningEnv): string { + const secret = env.SHARE_SIGNING_SECRET?.trim() || undefined; + if (secret !== undefined) { + if (secret.length < SHARE_SIGNING_SECRET_MIN) { + throw new Error( + `SHARE_SIGNING_SECRET must be at least ${SHARE_SIGNING_SECRET_MIN} characters.`, + ); + } + return secret; + } + if (env.APP_ENV === 'production') { + throw new Error( + 'SHARE_SIGNING_SECRET is not set, and APP_ENV=production does not fall ' + + 'back to a random one. Set it to at least 32 random characters.', + ); + } + new Logger('ShareAccess').warn( + 'SHARE_SIGNING_SECRET is not set: share unlocks are signed with a random ' + + 'value and end when the API restarts.', + ); + return randomBytes(32).toString('base64url'); +} + +export function shareCookieName(shareId: string): string { + return `${COOKIE_PREFIX}${shareId}`; +} + +/** + * The hash is signed too, so a later password change voids every unlock made + * with the old one. + */ +function sign(secret: string, share: LockedShare, exp: number): string { + return createHmac('sha256', secret) + .update(`${share.id}.${exp}.${share.passwordHash}`) + .digest('base64url'); +} + +export function accessValue( + secret: string, + share: LockedShare, + expiresAtMs: number, +): string { + const exp = Math.floor(expiresAtMs / 1000); + return `${exp}.${sign(secret, share, exp)}`; +} + +/** True for a value this secret signed for this share that has not expired. */ +export function accessAllows( + secret: string, + share: LockedShare, + value: string | undefined, + nowMs: number, +): boolean { + const match = value === undefined ? null : COOKIE_VALUE.exec(value); + if (!match) { + return false; + } + const exp = Number(match[1]); + if (exp * 1000 <= nowMs) { + return false; + } + const expected = Buffer.from(sign(secret, share, exp)); + const given = Buffer.from(match[2]); + return expected.length === given.length && timingSafeEqual(expected, given); +} + +/** + * The unlock cookie of a share with a password (spec #348, Decision 5): one + * per share, `df_share_`, HMAC-signed and good for 12 hours. + */ +@Injectable() +export class ShareAccess { + private readonly secret: string; + private readonly secure: boolean; + + constructor( + config: ConfigService, + private readonly clock: Clock, + ) { + const appEnv = config.get('APP_ENV'); + this.secret = resolveSigningSecret({ + APP_ENV: appEnv, + SHARE_SIGNING_SECRET: config.get('SHARE_SIGNING_SECRET'), + }); + // As the session cookie: staging and production run APP_ENV=production. + this.secure = appEnv === 'production'; + } + + grant(res: Response, share: LockedShare): void { + const options: CookieOptions = { + httpOnly: true, + secure: this.secure, + sameSite: 'lax', + path: '/', + maxAge: SHARE_ACCESS_MS, + }; + res.cookie( + shareCookieName(share.id), + accessValue( + this.secret, + share, + this.clock.now().getTime() + SHARE_ACCESS_MS, + ), + options, + ); + } + + allows( + cookies: Record | undefined, + share: LockedShare, + ): boolean { + const value = cookies?.[shareCookieName(share.id)]; + return accessAllows( + this.secret, + share, + typeof value === 'string' ? value : undefined, + this.clock.now().getTime(), + ); + } +} diff --git a/apps/api/src/shares/share-config.ts b/apps/api/src/shares/share-config.ts new file mode 100644 index 00000000..1079e33e --- /dev/null +++ b/apps/api/src/shares/share-config.ts @@ -0,0 +1,54 @@ +import { resolve } from 'node:path'; +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +/** 200 MB (spec #347, Decision 3). */ +export const DEFAULT_SHARE_MAX_FILE_BYTES = 209_715_200; +/** 2 GB per user. */ +export const DEFAULT_SHARE_USER_QUOTA_BYTES = 2_147_483_648; + +/** + * Where shared files live and how much of them a user may keep (spec #347, + * Configuration). A limit that is set but not a positive integer stops the + * API at boot rather than falling back silently. + */ +@Injectable() +export class ShareConfig { + /** Absolute: `SHARE_DIR` resolved against the API's working directory. */ + readonly dir: string; + readonly maxFileBytes: number; + readonly quotaBytes: number; + /** The web origin a share's link points at. */ + readonly webUrl: string; + + constructor(config: ConfigService) { + this.dir = resolve( + config.get('SHARE_DIR')?.trim() || './data/shares', + ); + this.maxFileBytes = bytes( + config, + 'SHARE_MAX_FILE_BYTES', + DEFAULT_SHARE_MAX_FILE_BYTES, + ); + this.quotaBytes = bytes( + config, + 'SHARE_USER_QUOTA_BYTES', + DEFAULT_SHARE_USER_QUOTA_BYTES, + ); + this.webUrl = ( + config.get('WEB_URL') ?? 'http://localhost:3670' + ).replace(/\/+$/, ''); + } +} + +function bytes(config: ConfigService, name: string, fallback: number): number { + const raw = config.get(name)?.trim(); + if (!raw) { + return fallback; + } + const value = Number(raw); + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer, got "${raw}"`); + } + return value; +} diff --git a/apps/api/src/shares/share-storage.ts b/apps/api/src/shares/share-storage.ts new file mode 100644 index 00000000..e60a0528 --- /dev/null +++ b/apps/api/src/shares/share-storage.ts @@ -0,0 +1,90 @@ +import { access, mkdir, readdir, rm } from 'node:fs/promises'; +import { join } from 'node:path'; +import { Injectable, Module } from '@nestjs/common'; +import type { RunFormat } from '../projects/projects.dto'; +import { ShareConfig } from './share-config'; + +/** A cuid from the database: never a string from a request (Decision 2). */ +const SHARE_ID = /^[a-z0-9]+$/; + +/** + * The share directory, `SHARE_DIR` (spec #347, Decision 2): one directory + * per share, holding `file.` and `poster.jpg`. Every path is built from + * a share id the database made, checked here once more before it touches + * the disk. + */ +@Injectable() +export class ShareStorage { + constructor(private readonly config: ShareConfig) {} + + dirOf(id: string): string { + if (!SHARE_ID.test(id)) { + throw new Error('Not a share id'); + } + return join(this.config.dir, id); + } + + /** `mp4`, `gif` or `webm`: the format without its orientation. */ + filePath(id: string, format: RunFormat): string { + return join(this.dirOf(id), `file.${extensionOf(format)}`); + } + + posterPath(id: string): string { + return join(this.dirOf(id), 'poster.jpg'); + } + + /** An empty directory for the share's files. */ + async prepare(id: string): Promise { + await mkdir(this.dirOf(id), { recursive: true }); + } + + async exists(path: string): Promise { + try { + await access(path); + return true; + } catch { + return false; + } + } + + /** Deletes the shares' directories; one already gone is no error. */ + async remove(ids: readonly string[]): Promise { + await Promise.all( + ids.map((id) => rm(this.dirOf(id), { recursive: true, force: true })), + ); + } + + /** The share ids that have a directory. Anything else in `SHARE_DIR` is left. */ + async list(): Promise { + try { + const entries = await readdir(this.config.dir, { withFileTypes: true }); + return entries + .filter((entry) => entry.isDirectory() && SHARE_ID.test(entry.name)) + .map((entry) => entry.name); + } catch (error) { + if (isMissing(error)) { + return []; + } + throw error; + } + } +} + +export function extensionOf(format: RunFormat): string { + const [extension] = format.split('-'); + return extension ?? format; +} + +function isMissing(error: unknown): boolean { + return error instanceof Error && 'code' in error && error.code === 'ENOENT'; +} + +/** + * Its own module so the run, project and account deletes can reach the + * directory without importing the shares module and its controllers. + */ +@Module({ + providers: [ShareConfig, ShareStorage], + exports: [ShareConfig, ShareStorage], +}) +export class ShareStorageModule {} diff --git a/apps/api/src/shares/share-sweeper.ts b/apps/api/src/shares/share-sweeper.ts new file mode 100644 index 00000000..f12c2932 --- /dev/null +++ b/apps/api/src/shares/share-sweeper.ts @@ -0,0 +1,149 @@ +import { + Injectable, + Logger, + type OnApplicationBootstrap, + type OnModuleDestroy, +} from '@nestjs/common'; +import { Clock } from '../common/clock'; +import { PrismaService } from '../database/prisma.service'; +import { RUN_FORMATS, type RunFormat } from '../projects/projects.dto'; +import { ShareStorage } from './share-storage'; + +/** Spec #347, Decision 9. */ +export const SHARE_SWEEP_INTERVAL_MS = 10 * 60 * 1000; +/** An upload still running this long after the share was created has failed. */ +export const SHARE_UPLOAD_STALE_MS = 15 * 60 * 1000; + +export interface SweepResult { + expired: number; + timedOut: number; + orphans: number; + lost: number; +} + +/** + * Keeps the share directory and the table in step (spec #347, Decision 9): + * expired shares go with their files, stale uploads fail, directories with + * no row go, and on boot a ready share whose file is gone is marked lost. + * The backstop for every cascade that deletes rows before their files. + */ +@Injectable() +export class ShareSweeper implements OnApplicationBootstrap, OnModuleDestroy { + private readonly logger = new Logger(ShareSweeper.name); + private timer: NodeJS.Timeout | undefined; + private running = false; + + constructor( + private readonly prisma: PrismaService, + private readonly storage: ShareStorage, + private readonly clock: Clock, + ) {} + + onApplicationBootstrap(): void { + void this.run(true); + this.timer = setInterval( + () => void this.run(false), + SHARE_SWEEP_INTERVAL_MS, + ); + this.timer.unref(); + } + + onModuleDestroy(): void { + clearInterval(this.timer); + } + + /** One pass. `boot` also checks that every ready share still has its file. */ + async sweep(boot: boolean): Promise { + const now = this.clock.now(); + const expired = await this.prisma.share.findMany({ + where: { expiresAt: { lte: now } }, + select: { id: true }, + }); + const expiredIds = expired.map(({ id }) => id); + await this.prisma.share.deleteMany({ + where: { id: { in: expiredIds }, expiresAt: { lte: now } }, + }); + await this.storage.remove(expiredIds); + + const stale = await this.prisma.share.findMany({ + where: { + status: 'uploading', + createdAt: { lt: new Date(now.getTime() - SHARE_UPLOAD_STALE_MS) }, + }, + select: { id: true }, + }); + const staleIds = stale.map(({ id }) => id); + await this.prisma.share.updateMany({ + where: { id: { in: staleIds }, status: 'uploading' }, + data: { status: 'failed', failure: 'timed-out' }, + }); + await this.storage.remove(staleIds); + + const orphans = await this.orphans(); + await this.storage.remove(orphans); + + const lost = boot ? await this.markLost() : []; + return { + expired: expiredIds.length, + timedOut: staleIds.length, + orphans: orphans.length, + lost: lost.length, + }; + } + + private async run(boot: boolean): Promise { + // A slow pass is not overlapped by the next one. + if (this.running) { + return; + } + this.running = true; + try { + const result = await this.sweep(boot); + this.logger.log( + JSON.stringify({ event: 'share.sweep', boot, ...result }), + ); + } catch (error) { + this.logger.error(error); + } finally { + this.running = false; + } + } + + /** Directories with no row. The row is created before its directory. */ + private async orphans(): Promise { + const ids = await this.storage.list(); + if (ids.length === 0) { + return []; + } + const rows = await this.prisma.share.findMany({ + where: { id: { in: ids } }, + select: { id: true }, + }); + const known = new Set(rows.map(({ id }) => id)); + return ids.filter((id) => !known.has(id)); + } + + /** Not backed up (ADR-0019 §4): the owner uploads a lost share again. */ + private async markLost(): Promise { + const ready = await this.prisma.share.findMany({ + where: { status: 'ready' }, + select: { id: true, format: true }, + }); + const lost: string[] = []; + for (const { id, format } of ready) { + const known = RUN_FORMATS.find((candidate) => candidate === format); + if (!known || !(await this.hasFile(id, known))) { + lost.push(id); + } + } + await this.prisma.share.updateMany({ + where: { id: { in: lost }, status: 'ready' }, + data: { status: 'lost' }, + }); + return lost; + } + + private hasFile(id: string, format: RunFormat): Promise { + return this.storage.exists(this.storage.filePath(id, format)); + } +} diff --git a/apps/api/src/shares/share-upload.service.ts b/apps/api/src/shares/share-upload.service.ts new file mode 100644 index 00000000..16f7a031 --- /dev/null +++ b/apps/api/src/shares/share-upload.service.ts @@ -0,0 +1,302 @@ +import { randomUUID } from 'node:crypto'; +import { + Injectable, + Logger, + type OnApplicationBootstrap, + type OnModuleDestroy, + type OnModuleInit, +} from '@nestjs/common'; +import { Clock } from '../common/clock'; +import { PrismaService } from '../database/prisma.service'; +import type { RunFormat } from '../projects/projects.dto'; +import type { RunRender } from '../projects/projects.service'; +import { + RunnerConnections, + type ShareSink, +} from '../runner/runner-connections.service'; +import { + MAX_STREAMS_PER_RUNNER, + SHARE_POSTER_MAX_BYTES, + type ShareMessage, +} from '../runner/runner-socket'; +import { + type UploadOutcome, + type UploadVideo, + VideoRelay, +} from '../videos/video-relay.service'; +import { ShareConfig } from './share-config'; +import { ShareStorage } from './share-storage'; + +/** No answer this long after `share.prepare`: the share fails (spec #347, Decision 4). */ +export const PREPARE_TIMEOUT_MS = 30 * 1000; +/** How often the timeout is checked. */ +const SWEEP_INTERVAL_MS = 1000; + +/** What the daemon answered to `share.prepare`. */ +interface Prepared { + description: string | null; + chapters: Array<{ title: string; atMs: number }>; +} + +type PrepareOutcome = + | { ok: true; prepared: Prepared } + | { ok: false; error: string }; + +interface Pending { + id: string; + runnerId: string; + /** Clock ms. */ + sentAt: number; + settle: (outcome: PrepareOutcome) => void; +} + +/** One share's upload: which machine, which file, and what the run reported. */ +export interface ShareUpload { + shareId: string; + userId: string; + runId: string; + format: RunFormat; + runnerId: string; + /** The run's `renders` entry for the format; null when the CLI did not report it. */ + render: RunRender | null; +} + +/** + * Pulls a share's files from the machine that recorded the run (spec #347, + * Decision 4, ADR-0019 §3): `share.prepare`, then the poster and the render + * through `VideoRelay` into the share's directory. The row ends `ready`, or + * `failed` with its files deleted. + * + * Works only on the instance that holds the Runner's socket, like the relay. + */ +@Injectable() +export class ShareUploadService + implements ShareSink, OnModuleInit, OnApplicationBootstrap, OnModuleDestroy +{ + private readonly logger = new Logger(ShareUploadService.name); + private readonly pending = new Map(); + private timer: NodeJS.Timeout | undefined; + + constructor( + private readonly connections: RunnerConnections, + private readonly relay: VideoRelay, + private readonly prisma: PrismaService, + private readonly storage: ShareStorage, + private readonly config: ShareConfig, + private readonly clock: Clock, + ) {} + + onModuleInit(): void { + this.connections.setShareSink(this); + } + + onApplicationBootstrap(): void { + this.timer = setInterval(() => this.tick(), SWEEP_INTERVAL_MS); + this.timer.unref(); + } + + onModuleDestroy(): void { + clearInterval(this.timer); + } + + /** + * Uploads the share's files and records the outcome. Never throws: a share + * that cannot be uploaded ends `failed`. Settles once the row says so. + */ + async upload(target: ShareUpload): Promise { + const startedAt = this.clock.now().getTime(); + let outcome: string; + try { + outcome = await this.pull(target); + } catch (error) { + this.logger.error(error); + outcome = 'error'; + } + if (outcome !== 'ready') { + await this.fail(target.shareId, outcome); + } + // Decision 10: never the token, the password, the title or the description. + this.logger.log( + JSON.stringify({ + event: 'share.upload', + shareId: target.shareId, + runId: target.runId, + outcome, + durationMs: this.clock.now().getTime() - startedAt, + }), + ); + } + + message(runnerId: string, message: ShareMessage): boolean { + const request = this.pending.get(message.requestId); + // Only the machine a request was sent to can answer it. + if (request?.runnerId !== runnerId) { + return false; + } + if (message.type === 'share.prepared') { + this.settle(request, { + ok: true, + prepared: { + description: message.description ?? null, + chapters: message.chapters.map(({ title, atMs }) => ({ + title, + atMs, + })), + }, + }); + } else { + this.settle(request, { ok: false, error: message.code }); + } + return true; + } + + /** Every request on the machine's socket ends with it. */ + closed(runnerId: string): void { + for (const request of [...this.pending.values()]) { + if (request.runnerId === runnerId) { + this.settle(request, { ok: false, error: 'runner-offline' }); + } + } + } + + /** The timeout, on the injected clock. Runs every `SWEEP_INTERVAL_MS`. */ + tick(): void { + const now = this.clock.now().getTime(); + for (const request of [...this.pending.values()]) { + if (now - request.sentAt >= PREPARE_TIMEOUT_MS) { + this.settle(request, { ok: false, error: 'runner-timeout' }); + } + } + } + + /** `ready`, or the code the share fails with. */ + private async pull(target: ShareUpload): Promise { + const { shareId, userId, runnerId, runId, format, render } = target; + await this.storage.prepare(shareId); + const prepare = await this.prepare(runnerId, runId, format); + if (!prepare.ok) { + return prepare.error; + } + + const poster = await this.stream({ + runnerId, + runId, + format, + asset: 'poster', + path: this.storage.posterPath(shareId), + accept: (size) => + size > SHARE_POSTER_MAX_BYTES ? 'poster-too-large' : null, + }); + if (!poster.ok) { + return poster.error; + } + + // Read once, so the head's check stays synchronous. A share created + // meanwhile reserved its bytes under the user's lock and is counted. + const others = await this.usedBytes(userId, shareId); + const file = await this.stream({ + runnerId, + runId, + format, + path: this.storage.filePath(shareId, format), + accept: (size) => { + if (size > this.config.maxFileBytes) { + return 'file-too-large'; + } + if (others + poster.size + size > this.config.quotaBytes) { + return 'quota-exceeded'; + } + return null; + }, + }); + if (!file.ok) { + return file.error; + } + + // Only an upload still running may finish: a revoked or swept share stays gone. + const { count } = await this.prisma.share.updateMany({ + where: { id: shareId, status: 'uploading' }, + data: { + status: 'ready', + failure: null, + sizeBytes: file.size, + posterBytes: poster.size, + description: prepare.prepared.description, + chapters: prepare.prepared.chapters, + width: render?.width ?? null, + height: render?.height ?? null, + durationMs: render?.durationMs ?? null, + }, + }); + if (count !== 1) { + await this.storage.remove([shareId]); + return 'gone'; + } + return 'ready'; + } + + /** As playback: a fifth stream to the machine is `runner-busy`. */ + private stream(upload: UploadVideo): Promise { + if (this.relay.openTo(upload.runnerId) >= MAX_STREAMS_PER_RUNNER) { + return Promise.resolve({ ok: false, error: 'runner-busy' }); + } + return this.relay.upload(upload); + } + + private prepare( + runnerId: string, + runId: string, + format: RunFormat, + ): Promise { + return new Promise((settle) => { + const request: Pending = { + id: randomUUID(), + runnerId, + sentAt: this.clock.now().getTime(), + settle, + }; + this.pending.set(request.id, request); + const sent = this.connections.sendTo(runnerId, { + type: 'share.prepare', + requestId: request.id, + runId, + format, + }); + if (!sent) { + this.settle(request, { ok: false, error: 'runner-offline' }); + } + }); + } + + private settle(request: Pending, outcome: PrepareOutcome): void { + this.pending.delete(request.id); + request.settle(outcome); + } + + /** The user's live shares other than this one: what the quota counts. */ + private async usedBytes(userId: string, exceptId: string): Promise { + const { _sum } = await this.prisma.share.aggregate({ + where: { + userId, + id: { not: exceptId }, + status: { in: ['uploading', 'ready'] }, + }, + _sum: { sizeBytes: true, posterBytes: true }, + }); + return (_sum.sizeBytes ?? 0) + (_sum.posterBytes ?? 0); + } + + /** The files go, and with them the bytes the quota counted. */ + private async fail(shareId: string, failure: string): Promise { + try { + await this.prisma.share.updateMany({ + where: { id: shareId, status: 'uploading' }, + data: { status: 'failed', failure: failure.slice(0, 64) }, + }); + await this.storage.remove([shareId]); + } catch (error) { + // The sweeper fails a stale upload and deletes its directory. + this.logger.error(error); + } + } +} diff --git a/apps/api/src/shares/shares.controller.ts b/apps/api/src/shares/shares.controller.ts new file mode 100644 index 00000000..0c3d7a09 --- /dev/null +++ b/apps/api/src/shares/shares.controller.ts @@ -0,0 +1,68 @@ +import { + Body, + Controller, + Delete, + Get, + HttpCode, + HttpStatus, + Param, + Post, + UseGuards, +} from '@nestjs/common'; +import type { User } from '@prisma/client'; +import { CurrentUser } from '../auth/current-user.decorator'; +import { SessionGuard } from '../auth/session.guard'; +import { CreateShareDto, ShareIdParamsDto } from './shares.dto'; +import { + type SharesList, + SharesService, + type ShareView, +} from './shares.service'; + +/** The owner's share links (spec #347, ADR-0019). */ +@Controller('shares') +@UseGuards(SessionGuard) +export class SharesController { + constructor(private readonly shares: SharesService) {} + + /** 202: the row exists, the upload runs in the background. */ + @Post() + @HttpCode(HttpStatus.ACCEPTED) + create( + @CurrentUser() user: User, + @Body() body: CreateShareDto, + ): Promise { + return this.shares.create(user.id, body); + } + + @Get() + list(@CurrentUser() user: User): Promise { + return this.shares.list(user.id); + } + + @Get(':id') + get( + @CurrentUser() user: User, + @Param() { id }: ShareIdParamsDto, + ): Promise { + return this.shares.get(user.id, id); + } + + @Post(':id/extend') + @HttpCode(HttpStatus.OK) + extend( + @CurrentUser() user: User, + @Param() { id }: ShareIdParamsDto, + ): Promise { + return this.shares.extend(user.id, id); + } + + @Delete(':id') + @HttpCode(HttpStatus.NO_CONTENT) + remove( + @CurrentUser() user: User, + @Param() { id }: ShareIdParamsDto, + ): Promise { + return this.shares.remove(user.id, id); + } +} diff --git a/apps/api/src/shares/shares.dto.ts b/apps/api/src/shares/shares.dto.ts new file mode 100644 index 00000000..270c361b --- /dev/null +++ b/apps/api/src/shares/shares.dto.ts @@ -0,0 +1,36 @@ +import { IsIn, IsOptional, IsString, IsUUID, Length } from 'class-validator'; +import { RUN_FORMATS, type RunFormat } from '../projects/projects.dto'; + +/** A share's lifetime in days (spec #347, Decision 5). */ +export const SHARE_LIFETIMES = [7, 30, 90] as const; +export type ShareLifetime = (typeof SHARE_LIFETIMES)[number]; +export const DEFAULT_SHARE_LIFETIME: ShareLifetime = 30; + +/** Decision 8. */ +export const SHARE_PASSWORD_MIN = 8; +export const SHARE_PASSWORD_MAX = 128; + +/** `POST /shares` and `POST /runner/shares`. */ +export class CreateShareDto { + @IsUUID() + runId!: string; + + @IsIn(RUN_FORMATS) + format!: RunFormat; + + @IsOptional() + @IsIn(SHARE_LIFETIMES) + expiresInDays?: ShareLifetime; + + /** Never trimmed: a password is what the person typed. */ + @IsOptional() + @IsString() + @Length(SHARE_PASSWORD_MIN, SHARE_PASSWORD_MAX) + password?: string; +} + +export class ShareIdParamsDto { + @IsString() + @Length(1, 64) + id!: string; +} diff --git a/apps/api/src/shares/shares.integration.spec.ts b/apps/api/src/shares/shares.integration.spec.ts new file mode 100644 index 00000000..9777f5ee --- /dev/null +++ b/apps/api/src/shares/shares.integration.spec.ts @@ -0,0 +1,764 @@ +import { randomBytes } from 'node:crypto'; +import { + mkdir, + mkdtemp, + readdir, + readFile, + rm, + writeFile, +} from 'node:fs/promises'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Logger } from '@nestjs/common'; +import request from 'supertest'; +import type { App } from 'supertest/types'; +import { generateToken } from '../auth/tokens'; +import { + type Paired, + pair, + type SignedIn, + signedInUser, +} from '../runner/runner.fixtures'; +import { eventually, TestSocket } from '../runner/runner-socket.fixtures'; +import { createTestApp, type TestApp } from '../test/test-app'; +import { HEAD_TIMEOUT_MS, VideoRelay } from '../videos/video-relay.service'; +import { ShareConfig } from './share-config'; +import { ShareStorage } from './share-storage'; +import { ShareSweeper } from './share-sweeper'; + +const FILE = randomBytes(5000); +const POSTER = randomBytes(300); +const MAX_FILE = 10_000; +const QUOTA = 20_000; +const DAY_MS = 24 * 60 * 60 * 1000; +const TITLE = 'Checkout flow 7f3a'; +const DESCRIPTION = 'Buys one item, secret description 9c2e.'; +const PASSWORD = 'hunter22-correct-horse'; + +interface ShareBody { + id: string; + url: string; + fileUrl: string | null; + status: string; + failure: string | null; + sizeBytes: number; + expiresAt: string; +} + +describe('share links (integration)', () => { + let t: TestApp; + let http: App; + let port: number; + let dir: string; + let alice: SignedIn; + let bob: SignedIn; + let machine: Paired; + let runId: string; + const sockets: TestSocket[] = []; + + beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), 'demoflow-shares-')); + const config: ShareConfig = { + dir, + maxFileBytes: MAX_FILE, + quotaBytes: QUOTA, + webUrl: 'http://web.test', + }; + t = await createTestApp([], [{ provide: ShareConfig, useValue: config }]); + await t.app.listen(0, '127.0.0.1'); + http = t.app.getHttpServer(); + port = (http.address() as AddressInfo).port; + alice = await signedInUser(t, 'alice'); + bob = await signedInUser(t, 'bob'); + machine = await pair(http, alice, 'Laptop'); + runId = await report(machine); + }); + + afterEach(async () => { + for (const socket of sockets.splice(0)) { + socket.ws.terminate(); + } + // Uploads still waiting settle on the closed sockets before the app goes. + await eventually(async () => { + const uploading = await t.prisma.share.count({ + where: { + run: { project: { userId: alice.user.id } }, + status: 'uploading', + }, + }); + return uploading === 0; + }).catch(() => undefined); + await t.app.close(); + await rm(dir, { recursive: true, force: true }); + }); + + /** A run with `mp4`, `webm` and `gif`; `sizes` sets what `renders` says. */ + async function report( + by: Paired, + sizes: Record = { mp4: 6000, webm: 6000, gif: 6000 }, + ): Promise { + const formats = Object.keys(sizes); + const res = await request(http) + .post('/runner/runs') + .set('Authorization', `Bearer ${by.token}`) + .send({ + rootPath: '/home/alice/shop', + projectName: 'shop', + scenarioName: TITLE, + scenarioPath: 'demos/checkout.yaml', + status: 'passed', + startedAt: t.clock.now().toISOString(), + durationMs: 12_345, + formats, + cliVersion: '0.7.0', + renders: formats.map((format) => ({ + format, + width: 1280, + height: 720, + sizeBytes: sizes[format], + durationMs: 12_000, + })), + }) + .expect(201); + return res.body.runId; + } + + /** A daemon on `by`'s token that announced `features`; `null` sends none. */ + async function daemon( + by: Paired = machine, + features: string[] | null = ['share'], + ): Promise { + const socket = await TestSocket.connect( + `ws://127.0.0.1:${port}/runner`, + by.token, + ); + sockets.push(socket); + socket.sync([], undefined, features ?? undefined); + await socket.roundTrip(); + return socket; + } + + function create( + body: Record, + cookie = alice.cookie, + ): request.Test { + return request(http).post('/shares').set('Cookie', cookie).send(body); + } + + /** Answers `share.prepare`, then serves the poster and the render. */ + async function serveShare(socket: TestSocket, file = FILE): Promise { + const prepare = await socket.nextShare(); + socket.send({ + type: 'share.prepared', + requestId: prepare.requestId, + description: DESCRIPTION, + chapters: [ + { title: 'Open the cart', atMs: 0 }, + { title: 'Pay', atMs: 4000 }, + ], + posterSize: POSTER.length, + }); + const poster = await socket.nextOpen(); + socket.serve(poster.streamId, POSTER); + const render = await socket.nextOpen(); + socket.serve(render.streamId, file); + } + + async function shareOf(id: string): Promise { + const res = await request(http) + .get(`/shares/${id}`) + .set('Cookie', alice.cookie) + .expect(200); + return res.body as ShareBody; + } + + async function settled(id: string, status: string): Promise { + await eventually(async () => (await shareOf(id)).status === status); + return shareOf(id); + } + + async function ready(format = 'mp4', run = runId): Promise { + const socket = sockets[0] ?? (await daemon()); + const res = await create({ runId: run, format }).expect(202); + await serveShare(socket); + await settled(res.body.id, 'ready'); + return res.body.id; + } + + /** Every entry left under the share's directory, `.part` files included. */ + async function filesOf(id: string): Promise { + try { + return (await readdir(join(dir, id))).sort(); + } catch { + return null; + } + } + + describe('upload', () => { + it('pulls the poster and the render, then the share is ready', async () => { + const socket = await daemon(); + const res = await create({ runId, format: 'mp4' }).expect(202); + expect(res.body).toMatchObject({ + status: 'uploading', + runId, + title: TITLE, + format: 'mp4', + hasPassword: false, + views: 0, + failure: null, + }); + const token = (res.body.url as string).replace('http://web.test/s/', ''); + expect(token).toMatch(/^[A-Za-z0-9_-]{43}$/); + expect(res.body.fileUrl).toBe(`${res.body.url}/file`); + expect(res.body).not.toHaveProperty('token'); + expect(res.body).not.toHaveProperty('passwordHash'); + + const prepare = await socket.nextShare(); + expect(prepare).toMatchObject({ runId, format: 'mp4' }); + socket.send({ + type: 'share.prepared', + requestId: prepare.requestId, + description: DESCRIPTION, + chapters: [{ title: 'Open the cart', atMs: 0 }], + posterSize: POSTER.length, + }); + const poster = await socket.nextOpen(); + expect(poster).toMatchObject({ + runId, + format: 'mp4', + range: null, + asset: 'poster', + }); + socket.serve(poster.streamId, POSTER); + const render = await socket.nextOpen(); + expect(render).toMatchObject({ runId, format: 'mp4', range: null }); + expect(render).not.toHaveProperty('asset'); + socket.serve(render.streamId, FILE); + + const share = await settled(res.body.id, 'ready'); + expect(share).toMatchObject({ + sizeBytes: FILE.length, + width: 1280, + height: 720, + durationMs: 12_000, + }); + expect(await filesOf(share.id)).toEqual(['file.mp4', 'poster.jpg']); + expect(await readFile(join(dir, share.id, 'file.mp4'))).toEqual(FILE); + expect(await readFile(join(dir, share.id, 'poster.jpg'))).toEqual(POSTER); + const row = await t.prisma.share.findUniqueOrThrow({ + where: { id: share.id }, + }); + expect(row).toMatchObject({ + description: DESCRIPTION, + chapters: [{ title: 'Open the cart', atMs: 0 }], + posterBytes: POSTER.length, + }); + + const list = await request(http) + .get('/shares') + .set('Cookie', alice.cookie) + .expect(200); + expect(list.body.shares.map((s: ShareBody) => s.id)).toEqual([share.id]); + expect(list.body.quota).toEqual({ + usedBytes: FILE.length + POSTER.length, + limitBytes: QUOTA, + maxFileBytes: MAX_FILE, + }); + }); + + it('keeps only the hash of a password and hides the file URL', async () => { + await daemon(); + const res = await create({ + runId, + format: 'mp4', + password: PASSWORD, + expiresInDays: 7, + }).expect(202); + expect(res.body).toMatchObject({ hasPassword: true, fileUrl: null }); + expect(new Date(res.body.expiresAt).getTime()).toBe( + t.clock.now().getTime() + 7 * DAY_MS, + ); + const row = await t.prisma.share.findUniqueOrThrow({ + where: { id: res.body.id }, + }); + expect(row.passwordHash).toMatch(/^\$argon2id\$/); + }); + + it('validates the body', async () => { + await create({ runId, format: 'mp4', expiresInDays: 14 }).expect(400); + await create({ runId, format: 'mp4', password: 'short' }).expect(400); + await create({ runId, format: 'mov' }).expect(400); + await create({ runId, format: 'mp4', extra: 1 }).expect(400); + }); + + it('answers share-exists for a second share of the same render', async () => { + await daemon(); + const first = await create({ runId, format: 'mp4' }).expect(202); + const second = await create({ runId, format: 'mp4' }).expect(409); + expect(second.body).toEqual({ + error: 'share-exists', + shareId: first.body.id, + }); + }); + + it('replaces a failed share of the same render', async () => { + const socket = await daemon(); + const first = await create({ runId, format: 'mp4' }).expect(202); + const prepare = await socket.nextShare(); + socket.send({ + type: 'share.error', + requestId: prepare.requestId, + code: 'io', + }); + await settled(first.body.id, 'failed'); + const second = await create({ runId, format: 'mp4' }).expect(202); + expect(second.body.id).not.toBe(first.body.id); + expect(await t.prisma.share.count({ where: { id: first.body.id } })).toBe( + 0, + ); + }); + + it('refuses a format the run did not render', async () => { + await daemon(); + const res = await create({ runId, format: 'webm-square' }).expect(409); + expect(res.body).toEqual({ error: 'format-not-rendered' }); + }); + }); + + describe('the machine', () => { + it('sends nothing to a daemon without `share`: runner-outdated', async () => { + const socket = await daemon(machine, null); + const res = await create({ runId, format: 'mp4' }).expect(409); + expect(res.body).toEqual({ error: 'runner-outdated' }); + await socket.roundTrip(); + expect(socket.received.map((m) => m.type)).not.toContain('share.prepare'); + expect(await t.prisma.share.count({ where: { runId } })).toBe(0); + }); + + it('answers runner-offline without a socket', async () => { + const res = await create({ runId, format: 'mp4' }).expect(409); + expect(res.body).toEqual({ error: 'runner-offline' }); + }); + + it('refuses a run whose files were deleted', async () => { + await daemon(); + await t.prisma.run.update({ + where: { id: runId }, + data: { artifactsDeletedAt: t.clock.now() }, + }); + const res = await create({ runId, format: 'mp4' }).expect(409); + expect(res.body).toEqual({ error: 'artifacts-deleted' }); + }); + + it('shows the features on the run page', async () => { + await daemon(); + const res = await request(http) + .get(`/runs/${runId}`) + .set('Cookie', alice.cookie) + .expect(200); + expect(res.body.machine.features).toEqual(['share']); + }); + }); + + describe('limits', () => { + it('answers 413 before any message for a render over the file limit', async () => { + const big = await report(machine, { mp4: MAX_FILE + 1 }); + const socket = await daemon(); + const res = await create({ runId: big, format: 'mp4' }).expect(413); + expect(res.body).toEqual({ + error: 'file-too-large', + maxFileBytes: MAX_FILE, + }); + await socket.roundTrip(); + expect(socket.received.map((m) => m.type)).not.toContain('share.prepare'); + }); + + it('cancels a head over the file limit and fails the share', async () => { + const socket = await daemon(); + const res = await create({ runId, format: 'mp4' }).expect(202); + const prepare = await socket.nextShare(); + socket.send({ + type: 'share.prepared', + requestId: prepare.requestId, + chapters: [], + posterSize: POSTER.length, + }); + const poster = await socket.nextOpen(); + socket.serve(poster.streamId, POSTER); + const render = await socket.nextOpen(); + socket.send({ + type: 'video.head', + streamId: render.streamId, + size: MAX_FILE + 1, + start: 0, + end: MAX_FILE, + }); + const cancel = await socket.next('video.cancel'); + expect(cancel.streamId).toBe(render.streamId); + const share = await settled(res.body.id, 'failed'); + expect(share.failure).toBe('file-too-large'); + expect(await filesOf(share.id)).toBeNull(); + }); + + it('answers quota-exceeded, and a delete frees the bytes', async () => { + await daemon(); + // 6000 bytes reserved per uploading share, from the run's renders. + const mp4 = await create({ runId, format: 'mp4' }).expect(202); + await create({ runId, format: 'webm' }).expect(202); + await create({ runId, format: 'gif' }).expect(202); + const other = await report(machine, { mp4: 6000 }); + const over = await create({ runId: other, format: 'mp4' }).expect(409); + expect(over.body).toEqual({ + error: 'quota-exceeded', + usedBytes: 18_000, + limitBytes: QUOTA, + }); + await request(http) + .delete(`/shares/${mp4.body.id}`) + .set('Cookie', alice.cookie) + .expect(204); + await create({ runId: other, format: 'mp4' }).expect(202); + }); + }); + + describe('failures', () => { + it('share.error fails the share', async () => { + const socket = await daemon(); + const res = await create({ runId, format: 'mp4' }).expect(202); + const prepare = await socket.nextShare(); + socket.send({ + type: 'share.error', + requestId: prepare.requestId, + code: 'no-render', + }); + const share = await settled(res.body.id, 'failed'); + expect(share.failure).toBe('no-render'); + expect(await filesOf(share.id)).toBeNull(); + }); + + it('a head timeout fails the share', async () => { + const socket = await daemon(); + const res = await create({ runId, format: 'mp4' }).expect(202); + const prepare = await socket.nextShare(); + socket.send({ + type: 'share.prepared', + requestId: prepare.requestId, + chapters: [], + posterSize: POSTER.length, + }); + await socket.nextOpen(); + t.clock.advance(HEAD_TIMEOUT_MS); + t.app.get(VideoRelay).tick(); + const share = await settled(res.body.id, 'failed'); + expect(share.failure).toBe('runner-timeout'); + expect(await filesOf(share.id)).toBeNull(); + }); + + it('a dropped socket mid-file fails the share and leaves no .part', async () => { + const socket = await daemon(); + const res = await create({ runId, format: 'mp4' }).expect(202); + const prepare = await socket.nextShare(); + socket.send({ + type: 'share.prepared', + requestId: prepare.requestId, + chapters: [], + posterSize: POSTER.length, + }); + const poster = await socket.nextOpen(); + socket.serve(poster.streamId, POSTER); + const render = await socket.nextOpen(); + socket.send({ + type: 'video.head', + streamId: render.streamId, + size: FILE.length, + start: 0, + end: FILE.length - 1, + }); + socket.sendChunk(render.streamId, FILE.subarray(0, 1000)); + await socket.roundTrip(); + await eventually(async () => + ((await filesOf(res.body.id)) ?? []).includes('file.mp4.part'), + ); + socket.ws.terminate(); + const share = await settled(res.body.id, 'failed'); + expect(share.failure).toBe('runner-offline'); + expect(await filesOf(share.id)).toBeNull(); + }); + }); + + describe('extend', () => { + it('adds 30 days, never beyond 90 days from now', async () => { + const id = await ready(); + const start = t.clock.now().getTime(); + const extend = () => + request(http).post(`/shares/${id}/extend`).set('Cookie', alice.cookie); + + const once = await extend().expect(200); + expect(new Date(once.body.expiresAt).getTime()).toBe(start + 60 * DAY_MS); + const twice = await extend().expect(200); + expect(new Date(twice.body.expiresAt).getTime()).toBe( + start + 90 * DAY_MS, + ); + const capped = await extend().expect(409); + expect(capped.body).toEqual({ error: 'at-max-expiry' }); + + t.clock.advance(10 * DAY_MS); + const later = await extend().expect(200); + expect(new Date(later.body.expiresAt).getTime()).toBe( + start + 100 * DAY_MS, + ); + }); + }); + + describe('authorization', () => { + it('hides every share of another account', async () => { + const id = await ready(); + for (const call of [ + request(http).get(`/shares/${id}`), + request(http).post(`/shares/${id}/extend`), + request(http).delete(`/shares/${id}`), + ]) { + const res = await call.set('Cookie', bob.cookie).expect(404); + expect(res.body).toEqual({ error: 'not-found' }); + } + const list = await request(http) + .get('/shares') + .set('Cookie', bob.cookie) + .expect(200); + expect(list.body.shares).toEqual([]); + const res = await create({ runId, format: 'webm' }, bob.cookie).expect( + 404, + ); + expect(res.body).toEqual({ error: 'not-found' }); + expect(await t.prisma.share.count({ where: { id } })).toBe(1); + }); + + it('lets a machine share only the runs it reported', async () => { + await daemon(); + const second = await pair(http, alice, 'Desktop'); + const bobs = await pair(http, bob, 'Bob laptop'); + const viaRunner = (by: Paired, body: Record) => + request(http) + .post('/runner/shares') + .set('Authorization', `Bearer ${by.token}`) + .send(body); + + await viaRunner(bobs, { runId, format: 'mp4' }).expect(404); + await viaRunner(second, { runId, format: 'mp4' }).expect(404); + const res = await viaRunner(machine, { runId, format: 'mp4' }).expect( + 202, + ); + expect(res.body.status).toBe('uploading'); + + const read = (by: Paired) => + request(http) + .get(`/runner/shares/${res.body.id}`) + .set('Authorization', `Bearer ${by.token}`); + await read(machine).expect(200); + await read(second).expect(404); + await read(bobs).expect(404); + }); + }); + + describe('deletes', () => { + it('revokes a share: row and directory', async () => { + const id = await ready(); + await request(http) + .delete(`/shares/${id}`) + .set('Cookie', alice.cookie) + .expect(204); + expect(await t.prisma.share.count({ where: { id } })).toBe(0); + expect(await filesOf(id)).toBeNull(); + await request(http) + .delete(`/shares/${id}`) + .set('Cookie', alice.cookie) + .expect(404); + }); + + it('a run delete takes its shares and their files', async () => { + const id = await ready(); + await request(http) + .delete(`/runs/${runId}`) + .set('Cookie', alice.cookie) + .expect(204); + expect(await t.prisma.share.count({ where: { id } })).toBe(0); + expect(await filesOf(id)).toBeNull(); + }); + + it('a bulk run delete takes their shares and files', async () => { + const id = await ready(); + const { projectId } = await t.prisma.run.findUniqueOrThrow({ + where: { id: runId }, + }); + await request(http) + .post(`/projects/${projectId}/runs/delete`) + .set('Cookie', alice.cookie) + .send({ runIds: [runId] }) + .expect(200); + expect(await t.prisma.share.count({ where: { id } })).toBe(0); + expect(await filesOf(id)).toBeNull(); + }); + + it('a project delete takes its shares and files', async () => { + const id = await ready(); + const { projectId } = await t.prisma.run.findUniqueOrThrow({ + where: { id: runId }, + }); + await request(http) + .delete(`/projects/${projectId}`) + .set('Cookie', alice.cookie) + .expect(204); + expect(await t.prisma.share.count({ where: { id } })).toBe(0); + expect(await filesOf(id)).toBeNull(); + }); + + it('an account delete takes its shares and files', async () => { + const id = await ready(); + await request(http) + .delete('/me') + .set('Cookie', alice.cookie) + .send({ email: alice.user.email }) + .expect(204); + expect(await t.prisma.share.count({ where: { id } })).toBe(0); + expect(await filesOf(id)).toBeNull(); + }); + + it('deleting the run files on the machine leaves the share', async () => { + const id = await ready(); + const socket = sockets[0]; + const deleting = request(http) + .delete(`/runs/${runId}/artifacts`) + .set('Cookie', alice.cookie) + .then((res) => res.status); + const ask = await socket.nextDelete(); + socket.send({ type: 'artifacts.deleted', requestId: ask.requestId }); + expect(await deleting).toBe(204); + expect((await shareOf(id)).status).toBe('ready'); + expect(await filesOf(id)).toEqual(['file.mp4', 'poster.jpg']); + }); + }); + + describe('sweeper', () => { + /** A share row with a directory, as an upload would have left it. */ + async function seed( + format: string, + fields: { + status?: 'uploading' | 'ready'; + expiresAt?: Date; + createdAt?: Date; + }, + files: string[], + ): Promise { + const share = await t.prisma.share.create({ + data: { + token: generateToken(), + userId: alice.user.id, + runId, + format, + title: TITLE, + status: fields.status ?? 'ready', + expiresAt: + fields.expiresAt ?? new Date(t.clock.now().getTime() + DAY_MS), + ...(fields.createdAt ? { createdAt: fields.createdAt } : {}), + }, + }); + await mkdir(join(dir, share.id), { recursive: true }); + for (const name of files) { + await writeFile(join(dir, share.id, name), 'x'); + } + return share.id; + } + + it('deletes expired and orphaned files, fails stale uploads, marks lost', async () => { + const now = t.clock.now().getTime(); + const expired = await seed('mp4', { expiresAt: new Date(now - 1000) }, [ + 'file.mp4', + 'poster.jpg', + ]); + const stale = await seed( + 'webm', + { status: 'uploading', createdAt: new Date(now - 16 * 60 * 1000) }, + ['file.webm.part'], + ); + const lost = await seed('gif', {}, ['poster.jpg']); + const orphan = 'cl0rphan0000000000000000'; + await mkdir(join(dir, orphan)); + await writeFile(join(dir, 'README'), 'not a share'); + + const sweeper = new ShareSweeper( + t.prisma, + t.app.get(ShareStorage), + t.clock, + ); + await sweeper.sweep(true); + + expect(await t.prisma.share.count({ where: { id: expired } })).toBe(0); + expect(await filesOf(expired)).toBeNull(); + expect( + await t.prisma.share.findUniqueOrThrow({ where: { id: stale } }), + ).toMatchObject({ status: 'failed', failure: 'timed-out' }); + expect(await filesOf(stale)).toBeNull(); + expect( + (await t.prisma.share.findUniqueOrThrow({ where: { id: lost } })) + .status, + ).toBe('lost'); + expect(await filesOf(orphan)).toBeNull(); + expect(await readdir(dir)).toContain('README'); + }); + + it('marks nothing lost outside the boot pass', async () => { + const lost = await seed('gif', {}, []); + const sweeper = new ShareSweeper( + t.prisma, + t.app.get(ShareStorage), + t.clock, + ); + await sweeper.sweep(false); + expect( + (await t.prisma.share.findUniqueOrThrow({ where: { id: lost } })) + .status, + ).toBe('ready'); + }); + }); + + describe('logging', () => { + it('never logs the token, the password, the title or the description', async () => { + const lines: string[] = []; + const spy = jest + .spyOn(Logger.prototype, 'log') + .mockImplementation((message: unknown) => { + lines.push(String(message)); + }); + try { + const socket = await daemon(); + const res = await create({ + runId, + format: 'mp4', + password: PASSWORD, + }).expect(202); + await serveShare(socket); + await settled(res.body.id, 'ready'); + await request(http) + .post(`/shares/${res.body.id}/extend`) + .set('Cookie', alice.cookie) + .expect(200); + await request(http) + .delete(`/shares/${res.body.id}`) + .set('Cookie', alice.cookie) + .expect(204); + + const row = lines.join('\n'); + expect(row).toContain('share.create'); + expect(row).toContain('share.upload'); + expect(row).toContain('share.delete'); + const token = (res.body.url as string).split('/').pop() ?? ''; + for (const secret of [token, PASSWORD, TITLE, DESCRIPTION]) { + expect(row).not.toContain(secret); + } + } finally { + spy.mockRestore(); + } + }); + }); +}); diff --git a/apps/api/src/shares/shares.module.ts b/apps/api/src/shares/shares.module.ts new file mode 100644 index 00000000..ae070132 --- /dev/null +++ b/apps/api/src/shares/shares.module.ts @@ -0,0 +1,35 @@ +import { Module } from '@nestjs/common'; +import { AuthModule } from '../auth/auth.module'; +import { RunnerModule } from '../runner/runner.module'; +import { VideosModule } from '../videos/videos.module'; +import { PublicSharesController } from './public-shares.controller'; +import { PublicSharesService } from './public-shares.service'; +import { RunnerSharesController } from './runner-shares.controller'; +import { ShareAccess } from './share-access'; +import { ShareStorageModule } from './share-storage'; +import { ShareSweeper } from './share-sweeper'; +import { ShareUploadService } from './share-upload.service'; +import { SharesController } from './shares.controller'; +import { SharesService } from './shares.service'; + +/** + * Share links (spec #347, ADR-0019): a copy of one render on the server's + * disk, pulled from the machine through the video relay, and opened by + * anyone with the token (spec #348). + */ +@Module({ + imports: [AuthModule, RunnerModule, VideosModule, ShareStorageModule], + controllers: [ + SharesController, + RunnerSharesController, + PublicSharesController, + ], + providers: [ + SharesService, + ShareUploadService, + ShareSweeper, + PublicSharesService, + ShareAccess, + ], +}) +export class SharesModule {} diff --git a/apps/api/src/shares/shares.service.ts b/apps/api/src/shares/shares.service.ts new file mode 100644 index 00000000..6f9da083 --- /dev/null +++ b/apps/api/src/shares/shares.service.ts @@ -0,0 +1,365 @@ +import { HttpException, HttpStatus, Injectable, Logger } from '@nestjs/common'; +import { Prisma, type Share, type ShareStatus } from '@prisma/client'; +import { PasswordHasher } from '../auth/password-hasher'; +import { generateToken } from '../auth/tokens'; +import { Clock } from '../common/clock'; +import { PrismaService } from '../database/prisma.service'; +import { parseRenders } from '../projects/projects.service'; +import { RunnerConnections } from '../runner/runner-connections.service'; +import { ShareConfig } from './share-config'; +import { ShareStorage } from './share-storage'; +import { ShareUploadService } from './share-upload.service'; +import { type CreateShareDto, DEFAULT_SHARE_LIFETIME } from './shares.dto'; + +const DAY_MS = 24 * 60 * 60 * 1000; +/** `POST /shares/:id/extend` adds this much (spec #347, Decision 5). */ +export const SHARE_EXTEND_DAYS = 30; +/** Never beyond this far from now. */ +export const SHARE_MAX_DAYS = 90; + +/** The statuses whose bytes count against the quota (Decision 3). */ +const LIVE: ShareStatus[] = ['uploading', 'ready']; + +type ShareWithRun = Share & { run: { projectId: string } }; + +const SHARE_INCLUDE = { + run: { select: { projectId: true } }, +} satisfies Prisma.ShareInclude; + +/** What the owner sees of a share. Never the token's hash, never the password. */ +export interface ShareView { + id: string; + url: string; + /** Null while a password is set: the file is served only after unlocking. */ + fileUrl: string | null; + runId: string; + projectId: string; + title: string; + format: string; + status: ShareStatus; + failure: string | null; + sizeBytes: number; + width: number | null; + height: number | null; + durationMs: number | null; + hasPassword: boolean; + views: number; + expiresAt: Date; + createdAt: Date; +} + +export interface SharesList { + shares: ShareView[]; + quota: { usedBytes: number; limitBytes: number; maxFileBytes: number }; +} + +/** `{ error, ...details }` with a status, as the runs API answers. */ +function shareError( + status: HttpStatus, + error: string, + details: Record = {}, +): HttpException { + return new HttpException({ error, ...details }, status); +} + +function notFound(): HttpException { + return shareError(HttpStatus.NOT_FOUND, 'not-found'); +} + +/** + * Share links (spec #347, ADR-0019): created by the run's owner on the + * website or by the machine that reported the run, uploaded in the + * background, extended and revoked by the owner. Every query is scoped by + * the user: another account's share or run answers 404. + */ +@Injectable() +export class SharesService { + private readonly logger = new Logger(SharesService.name); + + constructor( + private readonly prisma: PrismaService, + private readonly connections: RunnerConnections, + private readonly uploads: ShareUploadService, + private readonly storage: ShareStorage, + private readonly config: ShareConfig, + private readonly hasher: PasswordHasher, + private readonly clock: Clock, + ) {} + + /** + * Every check before anything goes to the machine, then the row, then the + * upload in the background. `runnerId` limits the run to one that machine + * reported (Decision 7). + */ + async create( + userId: string, + body: CreateShareDto, + runnerId?: string, + ): Promise { + const run = await this.prisma.run.findFirst({ + where: { + id: body.runId, + project: { userId }, + ...(runnerId === undefined ? {} : { runnerId }), + }, + include: { runner: { select: { id: true, revokedAt: true } } }, + }); + if (!run) { + throw notFound(); + } + if (!run.formats.includes(body.format)) { + throw shareError(HttpStatus.CONFLICT, 'format-not-rendered'); + } + await this.replaceFailed(run.id, body.format); + if (run.artifactsDeletedAt) { + throw shareError(HttpStatus.CONFLICT, 'artifacts-deleted'); + } + // A revoked machine can never serve again: it counts as offline. + const machine = run.runner?.revokedAt === null ? run.runner : null; + if (!machine || !this.connections.isOnline(machine.id)) { + throw shareError(HttpStatus.CONFLICT, 'runner-offline'); + } + if (!this.connections.features(machine.id).includes('share')) { + throw shareError(HttpStatus.CONFLICT, 'runner-outdated'); + } + const render = + parseRenders(run.renders)?.find(({ format }) => format === body.format) ?? + null; + // Unknown until the head when the CLI did not report it: checked there. + const reserved = render?.sizeBytes ?? 0; + if (reserved > this.config.maxFileBytes) { + throw shareError(HttpStatus.PAYLOAD_TOO_LARGE, 'file-too-large', { + maxFileBytes: this.config.maxFileBytes, + }); + } + + const passwordHash = + body.password === undefined + ? null + : await this.hasher.hash(body.password); + const now = this.clock.now(); + const days = body.expiresInDays ?? DEFAULT_SHARE_LIFETIME; + const share = await this.insert(userId, run.id, reserved, { + token: generateToken(), + format: body.format, + title: run.scenarioName, + sizeBytes: reserved, + passwordHash, + expiresAt: new Date(now.getTime() + days * DAY_MS), + }); + this.log('share.create', share.id, run.id, 'uploading'); + + void this.uploads.upload({ + shareId: share.id, + userId, + runId: run.id, + format: body.format, + runnerId: machine.id, + render, + }); + return this.toView(share); + } + + /** Newest first, with what the quota counts now. */ + async list(userId: string): Promise { + const [shares, usedBytes] = await Promise.all([ + this.prisma.share.findMany({ + where: { userId, expiresAt: { gt: this.clock.now() } }, + include: SHARE_INCLUDE, + orderBy: [{ createdAt: 'desc' }, { id: 'desc' }], + }), + this.usedBytes(this.prisma, userId), + ]); + return { + shares: shares.map((share) => this.toView(share)), + quota: { + usedBytes, + limitBytes: this.config.quotaBytes, + maxFileBytes: this.config.maxFileBytes, + }, + }; + } + + async get(userId: string, id: string, runnerId?: string): Promise { + return this.toView(await this.findOwned(userId, id, runnerId)); + } + + /** 30 more days, never beyond 90 days from now (Decision 5). */ + async extend(userId: string, id: string): Promise { + const share = await this.findOwned(userId, id); + const now = this.clock.now().getTime(); + const expiresAt = Math.min( + Math.max(share.expiresAt.getTime(), now) + SHARE_EXTEND_DAYS * DAY_MS, + now + SHARE_MAX_DAYS * DAY_MS, + ); + if (expiresAt <= share.expiresAt.getTime()) { + throw shareError(HttpStatus.CONFLICT, 'at-max-expiry'); + } + const updated = await this.prisma.share.update({ + where: { id: share.id }, + data: { expiresAt: new Date(expiresAt) }, + include: SHARE_INCLUDE, + }); + this.log('share.extend', share.id, share.runId, 'extended'); + return this.toView(updated); + } + + /** The link stops working at once; the files go after the row (ADR-0019 §5). */ + async remove(userId: string, id: string): Promise { + // Read first only for the log line: the delete itself is scoped again. + const share = await this.prisma.share.findFirst({ + where: { id, userId }, + select: { id: true, runId: true }, + }); + if (!share) { + throw notFound(); + } + const { count } = await this.prisma.share.deleteMany({ + where: { id: share.id, userId }, + }); + this.log( + 'share.delete', + share.id, + share.runId, + count === 1 ? 'deleted' : 'not-found', + ); + if (count !== 1) { + throw notFound(); + } + await this.storage.remove([share.id]); + } + + /** + * A failed or lost share of the same run and format is replaced by a new + * one: retrying must not need a revoke first. A live one is `share-exists`. + */ + private async replaceFailed(runId: string, format: string): Promise { + const existing = await this.prisma.share.findUnique({ + where: { runId_format: { runId, format } }, + select: { id: true, status: true }, + }); + if (!existing) { + return; + } + if (LIVE.includes(existing.status)) { + throw shareError(HttpStatus.CONFLICT, 'share-exists', { + shareId: existing.id, + }); + } + const { count } = await this.prisma.share.deleteMany({ + where: { id: existing.id, status: { notIn: LIVE } }, + }); + if (count === 1) { + await this.storage.remove([existing.id]); + this.log('share.delete', existing.id, runId, 'replaced'); + } + } + + /** + * The quota check and the insert under the user's row lock, so two shares + * created at once cannot both pass it (Decision 3). + */ + private async insert( + userId: string, + runId: string, + reserved: number, + data: Omit, + ): Promise { + try { + return await this.prisma.$transaction(async (tx) => { + await tx.$queryRaw`SELECT 1 FROM "User" WHERE "id" = ${userId} FOR UPDATE`; + const usedBytes = await this.usedBytes(tx, userId); + if (usedBytes + reserved > this.config.quotaBytes) { + throw shareError(HttpStatus.CONFLICT, 'quota-exceeded', { + usedBytes, + limitBytes: this.config.quotaBytes, + }); + } + return tx.share.create({ + data: { ...data, userId, runId }, + include: SHARE_INCLUDE, + }); + }); + } catch (error) { + // Another request shared the same run and format in between. + if ( + error instanceof Prisma.PrismaClientKnownRequestError && + error.code === 'P2002' + ) { + const existing = await this.prisma.share.findUnique({ + where: { runId_format: { runId, format: data.format } }, + select: { id: true }, + }); + throw shareError(HttpStatus.CONFLICT, 'share-exists', { + ...(existing ? { shareId: existing.id } : {}), + }); + } + throw error; + } + } + + private async usedBytes( + db: Prisma.TransactionClient, + userId: string, + ): Promise { + const { _sum } = await db.share.aggregate({ + where: { userId, status: { in: LIVE } }, + _sum: { sizeBytes: true, posterBytes: true }, + }); + return (_sum.sizeBytes ?? 0) + (_sum.posterBytes ?? 0); + } + + /** An expired share is gone for its owner too, before the sweeper deletes it. */ + private async findOwned( + userId: string, + id: string, + runnerId?: string, + ): Promise { + const share = await this.prisma.share.findFirst({ + where: { + id, + userId, + expiresAt: { gt: this.clock.now() }, + ...(runnerId === undefined ? {} : { run: { runnerId } }), + }, + include: SHARE_INCLUDE, + }); + if (!share) { + throw notFound(); + } + return share; + } + + private toView(share: ShareWithRun): ShareView { + const url = `${this.config.webUrl}/s/${share.token}`; + return { + id: share.id, + url, + fileUrl: share.passwordHash === null ? `${url}/file` : null, + runId: share.runId, + projectId: share.run.projectId, + title: share.title, + format: share.format, + status: share.status, + failure: share.failure, + sizeBytes: share.sizeBytes, + width: share.width, + height: share.height, + durationMs: share.durationMs, + hasPassword: share.passwordHash !== null, + views: share.views, + expiresAt: share.expiresAt, + createdAt: share.createdAt, + }; + } + + /** Decision 10: event, share id, run id, outcome. Nothing else. */ + private log( + event: string, + shareId: string, + runId: string | null, + outcome: string, + ): void { + this.logger.log(JSON.stringify({ event, shareId, runId, outcome })); + } +} diff --git a/apps/api/src/test/test-app.ts b/apps/api/src/test/test-app.ts index 48da4c89..13d58719 100644 --- a/apps/api/src/test/test-app.ts +++ b/apps/api/src/test/test-app.ts @@ -8,6 +8,7 @@ import { Clock } from '../common/clock'; import { PrismaService } from '../database/prisma.service'; import { Mailer } from '../mail/mailer'; import { RecordingMailer } from '../mail/recording.mailer'; +import { ShareSweeper } from '../shares/share-sweeper'; /** A clock the test moves by hand. */ export class FakeClock extends Clock { @@ -63,7 +64,12 @@ export async function createTestApp( .overrideProvider(Mailer) .useValue(mailer) .overrideProvider(Clock) - .useValue(clock); + .useValue(clock) + // The sweeper acts on every share in the shared test database: a pass on + // boot in each parallel suite would delete and mark other suites' shares. + // Its spec builds one and calls `sweep` itself. + .overrideProvider(ShareSweeper) + .useValue({}); for (const { provide, useValue } of overrides) { builder.overrideProvider(provide).useValue(useValue); } diff --git a/apps/api/src/users/account.service.ts b/apps/api/src/users/account.service.ts index b5e09c10..a9f98f80 100644 --- a/apps/api/src/users/account.service.ts +++ b/apps/api/src/users/account.service.ts @@ -3,6 +3,7 @@ import type { User } from '@prisma/client'; import { normalizeEmail } from '../auth/email'; import { PrismaService } from '../database/prisma.service'; import { RunnerConnections } from '../runner/runner-connections.service'; +import { ShareStorage } from '../shares/share-storage'; export const CONFIRMATION_MISMATCH = 'confirmation-mismatch'; @@ -17,6 +18,7 @@ export class AccountService { constructor( private readonly prisma: PrismaService, private readonly connections: RunnerConnections, + private readonly shareStorage: ShareStorage, ) {} /** `typedEmail` must be the account's email, trimmed, in any case. */ @@ -26,21 +28,32 @@ export class AccountService { throw new BadRequestException({ error: CONFIRMATION_MISMATCH }); } - const runnerIds = await this.prisma.$transaction(async (tx) => { - const runners = await tx.runner.findMany({ - where: { userId: user.id }, - select: { id: true }, - }); - // A plain column, no cascade: the codes this user approved. - await tx.deviceCode.deleteMany({ where: { approvedById: user.id } }); - await tx.user.delete({ where: { id: user.id } }); - return runners.map(({ id }) => id); - }); + const { runnerIds, shareIds } = await this.prisma.$transaction( + async (tx) => { + const runners = await tx.runner.findMany({ + where: { userId: user.id }, + select: { id: true }, + }); + const shares = await tx.share.findMany({ + where: { userId: user.id }, + select: { id: true }, + }); + // A plain column, no cascade: the codes this user approved. + await tx.deviceCode.deleteMany({ where: { approvedById: user.id } }); + await tx.user.delete({ where: { id: user.id } }); + return { + runnerIds: runners.map(({ id }) => id), + shareIds: shares.map(({ id }) => id), + }; + }, + ); // After the commit: a socket accepted meanwhile finds no row and closes. for (const runnerId of runnerIds) { this.connections.revoke(runnerId); } + // The shares cascaded with the user; their files go now (spec #347, Decision 9). + await this.shareStorage.remove(shareIds); this.logger.log( JSON.stringify({ event: 'account.deleted', userId: user.id }), ); diff --git a/apps/api/src/videos/run-details.integration.spec.ts b/apps/api/src/videos/run-details.integration.spec.ts index 20f8e006..04ed073f 100644 --- a/apps/api/src/videos/run-details.integration.spec.ts +++ b/apps/api/src/videos/run-details.integration.spec.ts @@ -178,6 +178,7 @@ describe('run details (integration)', () => { 'failedStep', 'cliVersion', 'artifactsDeletedAt', + 'kind', ].sort(), ); } diff --git a/apps/api/src/videos/runs.service.ts b/apps/api/src/videos/runs.service.ts index 3493b648..e6c3655e 100644 --- a/apps/api/src/videos/runs.service.ts +++ b/apps/api/src/videos/runs.service.ts @@ -6,7 +6,8 @@ import { toRunDetailView, } from '../projects/projects.service'; import { RunnerConnections } from '../runner/runner-connections.service'; -import type { RunOption } from '../runner/runner-socket'; +import type { Feature, RunOption } from '../runner/runner-socket'; +import { ShareStorage } from '../shares/share-storage'; const RUN_INCLUDE = { project: { select: { id: true, name: true } }, @@ -21,13 +22,15 @@ export interface RunPage { project: { id: string; name: string }; /** * Null when the machine was removed: deleted or revoked. `runOptions` is - * what it accepts on a start, none while offline (spec #325, Decision 7). + * what it accepts on a start, none while offline (spec #325, Decision 7); + * `features` what its daemon announced, as `share` (spec #347, Decision 6). */ machine: { id: string; name: string; online: boolean; runOptions: RunOption[]; + features: Feature[]; } | null; } @@ -44,6 +47,7 @@ export class RunsService { constructor( private readonly prisma: PrismaService, private readonly connections: RunnerConnections, + private readonly shareStorage: ShareStorage, ) {} async find(userId: string, id: string): Promise { @@ -68,6 +72,7 @@ export class RunsService { name: machine.name, online: this.connections.isOnline(machine.id), runOptions: [...this.connections.runOptions(machine.id)], + features: [...this.connections.features(machine.id)], }, }; } @@ -75,8 +80,14 @@ export class RunsService { /** * Deletes the run's row, never its files (spec #280, Decisions 2–4). One * statement: ownership and delete cannot race. Already gone is 404 too. + * Its shares cascade; their directories go once the row is gone (spec + * #347, Decision 9). */ async remove(userId: string, id: string): Promise { + const shares = await this.prisma.share.findMany({ + where: { runId: id, run: { project: { userId } } }, + select: { id: true }, + }); const { count } = await this.prisma.run.deleteMany({ where: { id, project: { userId } }, }); @@ -88,6 +99,7 @@ export class RunsService { if (count !== 1) { throw runError(HttpStatus.NOT_FOUND, 'not-found'); } + await this.shareStorage.remove(shares.map((share) => share.id)); } } diff --git a/apps/api/src/videos/video-relay.service.ts b/apps/api/src/videos/video-relay.service.ts index 5bbcaf7a..31b59020 100644 --- a/apps/api/src/videos/video-relay.service.ts +++ b/apps/api/src/videos/video-relay.service.ts @@ -1,4 +1,6 @@ import { randomUUID } from 'node:crypto'; +import { createWriteStream, type WriteStream } from 'node:fs'; +import { rename, rm } from 'node:fs/promises'; import { Injectable, Logger, @@ -14,6 +16,7 @@ import { type VideoSink, } from '../runner/runner-connections.service'; import type { + VideoAsset, VideoChunk, VideoMessage, VideoRange, @@ -52,23 +55,200 @@ const RUNNER_ERRORS: Record = { }; const UNKNOWN_RUNNER_ERROR = { status: 502, error: 'runner-error' }; +interface Head { + size: number; + start: number; + end: number; +} + +/** + * Where one stream's bytes go: a playback response, or a share's file (spec + * #347, Decision 4). The relay owns the protocol — spans, timeouts, credit — + * and a sink only takes what passed it. + */ +interface StreamSink { + /** Logged with the stream's outcome. */ + readonly kind: 'play' | 'download' | 'upload'; + /** The head passed the span check. An error code refuses it. */ + head(head: Head, expected: number): string | null; + /** In order. `taken` once the sink holds the bytes: they are acknowledged then. */ + write(bytes: Buffer, taken: () => void): void; + /** Every byte the head announced came. */ + end(): void; + /** Failed: `status` and `headers` count only for a response not yet started. */ + fail(status: number, error: string, headers?: Record): void; +} + +/** The HTTP response of `GET /runs/:id/video/:format` (spec #191). */ +class ResponseSink implements StreamSink { + readonly kind: 'play' | 'download'; + /** Chunks the response has not taken yet, acknowledged on `drain`. */ + private unacked: Array<() => void> = []; + + constructor( + private readonly res: Response, + private readonly format: RunFormat, + private readonly range: VideoRange | null, + /** Set for `?download=1`: the head says `attachment` (spec #299). */ + private readonly download: { fileName: string } | undefined, + ) { + this.kind = download ? 'download' : 'play'; + } + + head(head: Head, expected: number): null { + const partial = this.range !== null; + this.res.writeHead(partial ? 206 : 200, { + ...COMMON_HEADERS, + 'Content-Type': VIDEO_CONTENT_TYPES[this.format], + 'Content-Length': String(expected), + 'Content-Disposition': this.download + ? `attachment; filename="${this.download.fileName}"` + : 'inline', + ...(partial + ? { 'Content-Range': `bytes ${head.start}-${head.end}/${head.size}` } + : {}), + }); + return null; + } + + write(bytes: Buffer, taken: () => void): void { + const took = this.res.write(bytes); + if (took && this.unacked.length === 0) { + taken(); + return; + } + this.unacked.push(taken); + if (this.unacked.length === 1) { + this.res.once('drain', () => { + for (const ack of this.unacked.splice(0)) { + ack(); + } + }); + } + } + + end(): void { + this.res.end(); + } + + /** `{ error }` before the head; after it, the status is sent: destroy. */ + fail( + status: number, + error: string, + headers: Record = {}, + ): void { + if (this.res.headersSent) { + this.res.destroy(); + return; + } + this.res + .status(status) + .set({ ...COMMON_HEADERS, ...headers }) + .json({ error }); + } +} + +/** What an upload into a file settles with: the file's size, or an error code. */ +export type UploadOutcome = + | { ok: true; size: number } + | { ok: false; error: string }; + +export interface UploadVideo { + runnerId: string; + runId: string; + format: RunFormat; + /** The share's poster instead of the render. */ + asset?: VideoAsset; + /** Where the file lands. Bytes go to `.part` until the last one came. */ + path: string; + /** Checks the head's size before a byte is written; an error code refuses it. */ + accept: (size: number) => string | null; +} + +/** + * A share's file (spec #347, Decision 2): written to `.part`, renamed + * once every announced byte is on disk, deleted on anything else. + */ +class FileSink implements StreamSink { + readonly kind = 'upload'; + private file: WriteStream | undefined; + private failed = false; + + constructor( + private readonly path: string, + private readonly accept: (size: number) => string | null, + /** The disk failed under the stream: the relay cancels it. */ + private readonly broken: () => void, + private readonly settle: (outcome: UploadOutcome) => void, + ) {} + + private get part(): string { + return `${this.path}.part`; + } + + head({ size }: Head): string | null { + const refused = this.accept(size); + if (refused !== null) { + return refused; + } + this.file = createWriteStream(this.part); + this.file.once('error', () => this.broken()); + return null; + } + + write(bytes: Buffer, taken: () => void): void { + this.file?.write(bytes, (error) => { + if (!error) { + taken(); + } + }); + } + + end(): void { + const file = this.file; + if (!file) { + return; + } + // `close`, not `finish`: the descriptor is released before the rename. + file.once('close', () => { + if (this.failed) { + return; + } + rename(this.part, this.path).then( + () => this.settle({ ok: true, size: file.bytesWritten }), + () => this.discard('io'), + ); + }); + file.end(); + } + + fail(_status: number, error: string): void { + this.failed = true; + this.file?.destroy(); + this.discard(error); + } + + private discard(error: string): void { + rm(this.part, { force: true }).then( + () => this.settle({ ok: false, error }), + () => this.settle({ ok: false, error }), + ); + } +} + interface Stream { id: string; runnerId: string; runId: string; format: RunFormat; range: VideoRange | null; - /** Set for `?download=1`: the head says `attachment` (spec #299). */ - download: { fileName: string } | undefined; - res: Response; + sink: StreamSink; /** Clock ms. */ openedAt: number; lastChunkAt: number; /** `end - start + 1` of the head; null until it came. */ expected: number | null; relayed: number; - /** Sizes of chunks the response has not taken yet, acknowledged on `drain`. */ - unacked: number[]; } export interface OpenVideo { @@ -84,7 +264,7 @@ export interface OpenVideo { * Plays a run's video from the machine that recorded it (spec #191, ADR-0009 * §1): asks the Runner over its socket for the bytes and relays them into the * HTTP response, acknowledging each chunk once the response has taken it. - * Nothing is stored here. + * A share's upload (spec #347) pulls the same way into a file instead. * * Works only on the instance that holds the Runner's socket. Staging runs one * API instance; more would need sticky routing or a broker (Decision 10). @@ -115,7 +295,7 @@ export class VideoRelay clearInterval(this.timer); } - /** Streams open to the machine right now. */ + /** Streams open to the machine right now, uploads included. */ openTo(runnerId: string): number { let count = 0; for (const stream of this.streams.values()) { @@ -128,22 +308,13 @@ export class VideoRelay /** Sends `video.open`; the response is answered from here on. */ open({ runnerId, runId, format, range, download, res }: OpenVideo): void { - const now = this.clock.now().getTime(); - const stream: Stream = { - id: randomUUID(), + const stream = this.start( runnerId, runId, format, range, - download, - res, - openedAt: now, - lastChunkAt: now, - expected: null, - relayed: 0, - unacked: [], - }; - this.streams.set(stream.id, stream); + new ResponseSink(res, format, range, download), + ); // Seeked or left: the Runner stops reading (Decision 6). res.on('close', () => { if (this.streams.get(stream.id) === stream) { @@ -151,16 +322,37 @@ export class VideoRelay this.finish(stream, 'aborted'); } }); - const sent = this.connections.sendTo(runnerId, { - type: 'video.open', - streamId: stream.id, - runId, - format, - range, + this.send(stream, undefined); + } + + /** + * Pulls the whole file into `path` (spec #347, Decision 4), with the + * playback's timeouts and credit. Settles once the file is in place or gone. + */ + upload({ + runnerId, + runId, + format, + asset, + path, + accept, + }: UploadVideo): Promise { + return new Promise((settle) => { + let stream: Stream | undefined; + const sink = new FileSink( + path, + accept, + () => { + if (stream && this.streams.get(stream.id) === stream) { + this.cancel(stream); + this.fail(stream, 500, 'io'); + } + }, + settle, + ); + stream = this.start(runnerId, runId, format, null, sink); + this.send(stream, asset); }); - if (!sent) { - this.fail(stream, 503, 'runner-offline'); - } } message(runnerId: string, message: VideoMessage): void { @@ -194,20 +386,16 @@ export class VideoRelay // Bytes beyond the head's span must never pass as the file (Decision 7). if (stream.relayed + bytes.length > stream.expected) { this.cancel(stream); - this.destroy(stream, 'overrun'); + this.fail(stream, 502, 'overrun'); return true; } stream.relayed += bytes.length; stream.lastChunkAt = this.clock.now().getTime(); - const taken = stream.res.write(bytes); - if (taken && stream.unacked.length === 0) { - this.ack(stream, bytes.length); - return true; - } - stream.unacked.push(bytes.length); - if (stream.unacked.length === 1) { - stream.res.once('drain', () => this.drained(stream)); - } + stream.sink.write(bytes, () => { + if (this.streams.get(stream.id) === stream) { + this.ack(stream, bytes.length); + } + }); return true; } @@ -231,15 +419,51 @@ export class VideoRelay } } else if (now - stream.lastChunkAt >= CHUNK_TIMEOUT_MS) { this.cancel(stream); - this.destroy(stream, 'stalled'); + this.fail(stream, 504, 'stalled'); } } } - private head( - stream: Stream, - head: { size: number; start: number; end: number }, - ): void { + private start( + runnerId: string, + runId: string, + format: RunFormat, + range: VideoRange | null, + sink: StreamSink, + ): Stream { + const now = this.clock.now().getTime(); + const stream: Stream = { + id: randomUUID(), + runnerId, + runId, + format, + range, + sink, + openedAt: now, + lastChunkAt: now, + expected: null, + relayed: 0, + }; + this.streams.set(stream.id, stream); + return stream; + } + + /** `asset` only for an upload: a daemon without `share` never sees the field. */ + private send(stream: Stream, asset: VideoAsset | undefined): void { + const sent = this.connections.sendTo(stream.runnerId, { + type: 'video.open', + streamId: stream.id, + runId: stream.runId, + format: stream.format, + range: stream.range, + ...(asset ? { asset } : {}), + }); + if (!sent) { + this.fail(stream, 503, 'runner-offline'); + } + } + + private head(stream: Stream, head: Head): void { // A second head, or one that is not the span asked for, is a broken daemon. const span = spanOf(stream.range, head.size); if ( @@ -252,20 +476,15 @@ export class VideoRelay this.fail(stream, 502, 'runner-error'); return; } - stream.expected = head.end - head.start + 1; + const expected = head.end - head.start + 1; + const refused = stream.sink.head(head, expected); + if (refused !== null) { + this.cancel(stream); + this.fail(stream, 502, refused); + return; + } + stream.expected = expected; stream.lastChunkAt = this.clock.now().getTime(); - const partial = stream.range !== null; - stream.res.writeHead(partial ? 206 : 200, { - ...COMMON_HEADERS, - 'Content-Type': VIDEO_CONTENT_TYPES[stream.format], - 'Content-Length': String(stream.expected), - 'Content-Disposition': stream.download - ? `attachment; filename="${stream.download.fileName}"` - : 'inline', - ...(partial - ? { 'Content-Range': `bytes ${head.start}-${head.end}/${head.size}` } - : {}), - }); } private end(stream: Stream): void { @@ -274,11 +493,11 @@ export class VideoRelay return; } if (stream.relayed !== stream.expected) { - this.destroy(stream, 'short'); + this.fail(stream, 502, 'short'); return; } this.finish(stream, 'complete'); - stream.res.end(); + stream.sink.end(); } private runnerError( @@ -287,7 +506,7 @@ export class VideoRelay size: number | undefined, ): void { if (stream.expected !== null) { - this.destroy(stream, `runner ${code}`.slice(0, 80)); + this.fail(stream, 502, `runner ${code}`.slice(0, 80)); return; } if (code === 'bad-range' && size !== undefined) { @@ -300,15 +519,6 @@ export class VideoRelay this.fail(stream, status, error); } - private drained(stream: Stream): void { - if (this.streams.get(stream.id) !== stream) { - return; - } - for (const bytes of stream.unacked.splice(0)) { - this.ack(stream, bytes); - } - } - private ack(stream: Stream, bytes: number): void { this.connections.sendTo(stream.runnerId, { type: 'video.ack', @@ -324,28 +534,18 @@ export class VideoRelay }); } - /** Answers `{ error }` before the head; after it, the status is sent: destroy. */ + /** + * Ends the stream with `error`. A short or broken body must never pass as a + * complete one (Decision 7): a started response is destroyed, a file deleted. + */ private fail( stream: Stream, status: number, error: string, headers: Record = {}, ): void { - if (stream.res.headersSent) { - this.destroy(stream, error); - return; - } this.finish(stream, error); - stream.res - .status(status) - .set({ ...COMMON_HEADERS, ...headers }) - .json({ error }); - } - - /** A short or broken body must never pass as a complete one (Decision 7). */ - private destroy(stream: Stream, outcome: string): void { - this.finish(stream, outcome); - stream.res.destroy(); + stream.sink.fail(status, error, headers); } private finish(stream: Stream, outcome: string): void { @@ -356,7 +556,8 @@ export class VideoRelay event: 'video.stream', runId: stream.runId, format: stream.format, - download: stream.download !== undefined, + download: stream.sink.kind === 'download', + upload: stream.sink.kind === 'upload', bytes: stream.relayed, outcome, durationMs: this.clock.now().getTime() - stream.openedAt, diff --git a/apps/api/src/videos/videos.integration.spec.ts b/apps/api/src/videos/videos.integration.spec.ts index eed877ae..32e50bca 100644 --- a/apps/api/src/videos/videos.integration.spec.ts +++ b/apps/api/src/videos/videos.integration.spec.ts @@ -690,6 +690,7 @@ describe('run video through the runner relay (integration)', () => { failedStep: null, cliVersion: '0.4.0', artifactsDeletedAt: null, + kind: null, stages: null, renders: null, failure: null, @@ -700,6 +701,7 @@ describe('run video through the runner relay (integration)', () => { name: 'Laptop', online: true, runOptions: [], + features: [], }, }); }); @@ -711,6 +713,7 @@ describe('run video through the runner relay (integration)', () => { name: 'Laptop', online: false, runOptions: [], + features: [], }); }); @@ -725,6 +728,17 @@ describe('run video through the runner relay (integration)', () => { }); }); + it('shows the known features an online machine announced (spec #347)', async () => { + const socket = await daemon(); + socket.sync([], undefined, ['later', 'share']); + await socket.roundTrip(); + const res = await page().expect(200); + expect(res.body.machine).toMatchObject({ + online: true, + features: ['share'], + }); + }); + it('shows no run options once the announcing machine went offline', async () => { const socket = await daemon(); socket.sync([], ['formats']); diff --git a/apps/api/src/videos/videos.module.ts b/apps/api/src/videos/videos.module.ts index d666e803..8f1c5650 100644 --- a/apps/api/src/videos/videos.module.ts +++ b/apps/api/src/videos/videos.module.ts @@ -1,6 +1,7 @@ import { Module } from '@nestjs/common'; import { AuthModule } from '../auth/auth.module'; import { RunnerModule } from '../runner/runner.module'; +import { ShareStorageModule } from '../shares/share-storage'; import { ArtifactsService } from './artifacts.service'; import { RunsController } from './runs.controller'; import { RunsService } from './runs.service'; @@ -12,8 +13,10 @@ import { VideoRelay } from './video-relay.service'; * Deleting them goes through the same socket (spec #233). */ @Module({ - imports: [AuthModule, RunnerModule], + imports: [AuthModule, RunnerModule, ShareStorageModule], controllers: [RunsController], providers: [RunsService, VideoRelay, ArtifactsService], + // The share upload pulls through the same relay (spec #347, Decision 4). + exports: [VideoRelay], }) export class VideosModule {} diff --git a/apps/cli/README.md b/apps/cli/README.md index d7e779a3..e3dc9f35 100644 --- a/apps/cli/README.md +++ b/apps/cli/README.md @@ -140,6 +140,65 @@ are no longer listed. The website is not told, so its runs list keeps the first render's formats. A run that is unfinished or has no recording is refused with exit 2; a render that fails exits 3. +`demoflow run --check ` runs and verifies the scenario against the +live app like `run`, but records and renders nothing: the run directory holds +only `result.json`, with `recording` and `rendering` skipped. Without a +scenario, `demoflow run --check` runs every valid scenario of the project +(the files under `demoflow/`, sorted, at most 100) one after another against a +single start of `devCommand`, and prints one line per scenario, then +`N passed, M failed`. `--exclude ` leaves a scenario out of such a pass +(repeatable; `demoflow/login.demo.json` or `login.demo.json`; an id that is no +scenario is ignored). `--json` prints `{ "check": true, "passed", "failed", +"runs": [...] }`, each run as `run --json` prints it. It exits 0 when every run +passed and 1 when any failed. `--format` and `--headed` are refused with exit 2 +before anything runs. Each run is reported like any other, marked as a check. +Unlike `demoflow check`, which only inspects files and tools, this opens the +app. + +### Scheduled checks + +`schedule` makes the runner (`demoflow runner start`) run `demoflow run --check` +over the whole project on its own, at a set time: + +```json +{ + "baseUrl": "http://localhost:3000", + "schedule": { + "cadence": "weekly", + "weekday": "mon", + "at": "09:30", + "exclude": ["demoflow/wip.demo.json"] + } +} +``` + +- `cadence` is `daily`, `weekdays` (Monday to Friday) or `weekly`; `weekly` + needs `weekday` (`mon` … `sun`), the others refuse it. +- `at` is `HH:MM`, 24-hour, in the time zone of the machine the runner runs on. + A time the clock skips when daylight saving starts runs at the next valid + minute; a time that repeats when it ends runs once, the first time. +- `exclude` lists scenario ids the pass leaves out (at most 100), as + `--exclude` does. +- The runner watches the projects it has reported runs for and the projects + `demoflow run` ran in on this machine. It reads each `schedule` every + minute, so an edit needs no restart. A project with a config that does not + load is skipped and named in `runner.log`. +- One pass at a time, and never next to a run started from the website: a + pass that comes due waits for it. A pass is stopped after 2 hours, or after + 15 minutes on one scenario. +- A machine that was off or asleep at the time runs one pass when the runner + next looks, however many it missed. A new schedule waits for its first time: + it does not run at once. +- The passes are reported like any other check run (off without + `demoflow login`, under `CI`, or with `DEMOFLOW_NO_REPORT=1`). + +Without `demoflow login`, `runner start` and `runner install` still work when a +watched project has a schedule. The runner then runs in local mode: no +connection, scheduled checks only. `runner status` shows each scheduled +project, its next check (while the runner runs) and the last one, with its +result. A schedule runs only while the runner does: use +[`demoflow runner install`](#start-at-login) so it starts at every login. + `demoflow init [--base-url ]` writes `demoflow.config.json` (default `baseUrl` `http://localhost:3000`) and `demoflow/first.demo.json`, a scenario that opens `/` and passes against any app at `baseUrl`, in the current folder. @@ -185,7 +244,9 @@ typed value. Secret fills are left as recorded. | `demoflow record [name]` | record a scenario by using the app | | `demoflow validate ` | check a scenario against the schema | | `demoflow run ` | run it in a browser and render the video | +| `demoflow run --check [scenario]` | run and verify one scenario, or all of them, without a video | | `demoflow render ` | render an existing run again with the current config | +| `demoflow share ` | share a render of a reported run as a link on the website | | `demoflow list` | list recent runs of the current project, newest first | | `demoflow runs list` | list every run on this machine, with its size | | `demoflow runs delete ` / `prune` | delete runs by id, or by age and state | @@ -197,6 +258,7 @@ typed value. Secret fills are left as recorded. | `demoflow runner start` | keep this machine connected, in the background | | `demoflow runner stop` / `status` / `logs` | stop it, check it, read its log | | `demoflow runner install` / `uninstall` | start the runner at every login, or stop doing so | +| `demoflow runner allow ` / `disallow ` / `roots` | choose the folders the website may name | ## Pair this machine with your account @@ -215,7 +277,7 @@ expires after 10 minutes. - `demoflow logout` revokes the token on the server, then deletes the file. If the server cannot be reached it still deletes the file; the machine stays listed under `/app/machines` until you revoke it there. -- The server is `https://demoflow-app.luna-realm.com` (staging) unless +- The server is `https://demoflow.luna-realm.com` (production) unless `--server ` or `DEMOFLOW_SERVER` says otherwise. It is the website's origin; the API is under `/api`. `logout` uses the server stored at login. - The token is in `credentials.json` (`{ server, token, runnerId }`, mode 0600) @@ -238,8 +300,9 @@ it was last seen. After `login` the CLI asks whether to start it the command). No jobs run over it yet. - `runner status` prints whether it runs (PID, uptime), the connection - (`connected`, `reconnecting in Ns`, `revoked`), the server and whether - autostart is installed. `--json` prints the same as one object. + (`connected`, `reconnecting in Ns`, `revoked`, or local mode), the server, + whether autostart is installed, and the next and last check of each + scheduled project. `--json` prints the same as one object. - `runner logs` prints `runner.log`; `--follow` keeps printing. - `runner stop` stops it; `logout` stops it first. - One runner per machine. A second `start` prints the running PID. @@ -247,8 +310,10 @@ the command). No jobs run over it yet. and exits for good when the machine is revoked on the website. - Every 30 days the server rotates the token over the connection; the runner rewrites `credentials.json`. -- `runner.log` (rotated at 5 MB, three files kept), `runner.lock` and - `runner.state.json` sit next to `credentials.json`. None holds the token. +- `runner.log` (rotated at 5 MB, three files kept), `runner.lock`, + `runner.state.json`, `schedule.state.json` (each scheduled project's last + pass) and `local-roots.jsonl` (the projects `demoflow run` ran in, the last + 500) sit next to `credentials.json`. None holds the token. ### Start at login @@ -257,7 +322,8 @@ npx demoflow runner install ``` Registers the runner for your user, with no admin rights, and starts it now. -It needs `demoflow login` first. The unit runs the same `node` and CLI that +It needs `demoflow login` first, unless a watched project has a `schedule` +(local mode, see [Scheduled checks](#scheduled-checks)). The unit runs the same `node` and CLI that ran `install`, by absolute path; if you move or upgrade the CLI, run `install` again. `runner uninstall` removes it; a running runner keeps running until `runner stop`. @@ -272,6 +338,36 @@ A crash is restarted (systemd, launchd). A runner stopped on purpose, or revoked on the website, exits cleanly and stays down. Windows does not restart a crash; the next logon starts it again. Neither file holds the token. +### Folders the website may name + +From the website you can add a project by naming a folder on this machine. +The runner then reads that folder (whether it has `demoflow.config.json`, +its scenarios' ids and names, the GitHub `origin` of `.git/config`), can +create the `demoflow init` files in it, and can register it so the website +may start runs there. It never sends a file's contents or a config value. + +A folder qualifies only inside an allowed root. By default the only root is +your home directory. + +```sh +npx demoflow runner allow ~/code # the first allow replaces the home directory +npx demoflow runner disallow ~/code # removing the last root allows nothing +npx demoflow runner roots # the roots in force; --json for a script +``` + +- The roots are in `allowed-roots.json` (`{ "roots": [...] }`, at most 32, + mode 0600) next to `credentials.json`. A file that does not parse refuses + every folder until you fix or delete it; `roots` says so. +- The folder must exist, be a directory, and be reached without a link that + leads out of its root. No folder below the root may start with `.`, so + `~/.ssh` or `~/.config` are never offered. +- On Windows a drive path is required (`C:\…`); network (`\\server\…`) and + device (`\\?\…`) paths are refused. Windows and macOS compare paths without + case. +- The folders registered from the website are kept in + `website-projects.jsonl` next to `credentials.json` (the last 1,000). A + folder whose root you disallow stops being runnable from the website. + ## Runs on the website On a paired machine, `demoflow run` tells the website that the run happened, so @@ -289,12 +385,41 @@ fragments of URLs are removed, the project path becomes ``, the home directory becomes `~`, and the text is cut to 1,000 characters. It can still contain text from the page under test or other paths. -Videos, frames and step values never leave the machine. +Videos, frames and step values never leave the machine, unless you share a +render (below). The report waits at most two seconds and fails silently; the run's output and exit code are never affected. It is off without `demoflow login`, under `CI`, and with `DEMOFLOW_NO_REPORT=1`. +## Share a render + +```sh +npx demoflow share +``` + +Makes a link to one render of a run that this machine reported, so anyone with +the link can watch it without an account. The runner uploads the file, so it +must be running (`demoflow runner start`). The command prints the share page's +link and, without a password, the file's own link, then exits 0 once the +upload is done. + +- `--format ` picks the render (default: the run's first one). +- `--expires `: 7, 30 (default) or 90 days. +- `--password` asks for a password twice on the terminal, without echo; + without a terminal it reads `DEMOFLOW_SHARE_PASSWORD`. It is never a flag + value, so it never lands in your shell history. A protected share has no file + link (it would skip the password), so only the page link is printed. +- `--json` prints the share as one object on stdout. +- A render already shared keeps its link; the command prints it. Revoke it on + the website to share the render again. +- Only that render leaves the machine, with its poster frame, the chapters + (the scene cards) and the scenario's `description`. +- It exits 2 when the share cannot be made (not logged in, a run this + machine did not report, no such render, the runner offline or too old, the + file over the size limit, the quota full), 1 when the upload fails or is + still going after 10 minutes, and 3 when the server cannot be reached. + ## With Claude Code ```sh @@ -302,7 +427,18 @@ claude mcp add demoflow -- npx demoflow mcp ``` The server offers `generate_demo`, `get_task`, `cancel_task`, -`validate_scenario`, `list_demos` and `check_project`. +`validate_scenario`, `list_demos`, `check_project` and `share_demo`. + +`share_demo` shares a render of a reported run, like `demoflow share` without +a password. It is refused (`share-not-allowed`) unless the run's project allows +it in `demoflow.config.json`, which you commit: + +```json +{ "share": { "allowMcp": true } } +``` + +It waits up to 50 seconds for the upload; when the answer is still +`uploading`, call it again later for the same link. ## Updates diff --git a/apps/cli/package.json b/apps/cli/package.json index f72a9006..a6ecd25b 100644 --- a/apps/cli/package.json +++ b/apps/cli/package.json @@ -1,6 +1,6 @@ { "name": "@demoflow/cli", - "version": "0.7.0", + "version": "0.7.1", "private": true, "description": "Turn a scenario into a demo video: a browser runs it, FFmpeg renders it. CLI and MCP server.", "keywords": [ diff --git a/apps/cli/scripts/smoke.mjs b/apps/cli/scripts/smoke.mjs index 0084710a..13438069 100644 --- a/apps/cli/scripts/smoke.mjs +++ b/apps/cli/scripts/smoke.mjs @@ -46,6 +46,7 @@ const TOOLS = [ 'generate_demo', 'get_task', 'list_demos', + 'share_demo', 'validate_scenario', ]; const browsers = process.argv.includes('--browsers'); diff --git a/apps/cli/src/auth/client.ts b/apps/cli/src/auth/client.ts index b1908e0d..53c71982 100644 --- a/apps/cli/src/auth/client.ts +++ b/apps/cli/src/auth/client.ts @@ -133,7 +133,8 @@ export async function revokeSelf( throw unexpected(response.status); } -async function send( +/** One request to the API, with the device token when given. */ +export async function send( fetchFn: Fetch, server: string, path: string, @@ -158,7 +159,10 @@ async function send( } } -async function readJson(response: Response): Promise> { +/** The JSON object a response carries, or a `ServerError`. */ +export async function readJson( + response: Response, +): Promise> { try { const body: unknown = await response.json(); if (typeof body === 'object' && body !== null && !Array.isArray(body)) { @@ -186,6 +190,6 @@ function numberField(body: Record, name: string): number { return value; } -function unexpected(status: number): ServerError { +export function unexpected(status: number): ServerError { return new ServerError(`the server answered ${status}`, false); } diff --git a/apps/cli/src/auth/server.ts b/apps/cli/src/auth/server.ts index 91591c0b..999a93fe 100644 --- a/apps/cli/src/auth/server.ts +++ b/apps/cli/src/auth/server.ts @@ -1,7 +1,7 @@ import { CliError } from '../config/index.ts'; -/** Staging, until production exists (spec 135, Decision 9). */ -export const DEFAULT_SERVER = 'https://demoflow-app.luna-realm.com'; +/** Production; staging is reachable with `--server` or `DEMOFLOW_SERVER`. */ +export const DEFAULT_SERVER = 'https://demoflow.luna-realm.com'; /** `--server`, then `DEMOFLOW_SERVER`, then the default. Returns an origin-and-path without a trailing slash. */ export function resolveServer( diff --git a/apps/cli/src/commands/errors.ts b/apps/cli/src/commands/errors.ts index 3bff6020..ab5fd021 100644 --- a/apps/cli/src/commands/errors.ts +++ b/apps/cli/src/commands/errors.ts @@ -14,7 +14,20 @@ export type CommandErrorCode = | 'no-recording' | 'invalid-port' | 'app-already-running' - | 'fresh-app-needs-dev-command'; + | 'fresh-app-needs-dev-command' + | 'missing-scenario' + | 'check-flag' + | 'invalid-expires' + | 'invalid-password' + // `demoflow share` (spec 349, Decision 6): the server's refusals. + | 'not-logged-in' + | 'run-not-reported' + | 'format-not-rendered' + | 'artifacts-deleted' + | 'runner-offline' + | 'runner-outdated' + | 'file-too-large' + | 'quota-exceeded'; /** Bad input to a command (spec 5: exit 2) that is not a config problem. */ export class CommandError extends Error { diff --git a/apps/cli/src/commands/init.ts b/apps/cli/src/commands/init.ts index 3753bc72..d47feed2 100644 --- a/apps/cli/src/commands/init.ts +++ b/apps/cli/src/commands/init.ts @@ -1,3 +1,4 @@ +import { realpathSync } from 'node:fs'; import { mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; import { parseProjectConfig } from '@demoflow/core'; @@ -50,13 +51,11 @@ export async function initCommand( return EXIT_INVALID_INPUT; } - const files: [string, unknown][] = [ - [PROJECT_CONFIG_FILE, config], - [FIRST_SCENARIO, FIRST_SCENARIO_TEMPLATE], - ]; - for (const [rel, content] of files) { - const created = await createExclusive(deps.cwd, rel, content); - io.stdout(`${created ? 'created' : 'kept'} ${rel}\n`); + const written = await writeInitTemplate(deps.cwd, { + baseUrl: config.baseUrl, + }); + for (const rel of INIT_FILES) { + io.stdout(`${written.created.includes(rel) ? 'created' : 'kept'} ${rel}\n`); } io.stdout( [ @@ -71,15 +70,64 @@ export async function initCommand( return 0; } +/** The files `demoflow init` writes, in the order it reports them. */ +export const INIT_FILES = [PROJECT_CONFIG_FILE, FIRST_SCENARIO] as const; + +export interface InitTemplateOptions { + baseUrl?: string; + /** + * Spec 377: for a folder the website named, a file is never written through + * a linked directory, which could point anywhere; it throws instead. + */ + noLinks?: boolean; +} + +export interface InitTemplateResult { + created: string[]; + kept: string[]; +} + +/** + * The file-writing half of `demoflow init`, shared with the runner's + * `project.init` (spec 377, Decision 7): the config and the first scenario, + * each created only when it does not exist yet. + */ +export async function writeInitTemplate( + root: string, + options: InitTemplateOptions = {}, +): Promise { + const contents: Record<(typeof INIT_FILES)[number], unknown> = { + [PROJECT_CONFIG_FILE]: { baseUrl: options.baseUrl ?? DEFAULT_BASE_URL }, + [FIRST_SCENARIO]: FIRST_SCENARIO_TEMPLATE, + }; + const result: InitTemplateResult = { created: [], kept: [] }; + for (const rel of INIT_FILES) { + const created = await createExclusive( + root, + rel, + contents[rel], + options.noLinks === true, + ); + (created ? result.created : result.kept).push(rel); + } + return result; +} + /** `false` when the file already exists; it is left untouched. */ async function createExclusive( cwd: string, rel: string, content: unknown, + noLinks: boolean, ): Promise { const file = path.join(cwd, ...rel.split('/')); - await mkdir(path.dirname(file), { recursive: true }); + const dir = path.dirname(file); + await mkdir(dir, { recursive: true }); + if (noLinks && realpathSync.native(dir) !== dir) { + throw new Error(`${path.dirname(rel)} is reached through a link`); + } try { + // `wx` also refuses a link in the file's place, dangling or not. await writeFile(file, `${JSON.stringify(content, null, 2)}\n`, { flag: 'wx', }); diff --git a/apps/cli/src/commands/output.ts b/apps/cli/src/commands/output.ts index cde7e592..70892696 100644 --- a/apps/cli/src/commands/output.ts +++ b/apps/cli/src/commands/output.ts @@ -40,3 +40,41 @@ function failedStep(error: NonNullable): string { : ` (repeat steps[${error.origin.repeatIndex}], iteration ${error.origin.iteration + 1})`; return `step ${error.stepIndex}${origin}: `; } + +/** + * One line per scenario of a check (spec 360, Decision 4): its verdict, its + * id and, when it failed, the first failed stage with its error. + */ +export function formatCheckLine(result: RunResult): string { + const verdict = result.status === 'failed' ? 'failed' : 'passed'; + const line = `${verdict} ${result.scenarioId}`; + if (result.status !== 'failed') return `${line}\n`; + const failed = STAGE_NAMES.find( + (name) => result.stages[name].status === 'failed', + ); + if (failed === undefined) return `${line}\n`; + const { error } = result.stages[failed]; + const detail = + error === undefined + ? '' + : ` ${failedStep(error)}${firstLine(error.message)}`; + return `${line} ${failed}:${detail}\n`; +} + +/** `--check --json`: every run of the check, each with its `runDir`. */ +export function formatCheckJson( + runs: readonly { result: RunResult; runDir: string }[], +): string { + const failed = runs.filter(({ result }) => result.status === 'failed').length; + return `${JSON.stringify({ + check: true, + passed: runs.length - failed, + failed, + runs: runs.map(({ result, runDir }) => ({ ...result, runDir })), + })}\n`; +} + +function firstLine(text: string): string { + const end = text.indexOf('\n'); + return end === -1 ? text : text.slice(0, end); +} diff --git a/apps/cli/src/commands/progress.ts b/apps/cli/src/commands/progress.ts index e4c52084..c31ac06c 100644 --- a/apps/cli/src/commands/progress.ts +++ b/apps/cli/src/commands/progress.ts @@ -49,3 +49,27 @@ export function progressObserver(io: Io): EngineObserver { }, }; } + +/** + * `demoflow run --check --progress` (spec 360): one line as each scenario of + * a check pass starts, `run` 1-based. The daemon restarts its per-scenario + * time limit on it. + */ +export interface CheckProgress { + run: number; + runs: number; +} + +/** The check line as printed on stdout, newline included. */ +export function formatCheckProgress(progress: CheckProgress): string { + return `${JSON.stringify({ type: 'progress', phase: 'check', ...progress })}\n`; +} + +/** Writes a check line; best effort, like {@link emitProgress}. */ +export function emitCheckProgress(io: Io, progress: CheckProgress): void { + try { + io.stdout(formatCheckProgress(progress)); + } catch { + // Progress is best effort. + } +} diff --git a/apps/cli/src/commands/render.ts b/apps/cli/src/commands/render.ts index 23351976..81a931a3 100644 --- a/apps/cli/src/commands/render.ts +++ b/apps/cli/src/commands/render.ts @@ -28,7 +28,7 @@ import { exitCodeFor } from './exit.ts'; import type { Io } from './io.ts'; import { formatJson, formatSummary } from './output.ts'; import { assertSafeRunId } from './run-id.ts'; -import { loadScenario } from './scenario-file.ts'; +import { scenarioDescription } from './scenario-file.ts'; import { readResult, settleRun } from './settle.ts'; export interface RenderFlags { @@ -78,22 +78,7 @@ export async function renderCommand( io: Io, deps: RenderDeps = defaultRenderDeps(), ): Promise { - const { host } = deps; - assertSafeRunId(runId); - const root = - flags.project === undefined - ? projectRootAbove(deps.cwd) - : projectRootAt(path.resolve(deps.cwd, flags.project)); - const config = loadProjectConfig(root); - const { home } = resolveHome({ - ...host, - out: flags.out, - cwd: deps.cwd, - projectRoot: root, - projectConfig: config, - globalConfig: loadGlobalConfig(host), - }); - const runDir = runDirFor(home, runId, host.platform); + const { root, config, runDir } = locateRun(runId, flags, deps); const result = await readRenderable(runDir); io.stderr('Rendering\n'); @@ -128,6 +113,34 @@ export async function renderCommand( return code === 0 && renderError !== undefined ? EXIT_STAGE_FAILED : code; } +/** + * Where `` lives (spec 255, Decisions 1–2): under the home of the + * project above the cwd, or of `--project`. Throws for an unsafe id or a + * missing project; the run directory itself may not exist. + */ +export function locateRun( + runId: string, + flags: { out?: string; project?: string }, + deps: { host: HostEnvironment; cwd: string }, +): { root: string; config: ProjectConfig; runDir: string } { + const { host } = deps; + assertSafeRunId(runId); + const root = + flags.project === undefined + ? projectRootAbove(deps.cwd) + : projectRootAt(path.resolve(deps.cwd, flags.project)); + const config = loadProjectConfig(root); + const { home } = resolveHome({ + ...host, + out: flags.out, + cwd: deps.cwd, + projectRoot: root, + projectConfig: config, + globalConfig: loadGlobalConfig(host), + }); + return { root, config, runDir: runDirFor(home, runId, host.platform) }; +} + /** * The run's `result.json`, if the run can be rendered: the directory exists, * the result is valid, the capture stages are finished and the source video @@ -174,24 +187,6 @@ async function readRenderable(runDir: string): Promise { return result; } -/** - * The intro card's subtitle: the scenario's `description`, read from the - * project when its file is still there and loads. An inline scenario of - * `demoflow mcp` is gone after its run, so its card has no subtitle. - */ -function scenarioDescription( - root: string, - scenarioId: string, -): string | undefined { - const file = path.join(root, scenarioId); - if (!existsSync(file)) return undefined; - try { - return loadScenario(file).scenario.description; - } catch { - return undefined; - } -} - /** * What `buildRenderPlan` gets from the project's `render.*`, for `run` and * `render` alike: a re-render is styled exactly like a first render would be @@ -204,6 +199,7 @@ export function renderPlanOptions( ): RenderPlanOptions { return { idle: config.render.idle, + endHoldMs: config.render.endHoldMs, output: { width: config.render.width, height: config.render.height, diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index 921b48cf..5012f28d 100644 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -6,6 +6,7 @@ import { parsePort, type RenderFormat, type RunResult, + type Scenario, } from '@demoflow/core'; import { createCaptureObserver, @@ -28,12 +29,18 @@ import { loadGlobalConfig, PROJECT_CONFIG_FILE, projectFragments, + projectRootAbove, projectRootAt, resolveHome, runDir as runDirFor, scenarioId, } from '../config/index.ts'; +import { nodeLaunchFs } from '../runner/launch/checks.ts'; +import { LIST_MAX_SCENARIOS, listScenarios } from '../runner/launch/list.ts'; +import { runnerPaths } from '../runner/paths.ts'; +import { recordLocalRoot } from '../runner/schedule/local-roots.ts'; import { + canonicalRoot, defaultReportDeps, type ReportDeps, reportRun, @@ -42,8 +49,17 @@ import { type BrowserDeps, ensureChromium } from './browser.ts'; import { CommandError, EXIT_STAGE_FAILED } from './errors.ts'; import { exitCodeFor } from './exit.ts'; import type { Io } from './io.ts'; -import { formatJson, formatSummary } from './output.ts'; -import { emitProgress, progressObserver } from './progress.ts'; +import { + formatCheckJson, + formatCheckLine, + formatJson, + formatSummary, +} from './output.ts'; +import { + emitCheckProgress, + emitProgress, + progressObserver, +} from './progress.ts'; import { renderPlanOptions } from './render.ts'; import { assertSafeRunId, generateRunId } from './run-id.ts'; import { loadScenario } from './scenario-file.ts'; @@ -66,6 +82,10 @@ export interface RunFlags { project?: string; /** Hidden: one JSON progress line on stdout per phase change and step (spec 302). */ progress?: boolean; + /** `--check`: run and verify, record and render nothing (spec 360). */ + check?: boolean; + /** `--exclude`: a scenario id a check pass leaves out, repeatable (spec 360). */ + exclude?: string[]; } export interface RunDeps { @@ -76,29 +96,59 @@ export interface RunDeps { browser?: BrowserDeps; /** The report to the server after the run; the real one by default. */ report?: ReportDeps; + /** Starts `devCommand`; the real start by default. */ + startApp?: typeof startApp; } export function defaultRunDeps(): RunDeps { return { host: currentHost(), cwd: process.cwd(), runId: generateRunId }; } +/** A scenario about to run, and the run it became. */ +interface Planned { + file: string; + id: string; + scenario: Scenario; +} + +interface Finished { + result: RunResult; + runDir: string; + /** A render crash before the stage was recorded (an ordinary run only). */ + renderError?: unknown; +} + /** * `demoflow run `: validate, load the config, resolve the home, run * the engine, render, print. Input errors are thrown (exit 2); a finished run * returns its exit code. + * + * With `--check` (spec 360) nothing is recorded or rendered, and without a + * scenario every scenario of the project runs, one after another, against one + * start of the app: a check pass. */ export async function runCommand( - file: string, + file: string | undefined, flags: RunFlags, io: Io, deps: RunDeps = defaultRunDeps(), ): Promise { const { host } = deps; + const check = flags.check === true; + if (check) assertCheckFlags(file, flags); + else if (file === undefined) { + throw new CommandError( + 'missing-scenario', + 'a scenario is required; only --check runs every scenario of the project', + ); + } const port = flags.port === undefined ? undefined : portFlag(flags.port); const root = - flags.project === undefined - ? findProjectRoot(file) - : projectRootAt(path.resolve(deps.cwd, flags.project)); + flags.project !== undefined + ? projectRootAt(path.resolve(deps.cwd, flags.project)) + : file !== undefined + ? findProjectRoot(file) + : projectRootAbove(deps.cwd); const app = loadAppConfig(root, { port, baseUrl: flags.baseUrl, @@ -118,32 +168,39 @@ export async function runCommand( projectConfig: config, globalConfig: loadGlobalConfig(host), }); - const { scenario, warnings } = loadScenario( - file, - projectFragments(root, config.fragmentsDir), - ); - for (const warning of warnings) { - io.stderr(`warning ${warning.path}: ${warning.message}\n`); - } + const files = + file !== undefined + ? [file] + : await passFiles(root, config.fragmentsDir, flags.exclude ?? [], io); + const fragments = projectFragments(root, config.fragmentsDir); + const planned: Planned[] = files.map((scenarioFile) => { + const { scenario, warnings } = loadScenario(scenarioFile, fragments); + for (const warning of warnings) { + io.stderr(`warning ${warning.path}: ${warning.message}\n`); + } + return { file: scenarioFile, id: scenarioId(root, scenarioFile), scenario }; + }); - const runId = flags.runId ?? deps.runId(); - if (flags.runId !== undefined) assertSafeRunId(flags.runId); - const runDir = runDirFor(home, runId, host.platform); - if (flags.runId !== undefined && existsSync(runDir)) { - throw new CommandError( - 'invalid-run-id', - `run directory already exists: ${runDir}`, - ); + if (flags.runId !== undefined) { + assertSafeRunId(flags.runId); + const runDir = runDirFor(home, flags.runId, host.platform); + if (existsSync(runDir)) { + throw new CommandError( + 'invalid-run-id', + `run directory already exists: ${runDir}`, + ); + } } const fresh = flags.freshApp === true; if (fresh) await assertNoAppRunning(config.devCommand); // After every input check: a bad scenario never costs a browser download // or an app start. The start overlaps with the download; the app's output - // is kept and shown only if the start fails. + // is kept and shown only if the start fails. A check pass starts the app + // once for all its scenarios (spec 360, Decision 3). const progress = flags.progress === true; if (progress) emitProgress(io, { phase: 'app' }); const tail = outputTail(); - const starting: Promise = startApp(config, root, { + const starting: Promise = (deps.startApp ?? startApp)(config, root, { log: (text) => tail.push(text), env: app.port === undefined ? undefined : { PORT: String(app.port) }, fresh, @@ -167,43 +224,79 @@ export async function runCommand( return EXIT_STAGE_FAILED; } const started = await starting; - if ('error' in started) { - if (!(started.error instanceof AppStartError)) throw started.error; - return await reportAppNotReady({ - error: started.error, - output: tail.lines(), - scenarioName: scenario.name, - scenarioFile: scenarioId(root, file), - runId, - runDir, - json: flags.json === true, - io, - }); + if ('error' in started && !(started.error instanceof AppStartError)) { + throw started.error; } - return await execute(); + const output = tail.lines(); + if ('error' in started && output.length > 0) { + io.stderr(`${output.join('\n')}\n`); + } + const finished: Finished[] = []; + for (const [index, run] of planned.entries()) { + if (check && progress) { + emitCheckProgress(io, { run: index + 1, runs: planned.length }); + } + const runId = flags.runId ?? deps.runId(); + const runDir = runDirFor(home, runId, host.platform); + const done: Finished = + 'error' in started + ? { + result: await writeAppNotReady({ + error: started.error as AppStartError, + output, + scenarioName: run.scenario.name, + scenarioFile: run.id, + runId, + runDir, + }), + runDir, + } + : await execute(run, runId, runDir); + // Spec 361, Decision 4: the daemon watches this root's schedule. + if (index === 0) await recordRoot(root, host); + // A run whose app never answered was never reported before spec 360; a + // check reports it, so the stale state sees a readiness failure too. + if (!('error' in started) || check) { + await reportRun( + { root, runDir, result: done.result, check }, + deps.report ?? defaultReportDeps(host), + ); + } + if (check && flags.json !== true) { + io.stdout(formatCheckLine(done.result)); + } + finished.push(done); + } + return check ? concludeCheck(finished) : concludeRun(finished); } finally { process.off('SIGINT', onInterrupt); process.off('SIGTERM', onTerminate); await stopApp(); } - async function execute(): Promise { - io.stderr(`Running ${scenarioId(root, file)} (${runId})\n`); + async function execute( + run: Planned, + runId: string, + runDir: string, + ): Promise { + io.stderr(`${check ? 'Checking' : 'Running'} ${run.id} (${runId})\n`); // Secrets come from the process environment only (spec 5). - await runScenario(scenario, { + await runScenario(run.scenario, { runDir, baseUrl: flags.baseUrl ?? config.baseUrl, allowedOrigins: config.allowedOrigins, env: host.env, headless: flags.headed !== true, + endHoldMs: check ? 0 : config.render.endHoldMs, + record: !check, observers: [ - createCaptureObserver({ scenarioId: scenarioId(root, file), runId }), + createCaptureObserver({ scenarioId: run.id, runId }), ...(progress ? [progressObserver(io)] : []), ], }); // A recorded run is rendered whatever the verdict: the video of a failed - // verification is the evidence (ADR-0004 §4). + // verification is the evidence (ADR-0004 §4). A check recorded nothing. let renderError: unknown; if (existsSync(path.join(runDir, RUN_FILES.video))) { io.stderr('Rendering\n'); @@ -211,7 +304,7 @@ export async function runCommand( try { const plan = await buildRenderPlan( runDir, - renderPlanOptions(config, flags.format, scenario.description), + renderPlanOptions(config, flags.format, run.scenario.description), ); await ffmpegRenderer.render(runDir, plan); } catch (error) { @@ -227,19 +320,115 @@ export async function runCommand( : String(renderError); io.stderr(`rendering failed: ${reason}\n`); } + return { result, runDir, renderError }; + } + + /** An ordinary run: its result as before spec 360, and its exit code. */ + function concludeRun([done]: Finished[]): number { + if (done === undefined) return EXIT_STAGE_FAILED; + const { result, runDir, renderError } = done; io.stdout( flags.json === true ? formatJson(result, runDir) : formatSummary(result, runDir), ); - await reportRun( - { root, runDir, result }, - deps.report ?? defaultReportDeps(host), - ); const code = exitCodeFor(result); // A render crash before the stage was recorded must not read as success. return code === 0 && renderError !== undefined ? EXIT_STAGE_FAILED : code; } + + /** Spec 360, Decision 4: every run, then `0` when all passed, else `1`. */ + function concludeCheck(finished: Finished[]): number { + const failed = finished.filter( + ({ result }) => result.status === 'failed', + ).length; + io.stdout( + flags.json === true + ? formatCheckJson(finished) + : `${finished.length - failed} passed, ${failed} failed\n`, + ); + return failed === 0 ? 0 : EXIT_CHECK_FAILED; + } +} + +/** A check run or pass that failed: any run of it failed, whatever the stage. */ +export const EXIT_CHECK_FAILED = 1; + +/** + * Spec 360, Decision 2: what a check refuses before anything runs. A check + * renders nothing and nobody watches it. + */ +function assertCheckFlags(file: string | undefined, flags: RunFlags): void { + if (flags.format !== undefined) { + throw new CommandError( + 'check-flag', + '--check renders nothing: --format cannot be used with it', + ); + } + if (flags.headed === true) { + throw new CommandError( + 'check-flag', + '--check runs headless: --headed cannot be used with it', + ); + } + if (file === undefined && flags.runId !== undefined) { + throw new CommandError( + 'check-flag', + '--run-id names one run: give a scenario, or leave --run-id out of a check pass', + ); + } + if (file !== undefined && (flags.exclude ?? []).length > 0) { + throw new CommandError( + 'check-flag', + '--exclude applies to a check pass: leave out the scenario, or --exclude', + ); + } +} + +/** + * The scenarios of a check pass: the daemon's own listing (spec 247), sorted + * by id, minus `--exclude`. An excluded id that is no scenario is ignored: a + * deleted scenario must not break a schedule. An id may leave out the + * leading `demoflow/`. + */ +async function passFiles( + root: string, + fragmentsDir: string | undefined, + exclude: readonly string[], + io: Io, +): Promise { + // The listing compares real paths: a Windows 8.3 root would list nothing. + const real = await canonicalRoot(root); + const listing = await listScenarios(real, nodeLaunchFs, fragmentsDir); + const excluded = new Set( + exclude.map((id) => + id.startsWith(SCENARIOS_PREFIX) ? id : `${SCENARIOS_PREFIX}${id}`, + ), + ); + if (listing.skipped > 0) { + io.stderr( + `warning: ${listing.skipped} scenario file(s) left out: not valid (run demoflow validate)\n`, + ); + } + if (listing.truncated) { + io.stderr( + `warning: the project has more scenarios than a check pass runs (${LIST_MAX_SCENARIOS})\n`, + ); + } + return listing.scenarios + .filter((scenario) => !excluded.has(scenario.id)) + .map((scenario) => path.join(root, ...scenario.id.split('/'))); +} + +const SCENARIOS_PREFIX = 'demoflow/'; + +/** Never throws: a run's outcome does not depend on the local roots file. */ +async function recordRoot(root: string, host: HostEnvironment): Promise { + try { + recordLocalRoot(runnerPaths(host).localRoots, await canonicalRoot(root)); + } catch { + // A root not recorded waits for the next run. + } } type Started = { app: RunningApp } | { error: unknown }; @@ -282,24 +471,21 @@ async function assertNoAppRunning( /** * The app never answered, so the engine never ran: the CLI writes the run's * `result.json` itself — readiness failed with the app's last output, the - * other stages skipped (spec 5) — and prints it like any finished run. + * other stages skipped (spec 5) — and settles it like any finished run. */ -async function reportAppNotReady(input: { +async function writeAppNotReady(input: { error: AppStartError; output: string[]; scenarioName: string; scenarioFile: string; runId: string; runDir: string; - json: boolean; - io: Io; -}): Promise { - const { error, output, runDir, io } = input; +}): Promise { + const { error, output, runDir } = input; const message = output.length === 0 ? error.message : `${error.message}\nLast output of devCommand:\n${output.join('\n')}`; - if (output.length > 0) io.stderr(`${output.join('\n')}\n`); const now = new Date().toISOString(); const skipped = { status: 'skipped' } as const; const pending = { status: 'pending' } as const; @@ -333,9 +519,5 @@ async function reportAppNotReady(input: { path.join(runDir, RUN_FILES.result), `${JSON.stringify(unsettled, null, 2)}\n`, ); - const result = await settleRun(runDir); - io.stdout( - input.json ? formatJson(result, runDir) : formatSummary(result, runDir), - ); - return exitCodeFor(result); + return settleRun(runDir); } diff --git a/apps/cli/src/commands/scenario-file.ts b/apps/cli/src/commands/scenario-file.ts index 4a9b4ccc..5aaab559 100644 --- a/apps/cli/src/commands/scenario-file.ts +++ b/apps/cli/src/commands/scenario-file.ts @@ -1,4 +1,5 @@ -import { readFileSync } from 'node:fs'; +import { existsSync, readFileSync } from 'node:fs'; +import path from 'node:path'; import { type FragmentSource, type Issue, @@ -40,3 +41,22 @@ export function loadScenario( } return { scenario: parsed.value, warnings: parsed.warnings }; } + +/** + * The scenario's `description`, read from the project when its file is still + * there and loads (spec 255, Decision 3): the intro card's subtitle and a + * share's description (spec 349). An inline scenario of `demoflow mcp` is + * gone after its run, so it has none. + */ +export function scenarioDescription( + root: string, + scenarioId: string, +): string | undefined { + const file = path.join(root, scenarioId); + if (!existsSync(file)) return undefined; + try { + return loadScenario(file).scenario.description; + } catch { + return undefined; + } +} diff --git a/apps/cli/src/commands/share.ts b/apps/cli/src/commands/share.ts new file mode 100644 index 00000000..65878d5e --- /dev/null +++ b/apps/cli/src/commands/share.ts @@ -0,0 +1,238 @@ +import { existsSync } from 'node:fs'; +import { createInterface } from 'node:readline/promises'; +import { Writable } from 'node:stream'; +import { setTimeout as delay } from 'node:timers/promises'; +import type { RenderFormat } from '@demoflow/core'; +import { ServerError } from '../auth/client.ts'; +import { readCredentials } from '../auth/credentials.ts'; +import { currentHost, type HostEnvironment } from '../config/index.ts'; +import { runnerPaths } from '../runner/paths.ts'; +import { + createShare, + DEFAULT_SHARE_LIFETIME, + PASSWORD_MAX_LENGTH, + PASSWORD_MIN_LENGTH, + reportedRunOf, + SHARE_LIFETIMES, + type Share, + type ShareLifetime, + ShareRefused, + shareableFormat, + type WaitDeps, + waitForShare, +} from '../share/publish.ts'; +import { CommandError, EXIT_STAGE_FAILED } from './errors.ts'; +import type { Io } from './io.ts'; +import { locateRun } from './render.ts'; + +/** Decision 6: the upload is polled for up to 10 minutes. */ +export const SHARE_WAIT_MS = 10 * 60_000; +/** How often a long upload says it is still going. */ +const PROGRESS_EVERY_MS = 15_000; +/** Decision 6: where `--password` reads the password without a terminal. */ +export const PASSWORD_ENV = 'DEMOFLOW_SHARE_PASSWORD'; + +export interface ShareFlags { + format?: RenderFormat; + /** `--expires`: 7, 30 or 90 days. */ + expires?: string; + /** `--password`: asked for, never a flag value (shell history). */ + password?: boolean; + json?: boolean; + out?: string; + /** Hidden: the project root, instead of searching up from the cwd. */ + project?: string; +} + +export interface ShareCommandDeps extends WaitDeps { + host: HostEnvironment; + cwd: string; + /** The password typed without echo, or `null` without a terminal. */ + askPassword(): Promise; +} + +export const defaultShareDeps = (): ShareCommandDeps => ({ + host: currentHost(), + cwd: process.cwd(), + fetch: (input, init) => fetch(input, init), + sleep: (ms) => delay(ms), + now: () => Date.now(), + askPassword: askPasswordOnTerminal, +}); + +/** + * `demoflow share ` (spec 349, Decision 6): a share link for a render + * of a run this machine reported. The runner uploads the file; this command + * asks the server for the share and waits until it is ready. Refusals are + * `CommandError`s (exit 2); an upload that fails exits 1. + */ +export async function shareCommand( + runId: string, + flags: ShareFlags, + io: Io, + deps: ShareCommandDeps = defaultShareDeps(), +): Promise { + const expiresInDays = parseLifetime(flags.expires); + const credentials = readCredentials(deps.host); + if (credentials === null) { + throw new CommandError( + 'not-logged-in', + 'this machine is not logged in: run demoflow login', + ); + } + const { runDir } = locateRun(runId, flags, deps); + if (!existsSync(runDir)) { + throw new CommandError('run-not-found', `no run directory at ${runDir}`); + } + + try { + const format = await shareableFormat(runDir, flags.format); + const reported = await reportedRunOf( + runnerPaths(deps.host).reportedRuns, + runDir, + credentials.server, + ); + const password = + flags.password === true ? await readPassword(deps) : undefined; + + io.stderr(`Sharing the ${format} render of ${runId}\n`); + const { share, existing } = await createShare( + credentials, + { + runId: reported.id, + format, + expiresInDays, + ...(password === undefined ? {} : { password }), + }, + deps.fetch, + ); + if (existing) { + io.stderr( + `This render is already shared ${share.view.hasPassword === true ? 'with' : 'without'} a password; that link is kept (revoke it on the website to share it again)\n`, + ); + } else if (share.status === 'uploading') { + io.stderr('The runner is uploading the render\n'); + } + + let reportedAt = 0; + const settled = await waitForShare(credentials, share, deps, { + timeoutMs: SHARE_WAIT_MS, + onPoll: (current, elapsedMs) => { + if (current.status !== 'uploading') return; + if (elapsedMs - reportedAt < PROGRESS_EVERY_MS) return; + reportedAt = elapsedMs; + io.stderr(`Still uploading (${Math.round(elapsedMs / 1000)} s)\n`); + }, + }); + return printShare(settled, flags, io); + } catch (error) { + if (error instanceof ShareRefused) { + throw new CommandError(error.code, error.message); + } + if (error instanceof ServerError) { + io.stderr(`${error.message}\n`); + return EXIT_STAGE_FAILED; + } + throw error; + } +} + +/** The link on stdout, and the file link when there is no password. */ +function printShare(share: Share, flags: ShareFlags, io: Io): number { + if (flags.json === true) { + io.stdout(`${JSON.stringify(share.view, null, 2)}\n`); + } + switch (share.status) { + case 'ready': + if (flags.json !== true) { + io.stdout(`${share.url}\n`); + if (share.fileUrl !== null) io.stdout(`${share.fileUrl}\n`); + } + io.stderr( + `Ready; the link works until ${share.expiresAt.slice(0, 10)}\n`, + ); + return 0; + case 'uploading': + io.stderr( + `The upload is still going after ${SHARE_WAIT_MS / 60_000} minutes; the link works once it is done: ${share.url}\n`, + ); + return 1; + case 'failed': + case 'lost': + io.stderr(`The upload failed: ${share.failure ?? share.status}\n`); + return 1; + } +} + +function parseLifetime(value: string | undefined): ShareLifetime { + if (value === undefined) return DEFAULT_SHARE_LIFETIME; + const days = SHARE_LIFETIMES.find((known) => String(known) === value); + if (days === undefined) { + throw new CommandError( + 'invalid-expires', + `--expires must be ${SHARE_LIFETIMES.join(', ')} days, not ${JSON.stringify(value)}`, + ); + } + return days; +} + +/** Decision 6: typed on a terminal, else `DEMOFLOW_SHARE_PASSWORD`. */ +async function readPassword(deps: ShareCommandDeps): Promise { + const typed = await deps.askPassword(); + const password = typed ?? deps.host.env[PASSWORD_ENV]; + if (password === undefined || password === '') { + throw new CommandError( + 'invalid-password', + typed === null + ? `--password without a terminal reads ${PASSWORD_ENV}, which is not set` + : 'the password is empty', + ); + } + if ( + password.length < PASSWORD_MIN_LENGTH || + password.length > PASSWORD_MAX_LENGTH + ) { + throw new CommandError( + 'invalid-password', + `the password must be ${PASSWORD_MIN_LENGTH}–${PASSWORD_MAX_LENGTH} characters`, + ); + } + return password; +} + +/** Typed twice without echo; no terminal, no question. */ +async function askPasswordOnTerminal(): Promise { + if (process.stdin.isTTY !== true || process.stderr.isTTY !== true) { + return null; + } + let muted = false; + // The prompt itself is shown; what is typed after it is not. + const output = new Writable({ + write(chunk, _encoding, done) { + if (!muted) process.stderr.write(chunk); + done(); + }, + }); + const prompt = createInterface({ + input: process.stdin, + output, + terminal: true, + }); + const ask = async (question: string): Promise => { + muted = false; + const answer = prompt.question(question); + muted = true; + const typed = await answer; + process.stderr.write('\n'); + return typed; + }; + try { + const password = await ask('Password for the share: '); + if ((await ask('Again: ')) !== password) { + throw new CommandError('invalid-password', 'the passwords differ'); + } + return password; + } finally { + prompt.close(); + } +} diff --git a/apps/cli/src/main.ts b/apps/cli/src/main.ts index 1140a8ec..73d57583 100644 --- a/apps/cli/src/main.ts +++ b/apps/cli/src/main.ts @@ -27,6 +27,11 @@ import { type RenderFlags, renderCommand } from './commands/render.ts'; import { runCommand } from './commands/run.ts'; import { registerRunsCommands } from './commands/runs.ts'; import { setupCommand } from './commands/setup.ts'; +import { + PASSWORD_ENV, + type ShareFlags, + shareCommand, +} from './commands/share.ts'; import { validateCommand } from './commands/validate.ts'; import { mcpCommand } from './mcp/command.ts'; import { @@ -67,8 +72,13 @@ export async function main( program .command('run') - .description('Run a scenario and render its video') - .argument('', 'path to a .demo.json file') + .description( + 'Run a scenario and render its video; with --check, run and verify only', + ) + .argument( + '[scenario]', + 'path to a .demo.json file; required unless --check, which then runs every scenario', + ) .option('--base-url ', 'override baseUrl from demoflow.config.json') .option( '--port ', @@ -88,6 +98,16 @@ export async function main( `a file to render, repeatable; replaces render.formats (${RENDER_FORMATS.join(', ')})`, ).argParser(collectFormat), ) + .option( + '--check', + 'run and verify against the live app, recording and rendering nothing (demoflow check only validates files)', + ) + .addOption( + new Option( + '--exclude ', + 'with --check and no scenario: leave out this scenario id, repeatable', + ).argParser(collectExclude), + ) .addOption(new Option('--run-id ', 'name the run directory').hideHelp()) .addOption(new Option('--project ', 'the project root').hideHelp()) .addOption( @@ -98,7 +118,7 @@ export async function main( ) .action( async ( - scenario: string, + scenario: string | undefined, flags: { baseUrl?: string; port?: string; @@ -107,11 +127,19 @@ export async function main( headed?: boolean; json?: boolean; format?: RenderFormat[]; + check?: boolean; + exclude?: string[]; runId?: string; project?: string; progress?: boolean; }, + command: Command, ) => { + if (scenario === undefined && flags.check !== true) { + command.error("error: missing required argument 'scenario'", { + code: 'commander.missingArgument', + }); + } exitCode = await runCommand(scenario, flags, io); }, ); @@ -133,6 +161,32 @@ export async function main( exitCode = await renderCommand(runId, flags, io); }); + program + .command('share') + .description('Share a render of a reported run as a link on the website') + .argument('', 'the run directory under /runs/') + .addOption( + new Option( + '--format ', + `the render to share (default: the run's first; ${RENDER_FORMATS.join(', ')})`, + ).argParser(parseFormat), + ) + .option( + '--expires ', + 'how long the link works: 7, 30 or 90 days', + '30', + ) + .option( + '--password', + `protect the link with a password, asked for on the terminal (else ${PASSWORD_ENV})`, + ) + .option('--json', 'print the share as one JSON object on stdout') + .option('--out ', 'artifacts directory (the home of runs/)') + .addOption(new Option('--project ', 'the project root').hideHelp()) + .action(async (runId: string, flags: ShareFlags) => { + exitCode = await shareCommand(runId, flags, io); + }); + program .command('list') .description('List recent runs of the current project, newest first') @@ -195,7 +249,7 @@ export async function main( .description('Pair this machine with your DemoFlow account') .option( '--server ', - 'the DemoFlow server (default: DEMOFLOW_SERVER or staging)', + 'the DemoFlow server (default: DEMOFLOW_SERVER or production)', ) .option('--force', 'pair again, revoking the stored token first') .option('--start', 'start the runner afterwards, without asking') @@ -265,6 +319,15 @@ function collectFormat( return [...previous, format]; } +/** `--format` of `share`: one known format. */ +function parseFormat(value: string): RenderFormat { + return collectFormat(value)[0] as RenderFormat; +} + +function collectExclude(value: string, previous: string[] = []): string[] { + return [...previous, value]; +} + function isCommanderError(error: unknown): error is CommanderError { return error instanceof CommanderError; } diff --git a/apps/cli/src/mcp/errors.ts b/apps/cli/src/mcp/errors.ts index 05665b58..0a8529f3 100644 --- a/apps/cli/src/mcp/errors.ts +++ b/apps/cli/src/mcp/errors.ts @@ -1,4 +1,5 @@ import type { Issue } from '@demoflow/core'; +import type { ShareRefusalCode } from '../share/publish.ts'; export type ToolErrorCode = | 'invalid-input' @@ -8,7 +9,12 @@ export type ToolErrorCode = | 'invalid-scenario' | 'scenario-unreadable' | 'file-not-in-project' - | 'task-not-found'; + | 'task-not-found' + // `share_demo` (spec 349, Decision 7). + | 'share-not-allowed' + | 'share-failed' + | 'server-error' + | ShareRefusalCode; /** What an agent gets back from a failed tool call (spec #68, Decision 9). */ export interface ToolErrorPayload { diff --git a/apps/cli/src/mcp/server.ts b/apps/cli/src/mcp/server.ts index 1ab8d83c..9e3fb975 100644 --- a/apps/cli/src/mcp/server.ts +++ b/apps/cli/src/mcp/server.ts @@ -1,6 +1,7 @@ import { existsSync } from 'node:fs'; import { mkdir, writeFile } from 'node:fs/promises'; import path from 'node:path'; +import { setTimeout as delay } from 'node:timers/promises'; import { formatPath, type Issue, @@ -15,6 +16,8 @@ import { type Tool, } from '@modelcontextprotocol/sdk/types.js'; import { z } from 'zod'; +import { ServerError } from '../auth/client.ts'; +import { readCredentials } from '../auth/credentials.ts'; import { CommandError } from '../commands/errors.ts'; import { assertSafeRunId, generateRunId } from '../commands/run-id.ts'; import { readResult } from '../commands/settle.ts'; @@ -22,6 +25,7 @@ import { CliError, type HostEnvironment, loadGlobalConfig, + loadProjectConfig, osDataDir, projectFragments, runDir as runDirFor, @@ -31,6 +35,17 @@ import { pageUrlFor, readReportedRunsByDir, } from '../runs-report/reported-runs.ts'; +import { + createShare, + DEFAULT_SHARE_LIFETIME, + reportedRunOf, + type ShareRefusalCode, + ShareRefused, + type ShareStatus, + shareableFormat, + type WaitDeps, + waitForShare, +} from '../share/publish.ts'; import { packageVersion } from '../version.ts'; import { ToolError } from './errors.ts'; import { @@ -155,6 +170,40 @@ const CheckProjectInput = z.strictObject({ project: z.string().optional().describe(PROJECT_DESCRIPTION), }); +const ShareDemoInput = z.strictObject({ + runId: z + .string() + .describe('The task ID of a finished run (generate_demo, list_demos).'), + format: z + .enum(RENDER_FORMATS) + .optional() + .describe("The render to share; default: the run's first render."), + expiresInDays: z + .union([z.literal(7), z.literal(30), z.literal(90)]) + .optional() + .describe( + `How long the link works, in days; default ${DEFAULT_SHARE_LIFETIME}.`, + ), +}); + +/** What an agent can do about each refusal of the share API. */ +const SHARE_HINTS: Record = { + 'not-logged-in': 'Ask a person to run `demoflow login` on this machine.', + 'invalid-result': 'The run directory is damaged; generate the demo again.', + 'run-not-reported': + 'Only a run made while logged in, with reporting on, can be shared: generate the demo again after `demoflow login`.', + 'format-not-rendered': + 'Pass a format the run rendered (get_task lists the files), or generate the demo again with that format.', + 'artifacts-deleted': + "The run's files were deleted on the website; generate the demo again.", + 'runner-offline': + 'Ask a person to start the runner: `demoflow runner start`.', + 'runner-outdated': + 'Ask a person to restart the runner: `demoflow runner stop`, then `demoflow runner start`.', + 'file-too-large': 'Share a smaller render, or make a shorter demo.', + 'quota-exceeded': 'Ask a person to revoke old shares on the website.', +}; + const TOOLS: Tool[] = [ { name: 'generate_demo', @@ -191,6 +240,11 @@ const TOOLS: Tool[] = [ 'Whether a project is ready to run demos: its scenario files, a valid config, and FFmpeg and browser availability.', inputSchema: inputSchema(CheckProjectInput), }, + { + name: 'share_demo', + description: `Publish a finished run's render as a link on the DemoFlow website: anyone with the link can watch it, until it expires. Refused with share-not-allowed unless the project's committed demoflow.config.json sets "share": { "allowMcp": true }. Needs a logged-in machine with its runner running. Waits up to ${MAX_WAIT_SECONDS} s for the upload; on status "uploading", call again with the same arguments to wait more — it returns the same share.`, + inputSchema: inputSchema(ShareDemoInput), + }, ]; export interface McpServerDeps { @@ -199,12 +253,21 @@ export interface McpServerDeps { /** The default project, and where `DEMOFLOW_HOME` is resolved. */ cwd: string; newTaskId?: () => string; + /** `share_demo`'s requests and clock; tests replace them. */ + share?: WaitDeps; } +const processShareDeps: WaitDeps = { + fetch: (input, init) => fetch(input, init), + sleep: (ms) => delay(ms), + now: () => Date.now(), +}; + /** The `demoflow mcp` server, before it is connected to a transport. */ export function createMcpServer(deps: McpServerDeps): Server { const { queue, host, cwd } = deps; const newTaskId = deps.newTaskId ?? (() => generateRunId()); + const shareDeps = deps.share ?? processShareDeps; /** Homes of the projects used by this server: where a finished task is found. */ const homes = new Set(); @@ -235,6 +298,8 @@ export function createMcpServer(deps: McpServerDeps): Server { return ok( await checkProject(parse(CheckProjectInput, args), cwd, host), ); + case 'share_demo': + return ok(await shareDemo(parse(ShareDemoInput, args))); default: throw new ToolError( 'invalid-input', @@ -386,6 +451,100 @@ export function createMcpServer(deps: McpServerDeps): Server { ); } + /** + * Spec 349, Decision 7: what `demoflow share` does, without a password, + * and only for a project whose committed config allows it. + */ + async function shareDemo(input: z.infer): Promise<{ + shareId: string; + url: string; + fileUrl: string | null; + expiresAt: string; + status: ShareStatus; + existing: boolean; + }> { + checkTaskId(input.runId); + const runDir = locateRunDir(input.runId); + const indexFile = runnerPaths(host).reportedRuns; + const entry = (await readReportedRunsByDir(indexFile)).get( + path.resolve(runDir), + ); + if (entry === undefined) { + throw shareRefusal( + new ShareRefused( + 'run-not-reported', + 'the run was never reported to the server: it was made signed out or with reporting off', + ), + ); + } + checkShareAllowed(entry.root); + try { + const credentials = readCredentials(host); + if (credentials === null) { + throw new ShareRefused( + 'not-logged-in', + 'this machine is not logged in', + ); + } + await reportedRunOf(indexFile, runDir, credentials.server); + const format = await shareableFormat(runDir, input.format); + const created = await createShare( + credentials, + { + runId: entry.id, + format, + expiresInDays: input.expiresInDays ?? DEFAULT_SHARE_LIFETIME, + }, + shareDeps.fetch, + ); + const share = await waitForShare(credentials, created.share, shareDeps, { + timeoutMs: MAX_WAIT_SECONDS * 1000, + }); + if (share.status === 'failed' || share.status === 'lost') { + throw new ToolError( + 'share-failed', + `the upload failed: ${share.failure ?? share.status}`, + 'Call share_demo again to retry: a failed share is replaced.', + ); + } + return { + shareId: share.id, + url: share.url, + fileUrl: share.fileUrl, + expiresAt: share.expiresAt, + status: share.status, + existing: created.existing, + }; + } catch (error) { + if (error instanceof ShareRefused) throw shareRefusal(error); + if (error instanceof ServerError || error instanceof CliError) { + throw new ToolError( + 'server-error', + error.message, + error instanceof ServerError && error.unreachable + ? 'Check the network and the DemoFlow server, then call again.' + : 'Ask a person to check `demoflow login` on this machine.', + ); + } + throw error; + } + } + + /** The run directory of a task of this server, or of an earlier one. */ + function locateRunDir(taskId: string): string { + const task = queue.get(taskId); + if (task !== undefined) return task.spec.runDir; + for (const home of candidateHomes()) { + const runDir = runDirFor(home, taskId, host.platform); + if (existsSync(runDir)) return runDir; + } + throw new ToolError( + 'task-not-found', + `no run ${taskId}`, + 'Use a task ID returned by generate_demo or list_demos.', + ); + } + /** Every home this server wrote to, then where an earlier one would have. */ function candidateHomes(): Set { const env = host.env.DEMOFLOW_HOME; @@ -404,6 +563,34 @@ export function createMcpServer(deps: McpServerDeps): Server { return server; } +/** + * ADR-0019 §6: an agent may publish a project's videos only when the + * project's own, reviewed config says so. A run without a root, or a config + * that cannot be read, is refused the same way. + */ +function checkShareAllowed(root: string | undefined): void { + let allowed = false; + if (root !== undefined) { + try { + allowed = loadProjectConfig(root).share?.allowMcp === true; + } catch (error) { + if (!(error instanceof CliError)) throw error; + } + } + if (allowed) return; + throw new ToolError( + 'share-not-allowed', + root === undefined + ? 'the run was reported without its project root, so no project allows sharing it' + : `the project at ${root} does not let the MCP server share its demos`, + 'A person can share it with `demoflow share `. To let agents share this project\'s demos, a person sets "share": { "allowMcp": true } in its demoflow.config.json and commits it.', + ); +} + +function shareRefusal(error: ShareRefused): ToolError { + return new ToolError(error.code, error.message, SHARE_HINTS[error.code]); +} + /** Task IDs become directory names: no separators, no `..`. */ function checkTaskId(taskId: string): void { try { diff --git a/apps/cli/src/runner/command.ts b/apps/cli/src/runner/command.ts index 5b61c93f..de493e83 100644 --- a/apps/cli/src/runner/command.ts +++ b/apps/cli/src/runner/command.ts @@ -10,6 +10,11 @@ import { } from './control.ts'; import { defaultDaemonDeps, runForeground } from './daemon.ts'; import { installCommand, uninstallCommand } from './install/commands.ts'; +import { + allowCommand, + disallowCommand, + rootsCommand, +} from './roots/commands.ts'; /** * `demoflow runner`: the background daemon of spec 151. The hidden @@ -79,5 +84,31 @@ export function registerRunnerCommands( setExit(uninstallCommand(io, defaultControlDeps())); }); + const rootsDeps = () => ({ host: currentHost(), cwd: process.cwd() }); + + runner + .command('allow') + .description('Let the website name folders inside this directory') + .argument('', 'an existing directory') + .action((dir: string) => { + setExit(allowCommand(dir, io, rootsDeps())); + }); + + runner + .command('disallow') + .description('Stop the website naming folders inside this directory') + .argument('', 'an allowed root') + .action((dir: string) => { + setExit(disallowCommand(dir, io, rootsDeps())); + }); + + runner + .command('roots') + .description('List the directories the website may name folders in') + .option('--json', 'print the roots as one JSON object') + .action((flags: { json?: boolean }) => { + setExit(rootsCommand(flags, io, rootsDeps())); + }); + return runner; } diff --git a/apps/cli/src/runner/connection.ts b/apps/cli/src/runner/connection.ts index 38d1d803..e8af70e8 100644 --- a/apps/cli/src/runner/connection.ts +++ b/apps/cli/src/runner/connection.ts @@ -7,6 +7,8 @@ import { } from './launch/index.ts'; import type { Logger } from './log.ts'; import type { RelaySender, VideoRelayPort } from './relay/index.ts'; +import type { ProjectsPort } from './roots/index.ts'; +import type { SharePort } from './share/prepare.ts'; import type { ConnectionState } from './state.ts'; /** Decision 6: each side pings every 25 s and gives up after 60 s of silence. */ @@ -77,6 +79,18 @@ export interface ConnectionDeps { artifacts?: ArtifactsPort; /** Answers `scenarios.list` and `run.*` (spec 248); without it, logged as unknown. */ launches?: LaunchPort; + /** + * Spec 361, Decision 6: the current `schedules.sync`, sent after + * `launches.sync`; `null` while the scheduler has not looked yet. + */ + schedules?: () => Record | null; + /** Answers `project.*` (spec 377); without it, logged as unknown. */ + projects?: ProjectsPort; + /** + * Answers `share.prepare` (spec 349); without it, logged as unknown. Its + * poster goes out through `relay`, so it is announced only with one. + */ + share?: SharePort; } export type ConnectionOutcome = 'stopped' | 'revoked'; @@ -149,6 +163,13 @@ export class RunnerConnection { return this.done; } + /** Spec 361, Decision 6: a watched schedule changed; tell an open socket. */ + syncSchedules(): void { + if (!this.opened || this.stopping) return; + const frame = this.deps.schedules?.() ?? null; + if (frame !== null) this.send(frame); + } + private connect(): void { if (this.stopping) return; this.opened = false; @@ -183,11 +204,26 @@ export class RunnerConnection { // Spec 326, Decision 1: a connection without a launcher ignores run.start, // so it claims no options. const runOptions = this.deps.launches !== undefined ? [...RUN_OPTIONS] : []; - this.send({ type: 'launches.sync', running, runOptions }); + // Spec 377, Decision 9: what else this daemon answers. + const features: string[] = []; + if (this.deps.projects !== undefined) features.push('projects'); + // Spec 349, Decision 5: a share needs `share.prepare` and the poster's stream. + if (this.deps.share !== undefined && this.deps.relay !== undefined) { + features.push('share'); + } + this.send({ + type: 'launches.sync', + running, + runOptions, + features, + // Spec 360, Decision 6: a daemon that handles launches runs check passes. + ...(this.deps.launches !== undefined ? { checks: true } : {}), + }); this.deps.logger.log( 'info', `announced ${running.length} running launch${running.length === 1 ? '' : 'es'}`, ); + this.syncSchedules(); this.resetIdle(); this.schedulePing(); } @@ -254,6 +290,20 @@ export class RunnerConnection { return; } break; + case 'project.inspect': + case 'project.init': + case 'project.register': + if (this.deps.projects !== undefined) { + this.deps.projects.handle(fields, this.relaySender()); + return; + } + break; + case 'share.prepare': + if (this.deps.share !== undefined) { + this.deps.share.handle(fields, this.relaySender()); + return; + } + break; } this.deps.logger.log( 'warn', diff --git a/apps/cli/src/runner/control.ts b/apps/cli/src/runner/control.ts index fba7ffb1..f6d7e1a6 100644 --- a/apps/cli/src/runner/control.ts +++ b/apps/cli/src/runner/control.ts @@ -11,6 +11,7 @@ import { writeFileSync, } from 'node:fs'; import { setTimeout as delay } from 'node:timers/promises'; +import { nextDueSlot, type ScheduleConfig } from '@demoflow/core'; import { readCredentials } from '../auth/credentials.ts'; import type { Io } from '../commands/io.ts'; import { currentHost, type HostEnvironment } from '../config/platform.ts'; @@ -20,6 +21,8 @@ import { platformAutostart } from './install/autostart.ts'; import { isAlive, type LockHolder, readLock, releaseLock } from './lock.ts'; import { redact } from './log.ts'; import { type RunnerPaths, runnerPaths } from './paths.ts'; +import { hasScheduledRoot } from './schedule/roots.ts'; +import { type LastPass, readScheduleState } from './schedule/state.ts'; import { type ConnectionState, type RunnerState, readState } from './state.ts'; /** `start` waits this long for the child to take the lock. */ @@ -64,7 +67,11 @@ export const defaultControlDeps = (): ControlDeps => ({ /** `demoflow runner start`: spawns the daemon detached, its output into the log. */ export async function startRunner(io: Io, deps: ControlDeps): Promise { const paths = runnerPaths(deps.host); - if (readCredentials(deps.host) === null) { + // Spec 361, Decision 5: a scheduled root runs without an account. + if ( + readCredentials(deps.host) === null && + !hasScheduledRoot(paths, deps.host.env) + ) { io.stderr('Not logged in; run demoflow login first\n'); return 1; } @@ -183,6 +190,18 @@ export interface RunnerStatus { retryInSeconds: number | null; server: string | null; autostart: boolean; + /** Spec 361: each scheduled project, as the daemon last saw it. */ + schedules: ScheduledProject[]; +} + +export interface ScheduledProject { + rootPath: string; + schedule: ScheduleConfig; + /** The machine's zone the times are in. */ + zone: string; + /** `null` while the runner is not running: nothing will run it. */ + nextCheckAt: string | null; + lastCheck: LastPass | null; } export function runnerStatus(deps: ControlDeps): RunnerStatus { @@ -215,11 +234,40 @@ export function runnerStatus(deps: ControlDeps): RunnerStatus { : Math.max(0, Math.floor((deps.now() - started) / 1000)), connection, retryInSeconds, - server: credentials?.server ?? (own ? state.server : null), + server: credentials?.server ?? (own ? (state.server ?? null) : null), autostart: deps.autostart.installed(deps.host), + schedules: scheduledProjects(paths, holder !== null, deps.now()), }; } +/** From `schedule.state.json`: the daemon's view, no config is read here. */ +function scheduledProjects( + paths: RunnerPaths, + running: boolean, + now: number, +): ScheduledProject[] { + const saved = readScheduleState(paths.scheduleState); + if (saved === null) return []; + const projects: ScheduledProject[] = []; + for (const [rootPath, entry] of Object.entries(saved.roots)) { + if (entry.schedule === undefined) continue; + let next: number | null = null; + try { + next = running ? nextDueSlot(entry.schedule, now, saved.zone) : null; + } catch { + // A zone this Node does not know: no time to show. + } + projects.push({ + rootPath, + schedule: entry.schedule, + zone: saved.zone, + nextCheckAt: next === null ? null : new Date(next).toISOString(), + lastCheck: entry.last ?? null, + }); + } + return projects; +} + /** `demoflow runner status [--json]`. */ export function statusCommand( flags: { json?: boolean }, @@ -233,14 +281,66 @@ export function statusCommand( } const lines = [ `Runner: ${status.running ? `running (PID ${status.pid}${status.uptimeSeconds === null ? '' : `, up ${formatDuration(status.uptimeSeconds)}`})` : 'not running'}`, - `Connection: ${status.connection === 'reconnecting' && status.retryInSeconds !== null ? `reconnecting in ${status.retryInSeconds}s` : status.connection}`, + `Connection: ${connectionText(status)}`, `Server: ${status.server ?? 'not logged in'}`, `Autostart: ${status.autostart ? 'installed' : 'not installed'}`, ]; + for (const project of status.schedules) { + lines.push( + `Schedule: ${project.rootPath} (${scheduleText(project.schedule)})`, + ` next check: ${project.nextCheckAt === null ? 'when the runner is running' : localTime(project.nextCheckAt, project.zone)}`, + ` last check: ${lastCheckText(project.lastCheck, project.zone)}`, + ); + } io.stdout(`${redact(lines.join('\n'))}\n`); return 0; } +function connectionText(status: RunnerStatus): string { + if (status.connection === 'local') return 'local mode: scheduled checks only'; + if (status.connection === 'reconnecting' && status.retryInSeconds !== null) { + return `reconnecting in ${status.retryInSeconds}s`; + } + return status.connection; +} + +function scheduleText(schedule: ScheduleConfig): string { + const when = + schedule.cadence === 'weekly' + ? `weekly on ${schedule.weekday ?? '?'}` + : schedule.cadence; + const excluded = schedule.exclude?.length ?? 0; + return `${when} at ${schedule.at}${excluded === 0 ? '' : `, ${excluded} excluded`}`; +} + +function lastCheckText(last: LastPass | null, zone: string): string { + if (last === null) return 'none yet'; + const at = localTime(last.startedAt, zone); + if (last.outcome === undefined) return `${at}, no result yet`; + if (last.outcome !== 'finished') return `${at}, ${last.outcome}`; + return `${at}, ${last.passed ?? 0} passed, ${last.failed ?? 0} failed`; +} + +/** `YYYY-MM-DD HH:MM` in the daemon's zone; the ISO time when the zone is unknown. */ +function localTime(iso: string, zone: string): string { + try { + const parts = new Intl.DateTimeFormat('en-US', { + timeZone: zone, + hourCycle: 'h23', + year: 'numeric', + month: '2-digit', + day: '2-digit', + hour: '2-digit', + minute: '2-digit', + }).formatToParts(new Date(iso)); + const part = (type: Intl.DateTimeFormatPartTypes) => + parts.find((entry) => entry.type === type)?.value ?? ''; + return `${part('year')}-${part('month')}-${part('day')} ${part('hour')}:${part('minute')}`; + } catch { + return iso; + } +} + /** `demoflow runner logs [--follow]`: the current `runner.log`, then what is appended. */ export async function logsCommand( flags: { follow?: boolean }, diff --git a/apps/cli/src/runner/daemon.ts b/apps/cli/src/runner/daemon.ts index 9946fe4d..d6c7b330 100644 --- a/apps/cli/src/runner/daemon.ts +++ b/apps/cli/src/runner/daemon.ts @@ -18,6 +18,19 @@ import { acquireLock, releaseLock } from './lock.ts'; import { fileLogger, type Logger } from './log.ts'; import { runnerPaths } from './paths.ts'; import { nodeRelayFs, VideoRelay } from './relay/index.ts'; +import { + defaultWriteTemplate, + isWebsiteRegistered, + ProjectRequests, +} from './roots/index.ts'; +import { + hasScheduledRoot, + machineZone, + scheduleLoader, + watchedRoots, +} from './schedule/roots.ts'; +import { Scheduler } from './schedule/scheduler.ts'; +import { sharePreparer } from './share/prepare.ts'; import { type ConnectionState, writeState } from './state.ts'; /** How often the daemon looks for `runner.stop`. */ @@ -43,6 +56,8 @@ export interface DaemonDeps { spawn: SpawnRun; /** Mirror log lines here as well, e.g. a terminal running `--foreground`. */ echo?: (line: string) => void; + /** The machine's IANA zone for schedules (spec 361); the system's when absent. */ + zone?: () => string; } export const defaultDaemonDeps = (host: HostEnvironment): DaemonDeps => ({ @@ -60,12 +75,14 @@ export const defaultDaemonDeps = (host: HostEnvironment): DaemonDeps => ({ /** * `demoflow runner --foreground`: the daemon itself. `runner start` spawns it - * detached; the autostart units run it directly. One per machine. + * detached; the autostart units run it directly. One per machine. Without an + * account it runs scheduled checks only, when a watched root has a schedule + * (spec 361, Decision 5). */ export async function runForeground(io: Io, deps: DaemonDeps): Promise { const paths = runnerPaths(deps.host); const credentials = readCredentials(deps.host); - if (credentials === null) { + if (credentials === null && !hasScheduledRoot(paths, deps.host.env)) { io.stderr('Not logged in; run demoflow login first\n'); return EXIT_DAEMON_NOT_LOGGED_IN; } @@ -80,16 +97,19 @@ export async function runForeground(io: Io, deps: DaemonDeps): Promise { now: () => new Date(deps.clock.now()), ...(deps.echo === undefined ? {} : { echo: deps.echo }), }); - logger.log('info', `runner started (PID ${deps.pid})`); + logger.log( + 'info', + `runner started (PID ${deps.pid})${credentials === null ? ', local mode: scheduled checks only' : ''}`, + ); let current: { state: ConnectionState; retryAt?: number } = { - state: 'connecting', + state: credentials === null ? 'local' : 'connecting', }; const saveState = () => { writeState(paths.state, { pid: deps.pid, startedAt, - server: credentials.server, + ...(credentials === null ? {} : { server: credentials.server }), state: current.state, ...(current.retryAt === undefined ? {} @@ -99,54 +119,101 @@ export async function runForeground(io: Io, deps: DaemonDeps): Promise { }; saveState(); - const relay = new VideoRelay({ - indexFile: paths.reportedRuns, - fs: nodeRelayFs, - logger, - platform: deps.host.platform, - }); - const launcher = new Launcher({ - indexFile: paths.reportedRuns, - fs: nodeLaunchFs, - logger, + // Spec 361, Decision 5: without an account, no socket, relay or launches. + let connection: RunnerConnection | null = null; + let launcher: Launcher | null = null; + const scheduler = new Scheduler({ clock: deps.clock, - env: deps.host.env, + logger, + zone: deps.zone ?? machineZone, + roots: () => watchedRoots(paths), + loadSchedule: scheduleLoader(deps.host.env), + stateFile: paths.scheduleState, spawn: deps.spawn, - ...defaultLauncherDeps(deps.host), + // Spec 248: one launch slot per machine, shared with the website. + claim: () => (launcher === null ? () => {} : launcher.claim()), + onSchedules: () => connection?.syncSchedules(), + local: credentials === null, }); - const connection = new RunnerConnection({ - server: credentials.server, - token: credentials.token, - runnerId: credentials.runnerId, - connect: deps.connect, - clock: deps.clock, - random: deps.random, - logger, - saveToken: (token) => - writeCredentials(deps.host, { ...credentials, token }), - isRevoked: async (token) => - (await fetchSelf(credentials.server, token, deps.fetch)) === null, - onState: (state, retryAt) => { - current = retryAt === undefined ? { state } : { state, retryAt }; - saveState(); - }, - relay, - artifacts: new ArtifactsDeleter({ + if (credentials !== null) { + const relay = new VideoRelay({ indexFile: paths.reportedRuns, - fs: nodeArtifactsFs, + fs: nodeRelayFs, logger, platform: deps.host.platform, - homeDir: deps.host.homedir, - closeRun: (runId) => relay.closeRun(runId), - }), - launches: launcher, - }); + }); + const projectsFiles = { + allowedRootsFile: paths.allowedRoots, + websiteProjectsFile: paths.websiteProjects, + host: deps.host, + }; + const launcherDeps = defaultLauncherDeps(deps.host); + launcher = new Launcher({ + indexFile: paths.reportedRuns, + websiteRegistered: (rootPath) => + isWebsiteRegistered(rootPath, { ...projectsFiles, fs: nodeLaunchFs }), + fs: nodeLaunchFs, + logger, + clock: deps.clock, + env: deps.host.env, + spawn: deps.spawn, + onIdle: () => scheduler.wake(), + ...launcherDeps, + }); + connection = new RunnerConnection({ + server: credentials.server, + token: credentials.token, + runnerId: credentials.runnerId, + connect: deps.connect, + clock: deps.clock, + random: deps.random, + logger, + saveToken: (token) => + writeCredentials(deps.host, { ...credentials, token }), + isRevoked: async (token) => + (await fetchSelf(credentials.server, token, deps.fetch)) === null, + onState: (state, retryAt) => { + current = retryAt === undefined ? { state } : { state, retryAt }; + saveState(); + }, + relay, + artifacts: new ArtifactsDeleter({ + indexFile: paths.reportedRuns, + fs: nodeArtifactsFs, + logger, + platform: deps.host.platform, + homeDir: deps.host.homedir, + closeRun: (runId) => relay.closeRun(runId), + }), + launches: launcher, + projects: new ProjectRequests({ + ...projectsFiles, + fs: nodeLaunchFs, + logger, + clock: deps.clock, + loadProject: launcherDeps.loadProject, + writeTemplate: defaultWriteTemplate, + }), + schedules: () => scheduler.schedules(), + share: sharePreparer({ + indexFile: paths.reportedRuns, + logger, + platform: deps.host.platform, + }), + }); + } + let stopped = (): void => {}; + const stopping = new Promise((resolve) => { + stopped = resolve; + }); const stop = (why: string) => { logger.log('info', `stopping (${why})`); // Spec 248, Decision 11: the run is cancelled; `finally` waits for it. - void launcher.close(); - void connection.stop(); + void launcher?.close(); + void scheduler.stop(); + if (connection === null) stopped(); + else void connection.stop(); }; const watch = setInterval(() => { if (existsSync(paths.stop)) stop('runner stop'); @@ -159,6 +226,13 @@ export async function runForeground(io: Io, deps: DaemonDeps): Promise { process.on('SIGINT', onSignal); try { + scheduler.start(); + if (connection === null) { + await stopping; + rmSync(paths.state, { force: true }); + logger.log('info', 'stopped'); + return EXIT_DAEMON_DONE; + } const outcome = await connection.run(); if (outcome === 'revoked') { current = { state: 'revoked' }; @@ -173,7 +247,7 @@ export async function runForeground(io: Io, deps: DaemonDeps): Promise { return EXIT_DAEMON_CRASHED; } finally { // No child outlives the daemon, whatever ended it. - await launcher.close(); + await Promise.all([launcher?.close(), scheduler.stop()]); clearInterval(watch); clearInterval(tick); process.off('SIGTERM', onSignal); diff --git a/apps/cli/src/runner/install/commands.ts b/apps/cli/src/runner/install/commands.ts index 1976d070..3be8d4e0 100644 --- a/apps/cli/src/runner/install/commands.ts +++ b/apps/cli/src/runner/install/commands.ts @@ -2,6 +2,8 @@ import { readCredentials } from '../../auth/credentials.ts'; import type { Io } from '../../commands/io.ts'; import type { ControlDeps } from '../control.ts'; import { redact } from '../log.ts'; +import { runnerPaths } from '../paths.ts'; +import { hasScheduledRoot } from '../schedule/roots.ts'; /** * `demoflow runner install`: registers login autostart for this user. The @@ -9,7 +11,11 @@ import { redact } from '../log.ts'; * and carries no token — the daemon reads `credentials.json` itself. */ export function installCommand(io: Io, deps: ControlDeps): number { - if (readCredentials(deps.host) === null) { + // Spec 361, Decision 5: a scheduled root runs without an account. + if ( + readCredentials(deps.host) === null && + !hasScheduledRoot(runnerPaths(deps.host), deps.host.env) + ) { io.stderr('Not logged in; run demoflow login first\n'); return 1; } diff --git a/apps/cli/src/runner/launch/check.ts b/apps/cli/src/runner/launch/check.ts new file mode 100644 index 00000000..69b4130b --- /dev/null +++ b/apps/cli/src/runner/launch/check.ts @@ -0,0 +1,79 @@ +import { LIST_MAX_SCENARIOS } from './list.ts'; +import { isScenarioId } from './scenario-id.ts'; + +/** Spec 360, Decision 6: the launch cap, per scenario of a check pass. */ +export const CHECK_SCENARIO_MAX_MS = 15 * 60_000; +/** Spec 360, Decision 6: a whole check pass is killed after this. */ +export const CHECK_PASS_MAX_MS = 2 * 60 * 60_000; + +const CHECK_START_FIELDS = new Set([ + 'type', + 'requestId', + 'rootPath', + 'exclude', +]); + +export type CheckStart = + | { ok: true; requestId: string; rootPath: string; exclude: string[] } + | { + ok: false; + requestId: string; + code: 'request-invalid' | 'exclude-invalid'; + } + | { ok: false; requestId: null }; + +/** + * Spec 360, Decision 6: `check.start { requestId, rootPath, exclude }` and + * nothing else; `exclude` holds at most `LIST_MAX_SCENARIOS` scenario ids. + * Pure: a refused request touches no file. A request with no string + * `requestId` cannot be answered and is ignored. + */ +export function checkCheckStart(message: Record): CheckStart { + const { requestId, rootPath, exclude } = message; + if (typeof requestId !== 'string') return { ok: false, requestId: null }; + if ( + typeof rootPath !== 'string' || + Object.keys(message).some((key) => !CHECK_START_FIELDS.has(key)) + ) { + return { ok: false, requestId, code: 'request-invalid' }; + } + if ( + !Array.isArray(exclude) || + exclude.length > LIST_MAX_SCENARIOS || + !exclude.every(isScenarioId) + ) { + return { ok: false, requestId, code: 'exclude-invalid' }; + } + return { ok: true, requestId, rootPath, exclude: [...new Set(exclude)] }; +} + +function parsed(line: string): Record | null { + try { + const value: unknown = JSON.parse(line); + return typeof value === 'object' && value !== null && !Array.isArray(value) + ? (value as Record) + : null; + } catch { + return null; + } +} + +function isCount(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value >= 0; +} + +/** True for the line `run --check --progress` prints as each scenario starts. */ +export function isCheckProgressLine(line: string): boolean { + const value = parsed(line); + return value?.type === 'progress' && value.phase === 'check'; +} + +/** The totals of the pass's final `--json` object, or `null` for any other line. */ +export function checkTotals( + line: string, +): { passed: number; failed: number } | null { + const value = parsed(line); + if (value?.check !== true) return null; + const { passed, failed } = value; + return isCount(passed) && isCount(failed) ? { passed, failed } : null; +} diff --git a/apps/cli/src/runner/launch/checks.ts b/apps/cli/src/runner/launch/checks.ts index 1007cbad..888e6634 100644 --- a/apps/cli/src/runner/launch/checks.ts +++ b/apps/cli/src/runner/launch/checks.ts @@ -72,6 +72,11 @@ export type ProjectLoad = export interface CheckDeps { /** `reported-runs.jsonl`: the registry (Decision 1). */ indexFile: string; + /** + * Spec 377, Decision 8: is the root registered from the website and still + * allowed? The registry is the union of both files (ADR-0022 §4). + */ + websiteRegistered?(rootPath: string): Promise; fs: LaunchFs; loadProject(root: string): ProjectLoad; } @@ -90,7 +95,10 @@ export async function checkProject( deps: CheckDeps, ): Promise { const unknown = { ok: false, code: 'project-unknown' } as const; - if (!(await knownRoots(deps.indexFile)).has(rootPath)) return unknown; + const known = + (await knownRoots(deps.indexFile)).has(rootPath) || + ((await deps.websiteRegistered?.(rootPath)) ?? false); + if (!known) return unknown; try { if ((await deps.fs.realpath(rootPath)) !== rootPath) return unknown; if (!(await deps.fs.lstat(rootPath)).isDirectory()) return unknown; diff --git a/apps/cli/src/runner/launch/index.ts b/apps/cli/src/runner/launch/index.ts index 0eff5405..7b883f90 100644 --- a/apps/cli/src/runner/launch/index.ts +++ b/apps/cli/src/runner/launch/index.ts @@ -13,6 +13,13 @@ import { isReportOff } from '../../runs-report/report.ts'; import { readReportedRunsByDir } from '../../runs-report/reported-runs.ts'; import type { Clock } from '../connection.ts'; import type { Logger } from '../log.ts'; +import { + CHECK_PASS_MAX_MS, + CHECK_SCENARIO_MAX_MS, + checkCheckStart, + checkTotals, + isCheckProgressLine, +} from './check.ts'; import { type CheckDeps, checkProject, @@ -73,6 +80,8 @@ export interface LauncherDeps extends CheckDeps { newRunId(): string; /** The run's `result.json` status; throws when there is none. */ readStatus(runDir: string): Promise; + /** Called once a launch is over and the slot is free (spec 361, Decision 3). */ + onIdle?: () => void; } /** The project's config, loaded the way `demoflow mcp` loads it. */ @@ -132,7 +141,8 @@ class Refused extends Error { /** * Spec 248: the daemon's side of `scenarios.list`, `run.start` and - * `run.cancel`. One website-started run at a time, an ordinary `demoflow run` + * `run.cancel`, and (spec 360) `check.start`. One website-started run or + * check pass at a time, an ordinary `demoflow run` * child that reports itself (ADR-0016 §4). `runner.log` gets request ids, * outcomes, codes and run ids, never a path or a scenario id (Decision 13). */ @@ -140,6 +150,8 @@ export class Launcher implements LaunchPort { private readonly deps: LauncherDeps; private launch: Launch | null = null; private closing: Promise | null = null; + /** A scheduled check pass holds the slot (spec 361, Decision 3). */ + private claimed = false; constructor(deps: LauncherDeps) { this.deps = deps; @@ -171,6 +183,9 @@ export class Launcher implements LaunchPort { } this.start(requestId, rootPath, scenarioId, options, reply, current); return; + case 'check.start': + this.startCheck(message, reply, current); + return; case 'run.cancel': if (typeof requestId !== 'string') { logger.log('warn', 'ignored a malformed run.cancel'); @@ -199,6 +214,31 @@ export class Launcher implements LaunchPort { return [launch.requestId]; } + /** + * Spec 361, Decision 3: the machine's one launch slot, for a scheduled + * check pass. `null` while a launch runs or the daemon is closing; else a + * release, and until it is called a website start is refused `busy`. + */ + claim(): (() => void) | null { + if (this.launch !== null || this.closing !== null || this.claimed) { + return null; + } + this.claimed = true; + let released = false; + return () => { + if (released) return; + released = true; + this.claimed = false; + }; + } + + /** Frees the slot of a launch that is over and tells the scheduler. */ + private ended(launch: Launch): void { + if (this.launch !== launch) return; + this.launch = null; + this.deps.onIdle?.(); + } + /** Decision 11: cancels the running launch and resolves once its child is gone. */ close(): Promise { this.closing ??= (async () => { @@ -254,15 +294,7 @@ export class Launcher implements LaunchPort { ); reply.send({ type: 'run.error', requestId, code }); }; - // Decision 6: a start still in its checks counts as running. - const early = - this.launch !== null - ? 'busy' - : this.closing !== null - ? 'io' - : isReportOff(this.deps.env) - ? 'reporting-off' - : null; + const early = this.refusedEarly(); if (early !== null) { refuse(early); return; @@ -293,9 +325,142 @@ export class Launcher implements LaunchPort { `launch ${shown(requestId)}: failed after start`, ); }) - .finally(() => { - if (this.launch === launch) this.launch = null; + .finally(() => this.ended(launch)); + } + + /** Decision 6: a start still in its checks counts as running. */ + private refusedEarly(): string | null { + return this.launch !== null || this.claimed + ? 'busy' + : this.closing !== null + ? 'io' + : isReportOff(this.deps.env) + ? 'reporting-off' + : null; + } + + /** + * Spec 360, Decision 6: a check pass the website starts. Checked like a + * `run.start`, then one `run --check` child for the whole project; it is + * the machine's one launch and is cancelled with `run.cancel`. + */ + private startCheck( + message: Record, + reply: LaunchSender, + current: LaunchSender, + ): void { + const { logger } = this.deps; + const checked = checkCheckStart(message); + if (!checked.ok && checked.requestId === null) { + logger.log('warn', 'ignored a malformed check.start'); + return; + } + const { requestId } = checked; + const refuse = (code: string) => { + logger.log('info', `check ${shown(requestId)}: refused ${code}`); + reply.send({ type: 'check.error', requestId, code }); + }; + if (!checked.ok) { + refuse(checked.code); + return; + } + const early = this.refusedEarly(); + if (early !== null) { + refuse(early); + return; + } + const launch: Launch = { requestId, done: Promise.resolve() }; + this.launch = launch; + launch.done = this.runCheck( + launch, + checked.rootPath, + checked.exclude, + reply, + current, + ) + .catch((error: unknown) => { + if (error instanceof Refused) refuse(error.code); + else if (launch.child === undefined) refuse('io'); + else + logger.log('error', `check ${shown(requestId)}: failed after start`); + }) + .finally(() => this.ended(launch)); + } + + private async runCheck( + launch: Launch, + rootPath: string, + exclude: string[], + reply: LaunchSender, + current: LaunchSender, + ): Promise { + const { deps } = this; + const { requestId } = launch; + const project = await checkProject(rootPath, deps); + if (!project.ok) throw new Refused(project.code); + if (launch.ending !== undefined) { + deps.logger.log( + 'info', + `check ${shown(requestId)}: cancelled before start`, + ); + return; + } + let totals: { passed: number; failed: number } | null = null; + // The launch cap applies per scenario: each scenario's start resets it. + let cancelScenarioCap = (): void => {}; + const capScenario = () => { + cancelScenarioCap(); + cancelScenarioCap = deps.clock.after(CHECK_SCENARIO_MAX_MS, () => { + void this.stop(launch, 'timed-out'); }); + }; + // Nothing from the message but the checked root and scenario ids. + const child = deps.spawn( + [ + 'run', + '--check', + '--json', + '--progress', + '--project', + project.root, + ...exclude.flatMap((id) => ['--exclude', id]), + ], + project.root, + { + killGraceMs: LAUNCH_KILL_GRACE_MS, + onStdoutLine: (line) => { + if (isCheckProgressLine(line)) capScenario(); + totals = checkTotals(line) ?? totals; + }, + }, + ); + if (child.pid === undefined) { + await child.exited; + throw new Refused('io'); + } + launch.child = child; + reply.send({ type: 'check.started', requestId }); + deps.logger.log( + 'info', + `check ${shown(requestId)}: started, ${exclude.length} excluded`, + ); + capScenario(); + const cancelPassCap = deps.clock.after(CHECK_PASS_MAX_MS, () => { + void this.stop(launch, 'timed-out'); + }); + await child.exited; + cancelScenarioCap(); + cancelPassCap(); + // A pass that was ended, or crashed before its summary, verified nothing. + const { passed, failed } = + launch.ending === undefined && totals !== null + ? totals + : { passed: 0, failed: 0 }; + deps.logger.log( + 'info', + `check ${shown(requestId)}: ${launch.ending ?? 'finished'}, ${passed} passed, ${failed} failed`, + ); + current.send({ type: 'check.finished', requestId, passed, failed }); } private async run( diff --git a/apps/cli/src/runner/paths.ts b/apps/cli/src/runner/paths.ts index bae0a755..d887be37 100644 --- a/apps/cli/src/runner/paths.ts +++ b/apps/cli/src/runner/paths.ts @@ -14,6 +14,14 @@ export interface RunnerPaths { stop: string; /** Server run id → run directory, one JSON line per reported run (spec 192). */ reportedRuns: string; + /** Every root `demoflow run` ran in, one JSON line each (spec 361, Decision 4). */ + localRoots: string; + /** Per scheduled root: its schedule and last check pass (spec 361, Decision 3). */ + scheduleState: string; + /** The folders the website may name (spec 377, Decision 1). */ + allowedRoots: string; + /** Roots registered from the website, one JSON line each (spec 377, Decision 8). */ + websiteProjects: string; } export function runnerPaths(host: HostEnvironment): RunnerPaths { @@ -26,5 +34,9 @@ export function runnerPaths(host: HostEnvironment): RunnerPaths { state: p.join(dir, 'runner.state.json'), stop: p.join(dir, 'runner.stop'), reportedRuns: p.join(dir, 'reported-runs.jsonl'), + localRoots: p.join(dir, 'local-roots.jsonl'), + scheduleState: p.join(dir, 'schedule.state.json'), + allowedRoots: p.join(dir, 'allowed-roots.json'), + websiteProjects: p.join(dir, 'website-projects.jsonl'), }; } diff --git a/apps/cli/src/runner/relay/index.ts b/apps/cli/src/runner/relay/index.ts index 9599f8eb..35145ed3 100644 --- a/apps/cli/src/runner/relay/index.ts +++ b/apps/cli/src/runner/relay/index.ts @@ -7,6 +7,7 @@ import { import { pathFor } from '../../config/platform.ts'; import { findReportedRun } from '../../runs-report/reported-runs.ts'; import type { Logger } from '../log.ts'; +import { posterPath } from '../share/files.ts'; import { parseRange } from './range.ts'; import { type RelayFile, @@ -82,19 +83,36 @@ export function isInside( } /** - * The real path of a reported run's rendered file in `format`, or `null`. - * The run directory comes from the index, never from the message, and only a - * file `result.json` lists, inside that directory, can come out. + * Spec 349, Decision 4: what `video.open` may ask for besides the render. + * Anything else is `not-found`. + */ +export const VIDEO_ASSETS = ['poster'] as const; +export type VideoAsset = (typeof VIDEO_ASSETS)[number]; + +/** + * The real path of a reported run's rendered file in `format` — or, with + * `asset: 'poster'`, of the poster a share made for it — or `null`. The run + * directory comes from the index, never from the message, and only a file + * `result.json` lists, or the poster's fixed name, inside that directory, + * can come out. */ export async function locateVideo( runId: string, format: RenderFormat, deps: RelayDeps, + asset?: VideoAsset, ): Promise { const entry = await findReportedRun(deps.indexFile, runId); if (entry === null) return null; const p = pathFor(deps.platform); try { + if (asset === 'poster') { + const root = await deps.fs.realpath(entry.runDir); + const real = await deps.fs.realpath( + posterPath(entry.runDir, format, deps.platform), + ); + return isInside(root, real, deps.platform) ? real : null; + } const parsed = RunResultSchema.safeParse( JSON.parse(await deps.fs.readFile(p.join(entry.runDir, RESULT_FILE))), ); @@ -196,6 +214,11 @@ export class VideoRelay implements VideoRelayPort { return; } const videoFormat = format as RenderFormat; + const { asset } = message; + // An asset this daemon does not know opens, and fails `not-found`. + const known = + asset === undefined || VIDEO_ASSETS.includes(asset as VideoAsset); + const videoAsset = asset === undefined ? undefined : (asset as VideoAsset); if (this.streams.has(streamId)) { this.deps.logger.log( 'warn', @@ -203,9 +226,10 @@ export class VideoRelay implements VideoRelayPort { ); return; } + const what = known && videoAsset !== undefined ? ` ${videoAsset}` : ''; if (this.streams.size >= MAX_STREAMS) { sender.send({ type: 'video.error', streamId, code: 'busy' }); - this.logStream(runId, videoFormat, 0, 'busy'); + this.logStream(runId, videoFormat, what, 0, 'busy'); return; } const stream = new VideoStream(streamId, sender, (outcome, sent) => { @@ -213,12 +237,13 @@ export class VideoRelay implements VideoRelayPort { this.streams.delete(streamId); this.runIds.delete(streamId); } - this.logStream(runId, videoFormat, sent, outcome); + this.logStream(runId, videoFormat, what, sent, outcome); }); this.streams.set(streamId, stream); this.runIds.set(streamId, runId); void stream.begin(async () => { - const file = await locateVideo(runId, videoFormat, this.deps); + if (!known) return null; + const file = await locateVideo(runId, videoFormat, this.deps, videoAsset); return file === null ? null : this.deps.fs.open(file); }, range); } @@ -227,12 +252,13 @@ export class VideoRelay implements VideoRelayPort { private logStream( runId: string, format: RenderFormat, + asset: string, sent: number, outcome: StreamOutcome | 'busy', ): void { this.deps.logger.log( 'info', - `video ${runId} ${format}: ${sent} bytes sent, ${outcome}`, + `video ${runId} ${format}${asset}: ${sent} bytes sent, ${outcome}`, ); } } diff --git a/apps/cli/src/runner/roots/allowed.ts b/apps/cli/src/runner/roots/allowed.ts new file mode 100644 index 00000000..00d377ea --- /dev/null +++ b/apps/cli/src/runner/roots/allowed.ts @@ -0,0 +1,105 @@ +import { + chmodSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { type HostEnvironment, pathFor } from '../../config/platform.ts'; + +/** Decision 1: the list holds at most this many roots. */ +export const ALLOWED_ROOTS_MAX = 32; +/** Decision 3: a path longer than this is never allowed. */ +export const PATH_MAX_LENGTH = 4096; + +/** + * Decision 1: what `allowed-roots.json` says. `default` is the home directory + * of a machine without the file; `invalid` is a file that does not parse, + * which allows nothing. + */ +export type AllowedRoots = + | { source: 'default' | 'file'; roots: string[] } + | { source: 'invalid'; roots: [] }; + +function errorCode(error: unknown): unknown { + return typeof error === 'object' && error !== null && 'code' in error + ? error.code + : undefined; +} + +/** Absolute for the host, at most 4,096 characters, no NUL. */ +export function isAbsoluteFor( + value: unknown, + platform: NodeJS.Platform, +): value is string { + if ( + typeof value !== 'string' || + value.length === 0 || + value.length > PATH_MAX_LENGTH || + value.includes('\0') + ) { + return false; + } + // A drive letter only: `\foo` is drive-relative and UNC or device paths + // (`\\server`, `\\?\`) are refused. + if (platform === 'win32') return /^[A-Za-z]:[\\/]/.test(value); + return pathFor(platform).isAbsolute(value); +} + +/** + * The roots in force, read afresh on every call so the daemon needs no + * restart. Never throws; anything but a missing file that does not parse is + * `invalid`, so a broken file never widens access. + */ +export function readAllowedRoots( + file: string, + host: HostEnvironment, +): AllowedRoots { + let text: string; + try { + text = readFileSync(file, 'utf8'); + } catch (error) { + if (errorCode(error) === 'ENOENT') { + return { source: 'default', roots: [host.homedir] }; + } + return { source: 'invalid', roots: [] }; + } + try { + const parsed: unknown = JSON.parse(text); + if (typeof parsed !== 'object' || parsed === null) throw new Error(); + const { roots } = parsed as Record; + if ( + !Array.isArray(roots) || + roots.length > ALLOWED_ROOTS_MAX || + !roots.every((root) => isAbsoluteFor(root, host.platform)) + ) { + throw new Error(); + } + return { source: 'file', roots: roots as string[] }; + } catch { + return { source: 'invalid', roots: [] }; + } +} + +/** Written atomically with mode 600, like `credentials.json`. */ +export function writeAllowedRoots( + file: string, + roots: readonly string[], + platform: NodeJS.Platform, +): void { + const p = pathFor(platform); + mkdirSync(p.dirname(file), { recursive: true }); + const temp = `${file}.${process.pid}.tmp`; + try { + writeFileSync(temp, `${JSON.stringify({ roots }, null, 2)}\n`, { + mode: 0o600, + }); + // `mode` is masked by the umask and ignored on an existing file. + if (platform !== 'win32') chmodSync(temp, 0o600); + renameSync(temp, file); + } catch (error) { + rmSync(temp, { force: true }); + throw error; + } +} diff --git a/apps/cli/src/runner/roots/commands.ts b/apps/cli/src/runner/roots/commands.ts new file mode 100644 index 00000000..04476f31 --- /dev/null +++ b/apps/cli/src/runner/roots/commands.ts @@ -0,0 +1,149 @@ +import { realpathSync, statSync } from 'node:fs'; +import type { Io } from '../../commands/io.ts'; +import { EXIT_INVALID_INPUT } from '../../config/index.ts'; +import { type HostEnvironment, pathFor } from '../../config/platform.ts'; +import { runnerPaths } from '../paths.ts'; +import { + ALLOWED_ROOTS_MAX, + type AllowedRoots, + readAllowedRoots, + writeAllowedRoots, +} from './allowed.ts'; +import { samePath } from './path-rule.ts'; + +export interface RootsDeps { + host: HostEnvironment; + cwd: string; +} + +function invalidNote(file: string): string { + return `${file} is invalid: every folder is refused until it is fixed or removed`; +} + +/** + * `demoflow runner roots [--json]` (spec 377, Decision 2): the roots in + * force, the default marked. Exits 1 when the file is invalid. + */ +export function rootsCommand( + flags: { json?: boolean }, + io: Io, + deps: RootsDeps, +): number { + const file = runnerPaths(deps.host).allowedRoots; + const allowed = readAllowedRoots(file, deps.host); + const valid = allowed.source !== 'invalid'; + if (flags.json === true) { + io.stdout( + `${JSON.stringify({ roots: allowed.roots, default: allowed.source === 'default', valid })}\n`, + ); + return valid ? 0 : 1; + } + if (!valid) { + io.stderr(`${invalidNote(file)}\n`); + return 1; + } + if (allowed.roots.length === 0) { + io.stdout('No allowed roots: the website may name no folder\n'); + return 0; + } + for (const root of allowed.roots) { + io.stdout( + `${root}${allowed.source === 'default' ? ' (default: the home directory)' : ''}\n`, + ); + } + return 0; +} + +/** + * `demoflow runner allow `: adds the real path of an existing + * directory. The first `allow` on a machine without the file replaces the + * default home directory (Decision 2). + */ +export function allowCommand(dir: string, io: Io, deps: RootsDeps): number { + const { platform } = deps.host; + const resolved = pathFor(platform).resolve(deps.cwd, dir); + let real: string; + try { + if (!statSync(resolved).isDirectory()) throw new Error(); + real = realpathSync.native(resolved); + } catch { + io.stderr(`Not a directory: ${resolved}\n`); + return EXIT_INVALID_INPUT; + } + const file = runnerPaths(deps.host).allowedRoots; + const allowed = readAllowedRoots(file, deps.host); + const current = allowed.source === 'file' ? allowed.roots : []; + if (current.some((root) => samePath(root, real, platform))) { + io.stdout(`Already allowed: ${real}\n`); + return 0; + } + if (current.length >= ALLOWED_ROOTS_MAX) { + io.stderr( + `At most ${ALLOWED_ROOTS_MAX} allowed roots; disallow one first\n`, + ); + return EXIT_INVALID_INPUT; + } + writeAllowedRoots(file, [...current, real], platform); + io.stdout(`Allowed: ${real}\n`); + printReplaced(allowed, real, io, deps.host); + return 0; +} + +function printReplaced( + allowed: AllowedRoots, + added: string, + io: Io, + host: HostEnvironment, +): void { + if (allowed.source === 'invalid') { + io.stdout('The invalid allowed-roots file was replaced\n'); + } else if ( + allowed.source === 'default' && + !samePath(added, realOrResolved(host.homedir, host.platform), host.platform) + ) { + io.stdout( + `The home directory (${host.homedir}) is no longer allowed; allow it again to keep it\n`, + ); + } +} + +function realOrResolved(dir: string, platform: NodeJS.Platform): string { + try { + return realpathSync.native(dir); + } catch { + return pathFor(platform).resolve(dir); + } +} + +/** + * `demoflow runner disallow `: removes the entry equal to the folder's + * real path, or its resolved path once it is gone. Removing the last one + * allows nothing, never the default again (Decision 2). + */ +export function disallowCommand(dir: string, io: Io, deps: RootsDeps): number { + const { platform } = deps.host; + const target = realOrResolved( + pathFor(platform).resolve(deps.cwd, dir), + platform, + ); + const file = runnerPaths(deps.host).allowedRoots; + const allowed = readAllowedRoots(file, deps.host); + if (allowed.source === 'invalid') { + io.stderr(`${invalidNote(file)}\n`); + return 1; + } + const matches = (root: string) => + samePath(root, target, platform) || + samePath(realOrResolved(root, platform), target, platform); + const kept = allowed.roots.filter((root) => !matches(root)); + if (kept.length === allowed.roots.length) { + io.stderr(`Not an allowed root: ${target}\n`); + return 1; + } + writeAllowedRoots(file, kept, platform); + io.stdout(`Disallowed: ${target}\n`); + if (kept.length === 0) { + io.stdout('No allowed roots left: the website may name no folder\n'); + } + return 0; +} diff --git a/apps/cli/src/runner/roots/git-remote.ts b/apps/cli/src/runner/roots/git-remote.ts new file mode 100644 index 00000000..997d54b4 --- /dev/null +++ b/apps/cli/src/runner/roots/git-remote.ts @@ -0,0 +1,138 @@ +import { pathFor } from '../../config/platform.ts'; +import type { LaunchFs } from '../launch/checks.ts'; +import { realRoots, segmentsBelow } from './path-rule.ts'; + +/** Decision 6: a git config file is read only up to this size. */ +export const GIT_CONFIG_MAX_BYTES = 64 * 1024; +/** A `.git` file or `commondir` is one short line. */ +const GIT_POINTER_MAX_BYTES = 4096; + +const NAME = '[A-Za-z0-9._-]{1,100}'; +const GITHUB_REMOTES = [ + new RegExp(`^https://github\\.com/(${NAME})/(${NAME})/?$`), + new RegExp(`^git@github\\.com:(${NAME})/(${NAME})$`), + new RegExp(`^ssh://git@github\\.com/(${NAME})/(${NAME})/?$`), +]; + +/** + * Decision 6: `https://github.com//` for an https, scp-style or + * ssh remote on github.com; anything else — another host, a port, a URL with + * credentials — is `undefined`. + */ +export function githubUrlFrom(remote: string): string | undefined { + for (const pattern of GITHUB_REMOTES) { + const match = pattern.exec(remote); + if (match === null) continue; + const owner = match[1] as string; + const repo = (match[2] as string).replace(/\.git$/, ''); + if ([owner, repo].some((name) => /^\.*$/.test(name))) return undefined; + return `https://github.com/${owner}/${repo}`; + } + return undefined; +} + +/** The first `url` of `[remote "origin"]` in a git config file, unquoted. */ +export function originUrl(config: string): string | undefined { + let inOrigin = false; + for (const raw of config.split(/\r?\n/)) { + const line = raw.trim(); + const section = /^\[\s*([^\s\]"]+)(?:\s+"((?:[^"\\]|\\.)*)")?\s*\]/.exec( + line, + ); + if (section !== null) { + // Section names ignore case; subsection names do not. + inOrigin = + section[1]?.toLowerCase() === 'remote' && section[2] === 'origin'; + continue; + } + if (!inOrigin) continue; + const entry = /^url\s*=\s*(.*)$/i.exec(line); + if (entry === null) continue; + let value = (entry[1] as string).trim(); + if (value.startsWith('"')) { + const end = value.indexOf('"', 1); + value = end === -1 ? '' : value.slice(1, end); + } else { + value = value.replace(/\s[#;].*$/, '').trim(); + } + return value; + } + return undefined; +} + +async function readRegularFile( + file: string, + maxBytes: number, + fs: LaunchFs, +): Promise { + if (!(await fs.lstat(file)).isFile()) throw new Error('not a file'); + return fs.readFile(file, maxBytes); +} + +/** + * Decision 6: the GitHub URL of the canonical folder `root`'s `origin`, read + * from its git config without running git. A `.git` file is followed one + * level (to its `gitdir`, and from a worktree's gitdir to its `commondir`), + * and only to a directory inside an allowed root. Never throws. + */ +export async function githubUrlOf( + root: string, + roots: readonly string[], + platform: NodeJS.Platform, + fs: LaunchFs, +): Promise { + const p = pathFor(platform); + try { + const dotGit = p.join(root, '.git'); + const stats = await fs.lstat(dotGit); + let config: string; + if (stats.isDirectory()) { + config = await readRegularFile( + p.join(dotGit, 'config'), + GIT_CONFIG_MAX_BYTES, + fs, + ); + } else if (stats.isFile()) { + const allowed = await realRoots(roots, fs); + const follow = async (from: string, value: string) => { + const target = await fs.realpath(p.resolve(from, value.trim())); + if (!allowed.some((r) => segmentsBelow(target, r, platform) !== null)) { + throw new Error('outside the allowed roots'); + } + if (!(await fs.lstat(target)).isDirectory()) { + throw new Error('not a directory'); + } + return target; + }; + const pointer = await readRegularFile(dotGit, GIT_POINTER_MAX_BYTES, fs); + const gitdir = /^gitdir:\s*(.+)$/m.exec(pointer)?.[1]; + if (gitdir === undefined) return undefined; + const dir = await follow(root, gitdir); + try { + config = await readRegularFile( + p.join(dir, 'config'), + GIT_CONFIG_MAX_BYTES, + fs, + ); + } catch { + // A worktree's gitdir keeps its config in the common directory. + const common = await readRegularFile( + p.join(dir, 'commondir'), + GIT_POINTER_MAX_BYTES, + fs, + ); + config = await readRegularFile( + p.join(await follow(dir, common), 'config'), + GIT_CONFIG_MAX_BYTES, + fs, + ); + } + } else { + return undefined; + } + const url = originUrl(config); + return url === undefined ? undefined : githubUrlFrom(url); + } catch { + return undefined; + } +} diff --git a/apps/cli/src/runner/roots/index.ts b/apps/cli/src/runner/roots/index.ts new file mode 100644 index 00000000..c15ab88e --- /dev/null +++ b/apps/cli/src/runner/roots/index.ts @@ -0,0 +1,228 @@ +import { + type InitTemplateResult, + writeInitTemplate, +} from '../../commands/init.ts'; +import { PROJECT_CONFIG_FILE } from '../../config/index.ts'; +import { type HostEnvironment, pathFor } from '../../config/platform.ts'; +import type { Clock } from '../connection.ts'; +import type { LaunchFs, ProjectLoad } from '../launch/checks.ts'; +import type { Logger } from '../log.ts'; +import { readAllowedRoots } from './allowed.ts'; +import { fitted, type Inspection, inspectFolder } from './inspect.ts'; +import { isAllowedPath } from './path-rule.ts'; +import { registerWebsiteProject, websiteRoots } from './registry.ts'; + +/** Decision 10: a request not answered by then is answered `internal`. */ +export const PROJECT_REQUEST_MAX_MS = 5_000; + +export const PROJECT_REQUESTS = [ + 'project.inspect', + 'project.init', + 'project.register', +] as const; +type ProjectRequest = (typeof PROJECT_REQUESTS)[number]; + +/** Decision 4: the codes of `project.error`. */ +export type ProjectErrorCode = + | 'path-not-allowed' + | 'config-missing' + | 'config-invalid' + | 'busy' + | 'internal'; + +/** The socket, as the project requests see it: JSON text frames only. */ +export interface ProjectsSender { + send(message: Record): void; +} + +/** What the connection needs from the project requests. */ +export interface ProjectsPort { + handle(message: Record, sender: ProjectsSender): void; +} + +export interface ProjectsDeps { + /** `allowed-roots.json`, read on every request. */ + allowedRootsFile: string; + /** `website-projects.jsonl`. */ + websiteProjectsFile: string; + host: HostEnvironment; + fs: LaunchFs; + logger: Logger; + clock: Clock; + loadProject(root: string): ProjectLoad; + /** The `demoflow init` template, never through a link (Decision 7). */ + writeTemplate(root: string): Promise; +} + +export const defaultWriteTemplate = (root: string) => + writeInitTemplate(root, { noLinks: true }); + +/** A refusal that becomes `project.error` with its code. */ +class Refused extends Error { + readonly code: ProjectErrorCode; + + constructor(code: ProjectErrorCode) { + super(code); + this.code = code; + } +} + +/** A request id as `runner.log` shows it: one short line, whatever was sent. */ +function shown(requestId: string): string { + return requestId.slice(0, 64).replace(/[^A-Za-z0-9-]/g, '?'); +} + +/** + * Decision 8: a root the website registered counts for `checkProject` only + * while it is still in the file and still passes the path rule, as itself. + */ +export async function isWebsiteRegistered( + rootPath: string, + deps: Pick< + ProjectsDeps, + 'allowedRootsFile' | 'websiteProjectsFile' | 'host' + > & { + fs: Pick; + }, +): Promise { + if (!(await websiteRoots(deps.websiteProjectsFile)).has(rootPath)) { + return false; + } + const { roots } = readAllowedRoots(deps.allowedRootsFile, deps.host); + const check = await isAllowedPath( + rootPath, + roots, + deps.host.platform, + deps.fs, + ); + return check.ok && check.root === rootPath; +} + +/** + * Spec 377: the daemon's side of `project.inspect`, `project.init` and + * `project.register` (ADR-0022 §3). One at a time; each is answered within + * 5 s. `runner.log` gets request ids, outcomes and codes, never a path. + */ +export class ProjectRequests implements ProjectsPort { + private readonly deps: ProjectsDeps; + private inFlight = false; + + constructor(deps: ProjectsDeps) { + this.deps = deps; + } + + handle(message: Record, sender: ProjectsSender): void { + const { type, requestId, path } = message; + const { logger } = this.deps; + if (!PROJECT_REQUESTS.includes(type as ProjectRequest)) return; + if (typeof requestId !== 'string') { + logger.log('warn', `ignored a malformed ${type as string}`); + return; + } + const request = type as ProjectRequest; + const label = `${request} ${shown(requestId)}`; + const refuse = (code: ProjectErrorCode) => { + logger.log('info', `${label}: refused ${code}`); + sender.send({ type: 'project.error', requestId, code }); + }; + if (this.inFlight) { + refuse('busy'); + return; + } + this.inFlight = true; + let answered = false; + const answer = (send: () => void) => { + if (answered) return; + answered = true; + cancelTimer(); + send(); + }; + const cancelTimer = this.deps.clock.after(PROJECT_REQUEST_MAX_MS, () => { + answer(() => refuse('internal')); + }); + void this.run(request, path) + .then( + (reply) => + answer(() => { + logger.log('info', `${label}: answered`); + sender.send({ ...reply, requestId }); + }), + (error: unknown) => + answer(() => + refuse(error instanceof Refused ? error.code : 'internal'), + ), + ) + .finally(() => { + // Busy until the work itself is over, even after a timeout. + this.inFlight = false; + }); + } + + private async run( + request: ProjectRequest, + raw: unknown, + ): Promise> { + const { roots } = readAllowedRoots( + this.deps.allowedRootsFile, + this.deps.host, + ); + const check = await isAllowedPath( + raw, + roots, + this.deps.host.platform, + this.deps.fs, + ); + if (!check.ok) throw new Refused('path-not-allowed'); + const root = check.root; + switch (request) { + case 'project.inspect': + return fitted( + (inspection) => ({ type: 'project.inspected', inspection }), + await this.inspect(root, roots), + ); + case 'project.init': { + const { created, kept } = await this.deps.writeTemplate(root); + return fitted( + (inspection) => ({ + type: 'project.initialised', + created, + kept, + inspection, + }), + await this.inspect(root, roots), + ); + } + case 'project.register': + await this.register(root); + return { type: 'project.registered', rootPath: root }; + } + } + + private inspect(root: string, roots: readonly string[]): Promise { + return inspectFolder(root, roots, { + fs: this.deps.fs, + platform: this.deps.host.platform, + loadProject: this.deps.loadProject, + }); + } + + /** Decision 8: a config that exists and loads, then one line in the registry. */ + private async register(root: string): Promise { + const p = pathFor(this.deps.host.platform); + try { + const config = await this.deps.fs.lstat( + p.join(root, PROJECT_CONFIG_FILE), + ); + if (!config.isFile()) throw new Refused('config-missing'); + } catch (error) { + if (error instanceof Refused) throw error; + throw new Refused('config-missing'); + } + if (!this.deps.loadProject(root).ok) throw new Refused('config-invalid'); + registerWebsiteProject( + this.deps.websiteProjectsFile, + root, + new Date(this.deps.clock.now()).toISOString(), + ); + } +} diff --git a/apps/cli/src/runner/roots/inspect.ts b/apps/cli/src/runner/roots/inspect.ts new file mode 100644 index 00000000..703f94e3 --- /dev/null +++ b/apps/cli/src/runner/roots/inspect.ts @@ -0,0 +1,84 @@ +import { PROJECT_CONFIG_FILE } from '../../config/index.ts'; +import { pathFor } from '../../config/platform.ts'; +import type { LaunchFs, ProjectLoad } from '../launch/checks.ts'; +import { + LIST_MAX_FRAME_BYTES, + type ListedScenario, + listScenarios, +} from '../launch/list.ts'; +import { githubUrlOf } from './git-remote.ts'; + +/** + * Decision 5: everything an inspection says. Never a file's contents and + * never a config value such as `baseUrl` or `devCommand`. + */ +export interface Inspection { + rootPath: string; + name: string; + hasConfig: boolean; + configValid: boolean; + scenarios: ListedScenario[]; + githubUrl?: string; +} + +export interface InspectDeps { + fs: LaunchFs; + platform: NodeJS.Platform; + loadProject(root: string): ProjectLoad; +} + +/** + * Decision 5: inspects `root`, a folder that already passed the path rule. + * Scenarios are listed only for a config that loads, by the same rules and + * limits as `scenarios.list`. Throws only when listing them fails. + */ +export async function inspectFolder( + root: string, + roots: readonly string[], + deps: InspectDeps, +): Promise { + const p = pathFor(deps.platform); + let hasConfig = false; + try { + // `lstat`: a linked config is not this folder's config. + hasConfig = ( + await deps.fs.lstat(p.join(root, PROJECT_CONFIG_FILE)) + ).isFile(); + } catch { + // No config. + } + const loaded = hasConfig ? deps.loadProject(root) : null; + const configValid = loaded?.ok === true; + const scenarios = + loaded?.ok === true + ? (await listScenarios(root, deps.fs, loaded.fragmentsDir)).scenarios + : []; + const githubUrl = await githubUrlOf(root, roots, deps.platform, deps.fs); + return { + rootPath: root, + name: p.basename(root), + hasConfig, + configValid, + scenarios, + ...(githubUrl === undefined ? {} : { githubUrl }), + }; +} + +/** + * A reply frame with its inspection's scenarios dropped from the end until it + * is at most 60,000 bytes, as `scenarios.listed` is cut. + */ +export function fitted( + frame: (inspection: Inspection) => Record, + inspection: Inspection, +): Record { + const scenarios = [...inspection.scenarios]; + const build = () => frame({ ...inspection, scenarios }); + while ( + scenarios.length > 0 && + Buffer.byteLength(JSON.stringify(build()), 'utf8') > LIST_MAX_FRAME_BYTES + ) { + scenarios.pop(); + } + return build(); +} diff --git a/apps/cli/src/runner/roots/path-rule.ts b/apps/cli/src/runner/roots/path-rule.ts new file mode 100644 index 00000000..1e2a3830 --- /dev/null +++ b/apps/cli/src/runner/roots/path-rule.ts @@ -0,0 +1,100 @@ +import { pathFor } from '../../config/platform.ts'; +import type { LaunchFs } from '../launch/checks.ts'; +import { isAbsoluteFor } from './allowed.ts'; + +/** What the path rule touches: real paths and `lstat`, never a link followed. */ +export type PathFs = Pick; + +/** `ok` carries the canonical folder; a refusal says nothing more. */ +export type PathCheck = { ok: true; root: string } | { ok: false }; + +/** Decision 3: Windows and macOS compare paths without case. */ +function fold(value: string, platform: NodeJS.Platform): string { + return platform === 'win32' || platform === 'darwin' + ? value.toLowerCase() + : value; +} + +function segments(value: string, platform: NodeJS.Platform): string[] { + return value.split(platform === 'win32' ? /[\\/]/ : '/').filter(Boolean); +} + +export function samePath( + a: string, + b: string, + platform: NodeJS.Platform, +): boolean { + return fold(a, platform) === fold(b, platform); +} + +/** + * The segments of `child` below `parent`, compared segment-wise; `null` when + * `child` is neither `parent` nor below it. Both are real paths. + */ +export function segmentsBelow( + child: string, + parent: string, + platform: NodeJS.Platform, +): string[] | null { + const inner = segments(child, platform); + const outer = segments(parent, platform); + if (outer.length > inner.length) return null; + for (let i = 0; i < outer.length; i++) { + if (!samePath(outer[i] as string, inner[i] as string, platform)) { + return null; + } + } + return inner.slice(outer.length); +} + +/** The real paths of the roots that still resolve; a root that does not is skipped. */ +export async function realRoots( + roots: readonly string[], + fs: PathFs, +): Promise { + const real: string[] = []; + for (const root of roots) { + try { + real.push(await fs.realpath(root)); + } catch { + // A root that is gone allows nothing. + } + } + return real; +} + +/** + * Decision 3 (ADR-0022 §2): `raw` names a folder the website may use when it + * is an absolute path for the host, its real path is the path itself (no link + * or junction anywhere on the way, no `..` that leaves it), it is a + * directory, and it is an allowed root or lies below one with no segment + * below that root starting with `.`. Every failure is the same refusal. + */ +export async function isAllowedPath( + raw: unknown, + roots: readonly string[], + platform: NodeJS.Platform, + fs: PathFs, +): Promise { + const refused = { ok: false } as const; + if (!isAbsoluteFor(raw, platform)) return refused; + let real: string; + try { + // The OS resolves `..` after links, so `link/..` lands where the link + // points and differs from the lexical form below. + real = await fs.realpath(raw); + if (!samePath(real, pathFor(platform).resolve(raw), platform)) { + return refused; + } + if (!(await fs.lstat(real)).isDirectory()) return refused; + } catch { + return refused; + } + for (const root of await realRoots(roots, fs)) { + const below = segmentsBelow(real, root, platform); + if (below?.every((segment) => !segment.startsWith('.'))) { + return { ok: true, root: real }; + } + } + return refused; +} diff --git a/apps/cli/src/runner/roots/registry.ts b/apps/cli/src/runner/roots/registry.ts new file mode 100644 index 00000000..4cd5701c --- /dev/null +++ b/apps/cli/src/runner/roots/registry.ts @@ -0,0 +1,90 @@ +import { + appendFileSync, + chmodSync, + mkdirSync, + readFileSync, + renameSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import path from 'node:path'; + +/** Decision 8: beyond this many entries the oldest are dropped. */ +export const WEBSITE_PROJECTS_MAX = 1_000; + +/** One line of `website-projects.jsonl`: a root registered from the website. */ +export interface WebsiteProject { + rootPath: string; + registeredAt: string; +} + +function parseLine(line: string): WebsiteProject | null { + try { + const value: unknown = JSON.parse(line); + if (typeof value !== 'object' || value === null) return null; + const { rootPath, registeredAt } = value as Record; + if (typeof rootPath !== 'string' || rootPath === '') return null; + if (typeof registeredAt !== 'string') return null; + return { rootPath, registeredAt }; + } catch { + return null; + } +} + +function parse(text: string): WebsiteProject[] { + return text + .split('\n') + .map(parseLine) + .filter((entry) => entry !== null); +} + +/** + * Decision 8: the roots registered from the website, read once. Never throws: + * a missing or unreadable file registers nothing. + */ +export async function websiteRoots(file: string): Promise> { + try { + return new Set(parse(await readFile(file, 'utf8')).map((e) => e.rootPath)); + } catch { + return new Set(); + } +} + +/** + * Decision 8: appends `rootPath` once. A root already there is left as it + * is; past 1,000 entries the file is rewritten without the oldest. + */ +export function registerWebsiteProject( + file: string, + rootPath: string, + registeredAt: string, +): void { + let entries: WebsiteProject[] = []; + try { + entries = parse(readFileSync(file, 'utf8')); + } catch { + // A missing file starts empty. + } + if (entries.some((entry) => entry.rootPath === rootPath)) return; + const entry: WebsiteProject = { rootPath, registeredAt }; + mkdirSync(path.dirname(file), { recursive: true }); + if (entries.length < WEBSITE_PROJECTS_MAX) { + appendFileSync(file, `${JSON.stringify(entry)}\n`, { mode: 0o600 }); + return; + } + const kept = [...entries.slice(-(WEBSITE_PROJECTS_MAX - 1)), entry]; + const temp = `${file}.${process.pid}.tmp`; + try { + writeFileSync( + temp, + kept.map((line) => `${JSON.stringify(line)}\n`).join(''), + { mode: 0o600 }, + ); + if (process.platform !== 'win32') chmodSync(temp, 0o600); + renameSync(temp, file); + } catch (error) { + rmSync(temp, { force: true }); + throw error; + } +} diff --git a/apps/cli/src/runner/schedule/local-roots.ts b/apps/cli/src/runner/schedule/local-roots.ts new file mode 100644 index 00000000..e94c32a2 --- /dev/null +++ b/apps/cli/src/runner/schedule/local-roots.ts @@ -0,0 +1,76 @@ +import { + appendFileSync, + mkdirSync, + readFileSync, + renameSync, + writeFileSync, +} from 'node:fs'; +import path from 'node:path'; + +/** Spec 361, Decision 4: the file keeps the last 500 roots. */ +export const LOCAL_ROOTS_KEEP = 500; + +/** + * The roots of `local-roots.jsonl`, oldest first, once each. Never throws: a + * missing or unreadable file holds none, and malformed lines are skipped. + */ +export function readLocalRoots(file: string): string[] { + let text: string; + try { + text = readFileSync(file, 'utf8'); + } catch { + return []; + } + return parseRoots(text); +} + +function parseRoots(text: string): string[] { + const roots = new Set(); + for (const line of text.split('\n')) { + if (line.trim() === '') continue; + let value: unknown; + try { + value = JSON.parse(line); + } catch { + continue; + } + const root = + typeof value === 'object' && value !== null + ? (value as Record).root + : undefined; + if (typeof root !== 'string' || root === '') continue; + // The last line for a root is where it sits now. + roots.delete(root); + roots.add(root); + } + return [...roots]; +} + +/** + * Spec 361, Decision 4: `demoflow run` records its canonical root, so the + * daemon watches a project's schedule without an account. A root already + * listed is not appended again; past 500 roots the oldest go. Never throws: + * a run's outcome does not depend on this file. + */ +export function recordLocalRoot(file: string, root: string): void { + try { + const roots = readLocalRoots(file); + if (roots.includes(root)) return; + mkdirSync(path.dirname(file), { recursive: true }); + if (roots.length < LOCAL_ROOTS_KEEP) { + // One append of one short line: two runs together never interleave. + appendFileSync(file, `${JSON.stringify({ root })}\n`, { mode: 0o600 }); + return; + } + const kept = [...roots.slice(-(LOCAL_ROOTS_KEEP - 1)), root]; + const temp = `${file}.${process.pid}.tmp`; + writeFileSync( + temp, + kept.map((entry) => `${JSON.stringify({ root: entry })}\n`).join(''), + { mode: 0o600 }, + ); + renameSync(temp, file); + } catch { + // A root not recorded is a schedule that waits for the next run. + } +} diff --git a/apps/cli/src/runner/schedule/roots.ts b/apps/cli/src/runner/schedule/roots.ts new file mode 100644 index 00000000..d9b29ee0 --- /dev/null +++ b/apps/cli/src/runner/schedule/roots.ts @@ -0,0 +1,48 @@ +import type { ScheduleConfig } from '@demoflow/core'; +import { loadProjectConfig } from '../../config/project.ts'; +import { knownRootsSync } from '../../runs-report/reported-runs.ts'; +import type { RunnerPaths } from '../paths.ts'; +import { readLocalRoots } from './local-roots.ts'; + +type Env = Readonly>; + +/** Decision 4: the reported registry and the roots `demoflow run` ran in. */ +export function watchedRoots(paths: RunnerPaths): { + reported: Set; + local: string[]; +} { + return { + reported: knownRootsSync(paths.reportedRuns), + local: readLocalRoots(paths.localRoots), + }; +} + +/** The root's committed `schedule`; throws when its config does not load. */ +export function scheduleLoader( + env: Env, +): (root: string) => ScheduleConfig | undefined { + return (root) => loadProjectConfig(root, env).schedule; +} + +/** + * Decision 5: true when a watched root's config asks for a schedule, so a + * daemon without an account has something to do. A config that does not + * load asks for nothing. + */ +export function hasScheduledRoot(paths: RunnerPaths, env: Env): boolean { + const { reported, local } = watchedRoots(paths); + const load = scheduleLoader(env); + for (const root of new Set([...reported, ...local])) { + try { + if (load(root) !== undefined) return true; + } catch { + // Skipped, as the scheduler skips it. + } + } + return false; +} + +/** The machine's IANA zone name. */ +export function machineZone(): string { + return Intl.DateTimeFormat().resolvedOptions().timeZone; +} diff --git a/apps/cli/src/runner/schedule/scheduler.ts b/apps/cli/src/runner/schedule/scheduler.ts new file mode 100644 index 00000000..08c55ae0 --- /dev/null +++ b/apps/cli/src/runner/schedule/scheduler.ts @@ -0,0 +1,359 @@ +import path from 'node:path'; +import { lastDueSlot, type ScheduleConfig } from '@demoflow/core'; +import type { RunChild, SpawnRun } from '../../mcp/child.ts'; +import type { Clock } from '../connection.ts'; +import { + CHECK_PASS_MAX_MS, + CHECK_SCENARIO_MAX_MS, + checkTotals, + isCheckProgressLine, +} from '../launch/check.ts'; +import { LAUNCH_KILL_GRACE_MS } from '../launch/index.ts'; +import type { Logger } from '../log.ts'; +import { + type LastPass, + type PassOutcome, + readScheduleState, + type ScheduledRoot, + type ScheduleState, + writeScheduleState, +} from './state.ts'; + +/** Decision 3: the cadence is daily at most; a minute is the precision set. */ +export const SCHEDULE_TICK_MS = 60_000; + +/** Decision 6: what the website learns of the machine's schedules. */ +export type SchedulesSync = { + type: 'schedules.sync'; + zone: string; + projects: Array<{ rootPath: string; schedule: ScheduleConfig | null }>; +}; + +export interface SchedulerDeps { + clock: Clock; + logger: Logger; + /** The machine's IANA zone, asked on every tick: a laptop travels. */ + zone(): string; + /** + * Decision 4, read on every tick: the reported registry (the only roots + * `schedules.sync` names) and the roots `demoflow run` ran in. + */ + roots(): { reported: Iterable; local: Iterable }; + /** The root's `schedule`, `undefined` without one; throws when the config does not load. */ + loadSchedule(root: string): ScheduleConfig | undefined; + stateFile: string; + spawn: SpawnRun; + /** The daemon's one launch slot; `null` while a website launch holds it. */ + claim(): (() => void) | null; + /** A `schedules.sync` whose content changed since the last one. */ + onSchedules?: (frame: SchedulesSync) => void; + /** Decision 5: the daemon runs without an account. */ + local?: boolean; +} + +interface Pass { + root: string; + child?: RunChild; + ending?: 'cancelled' | 'timed-out'; + done: Promise; +} + +/** A root as `runner.log` names it: its folder name, never its path. */ +function label(root: string): string { + return path.basename(root) || 'root'; +} + +/** + * Spec 361, Decision 3: runs a check pass of each watched root at its due + * slot, one pass at a time and never alongside a website launch. A machine + * that missed several slots runs one pass (Decision 2). + */ +export class Scheduler { + private readonly deps: SchedulerDeps; + private state: ScheduleState = { zone: 'UTC', roots: {} }; + private readonly queue: string[] = []; + private pass: Pass | null = null; + private cancelTick: () => void = () => {}; + private stopping: Promise | null = null; + /** The config failure last logged per root, so each is logged once. */ + private readonly failures = new Map(); + private frame: SchedulesSync | null = null; + private sentFrame = ''; + + constructor(deps: SchedulerDeps) { + this.deps = deps; + } + + /** Reads the state file and ticks: on start, then once a minute. */ + start(): void { + const saved = readScheduleState(this.deps.stateFile); + if (saved !== null) this.state = saved; + this.tick(); + } + + /** Called when the launch slot frees up: a queued pass may start. */ + wake(): void { + this.pump(); + } + + /** The current `schedules.sync`, for the frame after `launches.sync`. */ + schedules(): SchedulesSync | null { + return this.frame; + } + + /** Stops ticking and kills a running pass; resolves once it is gone. */ + stop(): Promise { + this.stopping ??= (async () => { + this.cancelTick(); + this.queue.length = 0; + const pass = this.pass; + if (pass === null) return; + pass.ending ??= 'cancelled'; + await pass.child?.kill(); + await pass.done; + })(); + return this.stopping; + } + + private tick(): void { + if (this.stopping !== null) return; + const { clock } = this.deps; + const now = clock.now(); + try { + this.look(now); + } catch { + this.deps.logger.log('error', 'schedule: the tick failed'); + } + // On the minute, so 09:30 runs at 09:30 and not at 09:30:59. + const delay = SCHEDULE_TICK_MS - (clock.now() % SCHEDULE_TICK_MS); + this.cancelTick = clock.after(delay, () => this.tick()); + } + + private look(now: number): void { + const zone = this.deps.zone(); + const { reported, local } = this.deps.roots(); + const registry = new Set(reported); + const watched = [...new Set([...registry, ...local])].sort(); + const roots: Record = {}; + for (const root of watched) { + const previous = this.state.roots[root]; + const schedule = this.scheduleOf(root); + const entry: ScheduledRoot = {}; + if (previous?.last !== undefined) entry.last = previous.last; + if (schedule !== undefined) { + entry.schedule = schedule; + // A new schedule is not due for a slot before the daemon saw it. + entry.seenAt = previous?.seenAt ?? new Date(now).toISOString(); + } + if (entry.schedule !== undefined || entry.last !== undefined) { + roots[root] = entry; + } + } + const next: ScheduleState = { + zone, + ...(this.deps.local === true ? { local: true } : {}), + roots, + }; + if (JSON.stringify(next) !== JSON.stringify(this.state)) { + this.state = next; + this.save(); + } + for (const [root, entry] of Object.entries(roots)) { + if (this.isDue(root, entry, now, zone)) this.queue.push(root); + } + this.sync( + zone, + watched.filter((root) => registry.has(root)), + ); + this.pump(); + } + + private scheduleOf(root: string): ScheduleConfig | undefined { + try { + const schedule = this.deps.loadSchedule(root); + if (this.failures.delete(root)) { + this.deps.logger.log('info', `schedule: ${label(root)}: config loads`); + } + return schedule; + } catch (error) { + // Not the message: it may hold a path (spec 248, Decision 13). + const reason = error instanceof Error ? error.message : String(error); + if (this.failures.get(root) !== reason) { + this.failures.set(root, reason); + this.deps.logger.log( + 'warn', + `schedule: ${label(root)}: skipped, the config does not load`, + ); + } + return undefined; + } + } + + private isDue( + root: string, + entry: ScheduledRoot, + now: number, + zone: string, + ): boolean { + if (entry.schedule === undefined) return false; + if (this.pass?.root === root || this.queue.includes(root)) return false; + const since = Date.parse(entry.last?.startedAt ?? entry.seenAt ?? ''); + if (Number.isNaN(since)) return false; + try { + return lastDueSlot(entry.schedule, now, zone) > since; + } catch { + return false; + } + } + + private sync(zone: string, reported: string[]): void { + const frame: SchedulesSync = { + type: 'schedules.sync', + zone, + projects: reported.map((rootPath) => ({ + rootPath, + schedule: this.state.roots[rootPath]?.schedule ?? null, + })), + }; + this.frame = frame; + const text = JSON.stringify(frame); + if (text === this.sentFrame) return; + this.sentFrame = text; + this.deps.onSchedules?.(frame); + } + + private save(): void { + try { + writeScheduleState(this.deps.stateFile, this.state); + } catch { + this.deps.logger.log('warn', 'schedule: could not write the state file'); + } + } + + /** One pass at a time, in the slot the website's launches use. */ + private pump(): void { + if (this.pass !== null || this.stopping !== null) return; + for (;;) { + const root = this.queue[0]; + if (root === undefined) return; + const schedule = this.state.roots[root]?.schedule; + if (schedule === undefined) { + this.queue.shift(); + continue; + } + const release = this.deps.claim(); + if (release === null) return; + this.queue.shift(); + const pass: Pass = { root, done: Promise.resolve() }; + this.pass = pass; + pass.done = this.run(pass, schedule) + .catch(() => { + this.deps.logger.log( + 'error', + `schedule: ${label(root)}: pass failed`, + ); + }) + .finally(() => { + this.pass = null; + release(); + this.pump(); + }); + return; + } + } + + private async run(pass: Pass, schedule: ScheduleConfig): Promise { + const { clock, logger } = this.deps; + const { root } = pass; + // Decision 3: written before the spawn, so a crash does not repeat the slot. + const last: LastPass = { startedAt: new Date(clock.now()).toISOString() }; + this.record(root, last); + // Set from the child's stdout; an object, so the narrowing survives. + const seen: { totals: { passed: number; failed: number } | null } = { + totals: null, + }; + let cancelScenarioCap = (): void => {}; + const capScenario = () => { + cancelScenarioCap(); + cancelScenarioCap = clock.after(CHECK_SCENARIO_MAX_MS, () => { + void this.end(pass, 'timed-out'); + }); + }; + const exclude = schedule.exclude ?? []; + const child = this.deps.spawn( + [ + 'run', + '--check', + '--json', + '--progress', + '--project', + root, + ...exclude.flatMap((id) => ['--exclude', id]), + ], + root, + { + killGraceMs: LAUNCH_KILL_GRACE_MS, + onStdoutLine: (line) => { + if (isCheckProgressLine(line)) capScenario(); + seen.totals = checkTotals(line) ?? seen.totals; + }, + }, + ); + if (child.pid === undefined) { + await child.exited; + logger.log('warn', `schedule: ${label(root)}: the pass did not start`); + this.finish(root, last, 'not-started', null); + return; + } + pass.child = child; + logger.log( + 'info', + `schedule: ${label(root)}: check pass started, ${exclude.length} excluded`, + ); + capScenario(); + const cancelPassCap = clock.after(CHECK_PASS_MAX_MS, () => { + void this.end(pass, 'timed-out'); + }); + await child.exited; + cancelScenarioCap(); + cancelPassCap(); + const outcome: PassOutcome = pass.ending ?? 'finished'; + const counts = outcome === 'finished' ? seen.totals : null; + logger.log( + 'info', + `schedule: ${label(root)}: ${outcome}, ${counts?.passed ?? 0} passed, ${counts?.failed ?? 0} failed`, + ); + this.finish(root, last, outcome, counts); + } + + private async end( + pass: Pass, + reason: 'cancelled' | 'timed-out', + ): Promise { + pass.ending ??= reason; + await pass.child?.kill(); + } + + private finish( + root: string, + started: LastPass, + outcome: PassOutcome, + counts: { passed: number; failed: number } | null, + ): void { + this.record(root, { + ...started, + finishedAt: new Date(this.deps.clock.now()).toISOString(), + outcome, + ...(counts ?? {}), + }); + } + + private record(root: string, last: LastPass): void { + const entry = this.state.roots[root] ?? {}; + this.state = { + ...this.state, + roots: { ...this.state.roots, [root]: { ...entry, last } }, + }; + this.save(); + } +} diff --git a/apps/cli/src/runner/schedule/state.ts b/apps/cli/src/runner/schedule/state.ts new file mode 100644 index 00000000..a3fbb0a4 --- /dev/null +++ b/apps/cli/src/runner/schedule/state.ts @@ -0,0 +1,109 @@ +import { readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'; +import { type ScheduleConfig, ScheduleConfigSchema } from '@demoflow/core'; + +/** How a scheduled check pass ended. */ +export type PassOutcome = + | 'finished' + | 'timed-out' + | 'cancelled' + | 'not-started'; + +const OUTCOMES: readonly PassOutcome[] = [ + 'finished', + 'timed-out', + 'cancelled', + 'not-started', +]; + +/** The last scheduled pass of a root. Counts only once it finished. */ +export interface LastPass { + startedAt: string; + finishedAt?: string; + outcome?: PassOutcome; + passed?: number; + failed?: number; +} + +/** One root of `schedule.state.json`. */ +export interface ScheduledRoot { + /** The root's parsed `schedule`; absent once its config has none. */ + schedule?: ScheduleConfig; + /** + * When the daemon first saw this schedule: a slot before it is not due, so a + * schedule added today does not fire for yesterday. + */ + seenAt?: string; + last?: LastPass; +} + +/** + * `schedule.state.json` (spec 361, Decision 3): written by the daemon only, + * read by `runner status`. Never a token, never a scenario. + */ +export interface ScheduleState { + /** The machine's IANA zone when the daemon last looked. */ + zone: string; + /** `true` while the daemon runs without an account (Decision 5). */ + local?: boolean; + roots: Record; +} + +/** Through a temp file and a rename, so `status` never reads half a file. */ +export function writeScheduleState(file: string, state: ScheduleState): void { + const temp = `${file}.${process.pid}.tmp`; + try { + writeFileSync(temp, `${JSON.stringify(state)}\n`, { mode: 0o600 }); + renameSync(temp, file); + } catch (error) { + rmSync(temp, { force: true }); + throw error; + } +} + +/** A missing or malformed file is `null`; a malformed root is left out. */ +export function readScheduleState(file: string): ScheduleState | null { + let raw: unknown; + try { + raw = JSON.parse(readFileSync(file, 'utf8')); + } catch { + return null; + } + if (!isRecord(raw) || typeof raw.zone !== 'string' || !isRecord(raw.roots)) { + return null; + } + const roots: Record = {}; + for (const [root, value] of Object.entries(raw.roots)) { + if (!isRecord(value)) continue; + const entry: ScheduledRoot = {}; + const schedule = ScheduleConfigSchema.safeParse(value.schedule); + if (schedule.success) entry.schedule = schedule.data; + if (typeof value.seenAt === 'string') entry.seenAt = value.seenAt; + const last = lastPassOf(value.last); + if (last !== undefined) entry.last = last; + roots[root] = entry; + } + return { + zone: raw.zone, + ...(raw.local === true ? { local: true } : {}), + roots, + }; +} + +function lastPassOf(value: unknown): LastPass | undefined { + if (!isRecord(value) || typeof value.startedAt !== 'string') return undefined; + const last: LastPass = { startedAt: value.startedAt }; + if (typeof value.finishedAt === 'string') last.finishedAt = value.finishedAt; + const outcome = OUTCOMES.find((known) => known === value.outcome); + if (outcome !== undefined) last.outcome = outcome; + if (isCount(value.passed)) last.passed = value.passed; + if (isCount(value.failed)) last.failed = value.failed; + return last; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function isCount(value: unknown): value is number { + return typeof value === 'number' && Number.isInteger(value) && value >= 0; +} diff --git a/apps/cli/src/runner/share/files.ts b/apps/cli/src/runner/share/files.ts new file mode 100644 index 00000000..bc1cede6 --- /dev/null +++ b/apps/cli/src/runner/share/files.ts @@ -0,0 +1,14 @@ +import type { RenderFormat } from '@demoflow/core'; +import { pathFor } from '../../config/platform.ts'; + +/** Spec 349, Decision 3: what a share makes lives in `/share/`. */ +export const SHARE_DIR = 'share'; + +/** `/share/poster-.jpg`: one poster per rendered format. */ +export function posterPath( + runDir: string, + format: RenderFormat, + platform: NodeJS.Platform, +): string { + return pathFor(platform).join(runDir, SHARE_DIR, `poster-${format}.jpg`); +} diff --git a/apps/cli/src/runner/share/poster.ts b/apps/cli/src/runner/share/poster.ts new file mode 100644 index 00000000..353f9367 --- /dev/null +++ b/apps/cli/src/runner/share/poster.ts @@ -0,0 +1,87 @@ +import { mkdir, rename, rm, stat } from 'node:fs/promises'; +import { runFfmpeg } from '@demoflow/renderer'; +import { pathFor } from '../../config/platform.ts'; + +/** #347: the API refuses a poster over 1 MiB, so the daemon does too. */ +export const POSTER_MAX_BYTES = 1024 * 1024; +/** Decision 3: the frame is taken at most this far in. */ +export const POSTER_AT_MAX_MS = 1500; +/** Decision 3: the poster is at most this wide, keeping the aspect. */ +export const POSTER_MAX_WIDTH = 1280; + +/** FFmpeg, as the poster maker runs it; tests stub it. */ +export type RunFfmpeg = (args: readonly string[]) => Promise; + +export interface PosterRequest { + /** The render's file, already checked to be inside the run directory. */ + render: string; + /** Where the poster lives: `posterPath(runDir, format)`. */ + poster: string; + /** The render's `durationMs`. */ + durationMs: number; + platform: NodeJS.Platform; +} + +/** The FFmpeg arguments for one JPEG frame of `render` at `atMs`. */ +export function posterArgs( + render: string, + output: string, + atMs: number, +): string[] { + return [ + '-y', + '-i', + render, + // After `-i`: a GIF cannot be seeked on input, and the decode is short. + '-ss', + (atMs / 1000).toFixed(3), + '-frames:v', + '1', + '-an', + '-vf', + `scale='min(${POSTER_MAX_WIDTH},iw)':-2`, + '-q:v', + '4', + output, + ]; +} + +/** Decision 3: `min(1500, ⌊durationMs / 2⌋)`. */ +export function posterAtMs(durationMs: number): number { + return Math.min(POSTER_AT_MAX_MS, Math.floor(Math.max(0, durationMs) / 2)); +} + +/** + * Decision 3: the poster of a render, made with FFmpeg unless the one on + * disk is newer than the render, and its size in bytes. It is written to a + * temporary file and renamed, so a crash never leaves half a JPEG to reuse. + * Rejects when FFmpeg fails or the poster is over `POSTER_MAX_BYTES`. + */ +export async function makePoster( + request: PosterRequest, + ffmpeg: RunFfmpeg = runFfmpeg, +): Promise { + const { render, poster } = request; + const rendered = await stat(render); + const existing = await stat(poster).catch(() => null); + let size: number; + if (existing !== null && existing.mtimeMs > rendered.mtimeMs) { + size = existing.size; + } else { + const p = pathFor(request.platform); + await mkdir(p.dirname(poster), { recursive: true }); + const temp = `${poster.slice(0, -'.jpg'.length)}.tmp.jpg`; + try { + await ffmpeg(posterArgs(render, temp, posterAtMs(request.durationMs))); + await rename(temp, poster); + } catch (error) { + await rm(temp, { force: true }); + throw error; + } + size = (await stat(poster)).size; + } + if (size === 0 || size > POSTER_MAX_BYTES) { + throw new Error(`the poster is ${size} bytes`); + } + return size; +} diff --git a/apps/cli/src/runner/share/prepare.ts b/apps/cli/src/runner/share/prepare.ts new file mode 100644 index 00000000..dc2e131b --- /dev/null +++ b/apps/cli/src/runner/share/prepare.ts @@ -0,0 +1,181 @@ +import { readFile, realpath } from 'node:fs/promises'; +import { + type Chapter, + RENDER_FORMATS, + type RenderFormat, + RunResultSchema, +} from '@demoflow/core'; +import { scenarioDescription } from '../../commands/scenario-file.ts'; +import { pathFor } from '../../config/platform.ts'; +import { findReportedRun } from '../../runs-report/reported-runs.ts'; +import type { Logger } from '../log.ts'; +import { isInside, RUN_ID } from '../relay/index.ts'; +import { posterPath } from './files.ts'; +import { makePoster, type PosterRequest } from './poster.ts'; + +/** `RUN_FILES.result` of the engine, which the daemon does not load. */ +const RESULT_FILE = 'result.json'; +/** #347: the API keeps at most this much of a share's description. */ +export const DESCRIPTION_MAX_LENGTH = 2000; + +export interface ShareDeps { + /** `reported-runs.jsonl`. */ + indexFile: string; + logger: Logger; + platform: NodeJS.Platform; + /** Decision 3; tests stub FFmpeg through it. */ + makePoster(request: PosterRequest): Promise; +} + +/** The socket, as the preparer sees it: JSON text frames only. */ +export interface ShareSender { + send(message: Record): void; +} + +/** What the connection needs from the preparer. */ +export interface SharePort { + handle(message: Record, sender: ShareSender): void; +} + +/** What `share.prepare` answers, before the request id is added. */ +export type Prepared = + | { + ok: true; + description?: string; + chapters: Chapter[]; + posterSize: number; + } + | { ok: false; code: 'not-found' | 'no-render' | 'io' }; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * Spec 349, Decision 2: the daemon's side of `share.prepare`. It answers with + * the scenario's description, the render's chapters and the size of a poster + * it makes, which the relay then serves as `asset: 'poster'`. The server names + * a run by its id; no path is ever sent or logged. + */ +export class SharePreparer implements SharePort { + private readonly deps: ShareDeps; + /** One poster at a time per file: two shares of a render share it. */ + private readonly posters = new Map>(); + + constructor(deps: ShareDeps) { + this.deps = deps; + } + + handle(message: Record, sender: ShareSender): void { + const { requestId, runId, format } = message; + if ( + typeof requestId !== 'string' || + !UUID.test(requestId) || + typeof runId !== 'string' || + !RUN_ID.test(runId) || + !RENDER_FORMATS.includes(format as RenderFormat) + ) { + this.deps.logger.log('warn', 'ignored a malformed share.prepare'); + return; + } + void this.prepare(runId, format as RenderFormat) + .catch((): Prepared => ({ ok: false, code: 'io' })) + .then((prepared) => { + this.deps.logger.log( + 'info', + `share ${runId} ${format}: ${prepared.ok ? 'prepared' : prepared.code}`, + ); + if (!prepared.ok) { + sender.send({ type: 'share.error', requestId, code: prepared.code }); + return; + } + const { description, chapters, posterSize } = prepared; + sender.send({ + type: 'share.prepared', + requestId, + ...(description === undefined ? {} : { description }), + chapters, + posterSize, + }); + }); + } + + async prepare(runId: string, format: RenderFormat): Promise { + const { platform } = this.deps; + const p = pathFor(platform); + const entry = await findReportedRun(this.deps.indexFile, runId); + if (entry === null) return { ok: false, code: 'not-found' }; + let raw: unknown; + try { + raw = JSON.parse( + await readFile(p.join(entry.runDir, RESULT_FILE), 'utf8'), + ); + } catch { + return { ok: false, code: 'not-found' }; + } + const parsed = RunResultSchema.safeParse(raw); + if (!parsed.success) return { ok: false, code: 'not-found' }; + const render = parsed.data.renders.find((item) => item.format === format); + if (render === undefined) return { ok: false, code: 'no-render' }; + + // The same boundary as the relay: only a file inside the run directory. + let file: string; + try { + const root = await realpath(entry.runDir); + file = await realpath(p.resolve(entry.runDir, render.file)); + if (!isInside(root, file, platform)) { + return { ok: false, code: 'no-render' }; + } + } catch { + return { ok: false, code: 'no-render' }; + } + + const posterSize = await this.poster({ + render: file, + poster: posterPath(entry.runDir, format, platform), + durationMs: render.durationMs, + platform, + }); + const description = this.description(entry.root, parsed.data.scenarioId); + return { + ok: true, + ...(description === undefined ? {} : { description }), + chapters: render.chapters ?? [], + posterSize, + }; + } + + private poster(request: PosterRequest): Promise { + const pending = this.posters.get(request.poster); + if (pending !== undefined) return pending; + const making = this.deps.makePoster(request).finally(() => { + this.posters.delete(request.poster); + }); + this.posters.set(request.poster, making); + return making; + } + + /** + * The scenario's description, from its file in the project the run was + * reported from — and only from inside that project. + */ + private description( + root: string | undefined, + scenarioId: string, + ): string | undefined { + if (root === undefined) return undefined; + const p = pathFor(this.deps.platform); + if (!isInside(root, p.resolve(root, scenarioId), this.deps.platform)) { + return undefined; + } + const description = scenarioDescription(root, scenarioId)?.trim(); + return description === undefined || description === '' + ? undefined + : description.slice(0, DESCRIPTION_MAX_LENGTH); + } +} + +/** The daemon's preparer, with the real FFmpeg. */ +export function sharePreparer( + deps: Omit, +): SharePreparer { + return new SharePreparer({ ...deps, makePoster: (r) => makePoster(r) }); +} diff --git a/apps/cli/src/runner/state.ts b/apps/cli/src/runner/state.ts index 1dd425d6..bdbfa1c9 100644 --- a/apps/cli/src/runner/state.ts +++ b/apps/cli/src/runner/state.ts @@ -4,20 +4,24 @@ export type ConnectionState = | 'connecting' | 'connected' | 'reconnecting' - | 'revoked'; + | 'revoked' + /** Spec 361, Decision 5: no account, scheduled checks only. */ + | 'local'; const STATES: readonly ConnectionState[] = [ 'connecting', 'connected', 'reconnecting', 'revoked', + 'local', ]; /** `runner.state.json`: what the daemon tells `status`. Never the token. */ export interface RunnerState { pid: number; startedAt: string; - server: string; + /** Absent in local mode: there is no server (spec 361, Decision 5). */ + server?: string; state: ConnectionState; /** When the next attempt starts, while `reconnecting`. */ retryAt?: string; @@ -54,7 +58,7 @@ export function readState(file: string): RunnerState | null { if ( typeof pid !== 'number' || typeof startedAt !== 'string' || - typeof server !== 'string' || + (server !== undefined && typeof server !== 'string') || typeof updatedAt !== 'string' || known === undefined ) { @@ -63,7 +67,7 @@ export function readState(file: string): RunnerState | null { return { pid, startedAt, - server, + ...(typeof server === 'string' ? { server } : {}), state: known, ...(typeof retryAt === 'string' ? { retryAt } : {}), updatedAt, diff --git a/apps/cli/src/runs-report/report.ts b/apps/cli/src/runs-report/report.ts index 8438ce54..98dd4dd8 100644 --- a/apps/cli/src/runs-report/report.ts +++ b/apps/cli/src/runs-report/report.ts @@ -39,6 +39,8 @@ export interface RunReport { renders: ReportedRender[]; /** Decision 3: a failed run's first failed stage, its message redacted. */ failure?: { stage: StageName; message: string }; + /** Spec 360: `check` for a run of `demoflow run --check`; absent otherwise. */ + kind?: 'check'; } export interface ReportedRender { @@ -79,6 +81,7 @@ export function buildReport(input: { cliVersion: string; pathApi?: path.PlatformPath; home?: string; + check?: boolean; }): RunReport { const { root, result, cliVersion, pathApi = path, home = homedir() } = input; const failedStep = Object.values(result.stages) @@ -109,6 +112,8 @@ export function buildReport(input: { const failure = failureOf(result, { root, home, pathApi }); if (failure !== undefined) report.failure = failure; } + // An ordinary run sends no `kind`: an API before spec 360 refuses the field. + if (input.check === true) report.kind = 'check'; return report; } @@ -184,7 +189,7 @@ function projectIdOf(body: unknown): string | undefined { * `reported-runs.jsonl`, so the daemon can stream its video (spec 192). */ export async function reportRun( - input: { root: string; runDir: string; result: RunResult }, + input: { root: string; runDir: string; result: RunResult; check?: boolean }, deps: ReportDeps, ): Promise { try { @@ -198,6 +203,7 @@ export async function reportRun( cliVersion: deps.version(), pathApi: pathFor(deps.host.platform), home: deps.host.homedir, + check: input.check, }); const response = await deps.fetch( `${apiBase(credentials.server)}/runner/runs`, diff --git a/apps/cli/src/runs-report/reported-runs.ts b/apps/cli/src/runs-report/reported-runs.ts index c48e068f..be526268 100644 --- a/apps/cli/src/runs-report/reported-runs.ts +++ b/apps/cli/src/runs-report/reported-runs.ts @@ -63,15 +63,28 @@ export async function readReportedRunsByDir( * throws: a missing or unreadable index registers nothing. */ export async function knownRoots(file: string): Promise> { - const roots = new Set(); try { - const text = await readFile(file, 'utf8'); - for (const line of text.split('\n')) { - const root = parseLine(line)?.root; - if (root !== undefined) roots.add(root); - } + return rootsIn(await readFile(file, 'utf8')); } catch { // An unreadable index is an empty one. + return new Set(); + } +} + +/** `knownRoots` for a caller that cannot wait: the scheduler's tick, `runner status`. */ +export function knownRootsSync(file: string): Set { + try { + return rootsIn(readFileSync(file, 'utf8')); + } catch { + return new Set(); + } +} + +function rootsIn(text: string): Set { + const roots = new Set(); + for (const line of text.split('\n')) { + const root = parseLine(line)?.root; + if (root !== undefined) roots.add(root); } return roots; } diff --git a/apps/cli/src/share/publish.ts b/apps/cli/src/share/publish.ts new file mode 100644 index 00000000..9a2355c0 --- /dev/null +++ b/apps/cli/src/share/publish.ts @@ -0,0 +1,321 @@ +import path from 'node:path'; +import type { RenderFormat } from '@demoflow/core'; +import { + type Fetch, + readJson, + ServerError, + send, + unexpected, +} from '../auth/client.ts'; +import type { Credentials } from '../auth/credentials.ts'; +import { readResult } from '../commands/settle.ts'; +import { + type ReportedRun, + readReportedRunsByDir, +} from '../runs-report/reported-runs.ts'; + +/** #348: the lifetimes a share can be created with, in days. */ +export const SHARE_LIFETIMES = [7, 30, 90] as const; +export type ShareLifetime = (typeof SHARE_LIFETIMES)[number]; +export const DEFAULT_SHARE_LIFETIME: ShareLifetime = 30; +/** #348: the API's bounds on a share password. */ +export const PASSWORD_MIN_LENGTH = 8; +export const PASSWORD_MAX_LENGTH = 128; +/** Decision 6: `GET /runner/shares/:id` once a second while it uploads. */ +export const SHARE_POLL_MS = 1_000; + +export type ShareStatus = 'uploading' | 'ready' | 'failed' | 'lost'; +const SHARE_STATUSES: readonly string[] = [ + 'uploading', + 'ready', + 'failed', + 'lost', +]; + +/** What the CLI reads of a `ShareView`. */ +export interface Share { + id: string; + url: string; + /** `null` when the share has a password. */ + fileUrl: string | null; + status: ShareStatus; + failure: string | null; + expiresAt: string; + /** The whole `ShareView`, as the server sent it (`--json`). */ + view: Record; +} + +/** Why a share cannot be made; the command and the MCP tool name it so. */ +export type ShareRefusalCode = + | 'not-logged-in' + | 'invalid-result' + | 'run-not-reported' + | 'format-not-rendered' + | 'artifacts-deleted' + | 'runner-offline' + | 'runner-outdated' + | 'file-too-large' + | 'quota-exceeded'; + +export class ShareRefused extends Error { + readonly code: ShareRefusalCode; + + constructor(code: ShareRefusalCode, message: string) { + super(message); + this.name = 'ShareRefused'; + this.code = code; + } +} + +export interface ShareRequest { + /** The server's run id, from the reported-runs index. */ + runId: string; + format: RenderFormat; + expiresInDays: ShareLifetime; + password?: string; +} + +/** + * The format to share: the one asked for, which the run must have rendered, + * or the run's first render. The daemon reads the same `result.json`. + */ +export async function shareableFormat( + runDir: string, + format: RenderFormat | undefined, +): Promise { + let formats: RenderFormat[]; + try { + formats = (await readResult(runDir)).renders.map((render) => render.format); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + throw new ShareRefused( + 'invalid-result', + `cannot read the run's result.json: ${reason}`, + ); + } + const chosen = format ?? formats[0]; + if (chosen === undefined) { + throw new ShareRefused( + 'format-not-rendered', + 'the run has no render: make one with demoflow render ', + ); + } + if (!formats.includes(chosen)) { + throw new ShareRefused( + 'format-not-rendered', + `the run has no ${chosen} render (it has ${formats.join(', ')})`, + ); + } + return chosen; +} + +/** The run's line in the reported-runs index, if it went to `server`. */ +export async function reportedRunOf( + indexFile: string, + runDir: string, + server: string, +): Promise { + const entry = (await readReportedRunsByDir(indexFile)).get( + path.resolve(runDir), + ); + if (entry === undefined) { + throw new ShareRefused( + 'run-not-reported', + 'the run was never reported to the server: it was made signed out or with reporting off', + ); + } + if (entry.server !== undefined && entry.server !== server) { + throw new ShareRefused( + 'run-not-reported', + `the run was reported to ${entry.server}, not to ${server}`, + ); + } + return entry; +} + +/** + * `POST /runner/shares`. A live share of the same run and format is not an + * error: it is fetched and returned with `existing: true` (Decision 6). + */ +export async function createShare( + credentials: Credentials, + request: ShareRequest, + fetchFn: Fetch, +): Promise<{ share: Share; existing: boolean }> { + // A second try only for a `share-exists` that raced its own insert. + for (let attempt = 0; attempt < 2; attempt++) { + const response = await send(fetchFn, credentials.server, '/runner/shares', { + method: 'POST', + token: credentials.token, + body: { + runId: request.runId, + format: request.format, + expiresInDays: request.expiresInDays, + ...(request.password === undefined + ? {} + : { password: request.password }), + }, + }); + if (response.status === 202) { + return { share: toShare(await readJson(response)), existing: false }; + } + const body = await errorBody(response); + if (response.status === 409 && body.error === 'share-exists') { + if (typeof body.shareId !== 'string') continue; + return { + share: await fetchShare(credentials, body.shareId, fetchFn), + existing: true, + }; + } + throw refusal(response.status, body, request.format); + } + throw new ServerError('the server keeps answering share-exists', false); +} + +/** `GET /runner/shares/:id`. */ +export async function fetchShare( + credentials: Credentials, + id: string, + fetchFn: Fetch, +): Promise { + const response = await send( + fetchFn, + credentials.server, + `/runner/shares/${encodeURIComponent(id)}`, + { method: 'GET', token: credentials.token }, + ); + if (response.status === 200) return toShare(await readJson(response)); + if (response.status === 401) { + throw refusal(401, await errorBody(response), undefined); + } + throw unexpected(response.status); +} + +export interface WaitDeps { + fetch: Fetch; + sleep(ms: number): Promise; + now(): number; +} + +/** + * Polls a share until it leaves `uploading` or `timeoutMs` passes, and + * returns what it last read: still `uploading` on a timeout. + */ +export async function waitForShare( + credentials: Credentials, + share: Share, + deps: WaitDeps, + options: { + timeoutMs: number; + onPoll?(share: Share, elapsedMs: number): void; + }, +): Promise { + const started = deps.now(); + let current = share; + while (current.status === 'uploading') { + const elapsed = deps.now() - started; + if (elapsed + SHARE_POLL_MS > options.timeoutMs) break; + await deps.sleep(SHARE_POLL_MS); + current = await fetchShare(credentials, current.id, deps.fetch); + options.onPoll?.(current, deps.now() - started); + } + return current; +} + +/** `1.5 MB`: what a refusal names a size with. */ +export function formatSize(bytes: number): string { + if (bytes < 1000 * 1000) return `${Math.round(bytes / 1000)} kB`; + if (bytes < 1000 * 1000 * 1000) { + return `${(bytes / (1000 * 1000)).toFixed(1)} MB`; + } + return `${(bytes / (1000 * 1000 * 1000)).toFixed(2)} GB`; +} + +function refusal( + status: number, + body: Record, + format: RenderFormat | undefined, +): Error { + if (status === 401) { + return new ShareRefused( + 'not-logged-in', + 'the stored token was revoked: run demoflow login', + ); + } + if (status === 404) { + return new ShareRefused( + 'run-not-reported', + 'the server does not know this run on this machine: it was made signed out, with reporting off, or before the last login', + ); + } + switch (body.error) { + case 'format-not-rendered': + return new ShareRefused( + 'format-not-rendered', + `the server has no ${format ?? 'such'} render of this run: render it again with demoflow render while logged in`, + ); + case 'artifacts-deleted': + return new ShareRefused( + 'artifacts-deleted', + "the run's files were deleted on the website", + ); + case 'runner-offline': + return new ShareRefused( + 'runner-offline', + 'the runner is offline: start the runner: demoflow runner start', + ); + case 'runner-outdated': + return new ShareRefused( + 'runner-outdated', + 'the running runner cannot upload shares: restart the runner (demoflow runner stop, then demoflow runner start)', + ); + case 'file-too-large': + return new ShareRefused( + 'file-too-large', + typeof body.maxFileBytes === 'number' + ? `the render is over the ${formatSize(body.maxFileBytes)} a share can hold` + : 'the render is over the size a share can hold', + ); + case 'quota-exceeded': + return new ShareRefused( + 'quota-exceeded', + typeof body.usedBytes === 'number' && + typeof body.limitBytes === 'number' + ? `the share quota is full: ${formatSize(body.usedBytes)} of ${formatSize(body.limitBytes)} used; revoke a share on the website` + : 'the share quota is full; revoke a share on the website', + ); + } + return unexpected(status); +} + +/** An error body, or `{}` when it is not JSON: the status then decides. */ +async function errorBody(response: Response): Promise> { + try { + return await readJson(response); + } catch { + return {}; + } +} + +function toShare(body: Record): Share { + const { id, url, fileUrl, status, failure, expiresAt } = body; + if ( + typeof id !== 'string' || + typeof url !== 'string' || + !(typeof fileUrl === 'string' || fileUrl === null) || + typeof status !== 'string' || + !SHARE_STATUSES.includes(status) || + typeof expiresAt !== 'string' + ) { + throw new ServerError('the server sent a share it cannot read', false); + } + return { + id, + url, + fileUrl, + status: status as ShareStatus, + failure: typeof failure === 'string' ? failure : null, + expiresAt, + view: body, + }; +} diff --git a/apps/cli/test/auth/auth.spec.ts b/apps/cli/test/auth/auth.spec.ts index f2aefe30..48e5003a 100644 --- a/apps/cli/test/auth/auth.spec.ts +++ b/apps/cli/test/auth/auth.spec.ts @@ -22,7 +22,7 @@ import { readCredentials, writeCredentials, } from '../../src/auth/credentials.ts'; -import { DEFAULT_SERVER, resolveServer } from '../../src/auth/server.ts'; +import { resolveServer } from '../../src/auth/server.ts'; import type { Io } from '../../src/commands/io.ts'; import { type AuthDeps, loginCommand } from '../../src/commands/login.ts'; import { logoutCommand } from '../../src/commands/logout.ts'; @@ -235,11 +235,13 @@ describe('credentials', () => { }); describe('resolveServer', () => { - it('prefers the flag, then DEMOFLOW_SERVER, then staging', () => { + it('prefers the flag, then DEMOFLOW_SERVER, then production', () => { const env = { DEMOFLOW_SERVER: 'http://env.test/' }; expect(resolveServer('http://flag.test', env)).toBe('http://flag.test'); expect(resolveServer(undefined, env)).toBe('http://env.test'); - expect(resolveServer(undefined, {})).toBe(DEFAULT_SERVER); + expect(resolveServer(undefined, {})).toBe( + 'https://demoflow.luna-realm.com', + ); }); it('rejects anything but http(s)', () => { diff --git a/apps/cli/test/commands/check-run.spec.ts b/apps/cli/test/commands/check-run.spec.ts new file mode 100644 index 00000000..a132d82d --- /dev/null +++ b/apps/cli/test/commands/check-run.spec.ts @@ -0,0 +1,278 @@ +import { existsSync, readdirSync, readFileSync, realpathSync } from 'node:fs'; +import { cp, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import type { RunResult } from '@demoflow/core'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +// Read-only helper of the engine's tests: serves the built fixture on an ephemeral port. +import { + type FixtureServer, + startFixtureServer, +} from '../../../../packages/engine/test/execute/fixture-server.ts'; +import { AppStartError } from '../../src/app/dev-server.ts'; +import { CommandError } from '../../src/commands/errors.ts'; +import type { Io } from '../../src/commands/io.ts'; +import { + type RunDeps, + type RunFlags, + runCommand, +} from '../../src/commands/run.ts'; +import type { HostEnvironment } from '../../src/config/platform.ts'; +import { runnerPaths } from '../../src/runner/paths.ts'; +import { readLocalRoots } from '../../src/runner/schedule/local-roots.ts'; + +const REPO = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../../../..', +); +const FIXTURE_SCENARIOS = path.join(REPO, 'apps/fixture/demoflow'); + +let fixture: FixtureServer; +let project: string; + +beforeAll(async () => { + fixture = await startFixtureServer(); + project = await mkdtemp(path.join(tmpdir(), 'demoflow-check-project-')); + await writeFile( + path.join(project, 'demoflow.config.json'), + JSON.stringify({ baseUrl: fixture.origin }), + ); + await cp(FIXTURE_SCENARIOS, path.join(project, 'demoflow'), { + recursive: true, + }); +}); + +afterAll(async () => { + await fixture.close(); + await rm(project, { recursive: true, force: true }); +}); + +/** An in-process `demoflow run` with a start hook that counts app starts. */ +async function harness(start?: RunDeps['startApp']) { + const home = await mkdtemp(path.join(tmpdir(), 'demoflow-check-home-')); + const io: Io & { out: string; err: string } = { + out: '', + err: '', + stdout: (text) => { + io.out += text; + }, + stderr: (text) => { + io.err += text; + }, + }; + let starts = 0; + let stops = 0; + let runs = 0; + const host: HostEnvironment = { + platform: process.platform, + env: { + DEMOFLOW_HOME: home, + DEMOFLOW_NO_REPORT: '1', + // landing, login and phone sign in; any password does. + FIXTURE_PASSWORD: 'correct horse', + }, + homedir: path.join(home, 'user'), + }; + const run = (file: string | undefined, flags: RunFlags) => + runCommand(file, flags, io, { + host, + cwd: project, + runId: () => `check-${++runs}`, + startApp: async (...args) => { + starts++; + if (start !== undefined) return start(...args); + return { + started: true, + stop: async () => { + stops++; + }, + }; + }, + }); + return { + io, + run, + home, + host, + runsDir: path.join(home, 'runs'), + counts: () => ({ starts, stops }), + }; +} + +function scenario(name: string): string { + return path.join(project, 'demoflow', `${name}.demo.json`); +} + +function resultOf(runDir: string): RunResult { + return JSON.parse( + readFileSync(path.join(runDir, 'result.json'), 'utf8'), + ) as RunResult; +} + +async function refusal(promise: Promise): Promise { + const error = await promise.then( + () => undefined, + (reason: unknown) => reason, + ); + expect(error).toBeInstanceOf(CommandError); + return error as CommandError; +} + +describe('demoflow run --check', () => { + it('runs and verifies one scenario, recording and rendering nothing', async () => { + const { io, run, runsDir } = await harness(); + const code = await run(scenario('customers'), { check: true }); + expect(code).toBe(0); + const runDir = path.join(runsDir, 'check-1'); + const result = resultOf(runDir); + expect(result.status).not.toBe('failed'); + expect(result.stages.recording.status).toBe('skipped'); + expect(result.stages.rendering.status).toBe('skipped'); + expect(result.stages.verification.status).toBe('passed'); + expect(result.renders).toEqual([]); + expect( + readdirSync(runDir).filter((file) => file.endsWith('.webm')), + ).toEqual([]); + expect(readdirSync(runDir).filter((file) => file.endsWith('.mp4'))).toEqual( + [], + ); + expect(io.out).toBe( + 'passed demoflow/customers.demo.json\n1 passed, 0 failed\n', + ); + }); + + it('records the project root for the runner to watch, once (spec 361)', async () => { + const { run, host } = await harness(); + await run(scenario('customers'), { check: true }); + await run(scenario('customers'), { check: true }); + expect(readLocalRoots(runnerPaths(host).localRoots)).toEqual([ + realpathSync.native(project), + ]); + }); + + it('fails the check of a failing scenario: exit 1, verification failed', async () => { + const { io, run, runsDir } = await harness(); + const code = await run(scenario('failing'), { check: true, json: true }); + expect(code).toBe(1); + const result = resultOf(path.join(runsDir, 'check-1')); + expect(result.status).toBe('failed'); + expect(result.stages.verification.status).toBe('failed'); + const printed = JSON.parse(io.out) as { + check: true; + passed: number; + failed: number; + runs: { scenarioId: string; status: string; runDir: string }[]; + }; + expect(printed).toMatchObject({ check: true, passed: 0, failed: 1 }); + expect(printed.runs.map((entry) => entry.scenarioId)).toEqual([ + 'demoflow/failing.demo.json', + ]); + }); + + it('runs every scenario of the project against one app start, leaving out the excluded', async () => { + const { io, run, runsDir, counts } = await harness(); + const code = await run(undefined, { + check: true, + exclude: [ + 'login.demo.json', + 'demoflow/failing.demo.json', + 'gone.demo.json', + ], + }); + expect(counts()).toEqual({ starts: 1, stops: 1 }); + const expected = [ + 'demoflow/customers-repeat.demo.json', + 'demoflow/customers.demo.json', + 'demoflow/landing.demo.json', + 'demoflow/masked-elements.demo.json', + 'demoflow/phone.demo.json', + 'demoflow/tour.demo.json', + ]; + const lines = io.out.trimEnd().split('\n'); + expect(lines.slice(0, -1).map((line) => line.split(/\s+/)[1])).toEqual( + expected, + ); + expect(lines.at(-1), io.out).toBe(`${expected.length} passed, 0 failed`); + expect(code).toBe(0); + for (const runDir of readdirSync(runsDir)) { + const result = resultOf(path.join(runsDir, runDir)); + expect(result.stages.recording.status).toBe('skipped'); + expect(result.renders).toEqual([]); + } + expect(readdirSync(runsDir)).toHaveLength(expected.length); + }, 180_000); + + it('prints one check progress line per scenario of a pass', async () => { + const { io, run } = await harness(); + await run(undefined, { + check: true, + progress: true, + json: true, + exclude: [ + 'login.demo.json', + 'customers-repeat.demo.json', + 'landing.demo.json', + 'masked-elements.demo.json', + 'phone.demo.json', + 'tour.demo.json', + ], + }); + const checks = io.out + .split('\n') + .filter((line) => line.includes('"phase":"check"')) + .map((line) => JSON.parse(line) as unknown); + expect(checks).toEqual([ + { type: 'progress', phase: 'check', run: 1, runs: 2 }, + { type: 'progress', phase: 'check', run: 2, runs: 2 }, + ]); + }, 60_000); + + it.each<[string, string | undefined, RunFlags]>([ + ['--format', 'customers', { format: ['gif'] }], + ['--headed', 'customers', { headed: true }], + ['--run-id in a pass', undefined, { runId: 'mine' }], + ['--exclude with a scenario', 'customers', { exclude: ['x.demo.json'] }], + ])('refuses %s: exit 2, nothing started or written', async (_, name, extra) => { + const { io, run, runsDir, counts } = await harness(); + const file = name === undefined ? undefined : scenario(name); + const error = await refusal(run(file, { check: true, ...extra })); + expect(error.code).toBe('check-flag'); + expect(error.exitCode).toBe(2); + expect(counts().starts).toBe(0); + expect(io.out).toBe(''); + expect(existsSync(runsDir)).toBe(false); + }); + + it('fails every scenario of a pass whose app never answered, each with its own result', async () => { + const { io, run, runsDir } = await harness(async () => { + throw new AppStartError('the app did not answer'); + }); + const code = await run(undefined, { + check: true, + exclude: [ + 'customers-repeat.demo.json', + 'landing.demo.json', + 'login.demo.json', + 'masked-elements.demo.json', + 'phone.demo.json', + 'tour.demo.json', + ], + }); + expect(code).toBe(1); + expect(readdirSync(runsDir).sort()).toEqual(['check-1', 'check-2']); + for (const runDir of ['check-1', 'check-2']) { + const result = resultOf(path.join(runsDir, runDir)); + expect(result.stages.readiness.status).toBe('failed'); + expect(result.stages.recording.status).toBe('skipped'); + } + expect(io.out).toMatch(/0 passed, 2 failed\n$/); + }); + + it('refuses a run with no scenario and no --check', async () => { + const { run } = await harness(); + const error = await refusal(run(undefined, {})); + expect(error.code).toBe('missing-scenario'); + expect(error.exitCode).toBe(2); + }); +}); diff --git a/apps/cli/test/commands/cli.spec.ts b/apps/cli/test/commands/cli.spec.ts index 341465c2..c81de412 100644 --- a/apps/cli/test/commands/cli.spec.ts +++ b/apps/cli/test/commands/cli.spec.ts @@ -110,7 +110,8 @@ beforeAll(async () => { out = await mkdtemp(path.join(tmpdir(), 'demoflow-cli-out-')); await writeFile( path.join(project, 'demoflow.config.json'), - JSON.stringify({ baseUrl: fixture.origin }), + // No end hold (spec 365): it only lengthens every run of these tests. + JSON.stringify({ baseUrl: fixture.origin, render: { endHoldMs: 0 } }), ); await cp(FIXTURE_SCENARIOS, path.join(project, 'demoflow'), { recursive: true, @@ -291,13 +292,26 @@ describe('demoflow run on a paired machine', () => { status = 500; const env = pairedEnv(apiUrl()); const result = await demoflow( - ['run', scenario('customers'), '--json', '--out', out], + [ + 'run', + scenario('customers'), + '--json', + '--out', + out, + // The id of the Windows run that flaked (#387): a run started at + // 12:50:08 UTC prints "125008", "500" included, on stderr. + '--run-id', + '20261026-125008-xamdgj', + ], project, env, ); expect(result.code, result.stderr).toBe(0); expect(JSON.parse(result.stdout).status).toBe('completed'); - expect(result.stderr).not.toMatch(/report|500|runner\/runs/i); + // What a printed report failure would name: the status as a number of its + // own, the endpoint or the server. Bare digits match the run id (#387). + expect(result.stderr).not.toMatch(/report|\b500\b|runner\/runs/i); + expect(result.stderr).not.toContain(apiUrl()); const index = runnerPaths({ platform: process.platform, env, @@ -357,6 +371,7 @@ describe('demoflow run', () => { JSON.stringify({ baseUrl: fixture.origin, render: { + endHoldMs: 0, frame: { marginRatio: 0.2 }, background: { color: '#c03020' }, // Frame 0 is the page, not the intro card nor the first load's @@ -422,7 +437,7 @@ describe('demoflow run', () => { path.join(carded, 'demoflow.config.json'), JSON.stringify({ baseUrl: fixture.origin, - render: { background: { color: '#c03020' }, cards }, + render: { endHoldMs: 0, background: { color: '#c03020' }, cards }, }), ); const result = await demoflow( @@ -485,6 +500,7 @@ describe('demoflow run', () => { JSON.stringify({ baseUrl: fixture.origin, render: { + endHoldMs: 0, background: { color: '#c03020' }, cards: { intro: { enabled: false } }, }, diff --git a/apps/cli/test/commands/render.spec.ts b/apps/cli/test/commands/render.spec.ts index d7719e30..acd19c53 100644 --- a/apps/cli/test/commands/render.spec.ts +++ b/apps/cli/test/commands/render.spec.ts @@ -215,6 +215,8 @@ describe('demoflow render', () => { expect(d.plans[0]?.dir).toBe(runDir); expect(d.plans[0]?.options.formats).toEqual(['mp4', 'webm']); expect(d.plans[0]?.options.output?.fps).toBe(24); + // The default end hold, as `demoflow run` recorded it (spec 365). + expect(d.plans[0]?.options.endHoldMs).toBe(2500); expect(io.err).toContain('Rendering\n'); expect(io.out).toContain(RUN_ID); const written = await readResultFile(); diff --git a/apps/cli/test/commands/share.spec.ts b/apps/cli/test/commands/share.spec.ts new file mode 100644 index 00000000..aa1bc264 --- /dev/null +++ b/apps/cli/test/commands/share.spec.ts @@ -0,0 +1,526 @@ +import { mkdir, mkdtemp, realpath, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import type { RunResult } from '@demoflow/core'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Fetch } from '../../src/auth/client.ts'; +import { + credentialsPath, + writeCredentials, +} from '../../src/auth/credentials.ts'; +import { CommandError } from '../../src/commands/errors.ts'; +import type { Io } from '../../src/commands/io.ts'; +import { + type ShareCommandDeps, + type ShareFlags, + shareCommand, +} from '../../src/commands/share.ts'; +import { + type HostEnvironment, + PROJECT_CONFIG_FILE, +} from '../../src/config/index.ts'; +import { main } from '../../src/main.ts'; +import { runnerPaths } from '../../src/runner/paths.ts'; +import { appendReportedRun } from '../../src/runs-report/reported-runs.ts'; + +const RUN_ID = '20261026-101500-abcdef'; +const SERVER = 'https://demoflow.example'; +const TOKEN = 'device-token-123'; +const SERVER_RUN_ID = '0d6f4a52-5d0e-4c55-9a7f-3f6f3d5c1a01'; +const SHARE_ID = 'share-1'; + +let dir: string; +let project: string; +let runDir: string; +let host: HostEnvironment; + +beforeEach(async () => { + dir = await realpath(await mkdtemp(path.join(tmpdir(), 'demoflow-share-'))); + project = path.join(dir, 'project'); + runDir = path.join(project, 'artifacts', 'runs', RUN_ID); + host = { + platform: process.platform, + env: { XDG_CONFIG_HOME: dir, APPDATA: dir }, + homedir: dir, + }; + await mkdir(runDir, { recursive: true }); + await writeFile( + path.join(project, PROJECT_CONFIG_FILE), + JSON.stringify({ + baseUrl: 'http://localhost:3680', + artifactsDir: 'artifacts', + }), + ); + await writeResult(['mp4', 'gif']); + writeCredentials(host, { server: SERVER, token: TOKEN, runnerId: 'r-1' }); + appendReportedRun(runnerPaths(host).reportedRuns, { + id: SERVER_RUN_ID, + runDir, + at: '2026-10-26T10:15:05.000Z', + projectId: 'p-1', + server: SERVER, + root: project, + }); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +async function writeResult(formats: ('mp4' | 'gif' | 'webm')[]) { + const stage = { status: 'passed' }; + const result = { + schemaVersion: 1, + runId: RUN_ID, + scenarioId: 'demoflow/login.demo.json', + scenarioName: 'Sign in', + startedAt: '2026-10-26T10:15:00.000Z', + finishedAt: '2026-10-26T10:15:05.000Z', + status: 'completed', + stages: { + readiness: stage, + execution: stage, + verification: stage, + recording: stage, + rendering: stage, + storage: { status: 'skipped' }, + }, + renders: formats.map((format) => ({ + renderer: 'ffmpeg', + format, + file: `render/ffmpeg.${format}`, + durationMs: 4000, + sizeBytes: 1000, + width: 1280, + height: 720, + fps: 30, + })), + warnings: [], + } as unknown as RunResult; + await writeFile(path.join(runDir, 'result.json'), JSON.stringify(result)); +} + +function view(overrides: Record = {}) { + return { + id: SHARE_ID, + url: `${SERVER}/s/abc`, + fileUrl: `${SERVER}/s/abc/file.mp4`, + runId: SERVER_RUN_ID, + projectId: 'p-1', + title: 'Sign in', + format: 'mp4', + status: 'uploading', + failure: null, + sizeBytes: 1000, + width: 1280, + height: 720, + durationMs: 4000, + hasPassword: false, + views: 0, + expiresAt: '2026-11-25T10:00:00.000Z', + createdAt: '2026-10-26T10:00:00.000Z', + ...overrides, + }; +} + +interface Call { + method: string; + url: string; + authorization: string | undefined; + body: Record | undefined; +} + +type Answer = { status: number; body: unknown }; + +/** The API: the POST answers `post`, each GET the next of `gets` (the last repeats). */ +function fakeApi(post: Answer | Answer[], gets: Answer[] = []) { + const calls: Call[] = []; + const posts = Array.isArray(post) ? [...post] : [post]; + const fetchFn: Fetch = async (input, init) => { + const headers = (init?.headers ?? {}) as Record; + const body = + typeof init?.body === 'string' + ? (JSON.parse(init.body) as Record) + : undefined; + calls.push({ + method: init?.method ?? 'GET', + url: String(input), + authorization: headers.authorization, + body, + }); + const answer = + init?.method === 'POST' + ? posts.length > 1 + ? posts.shift() + : posts[0] + : gets.length > 1 + ? gets.shift() + : gets[0]; + if (answer === undefined) throw new Error('no answer'); + return new Response(JSON.stringify(answer.body), { + status: answer.status, + headers: { 'content-type': 'application/json' }, + }); + }; + return { calls, fetchFn }; +} + +function capture(): Io & { out: string; err: string } { + const io = { + out: '', + err: '', + stdout: (text: string) => { + io.out += text; + }, + stderr: (text: string) => { + io.err += text; + }, + }; + return io; +} + +function deps( + fetchFn: Fetch, + overrides: Partial = {}, +): ShareCommandDeps { + let now = 0; + return { + host, + cwd: project, + fetch: fetchFn, + sleep: async (ms) => { + now += ms; + }, + now: () => now, + askPassword: async () => null, + ...overrides, + }; +} + +async function share( + fetchFn: Fetch, + flags: ShareFlags = {}, + overrides: Partial = {}, +) { + const io = capture(); + const code = await shareCommand(RUN_ID, flags, io, deps(fetchFn, overrides)); + return { code, ...io }; +} + +async function refusal(promise: Promise): Promise { + try { + await promise; + } catch (error) { + if (error instanceof CommandError) return error; + throw error; + } + throw new Error('expected a CommandError'); +} + +describe('demoflow share', () => { + it('creates the share, waits for it and prints both links', async () => { + const api = fakeApi({ status: 202, body: view() }, [ + { status: 200, body: view() }, + { status: 200, body: view({ status: 'ready' }) }, + ]); + const { code, out, err } = await share(api.fetchFn); + expect(code).toBe(0); + expect(out).toBe(`${SERVER}/s/abc\n${SERVER}/s/abc/file.mp4\n`); + expect(err).toContain('Ready; the link works until 2026-11-25'); + expect(api.calls[0]).toEqual({ + method: 'POST', + url: `${SERVER}/api/runner/shares`, + authorization: `Bearer ${TOKEN}`, + body: { runId: SERVER_RUN_ID, format: 'mp4', expiresInDays: 30 }, + }); + expect(api.calls.slice(1).map((call) => call.url)).toEqual([ + `${SERVER}/api/runner/shares/${SHARE_ID}`, + `${SERVER}/api/runner/shares/${SHARE_ID}`, + ]); + }); + + it('shares the format and lifetime asked for', async () => { + const api = fakeApi({ status: 202, body: view({ status: 'ready' }) }); + const { code } = await share(api.fetchFn, { + format: 'gif', + expires: '90', + }); + expect(code).toBe(0); + expect(api.calls[0]?.body).toEqual({ + runId: SERVER_RUN_ID, + format: 'gif', + expiresInDays: 90, + }); + }); + + it('prints the ShareView with --json', async () => { + const api = fakeApi({ status: 202, body: view({ status: 'ready' }) }); + const { code, out } = await share(api.fetchFn, { json: true }); + expect(code).toBe(0); + expect(JSON.parse(out)).toEqual(view({ status: 'ready' })); + }); + + it('prints no file link for a share with a password', async () => { + const api = fakeApi({ + status: 202, + body: view({ status: 'ready', fileUrl: null, hasPassword: true }), + }); + const { out } = await share( + api.fetchFn, + { password: true }, + { askPassword: async () => 'correct horse' }, + ); + expect(out).toBe(`${SERVER}/s/abc\n`); + }); + + it('keeps an existing share and says so', async () => { + const api = fakeApi( + { status: 409, body: { error: 'share-exists', shareId: 'share-0' } }, + [{ status: 200, body: view({ id: 'share-0', status: 'ready' }) }], + ); + const { code, out, err } = await share(api.fetchFn); + expect(code).toBe(0); + expect(out).toContain(`${SERVER}/s/abc\n`); + expect(err).toContain('already shared without a password'); + expect(api.calls[1]?.url).toBe(`${SERVER}/api/runner/shares/share-0`); + }); + + it('asks again when share-exists comes without a share id', async () => { + const api = fakeApi( + [ + { status: 409, body: { error: 'share-exists' } }, + { status: 409, body: { error: 'share-exists', shareId: 'share-0' } }, + ], + [{ status: 200, body: view({ id: 'share-0', status: 'ready' }) }], + ); + const { code } = await share(api.fetchFn); + expect(code).toBe(0); + expect(api.calls.map((call) => call.method)).toEqual([ + 'POST', + 'POST', + 'GET', + ]); + }); + + it('exits 1 with the failure when the upload fails', async () => { + const api = fakeApi({ status: 202, body: view() }, [ + { status: 200, body: view({ status: 'failed', failure: 'runner-gone' }) }, + ]); + const { code, out, err } = await share(api.fetchFn); + expect(code).toBe(1); + expect(out).toBe(''); + expect(err).toContain('The upload failed: runner-gone'); + }); + + it('gives up after 10 minutes of uploading', async () => { + const api = fakeApi({ status: 202, body: view() }, [ + { status: 200, body: view() }, + ]); + const { code, err } = await share(api.fetchFn); + expect(code).toBe(1); + expect(err).toContain('still going after 10 minutes'); + expect(err).toContain('Still uploading (15 s)'); + // One GET a second, for 10 minutes. + expect(api.calls.length).toBe(1 + 600); + }); + + it('exits 3 when the server is unreachable', async () => { + const { code, err } = await share(async () => { + throw new TypeError('fetch failed'); + }); + expect(code).toBe(3); + expect(err).toContain(`cannot reach ${SERVER}`); + }); + + describe('refusals exit 2 with their code', () => { + it.each([ + [401, {}, 'not-logged-in'], + [404, { error: 'not-found' }, 'run-not-reported'], + [409, { error: 'format-not-rendered' }, 'format-not-rendered'], + [409, { error: 'artifacts-deleted' }, 'artifacts-deleted'], + [409, { error: 'runner-offline' }, 'runner-offline'], + [409, { error: 'runner-outdated' }, 'runner-outdated'], + [ + 413, + { error: 'file-too-large', maxFileBytes: 200_000_000 }, + 'file-too-large', + ], + [ + 409, + { + error: 'quota-exceeded', + usedBytes: 900_000_000, + limitBytes: 1_000_000_000, + }, + 'quota-exceeded', + ], + ])('%i %j → %s', async (status, body, code) => { + const api = fakeApi({ status, body }); + const error = await refusal(share(api.fetchFn)); + expect(error.code).toBe(code); + expect(error.exitCode).toBe(2); + }); + + it('names the hint of an offline runner, the size and the quota', async () => { + const offline = await refusal( + share( + fakeApi({ status: 409, body: { error: 'runner-offline' } }).fetchFn, + ), + ); + expect(offline.message).toContain('demoflow runner start'); + const outdated = await refusal( + share( + fakeApi({ status: 409, body: { error: 'runner-outdated' } }).fetchFn, + ), + ); + expect(outdated.message).toContain('restart the runner'); + const large = await refusal( + share( + fakeApi({ + status: 413, + body: { error: 'file-too-large', maxFileBytes: 200_000_000 }, + }).fetchFn, + ), + ); + expect(large.message).toContain('200.0 MB'); + const quota = await refusal( + share( + fakeApi({ + status: 409, + body: { + error: 'quota-exceeded', + usedBytes: 900_000_000, + limitBytes: 1_000_000_000, + }, + }).fetchFn, + ), + ); + expect(quota.message).toContain('900.0 MB of 1.00 GB used'); + }); + + it('not-logged-in without credentials, before any request', async () => { + await rm(credentialsPath(host)); + const api = fakeApi({ status: 500, body: {} }); + const error = await refusal(share(api.fetchFn)); + expect(error.code).toBe('not-logged-in'); + expect(error.message).toContain('demoflow login'); + expect(api.calls).toEqual([]); + }); + + it('run-not-found for a missing run directory', async () => { + const api = fakeApi({ status: 500, body: {} }); + const io = capture(); + const error = await refusal( + shareCommand('20261026-000000-ffffff', {}, io, deps(api.fetchFn)), + ); + expect(error.code).toBe('run-not-found'); + expect(api.calls).toEqual([]); + }); + + it('run-not-reported for a run missing from the index', async () => { + await writeFile(runnerPaths(host).reportedRuns, ''); + const api = fakeApi({ status: 500, body: {} }); + const error = await refusal(share(api.fetchFn)); + expect(error.code).toBe('run-not-reported'); + expect(api.calls).toEqual([]); + }); + + it('run-not-reported for a run reported to another server', async () => { + writeCredentials(host, { + server: 'https://staging.example', + token: TOKEN, + runnerId: 'r-1', + }); + const error = await refusal( + share(fakeApi({ status: 500, body: {} }).fetchFn), + ); + expect(error.code).toBe('run-not-reported'); + expect(error.message).toContain(SERVER); + }); + + it('format-not-rendered for a format the run lacks, or no render', async () => { + const api = fakeApi({ status: 500, body: {} }); + const missing = await refusal(share(api.fetchFn, { format: 'webm' })); + expect(missing.code).toBe('format-not-rendered'); + expect(missing.message).toContain('mp4, gif'); + await writeResult([]); + const none = await refusal(share(api.fetchFn)); + expect(none.code).toBe('format-not-rendered'); + expect(api.calls).toEqual([]); + }); + + it('invalid-expires for a lifetime the server does not offer', async () => { + const error = await refusal( + share(fakeApi({ status: 500, body: {} }).fetchFn, { expires: '14' }), + ); + expect(error.code).toBe('invalid-expires'); + }); + }); + + describe('--password', () => { + it('sends the typed password in the body only', async () => { + const api = fakeApi({ + status: 202, + body: view({ status: 'ready', fileUrl: null, hasPassword: true }), + }); + await share( + api.fetchFn, + { password: true }, + { askPassword: async () => 'typed secret' }, + ); + expect(api.calls[0]?.body?.password).toBe('typed secret'); + expect(api.calls[0]?.url).not.toContain('secret'); + }); + + it('reads DEMOFLOW_SHARE_PASSWORD without a terminal', async () => { + host = { + ...host, + env: { ...host.env, DEMOFLOW_SHARE_PASSWORD: 'from the env' }, + }; + const api = fakeApi({ + status: 202, + body: view({ status: 'ready', fileUrl: null, hasPassword: true }), + }); + await share(api.fetchFn, { password: true }); + expect(api.calls[0]?.body?.password).toBe('from the env'); + }); + + it('is refused without a terminal and without the variable', async () => { + const api = fakeApi({ status: 500, body: {} }); + const error = await refusal(share(api.fetchFn, { password: true })); + expect(error.code).toBe('invalid-password'); + expect(error.message).toContain('DEMOFLOW_SHARE_PASSWORD'); + expect(api.calls).toEqual([]); + }); + + it('is refused outside 8–128 characters', async () => { + const api = fakeApi({ status: 500, body: {} }); + const short = await refusal( + share( + api.fetchFn, + { password: true }, + { askPassword: async () => 'short' }, + ), + ); + expect(short.code).toBe('invalid-password'); + const long = await refusal( + share( + api.fetchFn, + { password: true }, + { askPassword: async () => 'x'.repeat(129) }, + ), + ); + expect(long.code).toBe('invalid-password'); + }); + + it('takes no value on the command line', async () => { + const io = capture(); + const code = await main( + ['share', RUN_ID, '--password', 'hunter2hunter2'], + io, + ); + expect(code).toBe(2); + expect(io.err).toContain('too many arguments'); + }); + }); +}); diff --git a/apps/cli/test/mcp/server.spec.ts b/apps/cli/test/mcp/server.spec.ts index cb3342a4..ffa6f1ce 100644 --- a/apps/cli/test/mcp/server.spec.ts +++ b/apps/cli/test/mcp/server.spec.ts @@ -216,7 +216,7 @@ afterAll(async () => { }); describe('demoflow mcp', () => { - it('lists its six tools with input schemas', async () => { + it('lists its seven tools with input schemas', async () => { const { tools } = await server.client.listTools(); expect(tools.map((tool) => tool.name).sort()).toEqual([ 'cancel_task', @@ -224,6 +224,7 @@ describe('demoflow mcp', () => { 'generate_demo', 'get_task', 'list_demos', + 'share_demo', 'validate_scenario', ]); for (const tool of tools) expect(tool.inputSchema.type).toBe('object'); @@ -243,6 +244,12 @@ describe('demoflow mcp', () => { expect(properties.scenario?.properties).toHaveProperty('steps'); const getTask = tools.find((tool) => tool.name === 'get_task'); expect(getTask?.inputSchema.required).toEqual(['taskId']); + const share = tools.find((tool) => tool.name === 'share_demo'); + expect(Object.keys(share?.inputSchema.properties ?? {}).sort()).toEqual([ + 'expiresInDays', + 'format', + 'runId', + ]); }); let finished: Record; diff --git a/apps/cli/test/mcp/share.spec.ts b/apps/cli/test/mcp/share.spec.ts new file mode 100644 index 00000000..a3259200 --- /dev/null +++ b/apps/cli/test/mcp/share.spec.ts @@ -0,0 +1,295 @@ +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { Fetch } from '../../src/auth/client.ts'; +import { writeCredentials } from '../../src/auth/credentials.ts'; +import { + type HostEnvironment, + runDir as runDirFor, +} from '../../src/config/index.ts'; +import { createMcpServer } from '../../src/mcp/server.ts'; +import { TaskQueue } from '../../src/mcp/tasks.ts'; +import { runnerPaths } from '../../src/runner/paths.ts'; +import { appendReportedRun } from '../../src/runs-report/reported-runs.ts'; + +const SERVER = 'https://demoflow.example'; +const TOKEN = 'device-token-123'; +const TASK_ID = 't1'; +const SERVER_RUN_ID = '0d6f4a52-5d0e-4c55-9a7f-3f6f3d5c1a01'; + +let dir: string; +let home: string; +let project: string; +let runDir: string; +let host: HostEnvironment; +let client: Client; +let queue: TaskQueue; +let calls: { method: string; url: string; body: unknown }[]; +let gets: { status: number; body: unknown }[]; +let post: { status: number; body: unknown }; + +beforeEach(async () => { + dir = realpathSync(mkdtempSync(path.join(tmpdir(), 'demoflow-mcp-share-'))); + home = path.join(dir, 'home'); + project = path.join(dir, 'project'); + host = { + platform: process.platform, + env: { DEMOFLOW_HOME: home, XDG_CONFIG_HOME: dir, APPDATA: dir }, + homedir: path.join(dir, 'user'), + }; + config({ share: { allowMcp: true } }); + runDir = runDirFor(home, TASK_ID, host.platform); + write(path.join(runDir, 'result.json'), JSON.stringify(result())); + writeCredentials(host, { server: SERVER, token: TOKEN, runnerId: 'r-1' }); + report({ root: project }); + + calls = []; + post = { status: 202, body: view({ status: 'ready' }) }; + gets = []; + const fetchFn: Fetch = async (input, init) => { + calls.push({ + method: init?.method ?? 'GET', + url: String(input), + body: typeof init?.body === 'string' ? JSON.parse(init.body) : undefined, + }); + const answer = + init?.method === 'POST' ? post : gets.length > 1 ? gets.shift() : gets[0]; + return new Response(JSON.stringify(answer?.body ?? {}), { + status: answer?.status ?? 500, + }); + }; + let now = 0; + queue = new TaskQueue({ + spawn: () => { + throw new Error('no runs here'); + }, + readResult: async () => { + throw new Error('no runs here'); + }, + removeFile: async () => undefined, + log: () => undefined, + }); + const server = createMcpServer({ + queue, + host, + cwd: project, + share: { + fetch: fetchFn, + sleep: async (ms) => { + now += ms; + }, + now: () => now, + }, + }); + const [clientSide, serverSide] = InMemoryTransport.createLinkedPair(); + client = new Client({ name: 'test', version: '0' }); + await Promise.all([server.connect(serverSide), client.connect(clientSide)]); +}); + +afterEach(async () => { + await queue.close(); + await client.close(); + rmSync(dir, { recursive: true, force: true }); +}); + +function write(file: string, content: string): void { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, content); +} + +function config(extra: Record): void { + write( + path.join(project, 'demoflow.config.json'), + JSON.stringify({ baseUrl: 'http://x', ...extra }), + ); +} + +function report(extra: { root?: string }): void { + appendReportedRun(runnerPaths(host).reportedRuns, { + id: SERVER_RUN_ID, + runDir, + at: '2026-10-26T10:15:05.000Z', + projectId: 'p-1', + server: SERVER, + ...extra, + }); +} + +function result() { + const stage = { status: 'passed' }; + return { + schemaVersion: 1, + runId: TASK_ID, + scenarioId: 'demoflow/tour.demo.json', + scenarioName: 'Tour', + startedAt: '2026-10-26T10:15:00.000Z', + finishedAt: '2026-10-26T10:15:05.000Z', + status: 'completed', + stages: { + readiness: stage, + execution: stage, + verification: stage, + recording: stage, + rendering: stage, + storage: { status: 'skipped' }, + }, + renders: ['mp4', 'gif'].map((format) => ({ + renderer: 'ffmpeg', + format, + file: `render/ffmpeg.${format}`, + durationMs: 4000, + sizeBytes: 1000, + width: 1280, + height: 720, + fps: 30, + })), + warnings: [], + }; +} + +function view(overrides: Record = {}) { + return { + id: 'share-1', + url: `${SERVER}/s/abc`, + fileUrl: `${SERVER}/s/abc/file.mp4`, + status: 'uploading', + failure: null, + hasPassword: false, + expiresAt: '2026-11-25T10:00:00.000Z', + ...overrides, + }; +} + +async function shareDemo(args: Record = {}) { + const answer = await client.callTool({ + name: 'share_demo', + arguments: { runId: TASK_ID, ...args }, + }); + return { + isError: answer.isError === true, + content: answer.structuredContent as Record, + }; +} + +describe('share_demo', () => { + it('creates the share when the project allows it', async () => { + const { isError, content } = await shareDemo(); + expect(isError).toBe(false); + expect(content).toEqual({ + shareId: 'share-1', + url: `${SERVER}/s/abc`, + fileUrl: `${SERVER}/s/abc/file.mp4`, + expiresAt: '2026-11-25T10:00:00.000Z', + status: 'ready', + existing: false, + }); + expect(calls).toEqual([ + { + method: 'POST', + url: `${SERVER}/api/runner/shares`, + body: { runId: SERVER_RUN_ID, format: 'mp4', expiresInDays: 30 }, + }, + ]); + }); + + it('passes the format and lifetime, and never a password', async () => { + await shareDemo({ format: 'gif', expiresInDays: 7 }); + expect(calls[0]?.body).toEqual({ + runId: SERVER_RUN_ID, + format: 'gif', + expiresInDays: 7, + }); + }); + + it.each([ + ['without a share block', {}], + ['with allowMcp false', { share: { allowMcp: false } }], + ])('is share-not-allowed %s', async (_name, extra) => { + config(extra); + const { isError, content } = await shareDemo(); + expect(isError).toBe(true); + expect(content.code).toBe('share-not-allowed'); + expect(content.hint).toContain('"allowMcp": true'); + expect(calls).toEqual([]); + }); + + it('is share-not-allowed for a run reported without a root', async () => { + rmSync(runnerPaths(host).reportedRuns); + report({}); + const { content } = await shareDemo(); + expect(content.code).toBe('share-not-allowed'); + expect(calls).toEqual([]); + }); + + it('is share-not-allowed when the config cannot be read', async () => { + write(path.join(project, 'demoflow.config.json'), '{not json'); + const { content } = await shareDemo(); + expect(content.code).toBe('share-not-allowed'); + }); + + it('waits for the upload, then answers uploading', async () => { + post = { status: 202, body: view() }; + gets = [{ status: 200, body: view() }]; + const { isError, content } = await shareDemo(); + expect(isError).toBe(false); + expect(content.status).toBe('uploading'); + // One GET a second, under the 60 s request timeout of MCP clients. + expect(calls.filter((call) => call.method === 'GET').length).toBe(50); + }); + + it('returns an existing share', async () => { + post = { status: 409, body: { error: 'share-exists', shareId: 'share-0' } }; + gets = [{ status: 200, body: view({ id: 'share-0', status: 'ready' }) }]; + const { content } = await shareDemo(); + expect(content.shareId).toBe('share-0'); + expect(content.existing).toBe(true); + }); + + it('is share-failed when the upload fails', async () => { + post = { status: 202, body: view() }; + gets = [{ status: 200, body: view({ status: 'failed', failure: 'io' }) }]; + const { isError, content } = await shareDemo(); + expect(isError).toBe(true); + expect(content.code).toBe('share-failed'); + expect(content.message).toContain('io'); + }); + + it('names the server refusal with a hint', async () => { + post = { status: 409, body: { error: 'runner-offline' } }; + const { content } = await shareDemo(); + expect(content.code).toBe('runner-offline'); + expect(content.hint).toContain('demoflow runner start'); + }); + + it('is not-logged-in without credentials', async () => { + rmSync(path.join(runnerPaths(host).dir, 'credentials.json')); + const { content } = await shareDemo(); + expect(content.code).toBe('not-logged-in'); + expect(calls).toEqual([]); + }); + + it('is run-not-reported for a run missing from the index', async () => { + rmSync(runnerPaths(host).reportedRuns); + const { content } = await shareDemo(); + expect(content.code).toBe('run-not-reported'); + }); + + it('is task-not-found for an unknown run', async () => { + const { content } = await shareDemo({ runId: 'nope' }); + expect(content.code).toBe('task-not-found'); + }); + + it('is invalid-input for a lifetime the server does not offer', async () => { + const { content } = await shareDemo({ expiresInDays: 14 }); + expect(content.code).toBe('invalid-input'); + }); +}); diff --git a/apps/cli/test/runner/connection.spec.ts b/apps/cli/test/runner/connection.spec.ts index 77567a51..ad868068 100644 --- a/apps/cli/test/runner/connection.spec.ts +++ b/apps/cli/test/runner/connection.spec.ts @@ -17,6 +17,8 @@ import type { RelaySender, VideoRelayPort, } from '../../src/runner/relay/index.ts'; +import type { ProjectsPort } from '../../src/runner/roots/index.ts'; +import type { SharePort } from '../../src/runner/share/prepare.ts'; import type { ConnectionState } from '../../src/runner/state.ts'; import { FakeClock, @@ -28,7 +30,12 @@ import { const TOKEN = 'dfr_old-token'; const NEW_TOKEN = 'dfr_new-token'; -const EMPTY_SYNC = { type: 'launches.sync', running: [], runOptions: [] }; +const EMPTY_SYNC = { + type: 'launches.sync', + running: [], + runOptions: [], + features: [], +}; let clock: FakeClock; let sockets: FakeSocket[]; @@ -45,6 +52,8 @@ function start( relay?: VideoRelayPort, artifacts?: ArtifactsPort, launches?: LaunchPort, + projects?: ProjectsPort, + share?: SharePort, ) { const fake = fakeSockets(); sockets = fake.sockets; @@ -70,6 +79,8 @@ function start( ...(relay === undefined ? {} : { relay }), ...(artifacts === undefined ? {} : { artifacts }), ...(launches === undefined ? {} : { launches }), + ...(projects === undefined ? {} : { projects }), + ...(share === undefined ? {} : { share }), }); void connection.run().then((value) => { outcome = value; @@ -491,6 +502,8 @@ describe('RunnerConnection and the video relay', () => { type: 'launches.sync', running: ids, runOptions: ['formats'], + features: [], + checks: true, }); expect(logger.text()).toContain('announced 1 running launch'); first.serverClose(1006); @@ -500,7 +513,13 @@ describe('RunnerConnection and the video relay', () => { last().open(); expect(last()).not.toBe(first); expect(last().sent).toEqual([ - { type: 'launches.sync', running: [], runOptions: ['formats'] }, + { + type: 'launches.sync', + running: [], + runOptions: ['formats'], + features: [], + checks: true, + }, ]); }); @@ -510,6 +529,89 @@ describe('RunnerConnection and the video relay', () => { expect(last().sent).toEqual([EMPTY_SYNC]); }); + it('announces the projects feature and routes project.* to it (spec 377)', () => { + const handled: Array> = []; + start(0.5, undefined, undefined, undefined, { + handle: (message, sender) => { + handled.push(message); + sender.send({ type: 'project.error', requestId: 'q', code: 'busy' }); + }, + }); + last().open(); + expect(last().sent[0]).toEqual({ ...EMPTY_SYNC, features: ['projects'] }); + for (const type of [ + 'project.inspect', + 'project.init', + 'project.register', + ]) { + last().receive({ type, requestId: 'q', path: '/x' }); + } + expect(handled.map((message) => message.type)).toEqual([ + 'project.inspect', + 'project.init', + 'project.register', + ]); + expect(last().sent).toContainEqual({ + type: 'project.error', + requestId: 'q', + code: 'busy', + }); + }); + + it('logs project.inspect as unknown without the projects port', () => { + start(); + last().open(); + last().receive({ type: 'project.inspect', requestId: 'q', path: '/x' }); + expect(logger.text()).toMatch( + /ignored a message of type "project.inspect"/, + ); + }); + + it('announces the share feature and routes share.prepare to it (spec 349)', () => { + const relay = recordingRelay(); + const handled: Array> = []; + const share: SharePort = { + handle(message, sender) { + handled.push(message); + sender.send({ type: 'share.error', requestId: 'q', code: 'io' }); + }, + }; + start(0.5, relay, undefined, undefined, undefined, share); + last().open(); + expect(last().sent[0]).toEqual({ ...EMPTY_SYNC, features: ['share'] }); + last().receive({ + type: 'share.prepare', + requestId: 'q', + runId: 'r', + format: 'mp4', + }); + expect(handled).toEqual([ + { type: 'share.prepare', requestId: 'q', runId: 'r', format: 'mp4' }, + ]); + expect(last().sent).toContainEqual({ + type: 'share.error', + requestId: 'q', + code: 'io', + }); + expect(logger.text()).not.toMatch(/ignored/); + }); + + it('does not announce share without the relay that streams its poster', () => { + start(0.5, undefined, undefined, undefined, undefined, { + handle: () => {}, + }); + last().open(); + expect(last().sent[0]).toEqual(EMPTY_SYNC); + }); + + it('logs share.prepare as unknown without the share port, as a daemon before #349 did', () => { + start(0.5, recordingRelay()); + last().open(); + last().receive({ type: 'share.prepare', requestId: 'q' }); + expect(logger.text()).toMatch(/ignored a message of type "share.prepare"/); + expect(last().sent).toEqual([EMPTY_SYNC]); + }); + it('logs run.start as unknown without a launcher, as a daemon before #248 did', () => { start(); last().open(); diff --git a/apps/cli/test/runner/daemon-launch.spec.ts b/apps/cli/test/runner/daemon-launch.spec.ts index 2582c682..457554a6 100644 --- a/apps/cli/test/runner/daemon-launch.spec.ts +++ b/apps/cli/test/runner/daemon-launch.spec.ts @@ -191,6 +191,8 @@ describe('the daemon with a website-started run', () => { type: 'launches.sync', running: [], runOptions: ['formats'], + features: ['projects', 'share'], + checks: true, }); first.receive({ type: 'run.start', @@ -230,6 +232,8 @@ describe('the daemon with a website-started run', () => { type: 'launches.sync', running: [REQUEST], runOptions: ['formats'], + features: ['projects', 'share'], + checks: true, }); await stop(next); }); @@ -240,8 +244,17 @@ describe('the daemon with a website-started run', () => { // Let the launcher settle: the run.finished has no socket and is dropped. await new Promise((r) => setTimeout(r, 50)); const next = await reconnect(); - expect(next.sent).toEqual([ - { type: 'launches.sync', running: [], runOptions: ['formats'] }, + // Spec 361: `schedules.sync` follows; it is not about this run. + expect( + next.sent.filter((frame) => frame.type !== 'schedules.sync'), + ).toEqual([ + { + type: 'launches.sync', + running: [], + runOptions: ['formats'], + features: ['projects', 'share'], + checks: true, + }, ]); }); }); diff --git a/apps/cli/test/runner/fakes.ts b/apps/cli/test/runner/fakes.ts index dc20cf67..b81ee70d 100644 --- a/apps/cli/test/runner/fakes.ts +++ b/apps/cli/test/runner/fakes.ts @@ -8,9 +8,14 @@ import type { Logger, LogLevel } from '../../src/runner/log.ts'; /** Timers that only fire when the test advances time. */ export class FakeClock implements Clock { - private time = 0; + private time: number; private timers: Array<{ at: number; run: () => void; live: boolean }> = []; + /** `start` is the epoch time `now()` begins at; a schedule needs a real date. */ + constructor(start = 0) { + this.time = start; + } + now(): number { return this.time; } diff --git a/apps/cli/test/runner/launch-check.spec.ts b/apps/cli/test/runner/launch-check.spec.ts new file mode 100644 index 00000000..ce514b75 --- /dev/null +++ b/apps/cli/test/runner/launch-check.spec.ts @@ -0,0 +1,336 @@ +import { + mkdirSync, + mkdtempSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { + ChildExit, + RunChild, + SpawnRun, + SpawnRunOptions, +} from '../../src/mcp/child.ts'; +import { + CHECK_PASS_MAX_MS, + CHECK_SCENARIO_MAX_MS, +} from '../../src/runner/launch/check.ts'; +import { nodeLaunchFs } from '../../src/runner/launch/checks.ts'; +import { Launcher, type LauncherDeps } from '../../src/runner/launch/index.ts'; +import { LIST_MAX_SCENARIOS } from '../../src/runner/launch/list.ts'; +import { appendReportedRun } from '../../src/runs-report/reported-runs.ts'; +import { FakeClock, memoryLogger } from './fakes.ts'; + +const REQUEST = '6f1c2b8e-3d4a-4b5c-9e7f-0a1b2c3d4e5f'; + +interface Spawned { + args: string[]; + options: SpawnRunOptions | undefined; + killed: number; + exit(code?: number | null): void; + print(...lines: string[]): void; +} + +function fakeSpawn(): { spawn: SpawnRun; calls: Spawned[] } { + const calls: Spawned[] = []; + const spawn: SpawnRun = (args, _cwd, options) => { + let resolve: (exit: ChildExit) => void = () => {}; + const exited = new Promise((r) => { + resolve = r; + }); + const child: RunChild = { + pid: 4242, + exited, + kill: async () => { + call.killed++; + resolve({ code: null, stderrTail: '' }); + await exited; + }, + }; + const call: Spawned = { + args, + options, + killed: 0, + exit: (code = 0) => resolve({ code, stderrTail: '' }), + print: (...lines) => { + for (const line of lines) options?.onStdoutLine?.(line); + }, + }; + calls.push(call); + return child; + }; + return { spawn, calls }; +} + +function recorder() { + const sent: Array> = []; + return { + sent, + send: (message: Record) => sent.push(message), + }; +} + +let tmp: string; +let root: string; +let indexFile: string; +let clock: FakeClock; +let logger: ReturnType; +let spawned: ReturnType; +let reply: ReturnType; +let current: ReturnType; +let env: Record; + +beforeEach(() => { + tmp = realpathSync.native( + mkdtempSync(path.join(tmpdir(), 'demoflow-launch-check-')), + ); + root = path.join(tmp, 'shop'); + indexFile = path.join(tmp, 'config', 'reported-runs.jsonl'); + mkdirSync(root); + writeFileSync( + path.join(root, 'demoflow.config.json'), + '{"baseUrl":"http://localhost:1"}', + ); + clock = new FakeClock(); + logger = memoryLogger(); + spawned = fakeSpawn(); + reply = recorder(); + current = recorder(); + env = {}; +}); +afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +function register(): void { + appendReportedRun(indexFile, { + id: 'srv-0', + runDir: path.join(tmp, 'home', 'runs', 'old'), + at: '2026-10-23T10:00:00.000Z', + root, + }); +} + +function launcher(overrides: Partial = {}): Launcher { + return new Launcher({ + indexFile, + fs: nodeLaunchFs, + logger, + clock, + platform: process.platform, + env, + spawn: spawned.spawn, + loadProject: () => ({ ok: true, home: path.join(tmp, 'home') }), + newRunId: () => 'local-1', + readStatus: async () => 'completed', + ...overrides, + }); +} + +function send(target: Launcher, message: Record): void { + target.handle(message, reply, current); +} + +const checkOf = (extra: Record = {}) => ({ + type: 'check.start', + requestId: REQUEST, + rootPath: root, + exclude: [], + ...extra, +}); + +async function answered(count = 1): Promise> { + await vi.waitFor(() => + expect(reply.sent.length).toBeGreaterThanOrEqual(count), + ); + return reply.sent[count - 1] as Record; +} + +async function finished(): Promise> { + await vi.waitFor(() => + expect(current.sent.some((frame) => frame.type === 'check.finished')).toBe( + true, + ), + ); + return current.sent.find((frame) => frame.type === 'check.finished') ?? {}; +} + +async function started(target: Launcher, extra = {}): Promise { + register(); + send(target, checkOf(extra)); + expect(await answered()).toEqual({ + type: 'check.started', + requestId: REQUEST, + }); + const call = spawned.calls[0]; + if (call === undefined) throw new Error('nothing spawned'); + return call; +} + +describe('check.start', () => { + it('spawns exactly run --check --json --progress --project and the checked excludes', async () => { + const call = await started(launcher(), { + exclude: ['demoflow/login.demo.json', 'demoflow/a/b.demo.json'], + }); + expect(call.args).toEqual([ + 'run', + '--check', + '--json', + '--progress', + '--project', + root, + '--exclude', + 'demoflow/login.demo.json', + '--exclude', + 'demoflow/a/b.demo.json', + ]); + call.print(JSON.stringify({ check: true, passed: 3, failed: 1, runs: [] })); + call.exit(1); + expect(await finished()).toEqual({ + type: 'check.finished', + requestId: REQUEST, + passed: 3, + failed: 1, + }); + }); + + it.each<[string, boolean, () => Record, string]>([ + ['an unregistered root', false, () => checkOf(), 'project-unknown'], + [ + 'an extra field', + true, + () => checkOf({ scenarioId: 'demoflow/a.demo.json' }), + 'request-invalid', + ], + [ + 'a root that is not a string', + true, + () => checkOf({ rootPath: 42 }), + 'request-invalid', + ], + [ + 'no exclude', + true, + () => ({ type: 'check.start', requestId: REQUEST, rootPath: root }), + 'exclude-invalid', + ], + [ + 'an exclude id outside demoflow/', + true, + () => checkOf({ exclude: ['../a.demo.json'] }), + 'exclude-invalid', + ], + [ + 'an exclude id that is not a string', + true, + () => checkOf({ exclude: [7] }), + 'exclude-invalid', + ], + [ + 'too many exclude ids', + true, + () => + checkOf({ + exclude: Array.from( + { length: LIST_MAX_SCENARIOS + 1 }, + (_, i) => `demoflow/s${i}.demo.json`, + ), + }), + 'exclude-invalid', + ], + ])('refuses %s with check.error and spawns nothing', async (_why, registered, message, code) => { + if (registered) register(); + send(launcher(), message()); + expect(await answered()).toEqual({ + type: 'check.error', + requestId: REQUEST, + code, + }); + expect(spawned.calls).toEqual([]); + }); + + it('ignores a check.start with no request id', () => { + send(launcher(), { ...checkOf(), requestId: 7 }); + expect(reply.sent).toEqual([]); + expect(logger.text()).toMatch(/ignored a malformed check\.start/); + }); + + it("is the machine's one launch: busy for a run.start, and refused with reporting off", async () => { + const target = launcher(); + const call = await started(target); + send(target, { + type: 'run.start', + requestId: 'next', + rootPath: root, + scenarioId: 'demoflow/a.demo.json', + }); + expect(await answered(2)).toEqual({ + type: 'run.error', + requestId: 'next', + code: 'busy', + }); + expect(target.running()).toEqual([REQUEST]); + call.exit(0); + await finished(); + + env.DEMOFLOW_NO_REPORT = '1'; + send(target, { ...checkOf(), requestId: 'off' }); + expect(await answered(3)).toEqual({ + type: 'check.error', + requestId: 'off', + code: 'reporting-off', + }); + }); + + it('is cancelled with run.cancel and the same request id', async () => { + const target = launcher(); + const call = await started(target); + send(target, { type: 'run.cancel', requestId: REQUEST }); + expect(await finished()).toEqual({ + type: 'check.finished', + requestId: REQUEST, + passed: 0, + failed: 0, + }); + expect(call.killed).toBe(1); + }); + + it('caps each scenario at 15 minutes: a new scenario restarts the limit', async () => { + const call = await started(launcher()); + const scenarioLine = (run: number) => + JSON.stringify({ type: 'progress', phase: 'check', run, runs: 3 }); + call.print(scenarioLine(1)); + await clock.advance(CHECK_SCENARIO_MAX_MS - 1); + call.print(scenarioLine(2)); + await clock.advance(CHECK_SCENARIO_MAX_MS - 1); + expect(call.killed).toBe(0); + await clock.advance(1); + expect(call.killed).toBe(1); + await finished(); + }); + + it('caps the whole pass at 2 hours', async () => { + const call = await started(launcher()); + const step = CHECK_SCENARIO_MAX_MS - 1; + for ( + let at = 0, run = 1; + at + step < CHECK_PASS_MAX_MS; + at += step, run++ + ) { + call.print( + JSON.stringify({ type: 'progress', phase: 'check', run, runs: 99 }), + ); + await clock.advance(step); + expect(call.killed).toBe(0); + } + call.print( + JSON.stringify({ type: 'progress', phase: 'check', run: 99, runs: 99 }), + ); + await clock.advance(step); + expect(call.killed).toBe(1); + await finished(); + }); +}); diff --git a/apps/cli/test/runner/relay.spec.ts b/apps/cli/test/runner/relay.spec.ts index 3a292913..20227d53 100644 --- a/apps/cli/test/runner/relay.spec.ts +++ b/apps/cli/test/runner/relay.spec.ts @@ -133,6 +133,7 @@ function open( format?: RenderFormat; range?: VideoRange | null; streamId?: string; + asset?: unknown; } = {}, ): string { const streamId = options.streamId ?? randomUUID(); @@ -143,6 +144,7 @@ function open( runId: options.runId ?? RUN_ID, format: options.format ?? 'mp4', range: options.range ?? null, + ...('asset' in options ? { asset: options.asset } : {}), }, sender, ); @@ -451,6 +453,82 @@ describe('VideoRelay: the ADR-0008 boundary', () => { }); }); +describe('VideoRelay: the poster asset (spec 349)', () => { + let poster: Buffer; + beforeEach(() => { + poster = randomBytes(CHUNK_BYTES + 77); + mkdirSync(path.join(runDir, 'share')); + writeFileSync(path.join(runDir, 'share', 'poster-mp4.jpg'), poster); + }); + + it('streams exactly the poster of the format, and logs it as one', async () => { + const sender = recordingSender(); + const streamId = open(sender, { asset: 'poster' }); + expect((await drain(sender, streamId)).equals(poster)).toBe(true); + expect(sender.messagesOf(streamId)[0]).toMatchObject({ + type: 'video.head', + size: poster.length, + }); + await until(() => logger.lines.length > 0, 'log line'); + expect(logger.lines[0]?.message).toBe( + `video ${RUN_ID} mp4 poster: ${poster.length} bytes sent, sent`, + ); + expect(logger.text()).not.toContain(home); + }); + + it('still serves the render without an asset', async () => { + const sender = recordingSender(); + const streamId = open(sender); + expect((await drain(sender, streamId)).equals(video)).toBe(true); + }); + + it('answers not-found for a format with no poster', async () => { + const sender = recordingSender(); + await expectNotFound( + sender, + open(sender, { asset: 'poster', format: 'gif' }), + ); + }); + + it('answers not-found for a run this machine never reported', async () => { + const sender = recordingSender(); + await expectNotFound( + sender, + open(sender, { asset: 'poster', runId: 'someone-else' }), + ); + }); + + it.each([ + 'thumbnail', + '../result.json', + 42, + null, + ])('answers not-found for the unknown asset %j', async (asset) => { + const sender = recordingSender(); + await expectNotFound(sender, open(sender, { asset })); + expect(counts.open).toBe(0); + }); + + it('answers not-found for a poster that links out of the run directory', async (ctx) => { + const secret = path.join(home, 'secret.jpg'); + writeFileSync(secret, 'secret'); + const link = path.join(runDir, 'share', 'poster-gif.jpg'); + try { + symlinkSync(secret, link, 'file'); + } catch (error) { + if (process.platform === 'win32') { + ctx.skip(`file symlinks need a privilege here: ${String(error)}`); + } + throw error; + } + const sender = recordingSender(); + await expectNotFound( + sender, + open(sender, { asset: 'poster', format: 'gif' }), + ); + }); +}); + describe('VideoRelay: flow and ends', () => { beforeEach(() => { // 2 MiB and a bit: four windows' worth. diff --git a/apps/cli/test/runner/roots.spec.ts b/apps/cli/test/runner/roots.spec.ts new file mode 100644 index 00000000..65b37620 --- /dev/null +++ b/apps/cli/test/runner/roots.spec.ts @@ -0,0 +1,817 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Io } from '../../src/commands/io.ts'; +import type { HostEnvironment } from '../../src/config/platform.ts'; +import { + checkProject, + type LaunchStats, + nodeLaunchFs, +} from '../../src/runner/launch/checks.ts'; +import { loadProjectFor } from '../../src/runner/launch/index.ts'; +import { runnerPaths } from '../../src/runner/paths.ts'; +import { + ALLOWED_ROOTS_MAX, + readAllowedRoots, +} from '../../src/runner/roots/allowed.ts'; +import { + allowCommand, + disallowCommand, + rootsCommand, +} from '../../src/runner/roots/commands.ts'; +import { + githubUrlFrom, + githubUrlOf, + originUrl, +} from '../../src/runner/roots/git-remote.ts'; +import { + defaultWriteTemplate, + isWebsiteRegistered, + PROJECT_REQUEST_MAX_MS, + ProjectRequests, + type ProjectsDeps, +} from '../../src/runner/roots/index.ts'; +import { + isAllowedPath, + type PathFs, +} from '../../src/runner/roots/path-rule.ts'; +import { + registerWebsiteProject, + WEBSITE_PROJECTS_MAX, + websiteRoots, +} from '../../src/runner/roots/registry.ts'; +import { FakeClock, memoryLogger } from './fakes.ts'; + +const REQUEST = '6f1c2b8e-3d4a-4b5c-9e7f-0a1b2c3d4e5f'; +const FIXTURE = readFileSync( + path.join( + import.meta.dirname, + '../../../fixture/demoflow/customers.demo.json', + ), + 'utf8', +); +const CONFIG = JSON.stringify({ + baseUrl: 'http://localhost:4123', + devCommand: { + command: 'secret-dev-command', + readyUrl: 'http://localhost:4123', + }, +}); + +let tmp: string; +let home: string; +let outside: string; +let host: HostEnvironment; +let clock: FakeClock; +let logger: ReturnType; +let sent: Array>; + +function write(file: string, text: string): void { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, text); +} + +function dir(...parts: string[]): string { + const target = path.join(...parts); + mkdirSync(target, { recursive: true }); + return target; +} + +/** A directory link; a junction on Windows, which needs no privilege. */ +function link(target: string, at: string): void { + symlinkSync(target, at, 'junction'); +} + +function files() { + return runnerPaths(host); +} + +function deps(overrides: Partial = {}): ProjectsDeps { + return { + allowedRootsFile: files().allowedRoots, + websiteProjectsFile: files().websiteProjects, + host, + fs: nodeLaunchFs, + logger, + clock, + loadProject: loadProjectFor(host), + writeTemplate: defaultWriteTemplate, + ...overrides, + }; +} + +function requests(overrides: Partial = {}): ProjectRequests { + return new ProjectRequests(deps(overrides)); +} + +async function ask( + type: string, + target: unknown, + handler: ProjectRequests = requests(), +): Promise> { + const before = sent.length; + handler.handle( + { type, requestId: REQUEST, path: target }, + { send: (message) => sent.push(message) }, + ); + await vi.waitFor(() => expect(sent.length).toBe(before + 1)); + return sent[before] as Record; +} + +const refused = { + type: 'project.error', + requestId: REQUEST, + code: 'path-not-allowed', +}; + +function memoryIo(): Io & { out: string; err: string } { + const io = { + out: '', + err: '', + stdout: (text: string) => { + io.out += text; + }, + stderr: (text: string) => { + io.err += text; + }, + }; + return io as Io & { out: string; err: string }; +} + +const commandDeps = () => ({ host, cwd: home }); + +beforeEach(() => { + tmp = realpathSync.native( + mkdtempSync(path.join(tmpdir(), 'demoflow-roots-')), + ); + home = dir(tmp, 'home'); + outside = dir(tmp, 'outside'); + host = { platform: process.platform, env: {}, homedir: home }; + clock = new FakeClock(); + logger = memoryLogger(); + sent = []; +}); +afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +describe('the path rule on this machine', () => { + it('allows a folder in the home directory by default, nothing hidden and nothing outside', async () => { + const app = dir(home, 'work', 'app'); + const ssh = dir(home, '.ssh'); + const hiddenParent = dir(home, '.config', 'app'); + const answer = await ask('project.inspect', app); + expect(answer).toMatchObject({ + type: 'project.inspected', + requestId: REQUEST, + inspection: { rootPath: app, name: 'app' }, + }); + for (const target of [ssh, hiddenParent, outside, tmp]) { + expect(await ask('project.inspect', target)).toEqual(refused); + } + }); + + it('refuses a link inside home that points outside, and a .. that leaves the root', async () => { + const at = path.join(home, 'out'); + link(outside, at); + expect(await ask('project.inspect', at)).toEqual(refused); + expect(await ask('project.inspect', path.join(at, '.'))).toEqual(refused); + expect( + await ask('project.inspect', `${home}${path.sep}..${path.sep}outside`), + ).toEqual(refused); + if (process.platform !== 'win32') { + // POSIX resolves `..` after the link: `jump/..` is `outside`, not `inner`. + dir(home, 'inner'); + link(dir(outside, 'deep'), path.join(home, 'inner', 'jump')); + expect( + await ask('project.inspect', `${path.join(home, 'inner', 'jump')}/..`), + ).toEqual(refused); + } + }); + + it('refuses a relative path, a file, a folder that does not exist, and a non-string', async () => { + write(path.join(home, 'note.txt'), 'x'); + for (const target of [ + 'work/app', + path.join(home, 'note.txt'), + path.join(home, 'missing'), + 42, + '', + 'x'.repeat(4097), + ]) { + expect(await ask('project.inspect', target)).toEqual(refused); + } + }); + + it('accepts another case of the path where the file system ignores case', async () => { + const app = dir(home, 'App'); + const variant = path.join(home, 'APP'); + const answer = await ask('project.inspect', variant); + if (process.platform === 'win32' || process.platform === 'darwin') { + expect(answer).toMatchObject({ + type: 'project.inspected', + inspection: { rootPath: realpathSync.native(variant) }, + }); + } else { + expect(answer).toEqual(refused); + } + expect(await ask('project.inspect', app)).toMatchObject({ + type: 'project.inspected', + }); + }); +}); + +/** A file system of directories only, for another platform's rules. */ +function fakeFs( + platform: NodeJS.Platform, + dirs: string[], + links: Record = {}, +): PathFs { + const fold = (value: string) => + platform === 'linux' ? value : value.toLowerCase(); + const p = platform === 'win32' ? path.win32 : path.posix; + const real = new Map(dirs.map((d) => [fold(d), d])); + const stats = (directory: boolean): LaunchStats => ({ + isDirectory: () => directory, + isFile: () => !directory, + isSymbolicLink: () => false, + }); + return { + realpath: async (file) => { + const resolved = p.resolve(file); + const linked = links[fold(resolved)]; + if (linked !== undefined) return linked; + const found = real.get(fold(resolved)); + if (found === undefined) throw new Error('ENOENT'); + return found; + }, + lstat: async (file) => { + if (!real.has(fold(file))) throw new Error('ENOENT'); + return stats(true); + }, + }; +} + +describe('the path rule on other platforms', () => { + it('compares without case and needs a drive letter on Windows', async () => { + const fs = fakeFs('win32', [ + 'C:\\Users\\Ann', + 'C:\\Users\\Ann\\Work', + 'C:\\Users\\Ann\\.ssh', + 'D:\\Work', + ]); + const roots = ['C:\\Users\\Ann']; + expect( + await isAllowedPath('c:\\users\\ann\\work', roots, 'win32', fs), + ).toEqual({ ok: true, root: 'C:\\Users\\Ann\\Work' }); + expect( + await isAllowedPath('C:/Users/Ann/Work/', roots, 'win32', fs), + ).toEqual({ ok: true, root: 'C:\\Users\\Ann\\Work' }); + for (const raw of [ + 'C:\\Users\\Ann\\.ssh', + 'D:\\Work', + '\\\\server\\share\\Work', + '\\\\?\\C:\\Users\\Ann\\Work', + '\\Users\\Ann\\Work', + 'Users\\Ann\\Work', + ]) { + expect(await isAllowedPath(raw, roots, 'win32', fs)).toEqual({ + ok: false, + }); + } + }); + + it('refuses a junction whose real path is elsewhere on Windows', async () => { + const fs = fakeFs('win32', ['C:\\Users\\Ann', 'D:\\Secret'], { + 'c:\\users\\ann\\link': 'D:\\Secret', + }); + expect( + await isAllowedPath( + 'C:\\Users\\Ann\\link', + ['C:\\Users\\Ann'], + 'win32', + fs, + ), + ).toEqual({ ok: false }); + }); + + it('compares without case on macOS and with case on Linux', async () => { + const dirs = ['/Users/ann', '/Users/ann/Work']; + expect( + await isAllowedPath( + '/users/ann/work', + ['/Users/ann'], + 'darwin', + fakeFs('darwin', dirs), + ), + ).toEqual({ ok: true, root: '/Users/ann/Work' }); + expect( + await isAllowedPath( + '/home/ann/Work', + ['/home/ann'], + 'linux', + fakeFs('linux', ['/home/ann', '/home/ann/Work']), + ), + ).toEqual({ ok: true, root: '/home/ann/Work' }); + expect( + await isAllowedPath( + '/home/ann/Work', + ['/home/Ann'], + 'linux', + fakeFs('linux', ['/home/ann', '/home/Ann', '/home/ann/Work']), + ), + ).toEqual({ ok: false }); + }); + + it('allows a hidden folder that is itself a root, and a folder below the root /', async () => { + const fs = fakeFs('linux', [ + '/', + '/home/ann', + '/home/ann/.config', + '/home/ann/.config/app', + '/srv/app', + ]); + expect( + await isAllowedPath( + '/home/ann/.config/app', + ['/home/ann', '/home/ann/.config'], + 'linux', + fs, + ), + ).toEqual({ ok: true, root: '/home/ann/.config/app' }); + expect(await isAllowedPath('/srv/app', ['/'], 'linux', fs)).toEqual({ + ok: true, + root: '/srv/app', + }); + }); +}); + +describe('the allowed roots commands', () => { + it('allow narrows from the home directory to one root, disallow of the last refuses everything', async () => { + const work = dir(home, 'work'); + const app = dir(work, 'app'); + const other = dir(home, 'other'); + const io = memoryIo(); + expect(allowCommand('work', io, commandDeps())).toBe(0); + expect(io.out).toContain(`Allowed: ${work}`); + expect(io.out).toContain('home directory'); + expect(readAllowedRoots(files().allowedRoots, host)).toEqual({ + source: 'file', + roots: [work], + }); + expect(await ask('project.inspect', other)).toEqual(refused); + expect(await ask('project.inspect', app)).toMatchObject({ + type: 'project.inspected', + }); + + expect(disallowCommand(work, memoryIo(), commandDeps())).toBe(0); + expect(readAllowedRoots(files().allowedRoots, host)).toEqual({ + source: 'file', + roots: [], + }); + expect(await ask('project.inspect', app)).toEqual(refused); + expect(await ask('project.inspect', other)).toEqual(refused); + }); + + it('writes the file with mode 600', () => { + dir(home, 'work'); + allowCommand('work', memoryIo(), commandDeps()); + if (process.platform !== 'win32') { + expect(statSync(files().allowedRoots).mode & 0o777).toBe(0o600); + } + }); + + it('lists the default marked, then the roots as JSON', () => { + const io = memoryIo(); + expect(rootsCommand({}, io, commandDeps())).toBe(0); + expect(io.out).toBe(`${home} (default: the home directory)\n`); + const work = dir(home, 'work'); + allowCommand(work, memoryIo(), commandDeps()); + const json = memoryIo(); + expect(rootsCommand({ json: true }, json, commandDeps())).toBe(0); + expect(JSON.parse(json.out)).toEqual({ + roots: [work], + default: false, + valid: true, + }); + }); + + it('refuses every path when the file is corrupt, and roots says so', async () => { + write(files().allowedRoots, '{ "roots": ['); + expect(await ask('project.inspect', dir(home, 'app'))).toEqual(refused); + const io = memoryIo(); + expect(rootsCommand({}, io, commandDeps())).toBe(1); + expect(io.err).toMatch(/is invalid/); + const json = memoryIo(); + rootsCommand({ json: true }, json, commandDeps()); + expect(JSON.parse(json.out)).toMatchObject({ roots: [], valid: false }); + }); + + it('treats a file with a relative or too many roots as invalid', () => { + write(files().allowedRoots, JSON.stringify({ roots: ['work'] })); + expect(readAllowedRoots(files().allowedRoots, host).source).toBe('invalid'); + write( + files().allowedRoots, + JSON.stringify({ + roots: Array.from({ length: ALLOWED_ROOTS_MAX + 1 }, () => home), + }), + ); + expect(readAllowedRoots(files().allowedRoots, host).source).toBe('invalid'); + }); + + it('refuses to allow a path that is not a directory, and reports a root it does not hold', () => { + const io = memoryIo(); + expect(allowCommand('missing', io, commandDeps())).not.toBe(0); + expect(io.err).toMatch(/Not a directory/); + const gone = memoryIo(); + expect(disallowCommand(outside, gone, commandDeps())).toBe(1); + expect(gone.err).toMatch(/Not an allowed root/); + }); + + it('disallows a root that no longer exists by its resolved path', () => { + const work = dir(home, 'work'); + allowCommand(work, memoryIo(), commandDeps()); + allowCommand(outside, memoryIo(), commandDeps()); + rmSync(work, { recursive: true }); + expect(disallowCommand(work, memoryIo(), commandDeps())).toBe(0); + expect(readAllowedRoots(files().allowedRoots, host).roots).toEqual([ + outside, + ]); + }); +}); + +describe('inspection', () => { + it('answers hasConfig false and no scenarios for a folder without config', async () => { + const app = dir(home, 'app'); + write(path.join(app, 'demoflow', 'a.demo.json'), FIXTURE); + expect(await ask('project.inspect', app)).toEqual({ + type: 'project.inspected', + requestId: REQUEST, + inspection: { + rootPath: app, + name: 'app', + hasConfig: false, + configValid: false, + scenarios: [], + }, + }); + }); + + it('lists the scenarios of a valid config and never a config value', async () => { + const app = dir(home, 'app'); + write(path.join(app, 'demoflow.config.json'), CONFIG); + write( + path.join(app, 'demoflow', 'b.demo.json'), + JSON.stringify({ ...JSON.parse(FIXTURE), name: 'Bee' }), + ); + write( + path.join(app, 'demoflow', 'a.demo.json'), + JSON.stringify({ ...JSON.parse(FIXTURE), name: 'Ay' }), + ); + const answer = await ask('project.inspect', app); + expect(answer).toMatchObject({ + inspection: { + hasConfig: true, + configValid: true, + scenarios: [ + { id: 'demoflow/a.demo.json', name: 'Ay' }, + { id: 'demoflow/b.demo.json', name: 'Bee' }, + ], + }, + }); + const text = JSON.stringify(answer); + expect(text).not.toContain('4123'); + expect(text).not.toContain('secret-dev-command'); + expect(text).not.toContain('baseUrl'); + }); + + it('answers configValid false and no scenarios for a config that does not load', async () => { + const app = dir(home, 'app'); + write(path.join(app, 'demoflow.config.json'), '{ "baseUrl": 7 }'); + write(path.join(app, 'demoflow', 'a.demo.json'), FIXTURE); + expect(await ask('project.inspect', app)).toMatchObject({ + inspection: { hasConfig: true, configValid: false, scenarios: [] }, + }); + }); + + it('fills githubUrl from the origin remote of the folder', async () => { + const app = dir(home, 'app'); + write( + path.join(app, '.git', 'config'), + '[core]\n\tbare = false\n[remote "origin"]\n\turl = git@github.com:acme/shop.git\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n', + ); + expect(await ask('project.inspect', app)).toMatchObject({ + inspection: { githubUrl: 'https://github.com/acme/shop' }, + }); + }); +}); + +describe('the GitHub URL', () => { + it.each([ + ['https://github.com/acme/shop', 'https://github.com/acme/shop'], + ['https://github.com/acme/shop.git', 'https://github.com/acme/shop'], + ['git@github.com:acme/shop.git', 'https://github.com/acme/shop'], + ['git@github.com:a.b-c/d_e', 'https://github.com/a.b-c/d_e'], + ['ssh://git@github.com/acme/shop.git', 'https://github.com/acme/shop'], + ])('reads %s', (remote, url) => { + expect(githubUrlFrom(remote)).toBe(url); + }); + + it.each([ + 'https://gitlab.com/acme/shop', + 'https://user:token@github.com/acme/shop.git', + 'https://token@github.com/acme/shop', + 'ssh://git@github.com:22/acme/shop', + 'git@github.com:acme/shop/extra', + 'https://github.com/acme', + 'https://github.com/../shop', + 'https://github.com.evil.test/acme/shop', + 'file:///srv/shop', + ])('ignores %s', (remote) => { + expect(githubUrlFrom(remote)).toBeUndefined(); + }); + + it('reads the first url of [remote "origin"] only', () => { + const config = [ + '[remote "upstream"]', + ' url = https://github.com/other/shop', + '[Remote "origin"]', + ' URL = "https://github.com/acme/shop" ; a comment', + ' url = https://github.com/acme/second', + '[branch "main"]', + ' url = https://github.com/never/this', + ].join('\n'); + expect(originUrl(config)).toBe('https://github.com/acme/shop'); + expect(originUrl('[remote "Origin"]\nurl = x\n')).toBeUndefined(); + }); + + it('never answers credentials from the config', async () => { + const app = dir(home, 'app'); + write( + path.join(app, '.git', 'config'), + '[remote "origin"]\n\turl = https://ann:ghp_secret@github.com/acme/shop.git\n', + ); + const answer = await ask('project.inspect', app); + expect(answer).toMatchObject({ type: 'project.inspected' }); + expect(JSON.stringify(answer)).not.toContain('ghp_secret'); + expect( + (answer.inspection as Record).githubUrl, + ).toBeUndefined(); + }); + + it('follows a .git file to a gitdir inside an allowed root, and a worktree to its commondir', async () => { + const main = dir(home, 'main'); + write( + path.join(main, '.git', 'config'), + '[remote "origin"]\n\turl = https://github.com/acme/shop\n', + ); + const worktreeDir = dir(main, '.git', 'worktrees', 'wt'); + write(path.join(worktreeDir, 'commondir'), '../..\n'); + const wt = dir(home, 'wt'); + write(path.join(wt, '.git'), `gitdir: ${worktreeDir}\n`); + expect(await githubUrlOf(wt, [home], process.platform, nodeLaunchFs)).toBe( + 'https://github.com/acme/shop', + ); + + const sub = dir(main, 'sub'); + const modules = dir(main, '.git', 'modules', 'sub'); + write( + path.join(modules, 'config'), + '[remote "origin"]\n\turl = git@github.com:acme/sub.git\n', + ); + write(path.join(sub, '.git'), 'gitdir: ../.git/modules/sub\n'); + expect(await githubUrlOf(sub, [home], process.platform, nodeLaunchFs)).toBe( + 'https://github.com/acme/sub', + ); + }); + + it('does not follow a gitdir outside the allowed roots', async () => { + write( + path.join(outside, 'repo', 'config'), + '[remote "origin"]\n\turl = https://github.com/acme/shop\n', + ); + const app = dir(home, 'app'); + write(path.join(app, '.git'), `gitdir: ${path.join(outside, 'repo')}\n`); + expect( + await githubUrlOf(app, [home], process.platform, nodeLaunchFs), + ).toBeUndefined(); + }); + + it('ignores a config over 64 KiB', async () => { + const app = dir(home, 'app'); + write( + path.join(app, '.git', 'config'), + `[remote "origin"]\n\turl = https://github.com/acme/shop\n${'#'.repeat(64 * 1024)}\n`, + ); + expect( + await githubUrlOf(app, [home], process.platform, nodeLaunchFs), + ).toBeUndefined(); + }); +}); + +describe('project.init', () => { + it('creates both files in an empty allowed folder, then keeps them byte for byte', async () => { + const app = dir(home, 'app'); + const first = await ask('project.init', app); + expect(first).toMatchObject({ + type: 'project.initialised', + requestId: REQUEST, + created: ['demoflow.config.json', 'demoflow/first.demo.json'], + kept: [], + inspection: { + rootPath: app, + hasConfig: true, + configValid: true, + scenarios: [{ id: 'demoflow/first.demo.json', name: 'First demo' }], + }, + }); + const config = readFileSync(path.join(app, 'demoflow.config.json')); + const scenario = readFileSync( + path.join(app, 'demoflow', 'first.demo.json'), + ); + expect(JSON.parse(config.toString())).toEqual({ + baseUrl: 'http://localhost:3000', + }); + + const second = await ask('project.init', app); + expect(second).toMatchObject({ + created: [], + kept: ['demoflow.config.json', 'demoflow/first.demo.json'], + }); + expect(readFileSync(path.join(app, 'demoflow.config.json'))).toEqual( + config, + ); + expect(readFileSync(path.join(app, 'demoflow', 'first.demo.json'))).toEqual( + scenario, + ); + }); + + it('writes nothing outside the path rule', async () => { + expect(await ask('project.init', outside)).toEqual(refused); + expect(() => + readFileSync(path.join(outside, 'demoflow.config.json')), + ).toThrow(); + }); + + it('never writes through a linked demoflow/ folder', async () => { + const app = dir(home, 'app'); + link(outside, path.join(app, 'demoflow')); + expect(await ask('project.init', app)).toEqual({ + type: 'project.error', + requestId: REQUEST, + code: 'internal', + }); + expect(() => readFileSync(path.join(outside, 'first.demo.json'))).toThrow(); + }); +}); + +describe('project.register', () => { + const ROOTS = () => ({ + allowedRootsFile: files().allowedRoots, + websiteProjectsFile: files().websiteProjects, + host, + fs: nodeLaunchFs, + }); + const checkDeps = () => ({ + indexFile: path.join(tmp, 'reported-runs.jsonl'), + websiteRegistered: (rootPath: string) => + isWebsiteRegistered(rootPath, ROOTS()), + fs: nodeLaunchFs, + loadProject: loadProjectFor(host), + }); + + it('answers config-missing for a folder without config, config-invalid for a broken one', async () => { + const app = dir(home, 'app'); + expect(await ask('project.register', app)).toEqual({ + type: 'project.error', + requestId: REQUEST, + code: 'config-missing', + }); + write(path.join(app, 'demoflow.config.json'), '{'); + expect(await ask('project.register', app)).toEqual({ + type: 'project.error', + requestId: REQUEST, + code: 'config-invalid', + }); + expect(await websiteRoots(files().websiteProjects)).toEqual(new Set()); + }); + + it('registers the canonical root once, which checkProject then accepts until its root is disallowed', async () => { + const work = dir(home, 'work'); + const app = dir(work, 'app'); + write(path.join(app, 'demoflow.config.json'), CONFIG); + expect((await checkProject(app, checkDeps())).ok).toBe(false); + + expect(await ask('project.register', app)).toEqual({ + type: 'project.registered', + requestId: REQUEST, + rootPath: app, + }); + await ask('project.register', `${app}${path.sep}`); + const lines = readFileSync(files().websiteProjects, 'utf8') + .trim() + .split('\n'); + expect(lines).toHaveLength(1); + expect(JSON.parse(lines[0] as string)).toEqual({ + rootPath: app, + registeredAt: new Date(0).toISOString(), + }); + expect(await checkProject(app, checkDeps())).toMatchObject({ + ok: true, + root: app, + }); + + allowCommand(work, memoryIo(), commandDeps()); + expect((await checkProject(app, checkDeps())).ok).toBe(true); + disallowCommand(work, memoryIo(), commandDeps()); + expect(await checkProject(app, checkDeps())).toEqual({ + ok: false, + code: 'project-unknown', + }); + }); + + it('keeps the last 1,000 roots', () => { + const file = files().websiteProjects; + for (let i = 0; i < WEBSITE_PROJECTS_MAX; i++) { + registerWebsiteProject(file, `/p/${i}`, 'at'); + } + registerWebsiteProject(file, '/p/new', 'at'); + const lines = readFileSync(file, 'utf8').trim().split('\n'); + expect(lines).toHaveLength(WEBSITE_PROJECTS_MAX); + expect(lines[0]).toContain('/p/1"'); + expect(lines.at(-1)).toContain('/p/new'); + }); +}); + +describe('one request at a time', () => { + it('answers busy while one is in flight, and internal after 5 s', async () => { + const app = dir(home, 'app'); + let release: (() => void) | undefined; + const handler = requests({ + writeTemplate: () => + new Promise((resolve) => { + release = () => resolve({ created: [], kept: [] }); + }), + }); + const reply = { + send: (message: Record) => sent.push(message), + }; + handler.handle({ type: 'project.init', requestId: 'a', path: app }, reply); + await vi.waitFor(() => expect(release).toBeDefined()); + handler.handle( + { type: 'project.inspect', requestId: 'b', path: app }, + reply, + ); + expect(sent).toEqual([ + { type: 'project.error', requestId: 'b', code: 'busy' }, + ]); + await clock.advance(PROJECT_REQUEST_MAX_MS); + expect(sent).toContainEqual({ + type: 'project.error', + requestId: 'a', + code: 'internal', + }); + // Still busy until the work itself ends; then answered once only. + handler.handle( + { type: 'project.inspect', requestId: 'c', path: app }, + reply, + ); + expect(sent.at(-1)).toEqual({ + type: 'project.error', + requestId: 'c', + code: 'busy', + }); + release?.(); + // Free once the late work ends; its answer is never sent. + await vi.waitFor(async () => { + expect(await ask('project.inspect', app, handler)).toMatchObject({ + type: 'project.inspected', + }); + }); + expect(sent.filter((m) => m.requestId === 'a')).toHaveLength(1); + }); + + it('ignores a request without a request id and logs no path', async () => { + const app = dir(home, 'secret-name'); + const handler = requests(); + handler.handle( + { type: 'project.inspect', path: app }, + { send: (m) => sent.push(m) }, + ); + expect(sent).toEqual([]); + await ask('project.inspect', app, handler); + await ask('project.register', app, handler); + expect(logger.text()).not.toContain('secret-name'); + expect(logger.text()).toContain(`project.inspect ${REQUEST}: answered`); + }); +}); diff --git a/apps/cli/test/runner/schedule-daemon.spec.ts b/apps/cli/test/runner/schedule-daemon.spec.ts new file mode 100644 index 00000000..68380711 --- /dev/null +++ b/apps/cli/test/runner/schedule-daemon.spec.ts @@ -0,0 +1,211 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { Io } from '../../src/commands/io.ts'; +import type { HostEnvironment } from '../../src/config/platform.ts'; +import type { ChildExit, RunChild, SpawnRun } from '../../src/mcp/child.ts'; +import { + type ControlDeps, + defaultControlDeps, + runnerStatus, + statusCommand, +} from '../../src/runner/control.ts'; +import { + EXIT_DAEMON_DONE, + EXIT_DAEMON_NOT_LOGGED_IN, + runForeground, +} from '../../src/runner/daemon.ts'; +import { runnerPaths } from '../../src/runner/paths.ts'; +import { recordLocalRoot } from '../../src/runner/schedule/local-roots.ts'; +import { writeScheduleState } from '../../src/runner/schedule/state.ts'; +import { readState } from '../../src/runner/state.ts'; +import { FakeClock } from './fakes.ts'; + +const T0 = Date.parse('2026-10-26T08:00:00.000Z'); +const DAY = 86_400_000; + +let tmp: string; +let host: HostEnvironment; +let root: string; +let out: string; +let err: string; +let spawned: Array<{ args: string[]; killed: number }>; + +const io: Io = { + stdout: (text) => { + out += text; + }, + stderr: (text) => { + err += text; + }, +}; + +const spawn: SpawnRun = (args) => { + let resolve: (exit: ChildExit) => void = () => {}; + const exited = new Promise((r) => { + resolve = r; + }); + const call = { args, killed: 0 }; + spawned.push(call); + const child: RunChild = { + pid: 4242, + exited, + kill: async () => { + call.killed++; + resolve({ code: null, stderrTail: '' }); + await exited; + }, + }; + return child; +}; + +beforeEach(() => { + tmp = mkdtempSync(path.join(tmpdir(), 'demoflow-schedule-daemon-')); + const env = { + ...process.env, + HOME: tmp, + USERPROFILE: tmp, + XDG_CONFIG_HOME: path.join(tmp, '.config'), + APPDATA: path.join(tmp, 'AppData', 'Roaming'), + }; + host = { platform: process.platform, env, homedir: tmp }; + root = path.join(tmp, 'shop'); + mkdirSync(root); + out = ''; + err = ''; + spawned = []; +}); +afterEach(() => { + rmSync(tmp, { recursive: true, force: true }); +}); + +function scheduledProject(): void { + writeFileSync( + path.join(root, 'demoflow.config.json'), + JSON.stringify({ + baseUrl: 'http://localhost:1', + schedule: { cadence: 'daily', at: '09:00' }, + }), + ); + const paths = runnerPaths(host); + mkdirSync(paths.dir, { recursive: true }); + recordLocalRoot(paths.localRoots, root); +} + +function control(overrides: Partial = {}): ControlDeps { + return { ...defaultControlDeps(), host, ...overrides }; +} + +function daemon(connects: string[]) { + return runForeground(io, { + host, + pid: process.pid, + connect: (url) => { + connects.push(url); + throw new Error('no socket in local mode'); + }, + clock: new FakeClock(T0), + random: () => 0.5, + fetch: () => Promise.reject(new Error('no network in local mode')), + spawn, + zone: () => 'UTC', + }); +} + +describe('the daemon without an account', () => { + it('exits as before when no watched root has a schedule', async () => { + writeFileSync( + path.join(root, 'demoflow.config.json'), + '{"baseUrl":"http://localhost:1"}', + ); + recordLocalRoot(runnerPaths(host).localRoots, root); + expect(await daemon([])).toBe(EXIT_DAEMON_NOT_LOGGED_IN); + expect(err).toContain('demoflow login'); + }); + + it('runs in local mode: no socket, the due pass runs, status names the mode', async () => { + scheduledProject(); + const paths = runnerPaths(host); + writeScheduleState(paths.scheduleState, { + zone: 'UTC', + roots: { + [root]: { + schedule: { cadence: 'daily', at: '09:00' }, + seenAt: new Date(T0 - 10 * DAY).toISOString(), + last: { startedAt: new Date(T0 - 3 * DAY).toISOString() }, + }, + }, + }); + const connects: string[] = []; + const done = daemon(connects); + await vi.waitFor(() => expect(spawned).toHaveLength(1)); + expect(spawned[0]?.args.slice(0, 2)).toEqual(['run', '--check']); + const state = readState(paths.state); + expect(state?.state).toBe('local'); + expect(state?.server).toBeUndefined(); + + const status = runnerStatus(control()); + expect(status.connection).toBe('local'); + expect(status.schedules).toEqual([ + expect.objectContaining({ + rootPath: root, + zone: 'UTC', + lastCheck: { startedAt: '2026-10-26T08:00:00.000Z' }, + }), + ]); + expect(status.schedules[0]?.nextCheckAt).not.toBeNull(); + statusCommand({}, io, control()); + expect(out).toContain('Connection: local mode: scheduled checks only'); + expect(out).toContain(`Schedule: ${root} (daily at 09:00)`); + expect(out).toContain('last check: 2026-10-26 08:00, no result yet'); + + writeFileSync(paths.stop, `${process.pid}\n`); + expect(await done).toBe(EXIT_DAEMON_DONE); + expect(connects).toEqual([]); + expect(spawned[0]?.killed).toBe(1); + }); +}); + +describe('runner status and schedules', () => { + it('shows the last check and says the next waits for the runner', () => { + const paths = runnerPaths(host); + mkdirSync(paths.dir, { recursive: true }); + writeScheduleState(paths.scheduleState, { + zone: 'Europe/Berlin', + roots: { + [root]: { + schedule: { + cadence: 'weekly', + weekday: 'mon', + at: '09:30', + exclude: ['demoflow/wip.demo.json'], + }, + last: { + startedAt: '2026-10-26T08:30:00.000Z', + finishedAt: '2026-10-26T08:41:00.000Z', + outcome: 'finished', + passed: 12, + failed: 1, + }, + }, + [path.join(tmp, 'gone')]: { + last: { startedAt: '2026-10-20T08:30:00.000Z' }, + }, + }, + }); + statusCommand({}, io, control()); + expect(out).toContain( + `Schedule: ${root} (weekly on mon at 09:30, 1 excluded)`, + ); + expect(out).toContain('next check: when the runner is running'); + expect(out).toContain('last check: 2026-10-26 09:30, 12 passed, 1 failed'); + expect(out).not.toContain('gone'); + out = ''; + statusCommand({ json: true }, io, control()); + const json = JSON.parse(out) as { schedules: unknown[] }; + expect(json.schedules).toEqual([ + expect.objectContaining({ rootPath: root, nextCheckAt: null }), + ]); + }); +}); diff --git a/apps/cli/test/runner/schedule.spec.ts b/apps/cli/test/runner/schedule.spec.ts new file mode 100644 index 00000000..b081c4d2 --- /dev/null +++ b/apps/cli/test/runner/schedule.spec.ts @@ -0,0 +1,417 @@ +import { + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import type { ScheduleConfig } from '@demoflow/core'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import type { + ChildExit, + RunChild, + SpawnRun, + SpawnRunOptions, +} from '../../src/mcp/child.ts'; +import { nodeLaunchFs } from '../../src/runner/launch/checks.ts'; +import { Launcher } from '../../src/runner/launch/index.ts'; +import { + LOCAL_ROOTS_KEEP, + readLocalRoots, + recordLocalRoot, +} from '../../src/runner/schedule/local-roots.ts'; +import { + Scheduler, + type SchedulerDeps, + type SchedulesSync, +} from '../../src/runner/schedule/scheduler.ts'; +import { + readScheduleState, + writeScheduleState, +} from '../../src/runner/schedule/state.ts'; +import { appendReportedRun } from '../../src/runs-report/reported-runs.ts'; +import { FakeClock, memoryLogger } from './fakes.ts'; + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; +/** A Monday, an hour before the 09:00 slot. */ +const T0 = Date.parse('2026-10-26T08:00:00.000Z'); +const DAILY: ScheduleConfig = { cadence: 'daily', at: '09:00' }; +const REQUEST = '6f1c2b8e-3d4a-4b5c-9e7f-0a1b2c3d4e5f'; + +interface Spawned { + args: string[]; + cwd: string; + options: SpawnRunOptions | undefined; + killed: number; + exit(code?: number | null): void; + print(...lines: string[]): void; +} + +function fakeSpawn(): { spawn: SpawnRun; calls: Spawned[] } { + const calls: Spawned[] = []; + const spawn: SpawnRun = (args, cwd, options) => { + let resolve: (exit: ChildExit) => void = () => {}; + const exited = new Promise((r) => { + resolve = r; + }); + const child: RunChild = { + pid: 4242, + exited, + kill: async () => { + call.killed++; + resolve({ code: null, stderrTail: '' }); + await exited; + }, + }; + const call: Spawned = { + args, + cwd, + options, + killed: 0, + exit: (code = 0) => resolve({ code, stderrTail: '' }), + print: (...lines) => { + for (const line of lines) options?.onStdoutLine?.(line); + }, + }; + calls.push(call); + return child; + }; + return { spawn, calls }; +} + +let tmp: string; +let root: string; +let stateFile: string; +let clock: FakeClock; +let logger: ReturnType; +let spawned: ReturnType; +let schedules: Map; +let reported: string[]; +let local: string[]; +let frames: SchedulesSync[]; +let running: Scheduler[]; + +beforeEach(() => { + tmp = realpathSync.native( + mkdtempSync(path.join(tmpdir(), 'demoflow-schedule-')), + ); + root = path.join(tmp, 'shop'); + mkdirSync(root); + stateFile = path.join(tmp, 'schedule.state.json'); + clock = new FakeClock(T0); + logger = memoryLogger(); + spawned = fakeSpawn(); + schedules = new Map([[root, DAILY]]); + reported = []; + local = [root]; + frames = []; + running = []; +}); +afterEach(async () => { + for (const scheduler of running) await scheduler.stop(); + rmSync(tmp, { recursive: true, force: true }); +}); + +function scheduler(overrides: Partial = {}): Scheduler { + const created = new Scheduler({ + clock, + logger, + zone: () => 'UTC', + roots: () => ({ reported, local }), + loadSchedule: (at) => { + const schedule = schedules.get(at); + if (schedule instanceof Error) throw schedule; + return schedule; + }, + stateFile, + spawn: spawned.spawn, + claim: () => () => {}, + onSchedules: (frame) => frames.push(frame), + ...overrides, + }); + running.push(created); + return created; +} + +const totals = (passed: number, failed: number) => + JSON.stringify({ check: true, passed, failed }); + +describe('the scheduler', () => { + it('spawns exactly one check pass when the slot passes while it runs', async () => { + scheduler().start(); + expect(spawned.calls).toHaveLength(0); + await clock.advance(HOUR - 1); + expect(spawned.calls).toHaveLength(0); + await clock.advance(1); + expect(spawned.calls).toHaveLength(1); + const [pass] = spawned.calls; + expect(pass?.args).toEqual([ + 'run', + '--check', + '--json', + '--progress', + '--project', + root, + ]); + expect(pass?.cwd).toBe(root); + // Written before the spawn: a crash now does not repeat the slot. + expect(readScheduleState(stateFile)?.roots[root]?.last).toEqual({ + startedAt: '2026-10-26T09:00:00.000Z', + }); + await clock.advance(10 * MINUTE); + expect(spawned.calls).toHaveLength(1); + pass?.print(totals(3, 1)); + pass?.exit(0); + await clock.advance(22 * HOUR); + expect(spawned.calls).toHaveLength(1); + expect(readScheduleState(stateFile)?.roots[root]?.last).toEqual({ + startedAt: '2026-10-26T09:00:00.000Z', + finishedAt: '2026-10-26T09:10:00.000Z', + outcome: 'finished', + passed: 3, + failed: 1, + }); + expect(logger.text()).toContain('shop: finished, 3 passed, 1 failed'); + expect(logger.text()).not.toContain(tmp); + }); + + it('runs one catch-up pass, not three, three days after the last one', async () => { + writeScheduleState(stateFile, { + zone: 'UTC', + roots: { + [root]: { + schedule: DAILY, + seenAt: new Date(T0 - 10 * DAY).toISOString(), + last: { startedAt: new Date(T0 - 3 * DAY + HOUR).toISOString() }, + }, + }, + }); + scheduler().start(); + expect(spawned.calls).toHaveLength(1); + spawned.calls[0]?.exit(0); + await clock.advance(50 * MINUTE); + expect(spawned.calls).toHaveLength(1); + }); + + it('does not fire for a slot before it first saw the schedule', async () => { + clock = new FakeClock(T0 + 3 * HOUR); + scheduler().start(); + await clock.advance(MINUTE); + expect(spawned.calls).toHaveLength(0); + expect(readScheduleState(stateFile)?.roots[root]?.seenAt).toBe( + '2026-10-26T11:00:00.000Z', + ); + }); + + it('passes the excluded scenario ids', async () => { + schedules.set(root, { + ...DAILY, + exclude: ['demoflow/wip.demo.json', 'demoflow/a/b.demo.json'], + }); + scheduler().start(); + await clock.advance(HOUR); + expect(spawned.calls[0]?.args.slice(6)).toEqual([ + '--exclude', + 'demoflow/wip.demo.json', + '--exclude', + 'demoflow/a/b.demo.json', + ]); + }); + + it('runs one pass at a time', async () => { + const other = path.join(tmp, 'blog'); + schedules.set(other, DAILY); + local = [root, other]; + scheduler().start(); + await clock.advance(HOUR); + expect(spawned.calls).toHaveLength(1); + spawned.calls[0]?.exit(0); + await clock.advance(1); + expect(spawned.calls).toHaveLength(2); + expect(new Set(spawned.calls.map((call) => call.cwd))).toEqual( + new Set([root, other]), + ); + }); + + it('kills a pass that outlives the pass cap and records it timed-out', async () => { + scheduler().start(); + await clock.advance(HOUR); + await clock.advance(2 * HOUR); + expect(spawned.calls[0]?.killed).toBe(1); + expect(readScheduleState(stateFile)?.roots[root]?.last?.outcome).toBe( + 'timed-out', + ); + }); + + it('kills the running pass on stop', async () => { + const created = scheduler(); + created.start(); + await clock.advance(HOUR); + await created.stop(); + expect(spawned.calls[0]?.killed).toBe(1); + expect(readScheduleState(stateFile)?.roots[root]?.last?.outcome).toBe( + 'cancelled', + ); + }); + + it('skips a root whose config does not load and logs it once', async () => { + schedules.set(root, new Error(`${root}/demoflow.config.json is invalid`)); + scheduler().start(); + await clock.advance(2 * HOUR); + expect(spawned.calls).toHaveLength(0); + const warnings = logger.lines.filter((line) => + line.message.includes('does not load'), + ); + expect(warnings).toHaveLength(1); + expect(logger.text()).not.toContain(tmp); + }); + + it('forgets a removed schedule, so adding it back starts fresh', async () => { + const created = scheduler(); + created.start(); + schedules.delete(root); + await clock.advance(2 * HOUR); + expect(spawned.calls).toHaveLength(0); + expect(readScheduleState(stateFile)?.roots[root]).toBeUndefined(); + }); +}); + +describe('the scheduler and website launches', () => { + it('starts no pass while a launch runs, and starts it when the launch ends', async () => { + const indexFile = path.join(tmp, 'reported-runs.jsonl'); + writeFileSync( + path.join(root, 'demoflow.config.json'), + '{"baseUrl":"http://localhost:1"}', + ); + appendReportedRun(indexFile, { + id: 'srv-0', + runDir: path.join(tmp, 'home', 'runs', 'old'), + at: '2026-10-23T10:00:00.000Z', + root, + }); + const replies: Array> = []; + const reply = { + send: (message: Record) => replies.push(message), + }; + let created: Scheduler | null = null; + const launcher = new Launcher({ + indexFile, + fs: nodeLaunchFs, + logger, + clock, + platform: process.platform, + env: {}, + spawn: spawned.spawn, + loadProject: () => ({ ok: true, home: path.join(tmp, 'home') }), + newRunId: () => 'local-1', + readStatus: async () => 'completed', + onIdle: () => created?.wake(), + }); + created = scheduler({ claim: () => launcher.claim() }); + created.start(); + await clock.advance(HOUR - 5 * MINUTE); + const check = { + type: 'check.start', + requestId: REQUEST, + rootPath: root, + exclude: [], + }; + launcher.handle(check, reply, reply); + // The launcher checks the root on disk before it spawns. + await vi.waitFor(() => expect(spawned.calls).toHaveLength(1)); + await clock.advance(10 * MINUTE); + // The slot passed at 09:00; the website's pass still holds the launch slot. + expect(spawned.calls).toHaveLength(1); + spawned.calls[0]?.print(totals(1, 0)); + spawned.calls[0]?.exit(0); + await clock.advance(1); + expect(spawned.calls).toHaveLength(2); + // And while the scheduled pass runs, the website is told `busy`. + launcher.handle( + { ...check, requestId: `${REQUEST.slice(0, -1)}0` }, + reply, + reply, + ); + expect(replies.at(-1)).toMatchObject({ type: 'check.error', code: 'busy' }); + expect(spawned.calls).toHaveLength(2); + spawned.calls[1]?.exit(0); + await clock.advance(1); + await launcher.close(); + }); +}); + +describe('schedules.sync', () => { + it('names only reported roots, with the schedule as parsed or null, and the zone', async () => { + const blog = path.join(tmp, 'blog'); + const docs = path.join(tmp, 'docs'); + reported = [root, blog]; + local = [docs]; + schedules.set(docs, DAILY); + const created = scheduler({ zone: () => 'Europe/Berlin' }); + created.start(); + const expected: SchedulesSync = { + type: 'schedules.sync', + zone: 'Europe/Berlin', + projects: [ + { rootPath: blog, schedule: null }, + { rootPath: root, schedule: DAILY }, + ], + }; + expect(frames).toEqual([expected]); + expect(created.schedules()).toEqual(expected); + // Unchanged on the next tick: nothing new is sent. + await clock.advance(MINUTE); + expect(frames).toHaveLength(1); + const weekly: ScheduleConfig = { + cadence: 'weekly', + weekday: 'fri', + at: '18:00', + }; + schedules.set(blog, weekly); + await clock.advance(MINUTE); + expect(frames).toHaveLength(2); + expect(frames[1]?.projects[0]).toEqual({ + rootPath: blog, + schedule: weekly, + }); + }); +}); + +describe('local-roots.jsonl', () => { + it('records each root once, oldest first', () => { + const file = path.join(tmp, 'runner', 'local-roots.jsonl'); + recordLocalRoot(file, '/a'); + recordLocalRoot(file, '/b'); + recordLocalRoot(file, '/a'); + expect(readLocalRoots(file)).toEqual(['/a', '/b']); + expect(readFileSync(file, 'utf8').trim().split('\n')).toHaveLength(2); + }); + + it(`keeps the last ${LOCAL_ROOTS_KEEP} roots`, () => { + const file = path.join(tmp, 'local-roots.jsonl'); + writeFileSync( + file, + Array.from( + { length: LOCAL_ROOTS_KEEP }, + (_, i) => `${JSON.stringify({ root: `/r${i}` })}\n`, + ).join(''), + ); + recordLocalRoot(file, '/new'); + const roots = readLocalRoots(file); + expect(roots).toHaveLength(LOCAL_ROOTS_KEEP); + expect(roots[0]).toBe('/r1'); + expect(roots.at(-1)).toBe('/new'); + }); + + it('skips malformed lines and never throws on a missing file', () => { + const file = path.join(tmp, 'local-roots.jsonl'); + expect(readLocalRoots(file)).toEqual([]); + writeFileSync(file, 'nope\n{"root":""}\n{"root":"/a"}\n[1]\n'); + expect(readLocalRoots(file)).toEqual(['/a']); + }); +}); diff --git a/apps/cli/test/runner/share.spec.ts b/apps/cli/test/runner/share.spec.ts new file mode 100644 index 00000000..14b664c9 --- /dev/null +++ b/apps/cli/test/runner/share.spec.ts @@ -0,0 +1,431 @@ +import { randomUUID } from 'node:crypto'; +import { + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + statSync, + symlinkSync, + utimesSync, + writeFileSync, +} from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import type { Chapter, RenderFormat, RunResult } from '@demoflow/core'; +import { runFfmpeg } from '@demoflow/renderer'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { posterPath } from '../../src/runner/share/files.ts'; +import { + makePoster, + POSTER_MAX_BYTES, + type PosterRequest, + posterArgs, + posterAtMs, +} from '../../src/runner/share/poster.ts'; +import { + DESCRIPTION_MAX_LENGTH, + SharePreparer, +} from '../../src/runner/share/prepare.ts'; +import { appendReportedRun } from '../../src/runs-report/reported-runs.ts'; +import { memoryLogger } from './fakes.ts'; + +const RUN_ID = 'srv-run-1'; +const SCENARIO = 'demoflow/a.demo.json'; + +let base: string; +let root: string; +let runDir: string; +let indexFile: string; +let logger: ReturnType; +let posters: PosterRequest[]; +let posterSize: number | Error; + +function writeResult( + renders: Array<{ + format: RenderFormat; + file: string; + chapters?: Chapter[]; + durationMs?: number; + }>, + scenarioId = SCENARIO, +): void { + const done = { status: 'passed' } as const; + const result: RunResult = { + schemaVersion: 1, + runId: 'local-run', + scenarioId, + scenarioName: 'A', + startedAt: '2026-10-22T10:00:00.000Z', + finishedAt: '2026-10-22T10:00:01.000Z', + status: 'completed', + stages: { + readiness: done, + execution: done, + verification: done, + recording: done, + rendering: done, + storage: { status: 'skipped' }, + }, + renders: renders.map((render) => ({ + renderer: 'ffmpeg', + durationMs: 4000, + sizeBytes: 1, + width: 1280, + height: 720, + fps: 30, + ...render, + })), + warnings: [], + }; + writeFileSync(path.join(runDir, 'result.json'), JSON.stringify(result)); +} + +function writeScenario(description?: string, file = SCENARIO): void { + const target = path.join(root, file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync( + target, + JSON.stringify({ + schemaVersion: 1, + name: 'A', + ...(description === undefined ? {} : { description }), + steps: [{ action: 'navigate', path: '/' }], + }), + ); +} + +function preparer(): SharePreparer { + return new SharePreparer({ + indexFile, + logger, + platform: process.platform, + makePoster: async (request) => { + posters.push(request); + if (posterSize instanceof Error) throw posterSize; + return posterSize; + }, + }); +} + +/** Sends one `share.prepare` and waits for its answer. */ +async function prepare( + message: Record = {}, + share = preparer(), +): Promise> { + return new Promise((resolve) => { + share.handle( + { + type: 'share.prepare', + requestId: '2f1f6d0e-7a53-4c1b-9d57-0c1f6f1d2a11', + runId: RUN_ID, + format: 'mp4', + ...message, + }, + { send: resolve }, + ); + }); +} + +beforeEach(() => { + // The preparer resolves with the native realpath, which expands the 8.3 + // short names of a Windows temp dir (RUNNER~1); the JS one does not. + base = realpathSync.native( + mkdtempSync(path.join(tmpdir(), 'demoflow-share-')), + ); + root = path.join(base, 'project'); + runDir = path.join(base, 'home', 'runs', 'local-run'); + mkdirSync(root, { recursive: true }); + mkdirSync(path.join(runDir, 'renders'), { recursive: true }); + indexFile = path.join(base, 'runner', 'reported-runs.jsonl'); + appendReportedRun(indexFile, { + id: RUN_ID, + runDir, + at: '2026-10-22T10:00:01.000Z', + root, + }); + writeFileSync(path.join(runDir, 'renders', 'demo.mp4'), 'mp4 bytes'); + logger = memoryLogger(); + posters = []; + posterSize = 1234; +}); + +afterEach(() => { + rmSync(base, { recursive: true, force: true }); +}); + +describe('SharePreparer (spec 349, Decision 2)', () => { + it('answers with the description, the chapters and the poster size, and no path', async () => { + const chapters = [ + { title: 'Customers', atMs: 0 }, + { title: 'Invoices', atMs: 4200 }, + ]; + writeResult([{ format: 'mp4', file: 'renders/demo.mp4', chapters }]); + writeScenario('Create a customer, then bill them.'); + const answer = await prepare(); + expect(answer).toEqual({ + type: 'share.prepared', + requestId: '2f1f6d0e-7a53-4c1b-9d57-0c1f6f1d2a11', + description: 'Create a customer, then bill them.', + chapters, + posterSize: 1234, + }); + expect(posters).toEqual([ + { + render: path.join(runDir, 'renders', 'demo.mp4'), + poster: posterPath(runDir, 'mp4', process.platform), + durationMs: 4000, + platform: process.platform, + }, + ]); + expect(logger.text()).toContain(`share ${RUN_ID} mp4: prepared`); + expect(logger.text()).not.toContain(base); + }); + + it('answers chapters [] and no description for an old render and a gone scenario', async () => { + writeResult([{ format: 'mp4', file: 'renders/demo.mp4' }]); + const answer = await prepare(); + expect(answer).toEqual({ + type: 'share.prepared', + requestId: '2f1f6d0e-7a53-4c1b-9d57-0c1f6f1d2a11', + chapters: [], + posterSize: 1234, + }); + }); + + it('cuts a long description to the API limit', async () => { + writeResult([{ format: 'mp4', file: 'renders/demo.mp4' }]); + writeScenario('x'.repeat(DESCRIPTION_MAX_LENGTH + 50)); + const answer = await prepare(); + expect(answer.description).toBe('x'.repeat(DESCRIPTION_MAX_LENGTH)); + }); + + it('reads no description for a run without a project root', async () => { + rmSync(indexFile); + appendReportedRun(indexFile, { id: RUN_ID, runDir, at: '' }); + writeResult([{ format: 'mp4', file: 'renders/demo.mp4' }]); + writeScenario('Hidden.'); + expect((await prepare()).description).toBeUndefined(); + }); + + it('reads no description from outside the project', async () => { + writeFileSync( + path.join(base, 'outside.json'), + JSON.stringify({ + schemaVersion: 1, + name: 'O', + description: 'Outside.', + steps: [{ action: 'navigate', path: '/' }], + }), + ); + writeResult( + [{ format: 'mp4', file: 'renders/demo.mp4' }], + '../outside.json', + ); + expect((await prepare()).description).toBeUndefined(); + }); + + it('answers not-found for an unreported run or an unreadable result', async () => { + writeResult([{ format: 'mp4', file: 'renders/demo.mp4' }]); + expect(await prepare({ runId: 'srv-other' })).toEqual({ + type: 'share.error', + requestId: '2f1f6d0e-7a53-4c1b-9d57-0c1f6f1d2a11', + code: 'not-found', + }); + writeFileSync(path.join(runDir, 'result.json'), '{ not json'); + expect((await prepare()).code).toBe('not-found'); + writeFileSync(path.join(runDir, 'result.json'), '{}'); + expect((await prepare()).code).toBe('not-found'); + expect(posters).toEqual([]); + }); + + it('answers no-render for an unrendered format, a missing file or one outside the run', async () => { + writeResult([{ format: 'mp4', file: 'renders/demo.mp4' }]); + expect((await prepare({ format: 'gif' })).code).toBe('no-render'); + + writeResult([{ format: 'mp4', file: 'renders/gone.mp4' }]); + expect((await prepare()).code).toBe('no-render'); + + writeFileSync(path.join(base, 'secret.mp4'), 'secret'); + writeResult([{ format: 'mp4', file: '../../../secret.mp4' }]); + expect((await prepare()).code).toBe('no-render'); + + if (process.platform !== 'win32') { + symlinkSync( + path.join(base, 'secret.mp4'), + path.join(runDir, 'renders', 'link.mp4'), + ); + writeResult([{ format: 'mp4', file: 'renders/link.mp4' }]); + expect((await prepare()).code).toBe('no-render'); + } + expect(posters).toEqual([]); + expect(logger.text()).toContain(`share ${RUN_ID} gif: no-render`); + }); + + it('answers io when the poster cannot be made', async () => { + writeResult([{ format: 'mp4', file: 'renders/demo.mp4' }]); + posterSize = new Error('ffmpeg exited with 1'); + expect((await prepare()).code).toBe('io'); + expect(logger.text()).toContain(`share ${RUN_ID} mp4: io`); + expect(logger.text()).not.toContain('ffmpeg exited'); + }); + + it('ignores a malformed share.prepare', () => { + const share = preparer(); + const sent: unknown[] = []; + for (const message of [ + { requestId: 'not-a-uuid', runId: RUN_ID, format: 'mp4' }, + { requestId: randomUUID(), runId: '../x', format: 'mp4' }, + { requestId: randomUUID(), runId: RUN_ID, format: 'avi' }, + { requestId: randomUUID(), format: 'mp4' }, + ]) { + share.handle( + { type: 'share.prepare', ...message }, + { send: (m) => sent.push(m) }, + ); + } + expect(sent).toEqual([]); + expect(logger.text()).toMatch(/ignored a malformed share.prepare/); + }); + + it('makes one poster for two shares of the same render at once', async () => { + writeResult([{ format: 'mp4', file: 'renders/demo.mp4' }]); + let release = () => {}; + const gate = new Promise((resolve) => { + release = resolve; + }); + const share = new SharePreparer({ + indexFile, + logger, + platform: process.platform, + makePoster: async (request) => { + posters.push(request); + await gate; + return 1234; + }, + }); + setTimeout(release, 200); + const answers = await Promise.all([ + prepare({}, share), + prepare({ requestId: randomUUID() }, share), + ]); + expect(answers.map((answer) => answer.type)).toEqual([ + 'share.prepared', + 'share.prepared', + ]); + expect(posters).toHaveLength(1); + }); +}); + +/** The width of a baseline or progressive JPEG, from its SOF marker. */ +function jpegWidth(bytes: Buffer): number { + expect(bytes.subarray(0, 2)).toEqual(Buffer.from([0xff, 0xd8])); + let at = 2; + while (at < bytes.length) { + const marker = bytes[at + 1] ?? 0; + const length = bytes.readUInt16BE(at + 2); + if (marker === 0xc0 || marker === 0xc2) return bytes.readUInt16BE(at + 7); + at += 2 + length; + } + throw new Error('no SOF marker'); +} + +describe('the poster (spec 349, Decision 3)', () => { + it('is taken at min(1500, durationMs / 2), after -i, at most 1280 wide', () => { + expect(posterAtMs(10_000)).toBe(1500); + expect(posterAtMs(1001)).toBe(500); + expect(posterAtMs(0)).toBe(0); + const args = posterArgs('in.gif', 'out.jpg', 500); + expect(args.indexOf('-ss')).toBeGreaterThan(args.indexOf('-i')); + expect(args).toContain('0.500'); + expect(args).toContain("scale='min(1280,iw)':-2"); + expect(args.slice(-3)).toEqual(['-q:v', '4', 'out.jpg']); + }); + + it.each([ + ['mp4', ['-c:v', 'libx264', '-pix_fmt', 'yuv420p']], + ['gif', []], + ] as const)( + 'makes a JPEG at most 1280 px wide from a %s render', + async (format, codec) => { + const render = path.join(runDir, 'renders', `demo.${format}`); + await runFfmpeg([ + '-y', + '-f', + 'lavfi', + '-i', + 'testsrc=size=1920x1080:rate=10:duration=1', + ...codec, + render, + ]); + const poster = posterPath(runDir, format, process.platform); + const size = await makePoster({ + render, + poster, + durationMs: 1000, + platform: process.platform, + }); + const bytes = readFileSync(poster); + expect(size).toBe(bytes.length); + expect(size).toBeLessThanOrEqual(POSTER_MAX_BYTES); + expect(jpegWidth(bytes)).toBe(1280); + }, + 60_000, + ); + + it('reuses a poster newer than its render, and remakes an older one', async () => { + const render = path.join(runDir, 'renders', 'demo.mp4'); + const poster = posterPath(runDir, 'mp4', process.platform); + const request = { + render, + poster, + durationMs: 4000, + platform: process.platform, + }; + const runs: string[][] = []; + const ffmpeg = async (args: readonly string[]) => { + runs.push([...args]); + writeFileSync(args.at(-1) ?? '', 'jpeg'); + }; + expect(await makePoster(request, ffmpeg)).toBe(4); + expect(runs).toHaveLength(1); + // Written to a temporary file, then renamed. + expect(runs[0]?.at(-1)).toBe(poster.replace(/\.jpg$/, '.tmp.jpg')); + + const later = new Date(statSync(render).mtimeMs + 10_000); + utimesSync(poster, later, later); + expect(await makePoster(request, ffmpeg)).toBe(4); + expect(runs).toHaveLength(1); + + const newer = new Date(later.getTime() + 10_000); + utimesSync(render, newer, newer); + await makePoster(request, ffmpeg); + expect(runs).toHaveLength(2); + }); + + it('rejects a poster over 1 MiB and leaves no temporary file when FFmpeg fails', async () => { + const render = path.join(runDir, 'renders', 'demo.mp4'); + const poster = posterPath(runDir, 'mp4', process.platform); + const request = { + render, + poster, + durationMs: 4000, + platform: process.platform, + }; + await expect( + makePoster(request, async (args) => { + writeFileSync(args.at(-1) ?? '', Buffer.alloc(POSTER_MAX_BYTES + 1)); + }), + ).rejects.toThrow(/bytes/); + + rmSync(poster); + await expect( + makePoster(request, async (args) => { + writeFileSync(args.at(-1) ?? '', 'half'); + throw new Error('ffmpeg exited with 1'); + }), + ).rejects.toThrow(/exited/); + expect(() => statSync(poster.replace(/\.jpg$/, '.tmp.jpg'))).toThrow(); + expect(() => statSync(poster)).toThrow(); + }); +}); diff --git a/apps/cli/test/runs-report/report.spec.ts b/apps/cli/test/runs-report/report.spec.ts index 5671e296..35873508 100644 --- a/apps/cli/test/runs-report/report.spec.ts +++ b/apps/cli/test/runs-report/report.spec.ts @@ -113,6 +113,23 @@ describe('buildReport', () => { }); }); + it('marks a check run with kind check, and an ordinary run with no kind (#360)', () => { + const check = buildReport({ + root: ROOT, + result: result(), + cliVersion: 'x', + check: true, + }); + expect(check.kind).toBe('check'); + const ordinary = buildReport({ + root: ROOT, + result: result(), + cliVersion: 'x', + check: false, + }); + expect('kind' in ordinary).toBe(false); + }); + it('is failed with the 0-based failed step', () => { const failed = result({ status: 'failed', @@ -382,6 +399,16 @@ describe('reportRun', () => { }); }); + it('sends kind check for a check run only (#360)', async () => { + pair(); + await reportRun({ ...input(), check: true }, deps(ok)); + await reportRun(input(), deps(ok)); + const bodies = calls.map( + (call) => JSON.parse(String(call.init.body)) as Record, + ); + expect(bodies.map((body) => body.kind)).toEqual(['check', undefined]); + }); + it('sends the canonical root: a symlinked folder is the same project', async () => { pair(); const real = path.join(home, 'real'); diff --git a/apps/web/Dockerfile b/apps/web/Dockerfile index 78bb96f2..f6bdf4e4 100644 --- a/apps/web/Dockerfile +++ b/apps/web/Dockerfile @@ -53,6 +53,9 @@ ENV HOSTNAME=0.0.0.0 COPY --from=builder /app/apps/web/.next/standalone ./ COPY --from=builder /app/apps/web/.next/static ./apps/web/.next/static +# The standalone output leaves public/ out; server.js serves it from next to +# itself, so without this every file under it (the landing video) is a 404. +COPY --from=builder /app/apps/web/public ./apps/web/public RUN mkdir -p /app/apps/web/.next/cache && chown -R node:node /app/apps/web/.next diff --git a/apps/web/next.config.ts b/apps/web/next.config.ts index c655fdc5..258e086d 100644 --- a/apps/web/next.config.ts +++ b/apps/web/next.config.ts @@ -11,10 +11,22 @@ const nextConfig: NextConfig = { transpilePackages: ['@demoflow/shared'], // The docs pages are `page.mdx` (spec #179). pageExtensions: ['ts', 'tsx', 'md', 'mdx'], + // DemoFlow records this site at 127.0.0.1 (demoflow.config.json). The dev + // server blocks its scripts and HMR socket for any host but localhost, so + // without this the page never hydrates and no client control responds. + allowedDevOrigins: ['127.0.0.1'], // `/api` is reserved for the API. Stripping the prefix matches Caddy, so // the browser sees one origin locally and on staging alike. + // A share's file and poster live on the share page's own path (spec #348), + // as Caddy routes them on staging and production. async rewrites() { - return [{ source: '/api/:path*', destination: `${API_URL}/:path*` }]; + return [ + { source: '/api/:path*', destination: `${API_URL}/:path*` }, + { + source: '/s/:token([A-Za-z0-9_-]{43})/:asset(file|poster)', + destination: `${API_URL}/s/:token/:asset`, + }, + ]; }, }; diff --git a/apps/web/package.json b/apps/web/package.json index e26e158c..9814ae11 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -7,10 +7,10 @@ "start": "next start -p 3670", "start:dev": "node scripts/dev.mjs", "lint": "biome lint src", - "test": "node --test src/lib/docs-nav.test.mjs src/lib/playback.test.mjs src/lib/launch.test.mjs src/lib/format-bytes.test.mjs src/lib/run-delete.test.mjs src/lib/player-choice.test.mjs" + "test": "node --test src/lib/docs-nav.test.mjs src/lib/playback.test.mjs src/lib/launch.test.mjs src/lib/format-bytes.test.mjs src/lib/run-delete.test.mjs src/lib/player-choice.test.mjs src/lib/nav.test.mjs src/lib/crumbs.test.mjs src/lib/theme.test.mjs src/lib/overview-stats.test.mjs src/lib/status.test.mjs src/lib/share-page.test.mjs src/lib/shares.test.mjs src/lib/schedule.test.mjs src/lib/add-project.test.mjs" }, "dependencies": { - "@andrewbehma/glass-ui": "0.34.0", + "@andrewbehma/glass-ui": "0.37.0", "@demoflow/shared": "workspace:*", "@mdx-js/loader": "3.1.1", "@mdx-js/react": "3.1.1", diff --git a/apps/web/src/app/app/device/page.tsx b/apps/web/src/app/app/device/page.tsx index ad877e80..6bbf6018 100644 --- a/apps/web/src/app/app/device/page.tsx +++ b/apps/web/src/app/app/device/page.tsx @@ -1,8 +1,7 @@ import { Banner } from '@andrewbehma/glass-ui/banner'; import { Card } from '@andrewbehma/glass-ui/card'; -import Link from 'next/link'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; import { redirect } from 'next/navigation'; -import { ResendButton } from '@/components/auth/resend-button'; import { DeviceApproval } from '@/components/device/device-approval'; import { DeviceCodeForm } from '@/components/device/device-code-form'; import { t } from '@/i18n'; @@ -11,7 +10,8 @@ import { normalizeUserCode } from '@/lib/user-code'; // Approve or deny a machine that ran `demoflow login` (spec #135, Decision 6). // `?code=ABCD-1234` comes from the link the CLI opens; without it the person -// types the code shown in their terminal. +// types the code shown in their terminal. One decision on one screen (spec +// #375, Decision 3). export default async function DevicePage({ searchParams, }: { @@ -24,60 +24,43 @@ export default async function DevicePage({ const request = code ? await getDevice(code) : undefined; return ( -
-
- - {t('device.back')} - -
- {user.emailVerified ? null : ( - -

- {t('device.verify.body', { email: user.email })} -

-
- -
-
- )} - - {request && typeof request === 'object' ? ( - <> -

- {t('device.approve.title')} -

-
- -
- - ) : ( - <> -

- {request === 'not_found' + // A single decision reads best narrower than the shell's column. +
+ {request && typeof request === 'object' ? ( + <> + {/* The shell's banner asks for verification; this says why it matters. */} + + + + + + ) : ( + <> + -
- {request === 'not_found' ? ( -

- {t('device.notFound.body')} -

- ) : request === 'error' ? ( - {t('device.unavailable')} - ) : ( -

{t('device.code.body')}

- )} - -
- - )} - -

+ : t('device.code.title') + } + description={t('device.code.body')} + /> + + {request === 'not_found' ? ( + {t('device.notFound.body')} + ) : request === 'error' ? ( + {t('device.unavailable')} + ) : null} + + + + )} + ); } diff --git a/apps/web/src/app/app/layout.tsx b/apps/web/src/app/app/layout.tsx index 210a2334..bb8d46a3 100644 --- a/apps/web/src/app/app/layout.tsx +++ b/apps/web/src/app/app/layout.tsx @@ -1,15 +1,28 @@ import { headers } from 'next/headers'; import { redirect } from 'next/navigation'; import type { ReactNode } from 'react'; +import { AppShell } from '@/components/shell/app-shell'; import { signInUrl } from '@/lib/return-to'; -import { getSessionUser } from '@/lib/session'; +import { getRunners, getSessionUser } from '@/lib/session'; import { RETURN_TO_HEADER } from '@/proxy'; // One guard for every present and future `/app/**` page (spec 124, decision 4). // A guest comes back to the page they asked for (spec 135, decision 6). export default async function AppLayout({ children }: { children: ReactNode }) { - if (!(await getSessionUser())) { + const user = await getSessionUser(); + if (!user) { redirect(signInUrl((await headers()).get(RETURN_TO_HEADER) ?? undefined)); } - return children; + // For the sidebar's machine status (spec #373): read when the layout + // renders, shared with the page through the request cache; no polling. + const runners = await getRunners(); + return ( + + {children} + + ); } diff --git a/apps/web/src/app/app/machines/page.tsx b/apps/web/src/app/app/machines/page.tsx index 79968c2b..32d4fa8d 100644 --- a/apps/web/src/app/app/machines/page.tsx +++ b/apps/web/src/app/app/machines/page.tsx @@ -1,6 +1,5 @@ import { Banner } from '@andrewbehma/glass-ui/banner'; -import { Card } from '@andrewbehma/glass-ui/card'; -import Link from 'next/link'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; import { MachineList } from '@/components/machines/machine-list'; import { t } from '@/i18n'; import { getRunners } from '@/lib/session'; @@ -8,22 +7,16 @@ import { getRunners } from '@/lib/session'; export default async function MachinesPage() { const runners = await getRunners(); return ( -
-
- - {t('machines.back')} - -
- -

{t('machines.title')}

-
- {runners ? ( - - ) : ( - {t('machines.loadFailed')} - )} -
-
-
+ <> + + {runners ? ( + + ) : ( + {t('machines.loadFailed')} + )} + ); } diff --git a/apps/web/src/app/app/page.tsx b/apps/web/src/app/app/page.tsx index 90ba9791..c8b690e0 100644 --- a/apps/web/src/app/app/page.tsx +++ b/apps/web/src/app/app/page.tsx @@ -1,53 +1,67 @@ -import { Banner } from '@andrewbehma/glass-ui/banner'; -import Link from 'next/link'; -import { redirect } from 'next/navigation'; -import { ResendButton } from '@/components/auth/resend-button'; -import { SignOutButton } from '@/components/auth/sign-out-button'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; +import { StatTile } from '@andrewbehma/glass-ui/stat-tile'; import { MachinesSummary } from '@/components/dashboard/machines-summary'; import { RecentProjects } from '@/components/dashboard/recent-projects'; +import { StaleDemos } from '@/components/dashboard/stale-demos'; import { t } from '@/i18n'; -import { getRunners, getSessionUser } from '@/lib/session'; -import { getProjects } from './projects/_data'; +import type { Project } from '@/lib/api'; +import { overviewStats } from '@/lib/overview-stats'; +import { getRunners } from '@/lib/session'; +import { getProjectSchedules, getProjects, getStale } from './projects/_data'; + +/** + * Whether any bound project has a schedule, so an empty stale list reads "All + * passing" (spec #363, Decision 4). Asked only when nothing is stale. + */ +async function hasSchedule(projects: Project[] | null): Promise { + const bound = (projects ?? []).filter((project) => project.rootPath !== null); + const schedules = await Promise.all( + bound.map((project) => getProjectSchedules(project.id)), + ); + return schedules.some((entry) => entry?.site || entry?.config); +} export default async function AppPage() { - const user = await getSessionUser(); - if (!user) redirect('/sign-in'); - const [projects, runners] = await Promise.all([getProjects(), getRunners()]); + const [projects, runners, stale] = await Promise.all([ + getProjects(), + getRunners(), + getStale(), + ]); + const showStale = + stale === null || stale.length > 0 || (await hasSchedule(projects)); + const stats = projects ? overviewStats(projects) : null; + const unavailable = t('dashboard.stats.unavailable'); return ( -
-
-

- {t('app.shell.signedInAs', { email: user.email })} -

-
- - {t('projects.nav')} - - - {t('app.shell.machines')} - - - {t('app.shell.settings')} - - + <> + +
+ + + +
+ {showStale ? : null} +
+
+
-
- {user.emailVerified ? null : ( - -

- {t('app.shell.verifyBody', { email: user.email })} -

-
- -
-
- )} -

{t('dashboard.title')}

- - -
+ + + ); } diff --git a/apps/web/src/app/app/projects/[id]/page.tsx b/apps/web/src/app/app/projects/[id]/page.tsx index 35726b63..2e73f4c4 100644 --- a/apps/web/src/app/app/projects/[id]/page.tsx +++ b/apps/web/src/app/app/projects/[id]/page.tsx @@ -1,11 +1,22 @@ import { Banner } from '@andrewbehma/glass-ui/banner'; import { Card } from '@andrewbehma/glass-ui/card'; -import Link from 'next/link'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; +import { SectionHeader } from '@andrewbehma/glass-ui/section-header'; import { notFound } from 'next/navigation'; +import { CheckSchedule } from '@/components/projects/check-schedule'; +import { MachinePill } from '@/components/projects/machine-pill'; +import { ProjectPath } from '@/components/projects/project-path'; import { RunsTable } from '@/components/projects/runs-table'; +import { ScenarioHealth } from '@/components/projects/scenario-health'; import { StartRun } from '@/components/projects/start-run'; +import { CrumbName } from '@/components/shell/crumb-names'; import { t } from '@/i18n'; -import { getFirstRuns, getProjects } from '../_data'; +import { + getFirstRuns, + getProjectHealth, + getProjectSchedules, + getProjects, +} from '../_data'; export default async function ProjectLibraryPage({ params, @@ -18,36 +29,43 @@ export default async function ProjectLibraryPage({ const { status: rawStatus } = await searchParams; const status = rawStatus === 'passed' || rawStatus === 'failed' ? rawStatus : undefined; - const [projects, runs] = await Promise.all([ + const [projects, runs, schedules, health] = await Promise.all([ getProjects(), getFirstRuns(id, status), + getProjectSchedules(id), + getProjectHealth(id), ]); if (runs === 'not_found') notFound(); const project = projects?.find((candidate) => candidate.id === id); if (projects && !project) notFound(); return ( -
-
- - {t('projects.back')} - -
- -

- {project?.name ?? t('projects.title')} -

- {project?.rootPath ? ( -
-

- {t('projects.library.path')}: {project.rootPath} -

-

- {t('projects.library.machine')}:{' '} - {project.runner?.name ?? t('projects.item.machineRemoved')} -

-
- ) : null} -
+ <> + {project ? : null} + + {project?.name ?? t('projects.title')} + + } + status={ + project?.rootPath ? : null + } + description={ + project?.rootPath ? ( + + ) : project ? ( + t('projects.item.noFolder') + ) : null + } + /> + {project ? ( + + ) : null} + {project ? ( ) : null} - - {runs === 'error' ? ( - {t('projects.library.pageFailed')} - ) : ( - - )} - -
+
+ + + {runs === 'error' ? ( + {t('projects.library.pageFailed')} + ) : ( + + )} + +
+ ); } diff --git a/apps/web/src/app/app/projects/[id]/runs/[runId]/page.tsx b/apps/web/src/app/app/projects/[id]/runs/[runId]/page.tsx index 57b52254..95b4c148 100644 --- a/apps/web/src/app/app/projects/[id]/runs/[runId]/page.tsx +++ b/apps/web/src/app/app/projects/[id]/runs/[runId]/page.tsx @@ -1,16 +1,20 @@ import { Banner } from '@andrewbehma/glass-ui/banner'; -import { Card } from '@andrewbehma/glass-ui/card'; import { KeyValueList } from '@andrewbehma/glass-ui/key-value-list'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; import { notFound } from 'next/navigation'; import { RunPlayback } from '@/components/player/run-playback'; import { DeleteRun } from '@/components/projects/delete-run'; +import { formatList } from '@/components/projects/format-choice'; +import { MachinePill } from '@/components/projects/machine-pill'; import { RunAgain } from '@/components/projects/run-again'; -import { RunBreadcrumbs } from '@/components/projects/run-breadcrumbs'; import { - RunDetails, - RunFailureBanner, + RunFailure, + RunFiles, + RunStages, } from '@/components/projects/run-details'; import { RunStatus } from '@/components/projects/run-status'; +import { ShareButton } from '@/components/shares/share-button'; +import { CrumbName } from '@/components/shell/crumb-names'; import { t } from '@/i18n'; import { formatDate } from '@/lib/format-date'; import { formatDuration } from '@/lib/format-duration'; @@ -27,11 +31,7 @@ export default async function RunPage({ const detail = await getRunDetail(runId); if (detail === 'not_found') notFound(); if (detail === 'error') { - return ( -
- {t('runPage.pageFailed')} -
- ); + return {t('runPage.pageFailed')}; } const { run, project, machine } = detail; if (project.id !== id) notFound(); @@ -43,102 +43,143 @@ export default async function RunPage({ machine, scenarioPath: run.scenarioPath, }); + const check = run.kind === 'check'; + const details = ( + <> + , + }, + { + key: 'scenario', + label: t('runPage.metadata.scenario'), + value: {run.scenarioName}, + }, + { + key: 'path', + label: t('runPage.metadata.path'), + value: ( + + {run.scenarioPath} + + ), + }, + { + key: 'when', + label: t('runPage.metadata.when'), + value: ( + + ), + }, + { + key: 'duration', + label: t('runPage.metadata.duration'), + value: formatDuration(run.durationMs), + }, + { + key: 'failedStep', + label: t('runPage.metadata.failedStep'), + value: + run.failedStep === null + ? t('runPage.metadata.noFailedStep') + : t('runPage.metadata.step', { + number: run.failedStep + 1, + }), + }, + { + key: 'machine', + label: t('runPage.metadata.machine'), + value: , + }, + { + key: 'formats', + label: t('runPage.metadata.formats'), + value: + run.formats.length > 0 + ? formatList(run.formats) + : t('runPage.metadata.noFormats'), + }, + { + key: 'cliVersion', + label: t('runPage.metadata.cliVersion'), + value: {run.cliVersion}, + }, + ]} + /> + + + + ); return ( -
- + + + + {t('runPage.title', { name: run.scenarioName })} + + } + description={project.name} + actions={ + + } /> - -

- {t('runPage.title', { name: run.scenarioName })} -

-
- , - }, - { - key: 'when', - label: t('runPage.metadata.when'), - value: ( - - ), - }, - { - key: 'duration', - label: t('runPage.metadata.duration'), - value: formatDuration(run.durationMs), - }, - { - key: 'failedStep', - label: t('runPage.metadata.failedStep'), - value: - run.failedStep === null - ? t('runPage.metadata.noFailedStep') - : t('runPage.metadata.step', { - number: run.failedStep + 1, - }), - }, - { - key: 'machine', - label: t('runPage.metadata.machine'), - value: machine?.name ?? t('projects.item.machineRemoved'), - }, - { - key: 'cliVersion', - label: t('runPage.metadata.cliVersion'), - value: run.cliVersion, - }, - ]} + + ) : null + } + deleteRun={ + 0 && run.artifactsDeletedAt === null + } /> -
- 0 && run.artifactsDeletedAt === null - } - /> -
- - {machine !== null && againState !== 'hidden' ? ( - - ) : null} - - - - -
+ } + /> + + ); } diff --git a/apps/web/src/app/app/projects/_data.ts b/apps/web/src/app/app/projects/_data.ts index 70653333..e62f2f19 100644 --- a/apps/web/src/app/app/projects/_data.ts +++ b/apps/web/src/app/app/projects/_data.ts @@ -1,14 +1,20 @@ import { cookies } from 'next/headers'; import { + getHealth, getRun, + getSchedule, listProjectRuns, listProjects, listRunners, + listStale, type Project, type ProjectRunsPage, + type ProjectSchedules, type RunDetail, type Runner, type RunStatusFilter, + type ScenarioHealth, + type StaleScenario, } from '@/lib/api'; async function target() { @@ -48,3 +54,23 @@ export async function getFirstRuns( if (result.ok) return result.data; return result.status === 404 ? 'not_found' : 'error'; } + +/** A project's check schedules (spec #363), or null when the API fails. */ +export async function getProjectSchedules( + id: string, +): Promise { + const result = await getSchedule(id, await target()); + return result.ok ? result.data : null; +} + +/** The per-scenario state of a project; empty when the API fails. */ +export async function getProjectHealth(id: string): Promise { + const result = await getHealth(id, await target()); + return result.ok ? result.data : []; +} + +/** The user's stale scenarios across projects, or null when the API fails. */ +export async function getStale(): Promise { + const result = await listStale(await target()); + return result.ok ? result.data : null; +} diff --git a/apps/web/src/app/app/projects/page.tsx b/apps/web/src/app/app/projects/page.tsx index edb700ff..83040c04 100644 --- a/apps/web/src/app/app/projects/page.tsx +++ b/apps/web/src/app/app/projects/page.tsx @@ -1,6 +1,5 @@ import { Banner } from '@andrewbehma/glass-ui/banner'; -import { Card } from '@andrewbehma/glass-ui/card'; -import Link from 'next/link'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; import { ProjectList } from '@/components/projects/project-list'; import { t } from '@/i18n'; import { getMachines, getProjects } from './_data'; @@ -11,22 +10,16 @@ export default async function ProjectsPage() { getMachines(), ]); return ( -
-
- - {t('projects.back')} - -
- -

{t('projects.title')}

-
- {projects ? ( - - ) : ( - {t('projects.loadFailed')} - )} -
-
-
+ <> + + {projects ? ( + + ) : ( + {t('projects.loadFailed')} + )} + ); } diff --git a/apps/web/src/app/app/settings/page.tsx b/apps/web/src/app/app/settings/page.tsx index 26bdef6e..d6717f20 100644 --- a/apps/web/src/app/app/settings/page.tsx +++ b/apps/web/src/app/app/settings/page.tsx @@ -1,8 +1,13 @@ import { Banner } from '@andrewbehma/glass-ui/banner'; import { buttonClassName } from '@andrewbehma/glass-ui/button'; import { Card } from '@andrewbehma/glass-ui/card'; -import Link from 'next/link'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; +import { SectionHeader } from '@andrewbehma/glass-ui/section-header'; +import { Separator } from '@andrewbehma/glass-ui/separator'; +import { StatusPill } from '@andrewbehma/glass-ui/status-pill'; +import { Palette } from 'lucide-react'; import { redirect } from 'next/navigation'; +import type { ReactNode } from 'react'; import { UnlinkGithubButton } from '@/components/auth/unlink-github-button'; import { DeleteAccount } from '@/components/settings/delete-account'; import { t } from '@/i18n'; @@ -10,9 +15,50 @@ import { githubStartUrl } from '@/lib/api'; import { githubErrorMessage } from '@/lib/github-errors'; import { getProviders, getSessionUser } from '@/lib/session'; -// Account settings: linking and unlinking GitHub (spec #126, Decision 1), and -// deleting the account (spec #177). A failed link comes back here with -// `?error=`. +/** One titled section of the page (spec #375, Decision 5). */ +function SettingsSection({ + id, + title, + description, + children, +}: { + id: string; + title: string; + description: string; + children: ReactNode; +}) { + return ( +
+ + {children} +
+ ); +} + +/** A labelled fact with its state under it and, optionally, its action. */ +function SettingRow({ + label, + children, + action, +}: { + label: string; + children: ReactNode; + action?: ReactNode; +}) { + return ( +
+
+

{label}

+
{children}
+
+ {action ?
{action}
: null} +
+ ); +} + +// Account settings: the profile, linking and unlinking GitHub (spec #126, +// Decision 1), and deleting the account (spec #177). A failed link comes back +// here with `?error=`. export default async function SettingsPage({ searchParams, }: { @@ -24,51 +70,81 @@ export default async function SettingsPage({ const error = githubErrorMessage((await searchParams).error); return ( -
-
- - {t('app.settings.back')} - -
+ // Forms read best narrower than the shell's column. +
+ {error ? {error} : null} - -

{t('app.settings.title')}

-
-
-
{t('app.settings.email')}
-
{user.email}
-
-
-
- {t('app.settings.github.title')} -
-
-

- {user.githubLinked - ? t('app.settings.github.linked') - : t('app.settings.github.notLinked')} -

- {user.githubLinked ? ( + + + + {user.email} + + + + +

+

+
+ + + + + {user.hasPassword + ? t('app.settings.password.set') + : t('app.settings.password.notSet')} + + + ) : github ? ( - - ) : ( -

- {t('app.settings.github.unavailable')} -

- )} -
-
-
-
- -
+ + {t('app.settings.github.link')} + + ) : undefined + } + > + {user.githubLinked + ? t('app.settings.github.linked') + : github + ? t('app.settings.github.notLinked') + : t('app.settings.github.unavailable')} + + + + + + + + ); } diff --git a/apps/web/src/app/app/shares/page.tsx b/apps/web/src/app/app/shares/page.tsx new file mode 100644 index 00000000..bf014d74 --- /dev/null +++ b/apps/web/src/app/app/shares/page.tsx @@ -0,0 +1,26 @@ +import { Banner } from '@andrewbehma/glass-ui/banner'; +import { PageHeader } from '@andrewbehma/glass-ui/page-header'; +import { cookies } from 'next/headers'; +import { ShareList } from '@/components/shares/share-list'; +import { t } from '@/i18n'; +import { apiUrl } from '@/lib/session'; +import { listShares } from '@/lib/shares-client'; + +/** The signed-in user's shares and quota (spec #350, Decision 3). */ +export default async function SharesPage() { + const cookie = (await cookies()).toString(); + const result = await listShares({ baseUrl: apiUrl(), cookie }); + return ( + <> + + {result.ok ? ( + + ) : ( + {t('shares.loadFailed')} + )} + + ); +} diff --git a/apps/web/src/app/docs/_components/docs-sidebar.tsx b/apps/web/src/app/docs/_components/docs-sidebar.tsx index 09720e1d..b3a04e3a 100644 --- a/apps/web/src/app/docs/_components/docs-sidebar.tsx +++ b/apps/web/src/app/docs/_components/docs-sidebar.tsx @@ -12,7 +12,7 @@ export function DocsSidebar() { aria-label={t('docs.nav.label')} className="glass rounded-surface p-3 lg:sticky lg:top-24" > -

+

{t('docs.nav.title')}

    diff --git a/apps/web/src/app/docs/_components/on-this-page.tsx b/apps/web/src/app/docs/_components/on-this-page.tsx index 1e08dd0a..36c77a3e 100644 --- a/apps/web/src/app/docs/_components/on-this-page.tsx +++ b/apps/web/src/app/docs/_components/on-this-page.tsx @@ -30,9 +30,9 @@ export function OnThisPage() { return (