diff --git a/docs/superpowers/plans/2026-07-14-existing-photo-workflow-repair.md b/docs/superpowers/plans/2026-07-14-existing-photo-workflow-repair.md new file mode 100644 index 000000000..ade10b717 --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-existing-photo-workflow-repair.md @@ -0,0 +1,29 @@ +# Existing Photo Workflow Repair Plan + +## Goal + +Make the active kernelCAD Studio generation path accept a sourced, scaled photo +reference for simple consumer-device CAD, then prove the result through the +current runtime and physical checks. + +## Tasks + +1. Repair the e-reader acceptance artifact: verified scale anchor, valid image + dimensions, and parametric static-assembly metadata. Add regressions first. +2. Add an explicit photo-reference request contract to the active Studio UI: + file, MIME/name/hash, and known dimension. Write client tests before UI code. +3. Add a matching hosted API/server contract that validates and materializes + the reference, then supplies a concise photo-device brief to the existing + authoring-and-gate loop. Write route/orchestrator tests before implementation. +4. Make trace confidence fail closed for the light-device/internal-screen case; + tracing is evidence, not a device generator. +5. Run focused client/server/runtime tests plus e-reader evaluate, interference, + and hidden-reference render inspection. Review every changed boundary before + handoff. + +## Out of scope + +- No client-side direct vision-chat bypass or legacy HeadlessKernel repair. +- No public Meshy/Tripo integration. +- No claim that a single image yields an exact branded, manufacturable, or + electronically functional device. diff --git a/docs/superpowers/plans/2026-07-14-photo-to-parametric-consumer-device.md b/docs/superpowers/plans/2026-07-14-photo-to-parametric-consumer-device.md new file mode 100644 index 000000000..bdb85379e --- /dev/null +++ b/docs/superpowers/plans/2026-07-14-photo-to-parametric-consumer-device.md @@ -0,0 +1,226 @@ +# Photo-to-Parametric Consumer Device Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make KernelCAD's existing reference-image capability usable for a simple consumer-electronics photo by shipping a tested, parameterized e-reader reconstruction and its agent workflow. + +**Architecture:** A new child skill under the existing `kernelcad-from-reference` orchestrator constrains what an agent may infer from a photo and points to the appropriate CAD checks. A checked-in public-domain reference image, provenance note, and `.kcad.ts` acceptance model demonstrate the flow. An integration test executes the actual script and guards the source-level contract. + +**Tech Stack:** TypeScript, KernelCAD authoring API, Vitest, Sharp-backed `referenceImage()` validation, KernelCAD CLI rendering. + +--- + +### Task 1: Lock the acceptance contract with a failing integration test + +**Files:** +- Create: `tests/integration/examples/photoReferenceEreader.test.ts` +- Create later: `examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts` +- Create later: `examples/from-reference/e-reader/kindle-2-reference.jpg` +- Create later: `examples/from-reference/e-reader/PROVENANCE.md` + +- [ ] **Step 1: Write the failing test** + +```ts +import { existsSync, readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import { evaluateAndBuildScript } from '../../../src/agent/cli/commands/evaluate'; +import { checkInterference } from '../../../src/agent/script-runtime/checkInterference'; + +const EXAMPLE = 'examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts'; +const REFERENCE = 'examples/from-reference/e-reader/kindle-2-reference.jpg'; + +describe('photo-reference e-reader example', () => { + it('turns a sourced photo into a parametric, evaluatable device model', async () => { + expect(existsSync(REFERENCE)).toBe(true); + const source = readFileSync(EXAMPLE, 'utf8'); + expect(source).toContain('// Real Object Brief'); + expect(source).toContain("referenceImage('./kindle-2-reference.jpg'"); + for (const name of ['bodyWidth', 'bodyHeight', 'bodyThickness', 'screenWidth', 'screenHeight']) { + expect(source).toContain(`param('${name}'`); + } + + const built = await evaluateAndBuildScript({ file: EXAMPLE }); + expect(built.evaluation.exitCode).toBe(0); + expect(built.evaluation.diagnostics.filter((d) => d.severity === 'error')).toEqual([]); + expect(built.evaluation.featureCount).toBeGreaterThanOrEqual(7); + const referenceRecord = built.model?.records.find((record) => record.kind === 'referenceImage'); + expect(referenceRecord).toBeDefined(); + expect((referenceRecord?.metadata?.path as string | undefined)?.endsWith('kindle-2-reference.jpg')).toBe(true); + expect(referenceRecord?.metadata?.diagnostics ?? []).toEqual([]); + + const interference = await checkInterference({ + code: source, + fileName: EXAMPLE, + scriptDir: dirname(resolve(EXAMPLE)), + epsilonMm3: 0.01, + }); + expect(interference.partCount).toBe(4); + expect(interference.pairs).toEqual([]); + }); +}); +``` + +- [ ] **Step 2: Run it to verify it fails for missing acceptance artifacts** + +Run: + +```bash +/home/andrii/.nvm/versions/node/v22.22.0/bin/node ./node_modules/vitest/vitest.mjs run tests/integration/examples/photoReferenceEreader.test.ts --reporter=dot +``` + +Expected: FAIL because the e-reader source and reference image do not yet exist. + +- [ ] **Step 3: Commit the red test only** + +```bash +git add tests/integration/examples/photoReferenceEreader.test.ts +git commit -m "test: define photo-reference e-reader contract" +``` + +### Task 2: Add source-owned reference evidence and the e-reader model + +**Files:** +- Create: `examples/from-reference/e-reader/kindle-2-reference.jpg` +- Create: `examples/from-reference/e-reader/PROVENANCE.md` +- Create: `examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts` + +- [ ] **Step 1: Download the public-domain Kindle 2 front image from Wikimedia Commons** + +Run: + +```bash +curl -L --fail --silent --show-error \ + 'https://commons.wikimedia.org/wiki/Special:FilePath/Amazon-kindle-gen2.jpg' \ + -o examples/from-reference/e-reader/kindle-2-reference.jpg +``` + +- [ ] **Step 2: Record immutable source facts in `PROVENANCE.md`** + +Include the Wikimedia file page, author Evan-Amos, public-domain license, download date, SHA-256, intended internal benchmark use, and the explicit trade-dress limitation. + +- [ ] **Step 3: Implement the minimal parameterized model** + +Use a Real Object Brief followed by these named parameters: `bodyWidth`, `bodyHeight`, `bodyThickness`, `cornerRadius`, `bezelWidth`, `screenWidth`, `screenHeight`, `screenRecess`, `controlDiameter`, `usbPortWidth`, and `usbPortHeight`. Use `referenceImage()` on the XZ plane, an `extrudeRoundedRect()` body, a shallow screen recess, a seated display plate, a recessed circular control, status LED, and a through USB-C opening. Apply PBR material to leaf geometry before every boolean or union. Return `assembly('photo-reference-e-reader').model()` with exactly four named, non-overlapping parts: housing, display, navigation control, and status LED. + +- [ ] **Step 4: Run the test and make it pass** + +Run the Task 1 command again. + +Expected: PASS with one file and no error diagnostics. + +- [ ] **Step 5: Commit source, reference, provenance, and green test** + +```bash +git add examples/from-reference/e-reader tests/integration/examples/photoReferenceEreader.test.ts +git commit -m "feat: add photo-reference e-reader example" +``` + +### Task 3: Make the workflow reusable for agents + +**Files:** +- Create: `src/agent/skills/kernelcad-from-reference/photo-to-device/SKILL.md` +- Modify: `src/agent/skills/kernelcad-from-reference/SKILL.md` +- Create: `tests/unit/skill/photoToDeviceSkill.test.ts` + +- [ ] **Step 1: Write the failing skill contract test** + +```ts +import { readFileSync } from 'node:fs'; +import { describe, expect, it } from 'vitest'; + +const SKILL = 'src/agent/skills/kernelcad-from-reference/photo-to-device/SKILL.md'; + +describe('photo-to-device skill', () => { + it('requires a scale anchor and labels mesh output as visual reference only', () => { + const source = readFileSync(SKILL, 'utf8'); + expect(source).toContain('known dimension'); + expect(source).toContain('visual_mesh_reference'); + expect(source).toContain('kernelcad evaluate'); + expect(source).toContain('kernelcad interference'); + }); +}); +``` + +- [ ] **Step 2: Run it to verify it fails for the missing skill** + +Run: + +```bash +/home/andrii/.nvm/versions/node/v22.22.0/bin/node ./node_modules/vitest/vitest.mjs run tests/unit/skill/photoToDeviceSkill.test.ts --reporter=dot +``` + +Expected: FAIL with `ENOENT` for the new skill path. + +- [ ] **Step 3: Implement the skill and route to it from the orchestrator** + +The new skill must cover: suitable categories, required scale anchor, visible-versus-inferred facts, e-reader/handheld enclosure feature checklist, required provenance, optional tracing, `visual_mesh_reference` terminology, and the deterministic evaluate/interference/render-inspect gates. Add it as the right-angled consumer-device path after `blockout-model` in the orchestrator. + +- [ ] **Step 4: Run the skill test and distribution regression tests** + +Run: + +```bash +/home/andrii/.nvm/versions/node/v22.22.0/bin/node ./node_modules/vitest/vitest.mjs run \ + tests/unit/skill/photoToDeviceSkill.test.ts \ + tests/unit/cli/walkSkillTree.test.ts \ + tests/unit/cli/skillInstallRecursion.test.ts \ + --reporter=dot +``` + +Expected: PASS; recursive discovery makes the new child available to installed clients. + +- [ ] **Step 5: Commit the reusable workflow** + +```bash +git add src/agent/skills/kernelcad-from-reference tests/unit/skill/photoToDeviceSkill.test.ts +git commit -m "feat: guide photo-to-parametric device builds" +``` + +### Task 4: Run the physical and visual proof packet + +**Files:** +- Read only: `examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts` +- Generated outside the repository: `/tmp/kernelcad-e-reader-inspect/` + +- [ ] **Step 1: Build the CLI** + +```bash +/home/andrii/.nvm/versions/node/v22.22.0/bin/node ./node_modules/typescript/bin/tsc -p tsconfig.cli.json --noEmit +/home/andrii/.nvm/versions/node/v22.22.0/bin/node scripts/build-cli.mjs +``` + +- [ ] **Step 2: Run deterministic gates** + +```bash +/home/andrii/.nvm/versions/node/v22.22.0/bin/node dist/cli/index.js evaluate examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts --json +/home/andrii/.nvm/versions/node/v22.22.0/bin/node dist/cli/index.js interference examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts --json +``` + +Expected: zero error diagnostics, four assembly parts, and zero interference pairs. + +- [ ] **Step 3: Create and inspect the render packet** + +```bash +/home/andrii/.nvm/versions/node/v22.22.0/bin/node dist/cli/index.js render inspect \ + examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts \ + /tmp/kernelcad-e-reader-inspect \ + --channels rgb,mask,depth,normals \ + --hide-reference-images +``` + +Open a canonical RGB output and verify the body, screen recess, circular control, LED, and port are visibly present and the object has non-trivial depth. + +- [ ] **Step 4: Run focused regression tests and typecheck** + +```bash +/home/andrii/.nvm/versions/node/v22.22.0/bin/node ./node_modules/vitest/vitest.mjs run \ + tests/integration/examples/photoReferenceEreader.test.ts \ + tests/unit/skill/photoToDeviceSkill.test.ts \ + tests/integration/agent/trace-from-image-smoke.test.ts \ + tests/unit/cli/skillInstallRecursion.test.ts \ + --reporter=dot +/home/andrii/.nvm/versions/node/v22.22.0/bin/node ./node_modules/typescript/bin/tsc -b --noEmit +``` + +Expected: all focused tests and typecheck pass. diff --git a/docs/superpowers/specs/2026-07-14-existing-photo-workflow-repair.md b/docs/superpowers/specs/2026-07-14-existing-photo-workflow-repair.md new file mode 100644 index 000000000..d586e3c41 --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-existing-photo-workflow-repair.md @@ -0,0 +1,76 @@ +# Existing Photo Workflow Repair + +## Status + +Re-scoped on 2026-07-14 after direct reproduction showed that kernelCAD already +has a photo button, but that button is not connected to the current generated +CAD and verification path. + +## Observed failure + +The legacy AI Assistant uploads a data URL directly to a vision chat request, +then returns markdown. It does not retain a reference asset, run +`trace_from_image`, or enter the current agent loop. Its Apply and Preview +actions run `.kcad.ts` source through the obsolete `HeadlessKernel`, which only +injects `replicad` and `console`; ordinary `param()`-based kernelCAD source +therefore fails before the Studio runtime can evaluate it. + +The active Studio surface (`StudioGenerate`) has the right staged-review and +modern-runtime path but currently accepts only text/mesh context, not a photo. + +## Decision + +Repair the active Studio generation path rather than revive the legacy panel. +A photo-assisted build must include: + +- a bounded, typed reference asset and its SHA-256 provenance; +- at least one `known dimension` supplied by the user; +- a restricted first-use scope: simple, front-on consumer electronics and + passive hobby enclosures; +- a photo-aware brief passed to the hosted authoring agent; +- the existing current-runtime evaluate/gate loop before a staged edit is + offered; and +- a clear distinction between observed image facts and inferred depth or + internals. + +`trace_from_image` remains an optional coordinate aid. It is not a device +generator and must not return a confident outer silhouette when it has only +found an internal dark screen. Any Meshy/Tripo-style result remains a +`visual_mesh_reference`, not authoritative CAD. + +## Architecture + +```text +Studio photo + known dimension + | + v +typed reference request (mime, filename, hash, scale) + | + v +hosted generate route materializes a managed reference asset + | + +--> optional trace_from_image evidence + | + v +photo-device brief + existing authoring agent + | + v +current evaluate / assembly-interference gates + | + v +staged edit -> setCode -> current Studio runtime +``` + +## Acceptance criteria + +- A selected image and known dimension reach the active `StudioGenerate` + request as structured reference input, not an untracked chat attachment. +- The server validates and materializes that input, records its hash, and + passes a photo-specific brief into the verified authoring loop. +- The first consumer-device acceptance source has a real cited scale anchor, + a reference image with positive decoded dimensions, connected parametric + assembly metadata, zero interference, and a reviewed render. +- The system fails honestly when a photo cannot establish a usable silhouette + or scale; it does not label an internal display as the outer device. +- Robots, mechanisms, electronics internals, and manufacture-ready claims + remain outside the one-photo consumer-device path. diff --git a/docs/superpowers/specs/2026-07-14-photo-to-parametric-consumer-device-design.md b/docs/superpowers/specs/2026-07-14-photo-to-parametric-consumer-device-design.md new file mode 100644 index 000000000..7ff12dcf2 --- /dev/null +++ b/docs/superpowers/specs/2026-07-14-photo-to-parametric-consumer-device-design.md @@ -0,0 +1,65 @@ +# Photo-to-Parametric Consumer Device Design + +## Status + +Approved for implementation by the user's explicit overnight request on 2026-07-14. This document records the bounded first milestone rather than waiting for an interactive review. + +## Problem + +KernelCAD already exposes `referenceImage()` and `trace_from_image`, but an agent receiving a product photograph has no focused, evidence-backed recipe for turning a simple consumer-electronics reference into editable CAD. A raw image-to-mesh result is useful visual evidence but cannot establish hidden geometry, dimensions, manufacturability, or electronics fit. + +## Decision + +Ship a native photo-to-parametric path for simple, slab-like consumer devices. The acceptance artifact is a public-domain Kindle 2 front photograph reconstructed as an editable e-reader enclosure with a display recess, navigation control, status LED, and USB-C opening. The model will explicitly state its inferred dimensions and hidden-side assumptions. + +The path is deliberately provider-neutral: + +- A photo plus at least one known dimension is the required input. +- `referenceImage()` stays available as the visual alignment aid. +- `trace_from_image` is optional for organic silhouettes, not required for rectangular devices. +- Any later Meshy, Tripo, Rodin, or photogrammetry output is named `visual_mesh_reference`; it must never be represented as a KernelCAD, STEP, BREP, or manufacturable result. + +## Scope + +1. Add a `photo-to-device` child skill beneath `kernelcad-from-reference` with the decision rules, provenance requirements, and deterministic gates for consumer electronics. +2. Add a checked-in, public-domain e-reader reference photograph and provenance file. +3. Add one parameterized e-reader `.kcad.ts` artifact with a complete Real Object Brief and conservative physical assumptions. +4. Add an integration test that proves the example, its reference asset, and its key editable parameters remain valid. +5. Produce an explicit render-inspection packet during verification. + +## Non-goals + +- No claim that one image recovers a real product's PCB, battery, fasteners, material stack, or exact dimensions. +- No public proto.cat upload surface or direct handoff to its current mesh endpoint. +- No external image-to-mesh provider integration, keys, billing, or network dependency. +- No attempt to use the visual acceptance model as a branded production clone; it is an internal reference-reconstruction benchmark. + +## Architecture + +```text +reference photo + known width + | + v +Real Object Brief and provenance + | + v +parametric KernelCAD source (.kcad.ts) + | + +--> evaluate / interference / DFM + | + +--> render inspect packet and visual review +``` + +The skill owns agent behavior. The example owns the concrete first-use-case source and proves the API surface. The integration test protects both from drift without making a network request. + +## Acceptance criteria + +- The e-reader source has named, bounded dimensions for body, display, bezel, control, and USB-C opening. +- It contains a Real Object Brief with known facts, inferred facts, and validation focus. +- The local reference photo is validated by `referenceImage()` during evaluation. +- `kernelcad evaluate` has no errors, the four-part static assembly reports no interference pairs, and the rendered model visibly reads as a real, layered e-reader rather than a flat card. +- The reusable skill instructs agents to ask for a known dimension, model visible geometry first, label inferred internals, and route mechanisms to the assembly/physical-review workflow. + +## Follow-on integration boundary + +proto.cat should eventually accept an authenticated, stored reference identifier rather than arbitrary image URLs, derive a device brief, and invoke the same parametric KernelCAD path. Its current visual mesh route is not the handoff contract: it needs a separate security and provenance hardening pass before it can be used for this feature. diff --git a/eval/cookbook.test.ts b/eval/cookbook.test.ts index 4069843f7..c4ff282c1 100644 --- a/eval/cookbook.test.ts +++ b/eval/cookbook.test.ts @@ -1,22 +1,14 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors import { describe, it, expect, beforeAll } from 'vitest'; -import { mkdtempSync, readFileSync, readdirSync, existsSync } from 'node:fs'; -import { join, resolve } from 'node:path'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { runTask } from './runner'; import { MockAgentClient } from './agent'; import { isKernelcadAvailable } from './oracle/kernelcad-client'; import { injectCookbook } from './cookbook-injector'; - -function loadCombinedSkillMd(): string { - const root = resolve('src/agent/skills'); - const dirs = readdirSync(root, { withFileTypes: true }) - .filter((e) => e.isDirectory() && existsSync(join(root, e.name, 'SKILL.md'))) - .map((e) => e.name) - .sort(); - return dirs.map((name) => readFileSync(join(root, name, 'SKILL.md'), 'utf8')).join('\n\n---\n\n'); -} +import { loadCombinedSkillMd } from './skillContext'; let kernelcadAvailable = false; diff --git a/eval/golden.test.ts b/eval/golden.test.ts index bb6a89448..e68bd3eb6 100644 --- a/eval/golden.test.ts +++ b/eval/golden.test.ts @@ -1,21 +1,13 @@ // SPDX-License-Identifier: MIT // Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors import { describe, it, expect, beforeAll } from 'vitest'; -import { mkdtempSync, readFileSync, readdirSync, existsSync } from 'node:fs'; -import { join, resolve } from 'node:path'; +import { mkdtempSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { runTask } from './runner'; import { MockAgentClient } from './agent'; import { isKernelcadAvailable } from './oracle/kernelcad-client'; - -function loadCombinedSkillMd(): string { - const root = resolve('src/agent/skills'); - const dirs = readdirSync(root, { withFileTypes: true }) - .filter((e) => e.isDirectory() && existsSync(join(root, e.name, 'SKILL.md'))) - .map((e) => e.name) - .sort(); - return dirs.map((name) => readFileSync(join(root, name, 'SKILL.md'), 'utf8')).join('\n\n---\n\n'); -} +import { loadCombinedSkillMd } from './skillContext'; const GOLDEN = 'eval/runs/golden-2026-05-02-bracket-holes'; @@ -33,6 +25,13 @@ beforeAll(async () => { }); describe('golden mock replay', () => { + it('includes nested photo-device asset policy in the aggregate prompt', () => { + const skillMd = loadCombinedSkillMd(); + + expect(skillMd).toContain('name: photo-to-device'); + expect(skillMd).toContain('An active Studio/server uploaded photo is managed'); + }); + it('replays the golden fixture and produces matching score (deterministic fields only)', async (ctx) => { if (!kernelcadAvailable) return ctx.skip(); diff --git a/eval/run.ts b/eval/run.ts index 06df6611e..a861da3b5 100644 --- a/eval/run.ts +++ b/eval/run.ts @@ -6,6 +6,7 @@ import { join, resolve } from 'node:path'; import { runTask, BEST_OF_N } from './runner'; import { AnthropicAgentClient, MockAgentClient } from './agent'; import { isKernelcadAvailable } from './oracle/kernelcad-client'; +import { loadCombinedSkillMd } from './skillContext'; import type { AgentClient, AgentResponse, TaskResult } from './types'; const MODEL = process.env.EVAL_MODEL ?? 'claude-sonnet-4-6'; @@ -13,16 +14,6 @@ const TASKS_DIR = resolve('eval/tasks'); const RUNS_DIR = resolve('eval/runs'); const SKILLS_ROOT = resolve('src/agent/skills'); -function loadCombinedSkillMd(): string { - const dirs = readdirSync(SKILLS_ROOT, { withFileTypes: true }) - .filter((e) => e.isDirectory() && existsSync(join(SKILLS_ROOT, e.name, 'SKILL.md'))) - .map((e) => e.name) - .sort(); - return dirs - .map((name) => readFileSync(join(SKILLS_ROOT, name, 'SKILL.md'), 'utf8')) - .join('\n\n---\n\n'); -} - function timestamp(): string { // YYYY-MM-DDTHH-MM-SS — filesystem-safe ISO. return new Date().toISOString().replace(/\..+$/, '').replace(/:/g, '-'); diff --git a/eval/skillContext.ts b/eval/skillContext.ts new file mode 100644 index 000000000..269a5dd91 --- /dev/null +++ b/eval/skillContext.ts @@ -0,0 +1,17 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors +import { resolve } from 'node:path'; +import { walkSkillTree } from '../src/agent/cli/lib/walkSkillTree'; + +const SKILLS_ROOT = resolve('src/agent/skills'); + +/** + * Build the complete skill context used by evaluation and portfolio agents. + * Keep this aligned with `kernelcad skill onefile`: nested child skills are + * executable policy, not optional documentation. + */ +export function loadCombinedSkillMd(): string { + return walkSkillTree(SKILLS_ROOT) + .map((entry) => entry.source) + .join('\n\n---\n\n'); +} diff --git a/examples/from-reference/e-reader/PROVENANCE.md b/examples/from-reference/e-reader/PROVENANCE.md new file mode 100644 index 000000000..3fc71f457 --- /dev/null +++ b/examples/from-reference/e-reader/PROVENANCE.md @@ -0,0 +1,23 @@ +# Kindle 2 reference provenance + +- Wikimedia file page: +- Creator: Evan-Amos. +- Rights status: public domain. +- Downloaded (UTC): 2026-07-14T02:06:12Z. +- Download URL: +- SHA-256 (`kindle-2-reference.jpg`): `5fb89746d43b96c6ccfbe5f5f6125061d59cc7fcc675439160c59f8fa4e11d02` + +## Physical scale anchor + +- Source: [Amazon Kindle User's Guide, 2nd Edition](https://kindle.s3.amazonaws.com/Kindle%20User%E2%80%99s%20Guide%2C%202nd%20Ed.-%20English.pdf), Appendix product specifications, model D00701. +- Published exterior envelope: **203.2 mm × 134.6 mm × 9.1 mm** (height × width × thickness). +- Photo calibration: the checked-in JPEG is 2100 px × 3000 px. Its visible outer-device envelope is 1843 px × 2774 px (x=134..1976, y=110..2883, measured against the pale background). The horizontal and vertical published-dimension ratios are averaged into `REFERENCE_MM_PER_PIXEL`, so `referenceImage()` uses a 153.60 mm full-frame width while preserving the native photo aspect ratio. That maps the visible enclosure to the published envelope within 0.2% on either axis. +- Use in `kindle-2-e-reader.kcad.ts`: `REFERENCE_WIDTH_MM = 134.6` supplies the `bodyWidth` default; `REFERENCE_IMAGE_WIDTH_MM` applies the measured photo calibration to the full `referenceImage()` plane. The accompanying image remains a visual aid only; its crop and perspective are not treated as metrology for internal features. + +This image supports an internal, parametric reference-reconstruction benchmark. +The accompanying model is a deliberately simplified, clean-room e-reader +study rather than an exact production clone or a mesh derived from the photo. +The source image's product trade dress, Amazon/Kindle trademarks, logos, text, +keyboard layout, and other brand-identifying details are not licensed or +claimed by this benchmark; it must not be presented as an affiliated product, +an exact reproduction, or manufacturing documentation. diff --git a/examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts b/examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts new file mode 100644 index 000000000..8531cc08c --- /dev/null +++ b/examples/from-reference/e-reader/kindle-2-e-reader.kcad.ts @@ -0,0 +1,301 @@ +// Real Object Brief +// Artifact: neutral front-facing e-reader benchmark — a four-part parametric +// consumer-device reconstruction informed by the supplied Kindle 2 photo. +// Reference: ./kindle-2-reference.jpg (front-facing public-domain photograph). +// Scale: millimetres. Amazon's Kindle User's Guide, 2nd Edition, lists model +// D00701 at 203.2 mm high × 134.6 mm wide × 9.1 mm thick: +// https://kindle.s3.amazonaws.com/Kindle%20User%E2%80%99s%20Guide%2C%202nd%20Ed.-%20English.pdf +// Scale anchor: the 2100 px × 3000 px source photo's device envelope measures +// 1843 px × 2774 px. REFERENCE_MM_PER_PIXEL balances the published width and +// height against that envelope, then expands the entire image plane while +// preserving the photo's native aspect ratio. The photo remains a visual aid, +// not a claim about hidden internal dimensions. +// Visible facts (from reference photo): +// - The front is a tall, pale rounded-rectangle enclosure with a much larger +// height than width and a shallow case depth. +// - A dark, inset e-paper display occupies the upper central front face, +// leaving a consistent surrounding bezel. +// - The photographed product also visibly has page controls, a keyboard, +// brand text, and a square navigation cluster; those trade-dress-specific +// interface details are intentionally abstracted out of this neutral, +// four-part benchmark. +// - The benchmark retains a recessed circular navigation control, a small +// status LED, and a bottom USB-C-scale opening as generic device cues. +// Hidden-side inference: +// - The unseen rear is inferred as a continuous shallow enclosure with a +// battery/PCB volume, fastening features, and a small front recess behind +// the display and controls; none are asserted as production internals. +// Validation focus: +// - Front view: tall rounded silhouette, inset dark display, circular lower +// control, and small status LED remain legible without the reference plane. +// - Right and iso views: body reads as a real 9.1 mm enclosure rather than +// a flat card, with display and controls recessed inside the front face. +// - Bottom view: the USB-C-scale opening crosses the housing bottom face. +// - Assembly gate: exactly four named parts have modest clearances and zero +// interference pairs. + +const REFERENCE_WIDTH_MM = 134.6; +const REFERENCE_HEIGHT_MM = 203.2; +const REFERENCE_THICKNESS_MM = 9.1; +const REFERENCE_IMAGE_PIXEL_WIDTH = 2100; +const REFERENCE_DEVICE_PIXEL_WIDTH = 1843; +const REFERENCE_DEVICE_PIXEL_HEIGHT = 2774; +const REFERENCE_MM_PER_PIXEL = ( + (REFERENCE_WIDTH_MM / REFERENCE_DEVICE_PIXEL_WIDTH) + + (REFERENCE_HEIGHT_MM / REFERENCE_DEVICE_PIXEL_HEIGHT) +) / 2; +const REFERENCE_IMAGE_WIDTH_MM = REFERENCE_IMAGE_PIXEL_WIDTH * REFERENCE_MM_PER_PIXEL; + +const bodyWidth = param('bodyWidth', REFERENCE_WIDTH_MM, { + min: 120, + max: 150, + description: 'published exterior width in mm; photo scale anchor', +}); +const bodyHeight = param('bodyHeight', REFERENCE_HEIGHT_MM, { + min: 185, + max: 220, + description: 'published exterior height in mm', +}); +const bodyThickness = param('bodyThickness', REFERENCE_THICKNESS_MM, { + min: 7, + max: 13, + description: 'published exterior thickness in mm', +}); +const cornerRadius = param('cornerRadius', 11, { + min: 6, + max: 18, + description: 'front-outline corner radius in mm', +}); +const bezelWidth = param('bezelWidth', 14, { + min: 8, + max: 24, + description: 'nominal front display bezel in mm', +}); +const screenWidth = param('screenWidth', 94, { + min: 76, + max: 106, + description: 'visible display width in mm', +}); +const screenHeight = param('screenHeight', 125, { + min: 104, + max: 145, + description: 'visible display height in mm', +}); +const screenRecess = param('screenRecess', 0.9, { + min: 0.45, + max: 1.8, + description: 'front display and control recess depth in mm', +}); +const controlDiameter = param('controlDiameter', 19, { + min: 13, + max: 26, + description: 'recessed circular navigation-control diameter in mm', +}); +const usbPortWidth = param('usbPortWidth', 10, { + min: 7, + max: 14, + description: 'bottom USB-C opening width in mm', +}); +const usbPortHeight = param('usbPortHeight', 4, { + min: 2.5, + max: 6, + description: 'bottom USB-C opening height in mm', +}); + +// Virtual visual aid only: the positive Y offset places the photo behind the +// front face, and scoring hides it so it cannot affect the resulting geometry. +referenceImage('./kindle-2-reference.jpg', { + plane: { plane: 'xz', offset: 8 }, + anchor: 'origin', + scale: REFERENCE_IMAGE_WIDTH_MM, + opacity: 0.24, +}); + +const HOUSING_MATERIAL = { + baseColor: '#e9e7e1', + metalness: 0, + roughness: 0.42, + clearcoat: 0.16, + clearcoatRoughness: 0.22, +}; +const DISPLAY_MATERIAL = { + baseColor: '#252a2b', + metalness: 0, + roughness: 0.31, + clearcoat: 0.18, + clearcoatRoughness: 0.2, +}; +const CONTROL_MATERIAL = { + baseColor: '#c9c7c0', + metalness: 0, + roughness: 0.5, + clearcoat: 0.12, + clearcoatRoughness: 0.3, +}; +const LED_MATERIAL = { + baseColor: '#c96a45', + metalness: 0, + roughness: 0.18, + clearcoat: 0.55, + clearcoatRoughness: 0.1, +}; + +const frontY = bodyThickness.divide(2).negate(); +const bottomZ = bodyHeight.divide(2).negate(); +const screenCenterZ = bodyHeight.divide(2) + .subtract(bezelWidth) + .subtract(screenHeight.divide(2)); +const navigationCenterZ = screenCenterZ + .subtract(screenHeight.divide(2)) + .subtract(bezelWidth) + .subtract(controlDiameter.divide(2)); +const ledCenterZ = navigationCenterZ + .subtract(controlDiameter.divide(2)) + .subtract(bezelWidth.divide(2)); +const displayCenterY = frontY.add(screenRecess.multiply(0.5)); +const navigationCenterY = frontY.add(0.58); +const ledCenterY = frontY.add(0.64); + +// The rounded XZ extrusion is centred after rotation; its PBR material is on +// the leaf before every sequential boolean below. +const housingLeaf = extrudeRoundedRect(bodyWidth, bodyHeight, cornerRadius, bodyThickness) + .material(HOUSING_MATERIAL) + .rotateX(-90) + .translate( + 0, + bodyThickness.divide(2).negate(), + 0, + ); + +const displayPocket = extrudeRoundedRect( + screenWidth.add(0.8), + screenHeight.add(0.8), + 2.8, + screenRecess.add(0.8), +) + .rotateX(-90) + .translate(0, frontY.subtract(0.4), screenCenterZ); + +const navigationPocket = cylinder( + screenRecess.add(1), + controlDiameter.divide(2).add(0.55), + 64, +) + .alongAxis([0, 1, 0]) + .translate(0, frontY.subtract(0.35), navigationCenterZ); + +const ledPocket = cylinder(screenRecess.add(1), 1.65, 32) + .alongAxis([0, 1, 0]) + .translate(0, frontY.subtract(0.35), ledCenterZ); + +// This cutter deliberately extends past the bottom/front/back faces, making +// the port opening unambiguous rather than relying on coincident geometry. +const usbPortCutter = box( + usbPortWidth, + bodyThickness.add(1), + usbPortHeight.add(1), + true, +).translate( + 0, + 0, + bottomZ.add(usbPortHeight.add(1).divide(2).subtract(0.2)), +); + +let housing = housingLeaf.subtract(displayPocket); +housing = housing.subtract(navigationPocket); +housing = housing.subtract(ledPocket); +housing = housing.subtract(usbPortCutter); + +const display = extrudeRoundedRect( + screenWidth, + screenHeight, + 2.3, + screenRecess.multiply(0.45), +) + .material(DISPLAY_MATERIAL) + .rotateX(-90) + .translate(0, displayCenterY, screenCenterZ); + +const navigationControl = cylinder( + screenRecess.multiply(0.42), + controlDiameter.divide(2), + 64, +) + .material(CONTROL_MATERIAL) + .alongAxis([0, 1, 0]) + .translate(0, navigationCenterY, navigationCenterZ); + +const statusLed = cylinder(screenRecess.multiply(0.28), 1.15, 32) + .material(LED_MATERIAL) + .alongAxis([0, 1, 0]) + .translate(0, ledCenterY, ledCenterZ); + +// Mate-style connectors currently use numeric Vec3 origins. These three +// default-state points are deliberately just inside each retained component's +// edge and within 1 mm of the matching housing pocket wall, so the solved +// fastened graph describes a physical retention relationship rather than an +// arbitrary origin-to-origin graph. The legacy seats below remain ParamRef +// expressions: editable dimensions move their inspectable placement metadata +// together with the visible geometry. +const DEFAULT_DISPLAY_FASTENER = [46.8, -4.1, 25.1]; +const DEFAULT_NAVIGATION_FASTENER = [9.3, -3.97, -60.9]; +const DEFAULT_LED_FASTENER = [0.95, -3.91, -77.4]; + +const reader = assembly('photo-reference-e-reader'); +// Static retention connectors share the same ParamRef expressions as their +// visible inset geometry. A body/screen edit therefore moves both the model +// and its inspectable assembly metadata instead of leaving stale numbers. +const housingPart = reader.part('housing', housing, { + connectors: { + displaySeat: { origin: [0, displayCenterY, screenCenterZ] }, + navigationSeat: { origin: [0, navigationCenterY, navigationCenterZ] }, + ledSeat: { origin: [0, ledCenterY, ledCenterZ] }, + }, +}); +const displayPart = reader.part('display', display, { + connectors: { mount: { origin: [0, displayCenterY, screenCenterZ] } }, + connect: { + connector: 'mount', + to: housingPart.connector('displaySeat'), + name: 'display-retained', + }, +}); +const navigationPart = reader.part('navigation-control', navigationControl, { + connectors: { mount: { origin: [0, navigationCenterY, navigationCenterZ] } }, + connect: { + connector: 'mount', + to: housingPart.connector('navigationSeat'), + name: 'navigation-retained', + }, +}); +const ledPart = reader.part('status-led', statusLed, { + connectors: { mount: { origin: [0, ledCenterY, ledCenterZ] } }, + connect: { + connector: 'mount', + to: housingPart.connector('ledSeat'), + name: 'led-retained', + }, +}); + +// The ParamRef-backed seats above preserve editable placement intent; these +// fastened frame pairs are the solver-facing graph that keeps the four parts +// connected for mechanism checks and downstream assembly consumers. +housingPart + .connector('display-fastener', { type: 'frame', origin: { kind: 'vec3', value: DEFAULT_DISPLAY_FASTENER } }) + .connector('navigation-fastener', { type: 'frame', origin: { kind: 'vec3', value: DEFAULT_NAVIGATION_FASTENER } }) + .connector('led-fastener', { type: 'frame', origin: { kind: 'vec3', value: DEFAULT_LED_FASTENER } }); +displayPart.connector('housing-fastener', { + type: 'frame', origin: { kind: 'vec3', value: DEFAULT_DISPLAY_FASTENER }, +}); +navigationPart.connector('housing-fastener', { + type: 'frame', origin: { kind: 'vec3', value: DEFAULT_NAVIGATION_FASTENER }, +}); +ledPart.connector('housing-fastener', { + type: 'frame', origin: { kind: 'vec3', value: DEFAULT_LED_FASTENER }, +}); + +reader.mate('display-fastened', 'housing.display-fastener', 'display.housing-fastener', 'fastened'); +reader.mate('navigation-fastened', 'housing.navigation-fastener', 'navigation-control.housing-fastener', 'fastened'); +reader.mate('led-fastened', 'housing.led-fastener', 'status-led.housing-fastener', 'fastened'); + +return reader.solvedModel({}); diff --git a/examples/from-reference/e-reader/kindle-2-reference.jpg b/examples/from-reference/e-reader/kindle-2-reference.jpg new file mode 100644 index 000000000..983b2d9ca Binary files /dev/null and b/examples/from-reference/e-reader/kindle-2-reference.jpg differ diff --git a/scripts/portfolioAttempt.ts b/scripts/portfolioAttempt.ts index 4fae285b6..fd22f200d 100644 --- a/scripts/portfolioAttempt.ts +++ b/scripts/portfolioAttempt.ts @@ -8,14 +8,15 @@ // // Usage: portfolioAttempt --slug --model [--notes ] // The prompt and harness are read from eval/portfolio/_tasks//. -import { resolve, join } from 'node:path'; -import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { existsSync, readFileSync } from 'node:fs'; import { runTask } from '../eval/runner'; import { appendPortfolioAttempt } from '../eval/portfolio/portfolioAttemptsLog'; import type { PortfolioAttempt, PortfolioAttemptStatus } from '../eval/portfolio/portfolioAttemptsLog'; import type { FailureModeTag } from '../eval/portfolio/failureMode'; import type { DiagnosticCode } from '../src/shared/diagnostics/codes'; import { makeAgent } from '../eval/run'; +import { loadCombinedSkillMd } from '../eval/skillContext'; interface Args { slug: string; model: string; notes: string; } @@ -55,13 +56,7 @@ async function main(): Promise { const runDir = resolve('eval/runs', `portfolio-${a.slug}-${startedAt}`); const agent = makeAgent(a.model); - const skillsRoot = resolve('src/agent/skills'); - const skillMd = readdirSync(skillsRoot, { withFileTypes: true }) - .filter((e) => e.isDirectory() && existsSync(join(skillsRoot, e.name, 'SKILL.md'))) - .map((e) => e.name) - .sort() - .map((name) => readFileSync(join(skillsRoot, name, 'SKILL.md'), 'utf8')) - .join('\n\n---\n\n'); + const skillMd = loadCombinedSkillMd(); // runTask writes score.json + transcript.md into runDir; we read score.json // back to classify the attempt rather than relying on the in-memory return diff --git a/src/agent/skills/kernelcad-from-reference/SKILL.md b/src/agent/skills/kernelcad-from-reference/SKILL.md index 4f4f35b7f..21af5984f 100644 --- a/src/agent/skills/kernelcad-from-reference/SKILL.md +++ b/src/agent/skills/kernelcad-from-reference/SKILL.md @@ -16,6 +16,11 @@ work through the stages. Have a written spec with numeric dimensions? → just read kernelcad-authoring + use-the-available-kernel, build single-pass, score. +Simple front-on consumer electronics or passive enclosures with a photo and a +known dimension? → read `photo-to-device/SKILL.md` before blockout. Its + hosted-vs-local reference-asset rule controls generic + `referenceImage()` advice for this path; it is not a mesh + concept or mechanism workflow. Have a reference photo only? ──┬─ Extract numeric dimensions FIRST │ (measure visually OR if STL available │ use trimesh to extract bbox+landmarks), @@ -44,10 +49,11 @@ Iteration mode: visual > scored > spec+photo (R1-R6 empirical). 1. `use-the-available-kernel/SKILL.md` — **the most important sub-skill.** Hard rules for which primitive to reach for. 2. `prepare-prompt/SKILL.md` — turn the user's ask into a Real Object Brief. -3. `blockout-model/SKILL.md` — coarse parametric blockout in canonical views. -4. `kernelcad-trace-from-image/SKILL.md` — convert pixel-space curves on a reference photo into normalized waypoints. Load only when an organic-curve outline (eyewear brow, ergonomic handle, sneaker midsole) would take more than ~60 seconds to eyeball off the photo. -5. `image-replicator/SKILL.md` — the render→score→iterate loop (caveat: see Rule 9). -6. `render-inspect/SKILL.md` — interpret diagnostic hints from `kernelcad evaluate`. +3. `photo-to-device/SKILL.md` — only for a scaled, simple front-on consumer device or passive enclosure. Load it before blockout: its hosted-vs-local asset rule controls `referenceImage()` use, the current Studio/server handoff, photo-only limits, and real-part rules. +4. `blockout-model/SKILL.md` — coarse parametric blockout in canonical views. +5. `kernelcad-trace-from-image/SKILL.md` — convert pixel-space curves on a reference photo into normalized waypoints. Load only when an organic-curve outline (eyewear brow, ergonomic handle, sneaker midsole) would take more than ~60 seconds to eyeball off the photo. +6. `image-replicator/SKILL.md` — the render→score→iterate loop (caveat: see Rule 9). +7. `render-inspect/SKILL.md` — interpret diagnostic hints from `kernelcad evaluate`. ## Hard rules across all sub-skills @@ -58,7 +64,9 @@ Iteration mode: visual > scored > spec+photo (R1-R6 empirical). - **Use the available kernel.** Variable fillet, mirror, nurbsSurface, surfaceFromCurves, PBR material, and referenceImage are all available today. Authoring with path+extrude+booleans alone is leaving 80% of the kernel on - the table. See `use-the-available-kernel/SKILL.md` for prescribed rules. + the table. For a scaled consumer-device upload, `photo-to-device/SKILL.md` + overrides generic reference-image overlay instructions. See + `use-the-available-kernel/SKILL.md` for the remaining prescribed rules. - **Score against the reference photo, not against a self-graded markdown.** The eval task's `harness.ts` runs `scoreAgainstReference` and produces honest numbers. Trust those numbers, not your prose summary of "does it look right." diff --git a/src/agent/skills/kernelcad-from-reference/photo-to-device/SKILL.md b/src/agent/skills/kernelcad-from-reference/photo-to-device/SKILL.md new file mode 100644 index 000000000..a49cc6024 --- /dev/null +++ b/src/agent/skills/kernelcad-from-reference/photo-to-device/SKILL.md @@ -0,0 +1,81 @@ +--- +name: photo-to-device +description: Use when rebuilding a simple front-on consumer electronic, passive enclosure, controller, instrument face, or hobby case from a photo with a known dimension and no moving mechanism. +--- + +# photo-to-device + +Build portable kernelCAD from photo evidence: one image establishes scale, not +hidden construction. + +## Use this route + +Use for a simple front-on e-reader, remote, meter, controller, display housing, or passive enclosure with one **known dimension**. Escalate robots and mechanisms to the dedicated assembly workflow; require multi-view contracts for joints, transmissions, hardware, and unknown internals. + +## Current active Studio/server boundary + +The current active Studio sends a typed photo reference to the hosted server, +validating MIME/size, computing SHA-256 provenance, and removing it after the +run. Do not use legacy chat/HeadlessKernel or a client-side hash. + +## Reference asset rule — overrides generic overlay guidance + +This rule controls generic `referenceImage()` advice from +`use-the-available-kernel` and `blockout-model`. + +- An active Studio/server uploaded photo is managed. Never emit + `referenceImage()` or its temporary/managed asset path in `.kcad.ts`. + Instead add a source comment with filename and server SHA-256 (never a path), + then model only from the validated photo brief. +- A checked-in/local source-owned asset may use + `referenceImage('./reference.jpg', opts)` as a durable overlay. + +Record photo reference provenance in the source comment and validated photo +brief: filename, MIME, server SHA-256, known dimension in millimetres, observed +facts, and inferred assumptions. + +## Core flow + +1. In `// Real Object Brief`, record **observed facts** (outline, screen, + seams, controls, ports, ratios, measured span) and **inferred facts** + (depth, back, wall, internals), marking every inference as an assumption. +2. Parameterize the known dimension and derived front face; fit the housing + envelope before controls or fillets. +3. Model real assemblies/parts: use a named static assembly for distinct + housing, screen/lens, controls, bezels, or inserts—not cosmetic overlaps. + A physically one-piece case may remain one part. +4. Add detail only after front, top, and iso views agree with the brief. + +## Photo-only limits + +A photo-only build cannot establish depth, PCB/battery layout, material stack, +brand dimensions, tolerances, or hidden fastening. It proves neither a +functional device nor a manufacturable copy or safe mechanism; ask for more +dimensions, views, vendor data, or escalation when needed. + +`trace_from_image` is optional only for organic silhouettes such as an +ergonomic grip. Never trace a rectangular enclosure, display, or button grid; +use dimensions, parameters, and ratios. Trace output is evidence, not the +object hypothesis. + +## Required gates + +Before handoff, run: + +```bash +kernelcad evaluate build.kcad.ts +kernelcad interference build.kcad.ts +``` + +`kernelcad evaluate` must have zero errors and `kernelcad interference` zero +unintended overlaps. With the reference hidden, render front, top, and iso: +every visible feature maps to a real part or intentional one-piece feature. + +## External mesh and proto.cat boundary + +Meshy or Tripo output is not CAD. Retain it only as a visual_mesh_reference for +composition/proportions, never as parts or a way around kernelCAD gates. + +For a proto.cat handoff, send asset identity, provenance, known dimension, +observed/inferred brief, and portable gated source—not a local temporary path, +browser data URL, Meshy/Tripo mesh as CAD, or proof of hidden construction. diff --git a/src/agent/vision/opencvBackend.test.ts b/src/agent/vision/opencvBackend.test.ts index f1e165b2e..02ed8980a 100644 --- a/src/agent/vision/opencvBackend.test.ts +++ b/src/agent/vision/opencvBackend.test.ts @@ -88,6 +88,27 @@ describe('extractSilhouettePolyline', () => { .toBuffer(); await expect(extractSilhouettePolyline(white, 12)).rejects.toThrow(/no foreground contour/); }); + + it('fails closed when a compact dark interior is nested in a light foreground that reaches the frame', async () => { + const sharp = (await import('sharp')).default; + const paleBody = await sharp({ + create: { width: 220, height: 220, channels: 3, background: { r: 232, g: 232, b: 232 } }, + }).png().toBuffer(); + const darkScreen = await sharp({ + create: { width: 100, height: 100, channels: 3, background: { r: 24, g: 24, b: 24 } }, + }).png().toBuffer(); + const ambiguous = await sharp({ + create: { width: 256, height: 256, channels: 3, background: { r: 255, g: 255, b: 255 } }, + }) + .composite([ + { input: paleBody, left: 0, top: 18 }, + { input: darkScreen, left: 68, top: 70 }, + ]) + .png() + .toBuffer(); + + await expect(extractSilhouettePolyline(ambiguous, 12)).rejects.toThrow(/ambiguous outer silhouette/); + }); }); describe('otsuThreshold', () => { diff --git a/src/agent/vision/opencvBackend.ts b/src/agent/vision/opencvBackend.ts index 457d491a0..53ca1dac7 100644 --- a/src/agent/vision/opencvBackend.ts +++ b/src/agent/vision/opencvBackend.ts @@ -14,7 +14,9 @@ // 2. Luma grayscale: 0.299R + 0.587G + 0.114B. // 3. Otsu threshold (256-bin histogram). Inverted (THRESH_BINARY_INV // semantics): foreground = pixels darker than the threshold, i.e. a dark -// subject on a bright background. +// subject on a bright background. Prefer a background-relative foreground +// mask when it isolates a pale device, so an internal dark screen is not +// mistaken for the outer housing. // 4. Largest 4-connected foreground component, then Moore-neighbour boundary // tracing (with Jacob's stopping criterion) of its outer boundary. // 5. arcLength = summed closed-loop segment lengths. @@ -30,6 +32,20 @@ import type { Vec2Normalized } from './types'; /** Pixel-space point used internally before normalization. */ type Point = { x: number; y: number }; +type Bounds = { minX: number; maxX: number; minY: number; maxY: number }; + +type ContourCandidate = { + boundary: Point[]; + bounds: Bounds; +}; + +/** A pale device must differ from a uniform white background by at least this + * much luma before we consider it as a broader outer-silhouette candidate. */ +const LIGHT_SUBJECT_LUMA_DELTA = 20; +const COMPACT_DARK_BBOX_AREA = 0.5; +const BROAD_OVER_DARK_BBOX_RATIO = 1.6; +const MAX_TRUSTED_BBOX_AREA = 0.95; + /** * Compute an Otsu threshold from a 256-bin grayscale histogram. * Returns the grayscale value `t` that maximizes between-class variance. @@ -186,6 +202,86 @@ function largestComponent( return { member, size: bestSize, seed: bestSeed }; } +function traceLargestContour( + mask: Uint8Array, + width: number, + height: number, +): ContourCandidate | null { + const { member, size, seed } = largestComponent(mask, width, height); + if (size <= 0 || !seed) return null; + + const boundary = mooreTrace(member, width, height, seed); + if (boundary.length < 3) return null; + + return { boundary, bounds: boundsOf(boundary) }; +} + +function boundsOf(points: Point[]): Bounds { + let minX = Number.POSITIVE_INFINITY; + let maxX = Number.NEGATIVE_INFINITY; + let minY = Number.POSITIVE_INFINITY; + let maxY = Number.NEGATIVE_INFINITY; + for (const point of points) { + minX = Math.min(minX, point.x); + maxX = Math.max(maxX, point.x); + minY = Math.min(minY, point.y); + maxY = Math.max(maxY, point.y); + } + return { minX, maxX, minY, maxY }; +} + +function bboxArea(candidate: ContourCandidate, width: number, height: number): number { + const { minX, maxX, minY, maxY } = candidate.bounds; + return ((maxX - minX + 1) / width) * ((maxY - minY + 1) / height); +} + +function touchesFrame(candidate: ContourCandidate, width: number, height: number): boolean { + const { minX, maxX, minY, maxY } = candidate.bounds; + return minX === 0 || minY === 0 || maxX === width - 1 || maxY === height - 1; +} + +function cornerBackgroundLuma(grey: Uint8Array, width: number, height: number): number { + const side = Math.max(1, Math.min(32, Math.floor(Math.min(width, height) / 8))); + const samples: number[] = []; + const corners: ReadonlyArray = [ + [0, 0], + [width - side, 0], + [0, height - side], + [width - side, height - side], + ]; + for (const [startX, startY] of corners) { + for (let y = startY; y < startY + side; y++) { + for (let x = startX; x < startX + side; x++) samples.push(grey[y * width + x]); + } + } + samples.sort((a, b) => a - b); + return samples[Math.floor(samples.length / 2)] ?? 0; +} + +function darkMaskAtOrBelow(grey: Uint8Array, threshold: number): Uint8Array { + const mask = new Uint8Array(grey.length); + for (let i = 0; i < grey.length; i++) mask[i] = grey[i] <= threshold ? 1 : 0; + return mask; +} + +function isIsolatedForegroundCandidate( + candidate: ContourCandidate, + width: number, + height: number, +): boolean { + return !touchesFrame(candidate, width, height) && bboxArea(candidate, width, height) <= MAX_TRUSTED_BBOX_AREA; +} + +function looksLikeUnisolatedLightOuterCandidate( + dark: ContourCandidate, + light: ContourCandidate | null, + width: number, + height: number, +): boolean { + if (light == null || bboxArea(dark, width, height) >= COMPACT_DARK_BBOX_AREA) return false; + return bboxArea(light, width, height) >= bboxArea(dark, width, height) * BROAD_OVER_DARK_BBOX_RATIO; +} + // Moore-neighbour offsets, clockwise starting from the west neighbour. The // canonical Moore order is the 8 neighbours walked clockwise; we start the // search from the pixel we entered the boundary from (backtrack) per Jacob's @@ -332,24 +428,39 @@ export async function extractSilhouettePolyline( // 3: Otsu threshold, inverted mask (foreground = darker than threshold). const t = otsuThreshold(histogram); - const mask = new Uint8Array(width * height); // OpenCV THRESH_BINARY_INV: dst = (src > t) ? 0 : maxval, i.e. foreground is // `src <= t`. Using `<=` (not `<`) is what makes a perfectly bimodal mask // (Otsu returns t=0 for black-on-white) still capture the dark subject. - for (let i = 0; i < grey.length; i++) { - mask[i] = grey[i] <= t ? 1 : 0; - } - - // 4: largest 4-connected foreground component → outer boundary trace. - const { member, size, seed } = largestComponent(mask, width, height); - if (size <= 0 || !seed) { - throw new Error('opencvBackend: no foreground contour found'); + // 4: Trace a background-relative foreground first. On a white product photo + // this includes a pale outer enclosure as well as dark details, while a + // dark-on-light square remains the same single component. It avoids spending + // a second full-image connected-component pass just to discover that an + // internal screen was the Otsu result. + const backgroundLuma = cornerBackgroundLuma(grey, width, height); + const lightThreshold = backgroundLuma - LIGHT_SUBJECT_LUMA_DELTA; + const lightCandidate = lightThreshold > t + ? traceLargestContour(darkMaskAtOrBelow(grey, lightThreshold), width, height) + : null; + + let candidate = lightCandidate != null && isIsolatedForegroundCandidate(lightCandidate, width, height) + ? lightCandidate + : null; + + if (candidate == null) { + const darkCandidate = traceLargestContour(darkMaskAtOrBelow(grey, t), width, height); + if (darkCandidate == null) { + throw new Error('opencvBackend: no foreground contour found'); + } + if (looksLikeUnisolatedLightOuterCandidate(darkCandidate, lightCandidate, width, height)) { + // Do not present a compact dark interior as a confident outer silhouette + // when a larger light foreground leaks into the image frame or otherwise + // cannot be isolated safely. + throw new Error('opencvBackend: ambiguous outer silhouette; broad light foreground cannot be isolated from the image background'); + } + candidate = darkCandidate; } - const boundary = mooreTrace(member, width, height, seed); - if (boundary.length < 3) { - throw new Error('opencvBackend: no foreground contour found'); - } + const boundary = candidate.boundary; // 5: arcLength of the closed boundary. const perimeter = arcLength(boundary, true); diff --git a/src/agent/vision/orchestrator.test.ts b/src/agent/vision/orchestrator.test.ts index 6af0cdddc..7214320fd 100644 --- a/src/agent/vision/orchestrator.test.ts +++ b/src/agent/vision/orchestrator.test.ts @@ -137,6 +137,40 @@ describe('traceFromImage orchestrator', () => { expect(out.diagnostics.some((d) => d.code === 'tool.trace-from-image.backend-failed')).toBe(true); }); + it('fails closed with a diagnostic when an outer light foreground cannot be isolated', async () => { + const sharp = (await import('sharp')).default; + const paleBody = await sharp({ + create: { width: 220, height: 220, channels: 3, background: { r: 232, g: 232, b: 232 } }, + }).png().toBuffer(); + const darkScreen = await sharp({ + create: { width: 100, height: 100, channels: 3, background: { r: 24, g: 24, b: 24 } }, + }).png().toBuffer(); + const ambiguous = await sharp({ + create: { width: 256, height: 256, channels: 3, background: { r: 255, g: 255, b: 255 } }, + }) + .composite([ + { input: paleBody, left: 0, top: 18 }, + { input: darkScreen, left: 68, top: 70 }, + ]) + .png() + .toBuffer(); + + const out = await traceFromImage({ + imageUrl: `data:image/png;base64,${ambiguous.toString('base64')}`, + backend: 'opencv', + features: [{ label: 'outer_housing', kind: 'silhouette' }], + }); + + expect(out.ok).toBe(false); + expect(out.features).toEqual([]); + expect(out.diagnostics).toEqual(expect.arrayContaining([ + expect.objectContaining({ + code: 'tool.trace-from-image.backend-failed', + message: expect.stringMatching(/ambiguous outer silhouette/), + }), + ])); + }); + it('emits trace-timeout when a backend hangs past the hard timeout', async () => { const hangingExtract = vi.fn( () => new Promise(() => {}), // never resolves diff --git a/src/funnel/hooks/useGeneration.test.ts b/src/funnel/hooks/useGeneration.test.ts index ac23cdf22..6f288df09 100644 --- a/src/funnel/hooks/useGeneration.test.ts +++ b/src/funnel/hooks/useGeneration.test.ts @@ -3,6 +3,7 @@ // @vitest-environment happy-dom import { describe, it, expect, vi, beforeEach } from 'vitest'; import { renderHook, act, waitFor } from '@testing-library/react'; +import type { GenerateRequest } from '../lib/generateClient'; const startGeneration = vi.fn(); const parseSseStream = vi.fn(); @@ -53,4 +54,28 @@ describe('useGeneration edit mode', () => { expect.objectContaining({ prompt: 'a bracket', mesh: { renderImageUrl: 'https://t/r.png', proportions: [1, 0.7, 0.6] } }), ); }); + + it('forwards a structured photo reference to startGeneration', async () => { + const { result } = renderHook(() => useGeneration()); + const referenceImage: NonNullable = { + dataUrl: 'data:image/png;base64,cGhvdG8=', + fileName: 'e-reader.png', + mimeType: 'image/png', + knownDimension: { label: 'overall height', valueMm: 203 }, + }; + + await act(async () => { + await result.current.submit( + 'model this simple e-reader', + undefined, + undefined, + referenceImage, + ); + }); + + expect(startGeneration).toHaveBeenCalledWith(expect.objectContaining({ + prompt: 'model this simple e-reader', + referenceImage, + })); + }); }); diff --git a/src/funnel/hooks/useGeneration.ts b/src/funnel/hooks/useGeneration.ts index 75a473f29..371b302f2 100644 --- a/src/funnel/hooks/useGeneration.ts +++ b/src/funnel/hooks/useGeneration.ts @@ -25,7 +25,12 @@ export function useGeneration() { const [phase, setPhase] = useState({ state: 'idle' }); const [events, setEvents] = useState([]); - const submit = useCallback(async (prompt: string, currentCode?: string, mesh?: GenerateRequest['mesh']) => { + const submit = useCallback(async ( + prompt: string, + currentCode?: string, + mesh?: GenerateRequest['mesh'], + referenceImage?: GenerateRequest['referenceImage'], + ) => { setEvents([]); setPhase({ state: 'running', @@ -34,7 +39,7 @@ export function useGeneration() { let res: Response; try { - res = await startGeneration({ prompt, currentCode, mesh }); + res = await startGeneration({ prompt, currentCode, mesh, referenceImage }); } catch (err) { const message = err instanceof Error ? err.message : String(err); setPhase({ state: 'error', code: 'network', message }); diff --git a/src/funnel/lib/generateClient.ts b/src/funnel/lib/generateClient.ts index a61ceee99..b14fc7e9f 100644 --- a/src/funnel/lib/generateClient.ts +++ b/src/funnel/lib/generateClient.ts @@ -25,6 +25,30 @@ export type GenerateEvent = | { kind: 'done'; artifact: Artifact; generationId: string; anonId: string; durationMs: number } | { kind: 'error'; code: 'llm_failed' | 'gate_failed' | 'eval_failed' | 'timeout'; message: string; generationId: string }; +/** Source-image limits for the Studio photo-reference request. A 4 MiB source + * file expands to roughly 5.4 MiB as a base64 data URL, which keeps the JSON + * request bounded while leaving the server responsible for decoding, hashing, + * and materializing the authoritative asset. */ +export const REFERENCE_IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/webp'] as const; +export type ReferenceImageMimeType = typeof REFERENCE_IMAGE_MIME_TYPES[number]; +export const MAX_REFERENCE_IMAGE_BYTES = 4 * 1024 * 1024; + +export function isReferenceImageMimeType(value: string): value is ReferenceImageMimeType { + return (REFERENCE_IMAGE_MIME_TYPES as readonly string[]).includes(value); +} + +/** A user-supplied, scaled image reference. The server recomputes the SHA-256 + * from `dataUrl`; no client-provided hash is trusted as provenance. */ +export interface ReferenceImage { + dataUrl: string; + fileName: string; + mimeType: ReferenceImageMimeType; + knownDimension: { + label: string; + valueMm: number; + }; +} + /** * Parse a Server-Sent Events stream from POST /api/v1/generate into typed events. * @@ -121,6 +145,9 @@ export interface GenerateRequest { * bounding-box proportions of an in-progress mesh-first build. Omit when * there is no mesh context (text-only funnel/landing generation). */ mesh?: { renderImageUrl?: string | null; proportions?: number[] | null }; + /** A bounded simple-device reference photo plus a real-world scale anchor. + * The hosted route validates, hashes, and materializes it before authoring. */ + referenceImage?: ReferenceImage; } export async function startGeneration(req: GenerateRequest): Promise { diff --git a/src/modeling/capture/imageDimensions.ts b/src/modeling/capture/imageDimensions.ts index 64ebf10bf..53fe8fb23 100644 --- a/src/modeling/capture/imageDimensions.ts +++ b/src/modeling/capture/imageDimensions.ts @@ -2,7 +2,8 @@ // Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors // src/modeling/capture/imageDimensions.ts // -// Minimal image-header parser. Reads only the first N bytes of a file to +// Minimal image-header parser. Reads only the small fixed headers for PNG and +// WEBP, and walks JPEG segment headers without loading their payloads, to // extract pixel dimensions. Supports PNG, JPEG, and WEBP. // Returns { width: 0, height: 0 } when parsing fails — callers treat this // as "unknown dimensions" and continue; render-time will surface bad files. @@ -15,11 +16,17 @@ export interface ImageDimensions { } const FAIL: ImageDimensions = { width: 0, height: 0 }; +// JPEG APP/COM metadata can legitimately be large, but this synchronous parser +// is called on user-controlled files. Bound both bytes skipped and individual +// marker headers so a crafted stream cannot monopolize the event loop. +const MAX_JPEG_HEADER_SCAN_BYTES = 1024 * 1024; +const MAX_JPEG_MARKERS = 4096; /** * Read pixel dimensions from a PNG / JPEG / WEBP file without loading the - * full image bytes. Reads at most ~128 bytes for PNG/WEBP; scans up to ~64 KB - * for JPEG SOF markers. + * full image bytes. Reads at most ~128 bytes for PNG/WEBP; for JPEG it walks + * segment headers until it reaches a Start Of Frame marker, without loading + * image payloads, subject to a generous finite header budget. */ export function imageDimensions(filePath: string): ImageDimensions { let fd = -1; @@ -124,51 +131,79 @@ function parseWebp(fd: number): ImageDimensions { } function parseJpeg(fd: number): ImageDimensions { - // Scan for SOF0 (FF C0), SOF1 (FF C1), SOF2 (FF C2) markers. - // Each marker: FF <2-byte-length> <1-byte-precision> <2-byte-height> <2-byte-width> - // Scan up to 64 KB to avoid reading huge files. - const MAX_SCAN = 65536; - const CHUNK = 4096; - let pos = 2; // skip SOI - while (pos < MAX_SCAN) { - const markerBuf = Buffer.allocUnsafe(CHUNK); - const n = readSync(fd, markerBuf, 0, CHUNK, pos); - if (n < 2) return FAIL; - let i = 0; - while (i < n - 1) { - if (markerBuf[i] !== 0xff) { - // Not aligned; skip forward — shouldn't happen in a well-formed JPEG - i++; - continue; - } - const marker = markerBuf[i + 1]; - if (marker === 0x00 || marker === 0xff) { - // Stuffed byte or padding; skip - i++; - continue; - } - // SOF markers: C0, C1, C2 (baseline, extended sequential, progressive) - if (marker === 0xc0 || marker === 0xc1 || marker === 0xc2) { - // Length: 2 bytes at i+2 (big-endian, includes itself but not FF+marker) - if (i + 8 >= n) return FAIL; - const height = (markerBuf[i + 5] << 8) | markerBuf[i + 6]; - const width = (markerBuf[i + 7] << 8) | markerBuf[i + 8]; - if (width === 0 || height === 0) return FAIL; - return { width, height }; - } - // Skip this segment: length field at i+2 (2 bytes BE, includes the 2 length bytes) - if (i + 3 >= n) return FAIL; - const segLen = (markerBuf[i + 2] << 8) | markerBuf[i + 3]; - if (segLen < 2) return FAIL; - // Jump to next marker: current pos + 2 (FF+marker) + segLen - pos = pos + i + 2 + segLen; - i = n; // exit inner loop — restart outer loop from new pos + // JPEG dimensions live in a Start Of Frame segment before the SOS marker. + // Walk segment headers directly so a valid file with large EXIF/ICC metadata + // does not require loading that metadata. Keep both byte and marker limits: + // this code is synchronous and a hostile stream can otherwise create an + // unbounded sequence of tiny marker segments or marker fill bytes. + let position = 2; // skip SOI + let markersRead = 0; + + while (position < MAX_JPEG_HEADER_SCAN_BYTES && markersRead < MAX_JPEG_MARKERS) { + const prefix = readExact(fd, position, 2); + if (!prefix || prefix[0] !== 0xff) return FAIL; + + let marker = prefix[1]; + position += 2; + markersRead += 1; + + // JPEG permits any number of 0xff fill bytes before a marker code. + while (marker === 0xff) { + if (markersRead >= MAX_JPEG_MARKERS || position >= MAX_JPEG_HEADER_SCAN_BYTES) return FAIL; + const fill = readExact(fd, position, 1); + if (!fill) return FAIL; + marker = fill[0]; + position += 1; + markersRead += 1; } - if (i < n) { - pos += i + 1; - } else { - pos += n; + + // A stuffed zero is valid only in entropy-coded scan data. We never scan + // that data: dimensions must be declared before SOS. + if (marker === 0x00) return FAIL; + if (marker === 0xd9 || marker === 0xda) return FAIL; + if (isStandaloneJpegMarker(marker)) continue; + + const lengthBuffer = readExact(fd, position, 2); + if (!lengthBuffer) return FAIL; + const segmentLength = lengthBuffer.readUInt16BE(0); + if (segmentLength < 2) return FAIL; + + const payloadStart = position + 2; + const payloadLength = segmentLength - 2; + + if (isJpegStartOfFrame(marker)) { + // SOF payload: precision (1), height (2), width (2), components (1)... + if (payloadStart + 5 > MAX_JPEG_HEADER_SCAN_BYTES) return FAIL; + const frame = readExact(fd, payloadStart, 5); + if (!frame) return FAIL; + const height = frame.readUInt16BE(1); + const width = frame.readUInt16BE(3); + if (width === 0 || height === 0) return FAIL; + return { width, height }; } + + // Skip exactly this segment's payload. The next loop starts at its marker. + position = payloadStart + payloadLength; } + return FAIL; } + +function readExact(fd: number, position: number, length: number): Buffer | undefined { + const buffer = Buffer.allocUnsafe(length); + let offset = 0; + while (offset < length) { + const read = readSync(fd, buffer, offset, length - offset, position + offset); + if (read <= 0) return undefined; + offset += read; + } + return buffer; +} + +function isStandaloneJpegMarker(marker: number): boolean { + return marker === 0x01 || marker === 0xd8 || (marker >= 0xd0 && marker <= 0xd7); +} + +function isJpegStartOfFrame(marker: number): boolean { + return marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc; +} diff --git a/src/studio/StudioGenerate.test.tsx b/src/studio/StudioGenerate.test.tsx index c2c4f20e6..f6c4b39bf 100644 --- a/src/studio/StudioGenerate.test.tsx +++ b/src/studio/StudioGenerate.test.tsx @@ -2,7 +2,7 @@ // Copyright (c) 2026 Andrii Shylenko and kernelCAD contributors // @vitest-environment happy-dom import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; -import { render, screen, fireEvent, cleanup } from '@testing-library/react'; +import { render, screen, fireEvent, cleanup, waitFor } from '@testing-library/react'; import '@testing-library/jest-dom/vitest'; import type { PreviewPhase } from '../funnel/hooks/useTextTo3dPreview'; @@ -37,15 +37,15 @@ beforeEach(() => { afterEach(() => cleanup()); describe('StudioGenerate — unified prompt', () => { - it('renders exactly ONE textarea driving both actions', () => { + it('renders exactly ONE prompt textarea driving both actions', () => { render(); - expect(screen.getAllByRole('textbox')).toHaveLength(1); + expect(document.querySelectorAll('textarea')).toHaveLength(1); expect(screen.getByRole('button', { name: /3d concept/i })).toBeInTheDocument(); }); it('sends the shared prompt to the concept preview', () => { render(); - fireEvent.change(screen.getByRole('textbox'), { target: { value: 'a hex planter' } }); + fireEvent.change(screen.getByLabelText('Generate prompt'), { target: { value: 'a hex planter' } }); fireEvent.click(screen.getByRole('button', { name: /3d concept/i })); expect(previewSubmit).toHaveBeenCalledWith('a hex planter'); }); @@ -60,12 +60,12 @@ describe('StudioGenerate — unified prompt', () => { previewPhase = { state: 'running', progress: 42 }; render(); expect(screen.getByRole('button', { name: /concept… 42%/i })).toBeDisabled(); - expect(screen.getByRole('textbox')).toBeDisabled(); + expect(screen.getByLabelText('Generate prompt')).toBeDisabled(); }); it('Build-as-CAD feeds the concept prompt into the agent submit', () => { const { rerender } = render(); - fireEvent.change(screen.getByRole('textbox'), { target: { value: 'a hex planter' } }); + fireEvent.change(screen.getByLabelText('Generate prompt'), { target: { value: 'a hex planter' } }); fireEvent.click(screen.getByRole('button', { name: /3d concept/i })); previewPhase = { state: 'done', glbUrl: 'https://t/x.glb', costUsd: null, renderImageUrl: null, proportions: null }; rerender(); @@ -76,7 +76,7 @@ describe('StudioGenerate — unified prompt', () => { it('Build-as-CAD is a FRESH generation even when the editor holds code (not an edit of it)', () => { mockCode = 'const base = box(60, 40, 5); return base;'; const { rerender } = render(); - fireEvent.change(screen.getByRole('textbox'), { target: { value: 'a hex planter' } }); + fireEvent.change(screen.getByLabelText('Generate prompt'), { target: { value: 'a hex planter' } }); fireEvent.click(screen.getByRole('button', { name: /3d concept/i })); previewPhase = { state: 'done', glbUrl: 'https://t/x.glb', costUsd: null, renderImageUrl: null, proportions: null }; rerender(); @@ -86,11 +86,56 @@ describe('StudioGenerate — unified prompt', () => { it('Build-as-CAD passes the concept mesh context into the agent submit', () => { const { rerender } = render(); - fireEvent.change(screen.getByRole('textbox'), { target: { value: 'a bracket' } }); + fireEvent.change(screen.getByLabelText('Generate prompt'), { target: { value: 'a bracket' } }); fireEvent.click(screen.getByRole('button', { name: /3d concept/i })); previewPhase = { state: 'done', glbUrl: 'https://t/x.glb', costUsd: null, renderImageUrl: 'https://t/r.png', proportions: [1, 0.7, 0.6] }; rerender(); fireEvent.click(screen.getByRole('button', { name: /build as parametric cad/i })); expect(generationSubmit).toHaveBeenCalledWith('a bracket', undefined, { renderImageUrl: 'https://t/r.png', proportions: [1, 0.7, 0.6] }); }); + + it('disables 3D concept mode when a reference photo is selected', async () => { + render(); + fireEvent.change(screen.getByLabelText('Generate prompt'), { target: { value: 'an e-reader enclosure' } }); + fireEvent.change(screen.getByLabelText('Reference photo'), { + target: { files: [new File(['photo-bytes'], 'e-reader.png', { type: 'image/png' })] }, + }); + + await waitFor(() => expect(screen.getByText('e-reader.png')).toBeInTheDocument()); + + expect(screen.getByRole('button', { name: /3d concept/i })).toBeDisabled(); + }); + + it('Build-as-CAD sends a selected photo without a stale concept mesh', async () => { + const { rerender } = render(); + fireEvent.change(screen.getByLabelText('Generate prompt'), { target: { value: 'an e-reader enclosure' } }); + fireEvent.click(screen.getByRole('button', { name: /3d concept/i })); + previewPhase = { + state: 'done', + glbUrl: 'https://t/x.glb', + costUsd: null, + renderImageUrl: 'https://t/r.png', + proportions: [1, 0.7, 0.6], + }; + rerender(); + + fireEvent.change(screen.getByLabelText('Reference photo'), { + target: { files: [new File(['photo-bytes'], 'e-reader.png', { type: 'image/png' })] }, + }); + await waitFor(() => expect(screen.getByText('e-reader.png')).toBeInTheDocument()); + fireEvent.change(screen.getByLabelText('Known dimension label'), { target: { value: 'overall height' } }); + fireEvent.change(screen.getByLabelText('Known dimension (mm)'), { target: { value: '203' } }); + + fireEvent.click(screen.getByRole('button', { name: /build as parametric cad/i })); + + expect(generationSubmit).toHaveBeenCalledWith( + 'an e-reader enclosure', + undefined, + undefined, + expect.objectContaining({ + fileName: 'e-reader.png', + knownDimension: { label: 'overall height', valueMm: 203 }, + }), + ); + }); }); diff --git a/src/studio/StudioGenerate.tsx b/src/studio/StudioGenerate.tsx index 50d329c25..c745acaf0 100644 --- a/src/studio/StudioGenerate.tsx +++ b/src/studio/StudioGenerate.tsx @@ -3,7 +3,12 @@ import React, { useEffect, useMemo, useState } from 'react'; import { DiffEditor } from '@monaco-editor/react'; import { useGeneration } from '../funnel/hooks/useGeneration'; -import type { GenerateEvent } from '../funnel/lib/generateClient'; +import { + isReferenceImageMimeType, + MAX_REFERENCE_IMAGE_BYTES, + type GenerateEvent, + type GenerateRequest, +} from '../funnel/lib/generateClient'; import { useTextTo3dPreview } from '../funnel/hooks/useTextTo3dPreview'; import { inAppAgentEnabled } from './agentAvailability'; import { ConceptResult } from './components/ConceptResult'; @@ -40,6 +45,22 @@ interface GenerationReviewSnapshot { readonly repairWorkflow: AgentRepairWorkflow | null; } +type PendingReferenceImage = Pick, 'dataUrl' | 'fileName' | 'mimeType'>; + +function photoReferenceFrom( + pending: PendingReferenceImage | null, + dimensionLabel: string, + dimensionMmText: string, +): GenerateRequest['referenceImage'] | null { + const valueMm = Number(dimensionMmText); + const label = dimensionLabel.trim(); + if (pending == null || !label || !Number.isFinite(valueMm) || valueMm <= 0) return null; + return { + ...pending, + knownDimension: { label, valueMm }, + }; +} + const StudioGenerateInner: React.FC = () => { const { phase, events, submit } = useGeneration(); const { code } = useCode(); @@ -55,6 +76,11 @@ const StudioGenerateInner: React.FC = () => { // (so the diff is stable even though `code` changes once we apply). const [baseline, setBaseline] = useState(''); const [reviewSnapshot, setReviewSnapshot] = useState(null); + const [pendingReferenceImage, setPendingReferenceImage] = useState(null); + const [knownDimensionLabel, setKnownDimensionLabel] = useState(''); + const [knownDimensionMm, setKnownDimensionMm] = useState(''); + const [referenceImageError, setReferenceImageError] = useState(null); + const [readingReferenceImage, setReadingReferenceImage] = useState(false); const prompt = agentDraftPrompt !== null && agentDraftPromptVersion !== acknowledgedDraftVersion @@ -80,28 +106,50 @@ const StudioGenerateInner: React.FC = () => { const reviewing = phase.state === 'done' && resolution?.generationId !== phase.generationId; const steps = useMemo(() => events.map(stepLabel).filter(Boolean) as string[], [events]); + const referenceImage = useMemo( + () => photoReferenceFrom(pendingReferenceImage, knownDimensionLabel, knownDimensionMm), + [knownDimensionLabel, knownDimensionMm, pendingReferenceImage], + ); + const photoReferenceSelected = pendingReferenceImage != null; + const referenceNeedsDimension = pendingReferenceImage != null && referenceImage == null; useEffect(() => { if (phase.state !== 'error' || agentRepairWorkflow?.state !== 'running') return; shellStore.setAgentRepairWorkflow({ ...agentRepairWorkflow, state: 'drafted' }); }, [agentRepairWorkflow, phase.state]); - const runAgent = (text: string, snapshot: GenerationReviewSnapshot) => { + const runAgent = ( + text: string, + snapshot: GenerationReviewSnapshot, + photoReference?: GenerateRequest['referenceImage'], + ) => { // Edit mode: hand the agent the current model so it iterates instead of // replacing. Empty editor → fresh generation. setBaseline(snapshot.fromCode); setReviewSnapshot(snapshot); if (snapshot.fromCode.trim()) { - void submit(text, snapshot.fromCode); + if (photoReference) { + void submit(text, snapshot.fromCode, undefined, photoReference); + } else { + void submit(text, snapshot.fromCode); + } return; } - void submit(text); + if (photoReference) { + void submit(text, undefined, undefined, photoReference); + } else { + void submit(text); + } }; const onSubmit = (e: React.FormEvent) => { e.preventDefault(); const trimmed = prompt.trim(); - if (!trimmed || busy) return; + if (!trimmed || busy || readingReferenceImage) return; + if (referenceNeedsDimension) { + setReferenceImageError('Add a visible measurement label and a positive millimetre value before generating from a photo.'); + return; + } const matchesDraftedRepair = agentRepairWorkflow != null && agentRepairWorkflow.state === 'drafted' && @@ -124,18 +172,68 @@ const StudioGenerateInner: React.FC = () => { promptText: trimmed, selectedFeatureId: runTargetId, repairWorkflow: repairWorkflowForRun, - }); + }, referenceImage ?? undefined); + }; + + const onReferenceImageSelect = (event: React.ChangeEvent) => { + const file = event.target.files?.[0]; + if (!file) return; + const mimeType = file.type; + + setReferenceImageError(null); + // A scale anchor belongs to a specific photo. Never silently reuse a + // measurement from the previous reference after the file changes. + setKnownDimensionLabel(''); + setKnownDimensionMm(''); + if (!isReferenceImageMimeType(mimeType)) { + setPendingReferenceImage(null); + setReferenceImageError('Use a PNG, JPEG, or WebP image for the reference photo.'); + return; + } + if (file.size === 0) { + setPendingReferenceImage(null); + setReferenceImageError('Reference photo is empty. Choose an image with visible device details.'); + return; + } + if (file.size > MAX_REFERENCE_IMAGE_BYTES) { + setPendingReferenceImage(null); + setReferenceImageError('Reference images must be 4 MiB or smaller.'); + return; + } + + setPendingReferenceImage(null); + setReadingReferenceImage(true); + const reader = new FileReader(); + reader.onload = () => { + const dataUrl = reader.result; + if (typeof dataUrl !== 'string' || !dataUrl.startsWith(`data:${mimeType};base64,`)) { + setReferenceImageError('Could not read that image as a safe data URL. Choose a PNG, JPEG, or WebP image.'); + setReadingReferenceImage(false); + return; + } + setPendingReferenceImage({ dataUrl, fileName: file.name, mimeType }); + setReadingReferenceImage(false); + }; + reader.onerror = () => { + setReferenceImageError('Could not read that reference photo. Try another image.'); + setReadingReferenceImage(false); + }; + reader.readAsDataURL(file); }; const onConcept = () => { const trimmed = prompt.trim(); - if (!trimmed || busy) return; + if (!trimmed || busy || photoReferenceSelected) return; setConceptPrompt(trimmed); void preview.submit(trimmed); }; const buildConceptAsCad = () => { - if (!conceptPrompt || busy) return; + if (!conceptPrompt || busy || readingReferenceImage) return; + if (referenceNeedsDimension) { + setReferenceImageError('Add a visible measurement label and a positive millimetre value before generating from a photo.'); + return; + } // Fresh generation, never an edit: framing the concept prompt as an // edit of whatever happens to sit in the editor (often the untouched // starter sample) lets the model return that code unchanged. The @@ -152,10 +250,16 @@ const StudioGenerateInner: React.FC = () => { // state). A done preview with no Tripo render/fingerprint yields // {renderImageUrl:null, proportions:null} — intentional and distinct from // "no mesh" (undefined); the server's nullish schema accepts it. - const mesh = preview.phase.state === 'done' + const mesh = referenceImage == null && preview.phase.state === 'done' ? { renderImageUrl: preview.phase.renderImageUrl, proportions: preview.phase.proportions } : undefined; - void submit(conceptPrompt, undefined, mesh); + if (referenceImage) { + // A photo is its own evidence mode. A preview that completed before + // the photo was selected must not make the request ambiguous. + void submit(conceptPrompt, undefined, undefined, referenceImage); + } else { + void submit(conceptPrompt, undefined, mesh); + } }; const stageGeneratedEdit = () => { @@ -197,6 +301,62 @@ const StudioGenerateInner: React.FC = () => {
Target: {selectedFeatureId ?? 'whole model'}
+
+
Simple-device photo reference
+
A photo needs one visible real-world measurement; it does not determine hidden depth or internals.
+ + {readingReferenceImage &&
Reading reference photo…
} + {pendingReferenceImage != null && ( +
+ {pendingReferenceImage.fileName} +
+ )} +
+ + +
+ {referenceNeedsDimension && ( +
Add a visible measurement label and positive millimetres to use this photo.
+ )} + {referenceImageError != null && ( +
{referenceImageError}
+ )} +