feat(call): keep you in the middle of your own picture

app/index.html

@@ -227,6 +227,12 @@ <h1 class="app-brand"><img src="/pwa-192x192.png" width="28" height="28" alt=""
                   <input id="blurStrength" type="range" min="0" max="100" step="1" value="60" />
                 </div>
                 <div class="segmented" id="backgroundChoices" role="radiogroup" aria-label="Picture" hidden></div>
+                <div class="effectRow" id="framingRow">
+                  <label for="framingToggle" title="Zooms in a little and follows you, so you stay in the middle of the picture everybody sees">
+                    <input id="framingToggle" type="checkbox" />
+                    Keep me in the middle of the picture
+                  </label>
+                </div>
                 <div class="effectRow" id="fishRow" hidden>
                   <label for="fishToggle">
                     <input id="fishToggle" type="checkbox" />

app/src/auto-frame.test.ts

@@ -0,0 +1,83 @@
+import { describe, expect, test } from 'vitest'
+import { AutoFramer, HOLD_FRAMES, MAX_ZOOM, WHOLE_FRAME, cropFor, personBox, type FrameBox } from './auto-frame.js'
+
+/** A mask with the person filling this box, in mask pixels. */
+function maskWith(width: number, height: number, person?: { x: number; y: number; width: number; height: number }) {
+  const data = new Float32Array(width * height)
+  if (person) {
+    for (let y = person.y; y < person.y + person.height; y++) {
+      for (let x = person.x; x < person.x + person.width; x++) data[y * width + x] = 0.9
+    }
+  }
+  return { width, height, data }
+}
+
+function settle(framer: AutoFramer, person: FrameBox | null, frames = 400): FrameBox {
+  let crop = WHOLE_FRAME
+  for (let i = 0; i < frames; i++) crop = framer.next(person)
+  return crop
+}
+
+describe('personBox', () => {
+  test('finds a person sitting off to the left', () => {
+    const box = personBox(maskWith(640, 480, { x: 0, y: 120, width: 200, height: 360 }))!
+    expect(box.x).toBeCloseTo(0, 1)
+    expect(box.width).toBeCloseTo(200 / 640, 1)
+    expect(box.y).toBeCloseTo(120 / 480, 1)
+  })
+
+  test('nobody, or a speck, is nobody', () => {
+    expect(personBox(null)).toBeNull()
+    expect(personBox(maskWith(640, 480))).toBeNull()
+    expect(personBox(maskWith(640, 480, { x: 300, y: 200, width: 20, height: 20 }))).toBeNull()
+  })
+
+  test('a stray blob at the edge does not stretch the box across the room', () => {
+    const mask = maskWith(640, 480, { x: 240, y: 100, width: 160, height: 380 })
+    for (let y = 0; y < 8; y++) for (let x = 630; x < 634; x++) mask.data[y * 640 + x] = 0.9
+    const box = personBox(mask)!
+    expect(box.x + box.width).toBeLessThan(420 / 640)
+  })
+})
+
+describe('cropFor', () => {
+  test('centres the person and keeps the picture its own shape', () => {
+    const crop = cropFor({ x: 0.3, y: 0.3, width: 0.2, height: 0.7 })
+    expect(crop.width).toBe(crop.height)
+    expect(crop.x + crop.width / 2).toBeCloseTo(0.4, 5)
+    expect(crop.y).toBeLessThan(0.3)
+  })
+
+  test('never zooms past the limit and never leaves the frame', () => {
+    const crop = cropFor({ x: 0, y: 0.2, width: 0.05, height: 0.3 })
+    expect(crop.width).toBeCloseTo(1 / MAX_ZOOM, 5)
+    expect(crop.x).toBe(0)
+    const wide = cropFor({ x: 0, y: 0, width: 1, height: 1 })
+    expect(wide).toEqual(WHOLE_FRAME)
+  })
+})
+
+describe('AutoFramer', () => {
+  test('drifts towards the person rather than jumping', () => {
+    const framer = new AutoFramer()
+    const person = { x: 0.1, y: 0.3, width: 0.25, height: 0.7 }
+    const first = framer.next(person)
+    expect(first.width).toBeGreaterThan(0.95)
+    const settled = settle(framer, person)
+    expect(settled).toEqual(cropFor(person))
+  })
+
+  test('ignores small movements, so the picture does not twitch', () => {
+    const framer = new AutoFramer()
+    const person = { x: 0.3, y: 0.3, width: 0.3, height: 0.7 }
+    const settled = settle(framer, person)
+    expect(settle(framer, { ...person, x: 0.31 }, 50)).toEqual(settled)
+  })
+
+  test('holds through a short gap in the mask, then eases back to the whole frame', () => {
+    const framer = new AutoFramer()
+    const settled = settle(framer, { x: 0.3, y: 0.3, width: 0.2, height: 0.7 })
+    expect(settle(framer, null, HOLD_FRAMES)).toEqual(settled)
+    expect(settle(framer, null)).toEqual(WHOLE_FRAME)
+  })
+})

app/src/auto-frame.ts

@@ -0,0 +1,132 @@
+import type { SegmentationMask } from '../../src/video-effects.js'
+
+/**
+ * Keeping the person in the middle of their own picture.
+ *
+ * The background effect already knows where the person is: its mask. This
+ * turns that mask into a crop of the finished frame, eased so the picture
+ * drifts after somebody rather than following every twitch. It crops pixels
+ * the effect has already painted, so it can only ever send less of the room,
+ * never more. With no mask to go on (effect off, loading or failed) the crop
+ * eases back out to the whole frame.
+ *
+ * Every rectangle here is a fraction of the frame, 0 to 1 on each axis, and a
+ * crop is the same fraction of both axes so the picture keeps its shape.
+ */
+export interface FrameBox { x: number; y: number; width: number; height: number }
+
+export const WHOLE_FRAME: FrameBox = { x: 0, y: 0, width: 1, height: 1 }
+
+/** Confidence above which a mask pixel counts as the person. */
+export const PERSON_CUTOFF = 0.5
+/** Less of the frame than this is a stray hand or a false positive. */
+export const MIN_PERSON_SHARE = 0.03
+/** Mask pixels at either end of the person ignored, so one stray blob at the
+ *  edge of the frame does not stretch the box across the room. */
+export const TRIM_SHARE = 0.02
+/** Never zoom in further than this: a webcam picture gets soft quickly. */
+export const MAX_ZOOM = 1.6
+/** How much of the crop's width the person should fill. */
+export const PERSON_WIDTH_SHARE = 0.55
+/** Space kept above the head, as a share of the crop's height. */
+export const HEADROOM = 0.1
+/** A new crop is only chased once it differs from the current aim by this. */
+export const DEAD_BAND = 0.05
+/** How far towards its aim the crop moves each frame. */
+export const EASE = 0.06
+/** Frames without a mask before the crop starts easing back out. */
+export const HOLD_FRAMES = 30
+
+/** Where the person is in this mask, or null when nobody clearly is. */
+export function personBox(mask: SegmentationMask | null, step = 4): FrameBox | null {
+  if (!mask || mask.width <= 0 || mask.height <= 0) return null
+  const columns = new Uint32Array(Math.ceil(mask.width / step))
+  const rows = new Uint32Array(Math.ceil(mask.height / step))
+  let total = 0
+  let sampled = 0
+  for (let y = 0, row = 0; y < mask.height; y += step, row++) {
+    const offset = y * mask.width
+    for (let x = 0, column = 0; x < mask.width; x += step, column++) {
+      sampled++
+      if ((mask.data[offset + x] ?? 0) < PERSON_CUTOFF) continue
+      columns[column]!++
+      rows[row]!++
+      total++
+    }
+  }
+  if (total === 0 || total / sampled < MIN_PERSON_SHARE) return null
+  const [left, right] = extent(columns, total)
+  const [top, bottom] = extent(rows, total)
+  const x = (left * step) / mask.width
+  const y = (top * step) / mask.height
+  return {
+    x, y,
+    width: Math.min(1, ((right + 1) * step) / mask.width) - x,
+    height: Math.min(1, ((bottom + 1) * step) / mask.height) - y,
+  }
+}
+
+function extent(counts: Uint32Array, total: number): [number, number] {
+  const trim = total * TRIM_SHARE
+  let first = 0
+  for (let seen = 0; first < counts.length - 1; first++) { seen += counts[first]!; if (seen > trim) break }
+  let last = counts.length - 1
+  for (let seen = 0; last > first; last--) { seen += counts[last]!; if (seen > trim) break }
+  return [first, last]
+}
+
+/** The crop that puts this person in the middle with room above their head. */
+export function cropFor(person: FrameBox): FrameBox {
+  const size = Math.min(1, Math.max(1 / MAX_ZOOM, person.width / PERSON_WIDTH_SHARE))
+  const centre = person.x + person.width / 2
+  return {
+    x: clamp(centre - size / 2, 0, 1 - size),
+    y: clamp(person.y - HEADROOM * size, 0, 1 - size),
+    width: size,
+    height: size,
+  }
+}
+
+/** Follows the person from frame to frame, slowly. */
+export class AutoFramer {
+  #current: FrameBox = WHOLE_FRAME
+  #aim: FrameBox = WHOLE_FRAME
+  #missed = 0
+
+  /** The crop for this frame, given where the person is (or null). */
+  next(person: FrameBox | null): FrameBox {
+    if (person) {
+      this.#missed = 0
+      const aim = cropFor(person)
+      if (distance(aim, this.#aim) > DEAD_BAND) this.#aim = aim
+    } else if (++this.#missed > HOLD_FRAMES) {
+      this.#aim = WHOLE_FRAME
+    }
+    this.#current = {
+      x: ease(this.#current.x, this.#aim.x),
+      y: ease(this.#current.y, this.#aim.y),
+      width: ease(this.#current.width, this.#aim.width),
+      height: ease(this.#current.height, this.#aim.height),
+    }
+    return this.#current
+  }
+
+  /** Straight back to the whole frame, as when framing is turned off. */
+  reset(): void {
+    this.#current = this.#aim = WHOLE_FRAME
+    this.#missed = 0
+  }
+}
+
+function ease(from: number, to: number): number {
+  const next = from + (to - from) * EASE
+  return Math.abs(to - next) < 0.0005 ? to : next
+}
+
+function distance(a: FrameBox, b: FrameBox): number {
+  return Math.max(Math.abs(a.x - b.x), Math.abs(a.y - b.y), Math.abs(a.width - b.width))
+}
+
+function clamp(value: number, low: number, high: number): number {
+  return Math.min(high, Math.max(low, value))
+}

app/src/main.ts

@@ -2496,6 +2496,12 @@ const FISH_STORAGE_KEY = 'kithmoot.fish.enabled'
 let fishEnabled = (() => {
   try { return localStorage.getItem(FISH_STORAGE_KEY) === 'true' } catch { return false }
 })()
+/** Keeping you in the middle of your own picture - see `auto-frame.ts`. On
+ *  unless turned off, remembered on this device. */
+const FRAMING_STORAGE_KEY = 'kithmoot.camera.framing'
+let framingEnabled = (() => {
+  try { return localStorage.getItem(FRAMING_STORAGE_KEY) !== 'false' } catch { return true }
+})()
 let videoInputs: MediaDeviceInfo[] = []
 
 const localPreviewEls = new Map<'camera' | 'screen', HTMLVideoElement>()
@@ -4236,6 +4242,7 @@ async function toggleCamera(): Promise<void> {
     const pipeline = new CameraPipeline({
       mode: savedEffectMode,
       strength: savedBlurStrength,
+      framing: framingEnabled,
       onStateChange: state => { if (generation === callGeneration) renderEffectState(state) },
       onSourceEnded: () => {
         if (generation !== callGeneration) return
@@ -4323,6 +4330,9 @@ function renderEffectState(state: VideoEffectState): void {
   $('backgroundChoices').hidden = state.mode !== 'replace'
   // Only over a sea. Offering fish over Slate would be a question with no
   // sensible answer.
+  // Framing follows the effect's picture of where you are, so there is
+  // nothing to follow with the effect off.
+  $('framingRow').hidden = state.mode === 'off'
   $('fishRow').hidden =
     state.mode !== 'replace' || !BACKGROUNDS.find((b) => b.id === backgroundId)?.sea
 
@@ -11906,6 +11916,13 @@ $('fishToggle').addEventListener('change', () => {
   camera?.setFish(fishEnabled).catch((err) => setStatus(describeError(err)))
 })
 
+;($('framingToggle') as HTMLInputElement).checked = framingEnabled
+$('framingToggle').addEventListener('change', () => {
+  framingEnabled = ($('framingToggle') as HTMLInputElement).checked
+  try { localStorage.setItem(FRAMING_STORAGE_KEY, String(framingEnabled)) } catch { /* Still applies to this visit. */ }
+  camera?.setFraming(framingEnabled)
+})
+
 $('switchCamera').addEventListener('click', () => {
   switchCamera().catch((err) => setStatus(describeError(err)))
 })

app/src/video-pipeline.ts

@@ -8,6 +8,7 @@ import {
 } from '../../src/video-effects.js'
 import { ReefBackground, type FishSprite } from '../../src/reef-scene.js'
 import { createSegmenter } from './mediapipe-segmenter.js'
+import { AutoFramer, personBox } from './auto-frame.js'
 
 /**
  * The camera, with the effect wired into it.
@@ -173,6 +174,8 @@ export interface CameraPipelineOptions {
   mode?: EffectMode
   /** Start at this blur strength rather than `DEFAULT_BLUR_STRENGTH`. */
   strength?: number
+  /** Keep the person in the middle of the picture - see `auto-frame.ts`. */
+  framing?: boolean
 }
 
 export interface CameraStats {
@@ -197,8 +200,14 @@ type FrameCallbackHost = HTMLVideoElement & {
 
 export class CameraPipeline {
   readonly #video: HTMLVideoElement
+  /** The published picture: a crop of `#stage`, never of the camera. */
   readonly #canvas: HTMLCanvasElement
+  readonly #canvasCtx: CanvasRenderingContext2D
+  /** What the effect paints: the whole frame, already safe to send. */
+  readonly #stage: HTMLCanvasElement
   readonly #effect: VideoEffect
+  readonly #framer = new AutoFramer()
+  #framing: boolean
   readonly #onStateChange?: (state: VideoEffectState) => void
   readonly #onSourceEnded?: () => void
 
@@ -237,9 +246,14 @@ export class CameraPipeline {
     this.#canvas = document.createElement('canvas')
     this.#canvas.width = 640
     this.#canvas.height = 480
+    const ctx = this.#canvas.getContext('2d')
+    if (!ctx) throw new Error('the camera canvas has no 2D context')
+    this.#canvasCtx = ctx
+    this.#stage = document.createElement('canvas')
+    this.#framing = opts.framing ?? true
 
     this.#effect = new VideoEffect({
-      output: this.#canvas,
+      output: this.#stage,
       createCanvas: (width, height) => {
         const canvas = document.createElement('canvas')
         canvas.width = width
@@ -401,6 +415,16 @@ export class CameraPipeline {
     this.#effect.setMode(mode)
   }
 
+  /** Keep the person centred, or send the whole frame. */
+  setFraming(on: boolean): void {
+    this.#framing = on
+    if (!on) this.#framer.reset()
+  }
+
+  get framing(): boolean {
+    return this.#framing
+  }
+
   setStrength(strength: number): void {
     this.#effect.setStrength(strength)
   }
@@ -566,6 +590,15 @@ export class CameraPipeline {
     })
   }
 
+  /** Copy the effect's finished frame out, cropped around the person. */
+  #frame(width: number, height: number): void {
+    if (this.#stage.width !== width || this.#stage.height !== height) return
+    if (this.#canvas.width !== width) this.#canvas.width = width
+    if (this.#canvas.height !== height) this.#canvas.height = height
+    const crop = this.#framing ? this.#framer.next(personBox(this.#effect.personMask)) : { x: 0, y: 0, width: 1, height: 1 }
+    this.#canvasCtx.drawImage(this.#stage, crop.x * width, crop.y * height, crop.width * width, crop.height * height, 0, 0, width, height)
+  }
+
   #cancelSchedule(): void {
     const host = this.#video as FrameCallbackHost
     if (this.#vfcHandle && host.cancelVideoFrameCallback) host.cancelVideoFrameCallback(this.#vfcHandle)
@@ -582,6 +615,7 @@ export class CameraPipeline {
 
     const startedAt = performance.now()
     const action = this.#effect.renderFrame(this.#video, width, height, startedAt)
+    this.#frame(width, height)
     const cost = performance.now() - startedAt
 
     this.#lastDrawAt = startedAt

src/video-effects.ts

@@ -934,6 +934,16 @@ export class VideoEffect {
     return mask ? { width: mask.width, height: mask.height } : null
   }
 
+  /**
+   * The mask the last frame was composited with, or null when the last frame
+   * was not a composite. For framing the person in the picture: it only ever
+   * says where somebody is, never what gets painted. The buffer is reused, so
+   * read it before the next `renderFrame`.
+   */
+  get personMask(): SegmentationMask | null {
+    return this.#lastAction === 'composite' ? this.#lastMask : null
+  }
+
   /** Resolves when the current load attempt has settled, whichever way. Not
    *  a success signal: check `status` for that. */
   ready(): Promise<void> {