From dabb71a5cd849ed0056c2d7b9b0cb5c9dba45894 Mon Sep 17 00:00:00 2001
From: TheCryptoDonkey <TheCryptoDonkey@users.noreply.github.com>
Date: Sun, 27 Sep 2026 21:29:00 +0100
Subject: [PATCH] feat(call): keep you in the middle of your own picture

With a background effect on, the pipeline now crops the finished frame
around the person the effect's mask finds, eased so it drifts rather than
twitches and zoomed no more than 1.6x. It crops pixels the effect has
already painted, so it can only send less of the room. With no mask to go
on it eases back to the whole frame. "Keep me in the middle of the
picture" in the effects panel turns it off; it is on by default and
remembered per device.

Claude-Session: https://claude.ai/code/session_01CG4pPCsd8pdySNvBpt8fTk
---
 app/index.html             |   6 ++
 app/src/auto-frame.test.ts |  83 +++++++++++++++++++++++
 app/src/auto-frame.ts      | 132 +++++++++++++++++++++++++++++++++++++
 app/src/main.ts            |  17 +++++
 app/src/video-pipeline.ts  |  36 +++++++++-
 src/video-effects.ts       |  10 +++
 6 files changed, 283 insertions(+), 1 deletion(-)
 create mode 100644 app/src/auto-frame.test.ts
 create mode 100644 app/src/auto-frame.ts

diff --git a/app/index.html b/app/index.html
index 7576eeeb..64c128b5 100644
--- a/app/index.html
+++ b/app/index.html
@@ -227,6 +227,12 @@ <h1 class="app-brand"><img src="/pwa-192x192.png" width="28" height="28" alt=""
                   <input id="blurStrength" type="range" min="0" max="100" step="1" value="60" />
                 </div>
                 <div class="segmented" id="backgroundChoices" role="radiogroup" aria-label="Picture" hidden></div>
+                <div class="effectRow" id="framingRow">
+                  <label for="framingToggle" title="Zooms in a little and follows you, so you stay in the middle of the picture everybody sees">
+                    <input id="framingToggle" type="checkbox" />
+                    Keep me in the middle of the picture
+                  </label>
+                </div>
                 <div class="effectRow" id="fishRow" hidden>
                   <label for="fishToggle">
                     <input id="fishToggle" type="checkbox" />
diff --git a/app/src/auto-frame.test.ts b/app/src/auto-frame.test.ts
new file mode 100644
index 00000000..3fb5c041
--- /dev/null
+++ b/app/src/auto-frame.test.ts
@@ -0,0 +1,83 @@
+import { describe, expect, test } from 'vitest'
+import { AutoFramer, HOLD_FRAMES, MAX_ZOOM, WHOLE_FRAME, cropFor, personBox, type FrameBox } from './auto-frame.js'
+
+/** A mask with the person filling this box, in mask pixels. */
+function maskWith(width: number, height: number, person?: { x: number; y: number; width: number; height: number }) {
+  const data = new Float32Array(width * height)
+  if (person) {
+    for (let y = person.y; y < person.y + person.height; y++) {
+      for (let x = person.x; x < person.x + person.width; x++) data[y * width + x] = 0.9
+    }
+  }
+  return { width, height, data }
+}
+
+function settle(framer: AutoFramer, person: FrameBox | null, frames = 400): FrameBox {
+  let crop = WHOLE_FRAME
+  for (let i = 0; i < frames; i++) crop = framer.next(person)
+  return crop
+}
+
+describe('personBox', () => {
+  test('finds a person sitting off to the left', () => {
+    const box = personBox(maskWith(640, 480, { x: 0, y: 120, width: 200, height: 360 }))!
+    expect(box.x).toBeCloseTo(0, 1)
+    expect(box.width).toBeCloseTo(200 / 640, 1)
+    expect(box.y).toBeCloseTo(120 / 480, 1)
+  })
+
+  test('nobody, or a speck, is nobody', () => {
+    expect(personBox(null)).toBeNull()
+    expect(personBox(maskWith(640, 480))).toBeNull()
+    expect(personBox(maskWith(640, 480, { x: 300, y: 200, width: 20, height: 20 }))).toBeNull()
+  })
+
+  test('a stray blob at the edge does not stretch the box across the room', () => {
+    const mask = maskWith(640, 480, { x: 240, y: 100, width: 160, height: 380 })
+    for (let y = 0; y < 8; y++) for (let x = 630; x < 634; x++) mask.data[y * 640 + x] = 0.9
+    const box = personBox(mask)!
+    expect(box.x + box.width).toBeLessThan(420 / 640)
+  })
+})
+
+describe('cropFor', () => {
+  test('centres the person and keeps the picture its own shape', () => {
+    const crop = cropFor({ x: 0.3, y: 0.3, width: 0.2, height: 0.7 })
+    expect(crop.width).toBe(crop.height)
+    expect(crop.x + crop.width / 2).toBeCloseTo(0.4, 5)
+    expect(crop.y).toBeLessThan(0.3)
+  })
+
+  test('never zooms past the limit and never leaves the frame', () => {
+    const crop = cropFor({ x: 0, y: 0.2, width: 0.05, height: 0.3 })
+    expect(crop.width).toBeCloseTo(1 / MAX_ZOOM, 5)
+    expect(crop.x).toBe(0)
+    const wide = cropFor({ x: 0, y: 0, width: 1, height: 1 })
+    expect(wide).toEqual(WHOLE_FRAME)
+  })
+})
+
+describe('AutoFramer', () => {
+  test('drifts towards the person rather than jumping', () => {
+    const framer = new AutoFramer()
+    const person = { x: 0.1, y: 0.3, width: 0.25, height: 0.7 }
+    const first = framer.next(person)
+    expect(first.width).toBeGreaterThan(0.95)
+    const settled = settle(framer, person)
+    expect(settled).toEqual(cropFor(person))
+  })
+
+  test('ignores small movements, so the picture does not twitch', () => {
+    const framer = new AutoFramer()
+    const person = { x: 0.3, y: 0.3, width: 0.3, height: 0.7 }
+    const settled = settle(framer, person)
+    expect(settle(framer, { ...person, x: 0.31 }, 50)).toEqual(settled)
+  })
+
+  test('holds through a short gap in the mask, then eases back to the whole frame', () => {
+    const framer = new AutoFramer()
+    const settled = settle(framer, { x: 0.3, y: 0.3, width: 0.2, height: 0.7 })
+    expect(settle(framer, null, HOLD_FRAMES)).toEqual(settled)
+    expect(settle(framer, null)).toEqual(WHOLE_FRAME)
+  })
+})
diff --git a/app/src/auto-frame.ts b/app/src/auto-frame.ts
new file mode 100644
index 00000000..cf21ae86
--- /dev/null
+++ b/app/src/auto-frame.ts
@@ -0,0 +1,132 @@
+import type { SegmentationMask } from '../../src/video-effects.js'
+
+/**
+ * Keeping the person in the middle of their own picture.
+ *
+ * The background effect already knows where the person is: its mask. This
+ * turns that mask into a crop of the finished frame, eased so the picture
+ * drifts after somebody rather than following every twitch. It crops pixels
+ * the effect has already painted, so it can only ever send less of the room,
+ * never more. With no mask to go on (effect off, loading or failed) the crop
+ * eases back out to the whole frame.
+ *
+ * Every rectangle here is a fraction of the frame, 0 to 1 on each axis, and a
+ * crop is the same fraction of both axes so the picture keeps its shape.
+ */
+export interface FrameBox { x: number; y: number; width: number; height: number }
+
+export const WHOLE_FRAME: FrameBox = { x: 0, y: 0, width: 1, height: 1 }
+
+/** Confidence above which a mask pixel counts as the person. */
+export const PERSON_CUTOFF = 0.5
+/** Less of the frame than this is a stray hand or a false positive. */
+export const MIN_PERSON_SHARE = 0.03
+/** Mask pixels at either end of the person ignored, so one stray blob at the
+ *  edge of the frame does not stretch the box across the room. */
+export const TRIM_SHARE = 0.02
+/** Never zoom in further than this: a webcam picture gets soft quickly. */
+export const MAX_ZOOM = 1.6
+/** How much of the crop's width the person should fill. */
+export const PERSON_WIDTH_SHARE = 0.55
+/** Space kept above the head, as a share of the crop's height. */
+export const HEADROOM = 0.1
+/** A new crop is only chased once it differs from the current aim by this. */
+export const DEAD_BAND = 0.05
+/** How far towards its aim the crop moves each frame. */
+export const EASE = 0.06
+/** Frames without a mask before the crop starts easing back out. */
+export const HOLD_FRAMES = 30
+
+/** Where the person is in this mask, or null when nobody clearly is. */
+export function personBox(mask: SegmentationMask | null, step = 4): FrameBox | null {
+  if (!mask || mask.width <= 0 || mask.height <= 0) return null
+  const columns = new Uint32Array(Math.ceil(mask.width / step))
+  const rows = new Uint32Array(Math.ceil(mask.height / step))
+  let total = 0
+  let sampled = 0
+  for (let y = 0, row = 0; y < mask.height; y += step, row++) {
+    const offset = y * mask.width
+    for (let x = 0, column = 0; x < mask.width; x += step, column++) {
+      sampled++
+      if ((mask.data[offset + x] ?? 0) < PERSON_CUTOFF) continue
+      columns[column]!++
+      rows[row]!++
+      total++
+    }
+  }
+  if (total === 0 || total / sampled < MIN_PERSON_SHARE) return null
+  const [left, right] = extent(columns, total)
+  const [top, bottom] = extent(rows, total)
+  const x = (left * step) / mask.width
+  const y = (top * step) / mask.height
+  return {
+    x, y,
+    width: Math.min(1, ((right + 1) * step) / mask.width) - x,
+    height: Math.min(1, ((bottom + 1) * step) / mask.height) - y,
+  }
+}
+
+function extent(counts: Uint32Array, total: number): [number, number] {
+  const trim = total * TRIM_SHARE
+  let first = 0
+  for (let seen = 0; first < counts.length - 1; first++) { seen += counts[first]!; if (seen > trim) break }
+  let last = counts.length - 1
+  for (let seen = 0; last > first; last--) { seen += counts[last]!; if (seen > trim) break }
+  return [first, last]
+}
+
+/** The crop that puts this person in the middle with room above their head. */
+export function cropFor(person: FrameBox): FrameBox {
+  const size = Math.min(1, Math.max(1 / MAX_ZOOM, person.width / PERSON_WIDTH_SHARE))
+  const centre = person.x + person.width / 2
+  return {
+    x: clamp(centre - size / 2, 0, 1 - size),
+    y: clamp(person.y - HEADROOM * size, 0, 1 - size),
+    width: size,
+    height: size,
+  }
+}
+
+/** Follows the person from frame to frame, slowly. */
+export class AutoFramer {
+  #current: FrameBox = WHOLE_FRAME
+  #aim: FrameBox = WHOLE_FRAME
+  #missed = 0
+
+  /** The crop for this frame, given where the person is (or null). */
+  next(person: FrameBox | null): FrameBox {
+    if (person) {
+      this.#missed = 0
+      const aim = cropFor(person)
+      if (distance(aim, this.#aim) > DEAD_BAND) this.#aim = aim
+    } else if (++this.#missed > HOLD_FRAMES) {
+      this.#aim = WHOLE_FRAME
+    }
+    this.#current = {
+      x: ease(this.#current.x, this.#aim.x),
+      y: ease(this.#current.y, this.#aim.y),
+      width: ease(this.#current.width, this.#aim.width),
+      height: ease(this.#current.height, this.#aim.height),
+    }
+    return this.#current
+  }
+
+  /** Straight back to the whole frame, as when framing is turned off. */
+  reset(): void {
+    this.#current = this.#aim = WHOLE_FRAME
+    this.#missed = 0
+  }
+}
+
+function ease(from: number, to: number): number {
+  const next = from + (to - from) * EASE
+  return Math.abs(to - next) < 0.0005 ? to : next
+}
+
+function distance(a: FrameBox, b: FrameBox): number {
+  return Math.max(Math.abs(a.x - b.x), Math.abs(a.y - b.y), Math.abs(a.width - b.width))
+}
+
+function clamp(value: number, low: number, high: number): number {
+  return Math.min(high, Math.max(low, value))
+}
diff --git a/app/src/main.ts b/app/src/main.ts
index 43c21532..aa12b0e9 100644
--- a/app/src/main.ts
+++ b/app/src/main.ts
@@ -2496,6 +2496,12 @@ const FISH_STORAGE_KEY = 'kithmoot.fish.enabled'
 let fishEnabled = (() => {
   try { return localStorage.getItem(FISH_STORAGE_KEY) === 'true' } catch { return false }
 })()
+/** Keeping you in the middle of your own picture - see `auto-frame.ts`. On
+ *  unless turned off, remembered on this device. */
+const FRAMING_STORAGE_KEY = 'kithmoot.camera.framing'
+let framingEnabled = (() => {
+  try { return localStorage.getItem(FRAMING_STORAGE_KEY) !== 'false' } catch { return true }
+})()
 let videoInputs: MediaDeviceInfo[] = []
 
 const localPreviewEls = new Map<'camera' | 'screen', HTMLVideoElement>()
@@ -4236,6 +4242,7 @@ async function toggleCamera(): Promise<void> {
     const pipeline = new CameraPipeline({
       mode: savedEffectMode,
       strength: savedBlurStrength,
+      framing: framingEnabled,
       onStateChange: state => { if (generation === callGeneration) renderEffectState(state) },
       onSourceEnded: () => {
         if (generation !== callGeneration) return
@@ -4323,6 +4330,9 @@ function renderEffectState(state: VideoEffectState): void {
   $('backgroundChoices').hidden = state.mode !== 'replace'
   // Only over a sea. Offering fish over Slate would be a question with no
   // sensible answer.
+  // Framing follows the effect's picture of where you are, so there is
+  // nothing to follow with the effect off.
+  $('framingRow').hidden = state.mode === 'off'
   $('fishRow').hidden =
     state.mode !== 'replace' || !BACKGROUNDS.find((b) => b.id === backgroundId)?.sea
 
@@ -11906,6 +11916,13 @@ $('fishToggle').addEventListener('change', () => {
   camera?.setFish(fishEnabled).catch((err) => setStatus(describeError(err)))
 })
 
+;($('framingToggle') as HTMLInputElement).checked = framingEnabled
+$('framingToggle').addEventListener('change', () => {
+  framingEnabled = ($('framingToggle') as HTMLInputElement).checked
+  try { localStorage.setItem(FRAMING_STORAGE_KEY, String(framingEnabled)) } catch { /* Still applies to this visit. */ }
+  camera?.setFraming(framingEnabled)
+})
+
 $('switchCamera').addEventListener('click', () => {
   switchCamera().catch((err) => setStatus(describeError(err)))
 })
diff --git a/app/src/video-pipeline.ts b/app/src/video-pipeline.ts
index 32d153af..3d5c2eb8 100644
--- a/app/src/video-pipeline.ts
+++ b/app/src/video-pipeline.ts
@@ -8,6 +8,7 @@ import {
 } from '../../src/video-effects.js'
 import { ReefBackground, type FishSprite } from '../../src/reef-scene.js'
 import { createSegmenter } from './mediapipe-segmenter.js'
+import { AutoFramer, personBox } from './auto-frame.js'
 
 /**
  * The camera, with the effect wired into it.
@@ -173,6 +174,8 @@ export interface CameraPipelineOptions {
   mode?: EffectMode
   /** Start at this blur strength rather than `DEFAULT_BLUR_STRENGTH`. */
   strength?: number
+  /** Keep the person in the middle of the picture - see `auto-frame.ts`. */
+  framing?: boolean
 }
 
 export interface CameraStats {
@@ -197,8 +200,14 @@ type FrameCallbackHost = HTMLVideoElement & {
 
 export class CameraPipeline {
   readonly #video: HTMLVideoElement
+  /** The published picture: a crop of `#stage`, never of the camera. */
   readonly #canvas: HTMLCanvasElement
+  readonly #canvasCtx: CanvasRenderingContext2D
+  /** What the effect paints: the whole frame, already safe to send. */
+  readonly #stage: HTMLCanvasElement
   readonly #effect: VideoEffect
+  readonly #framer = new AutoFramer()
+  #framing: boolean
   readonly #onStateChange?: (state: VideoEffectState) => void
   readonly #onSourceEnded?: () => void
 
@@ -237,9 +246,14 @@ export class CameraPipeline {
     this.#canvas = document.createElement('canvas')
     this.#canvas.width = 640
     this.#canvas.height = 480
+    const ctx = this.#canvas.getContext('2d')
+    if (!ctx) throw new Error('the camera canvas has no 2D context')
+    this.#canvasCtx = ctx
+    this.#stage = document.createElement('canvas')
+    this.#framing = opts.framing ?? true
 
     this.#effect = new VideoEffect({
-      output: this.#canvas,
+      output: this.#stage,
       createCanvas: (width, height) => {
         const canvas = document.createElement('canvas')
         canvas.width = width
@@ -401,6 +415,16 @@ export class CameraPipeline {
     this.#effect.setMode(mode)
   }
 
+  /** Keep the person centred, or send the whole frame. */
+  setFraming(on: boolean): void {
+    this.#framing = on
+    if (!on) this.#framer.reset()
+  }
+
+  get framing(): boolean {
+    return this.#framing
+  }
+
   setStrength(strength: number): void {
     this.#effect.setStrength(strength)
   }
@@ -566,6 +590,15 @@ export class CameraPipeline {
     })
   }
 
+  /** Copy the effect's finished frame out, cropped around the person. */
+  #frame(width: number, height: number): void {
+    if (this.#stage.width !== width || this.#stage.height !== height) return
+    if (this.#canvas.width !== width) this.#canvas.width = width
+    if (this.#canvas.height !== height) this.#canvas.height = height
+    const crop = this.#framing ? this.#framer.next(personBox(this.#effect.personMask)) : { x: 0, y: 0, width: 1, height: 1 }
+    this.#canvasCtx.drawImage(this.#stage, crop.x * width, crop.y * height, crop.width * width, crop.height * height, 0, 0, width, height)
+  }
+
   #cancelSchedule(): void {
     const host = this.#video as FrameCallbackHost
     if (this.#vfcHandle && host.cancelVideoFrameCallback) host.cancelVideoFrameCallback(this.#vfcHandle)
@@ -582,6 +615,7 @@ export class CameraPipeline {
 
     const startedAt = performance.now()
     const action = this.#effect.renderFrame(this.#video, width, height, startedAt)
+    this.#frame(width, height)
     const cost = performance.now() - startedAt
 
     this.#lastDrawAt = startedAt
diff --git a/src/video-effects.ts b/src/video-effects.ts
index 7845d1c9..5df8d0e8 100644
--- a/src/video-effects.ts
+++ b/src/video-effects.ts
@@ -934,6 +934,16 @@ export class VideoEffect {
     return mask ? { width: mask.width, height: mask.height } : null
   }
 
+  /**
+   * The mask the last frame was composited with, or null when the last frame
+   * was not a composite. For framing the person in the picture: it only ever
+   * says where somebody is, never what gets painted. The buffer is reused, so
+   * read it before the next `renderFrame`.
+   */
+  get personMask(): SegmentationMask | null {
+    return this.#lastAction === 'composite' ? this.#lastMask : null
+  }
+
   /** Resolves when the current load attempt has settled, whichever way. Not
    *  a success signal: check `status` for that. */
   ready(): Promise<void> {