From de51140596dcaee5c131004fe8291fa5f7283711 Mon Sep 17 00:00:00 2001 From: Aidan Foster Date: Wed, 5 Aug 2026 18:35:46 -0700 Subject: [PATCH] Clarify global conditioning input semantics Signed-off-by: Aidan Foster --- docs/inference_runtime_api_design.md | 65 ++++--- ...inference_runtime_inputs_implementation.md | 97 +++++----- ...ence_runtime_supported_inputs_inventory.md | 177 +++++++++--------- flashdreams/flashdreams/runtime/__init__.py | 2 - flashdreams/flashdreams/runtime/inputs.py | 114 +++-------- flashdreams/flashdreams/runtime/interfaces.py | 4 +- flashdreams/flashdreams/runtime/mapping.py | 33 ++-- flashdreams/flashdreams/runtime/types.py | 9 +- .../tests/test_inference_runtime_api.py | 19 +- flashdreams/tests/test_runtime_canonical.py | 11 +- .../tests/test_runtime_input_mapping.py | 167 +++++------------ 11 files changed, 293 insertions(+), 405 deletions(-) diff --git a/docs/inference_runtime_api_design.md b/docs/inference_runtime_api_design.md index f70fbd89..b6314a20 100644 --- a/docs/inference_runtime_api_design.md +++ b/docs/inference_runtime_api_design.md @@ -119,7 +119,7 @@ InferenceRuntime | v InferenceSession - one rollout/stream: prompt/initial inputs, cache/state, current step, reset + one rollout/stream: global conditioning, cache/state, current step, reset keeps per-run state from leaking across prompts, clients, or benchmark repeats | v @@ -187,11 +187,11 @@ local model implementation, a Dynamo-like backend, or a hosted service. | --- | --- | --- | | Model/preset registry | Lists what can run: model/preset slugs, scenarios, capabilities, resource hints, and supported output modes. | Must remain cheap to query and must not load checkpoints. | | App / integration / benchmark / transport | Owns the user-facing mode: CLI, native integration, WebRTC, benchmark, hosted request, or replay. | Supplies run setup, user inputs, model inputs, and output target selection. | -| User input library | Normalizes live or replayed controls into FlashDreams-supported user input events/windows. | Shared primitives for keyboard, reset, prompt/image updates, traces, and future scalar controls. | -| Input mapping | Converts user/app inputs plus initial model inputs into the model-specific inputs needed by the session. | A model adapter may provide a default mapping; runtimes, applications, benchmarks, and replay tools may override it without changing the model step. | +| User input library | Normalizes live or replayed controls into FlashDreams-supported user input events/windows. | Shared primitives for keyboard, reset, prompt/image selection, traces, and future scalar controls. | +| Input mapping | Converts user/app inputs plus global conditioning into the model-specific inputs needed by the session. | A model adapter may provide a default mapping; runtimes, applications, benchmarks, and replay tools may override it without changing the model step. | | ModelRunner / standard loop | Orchestrates one run from setup through runtime initialization, stepping, output, metrics, and teardown. | Shared orchestration layer used by CLIs, benchmarks, MP4 runs, and simple realtime flows. | | InferenceRuntime | Owns heavyweight lifecycle: distributed init, model construction, checkpoint loading, compile/capture, warmup, hosted-service connection, and teardown. | Long-lived reusable runtime created from `InferenceConfig`; lets FlashDreams load/warm once and create sessions sequentially unless the backend supports concurrency. | -| InferenceSession | Owns one rollout or stream: initial inputs, cache state, current step, reset behavior, step requirements, and step execution. | Per-rollout interface consumed by the standard loop; keeps state isolated across prompts, browser clients, replay scenarios, or benchmark repeats. | +| InferenceSession | Owns one rollout or stream: global conditioning, cache state, current step, reset behavior, step requirements, and step execution. | Per-rollout interface consumed by the standard loop; keeps state isolated across prompts, browser clients, replay scenarios, or benchmark repeats. | | Model implementation / inference pipeline | Implements encode, model step, decode, cache updates, and model-specific optimizations. | FlashDreams wraps this boundary; it should not replace every model implementation. | | Output target | Consumes generated outputs and handles presentation or persistence. | Separate from model execution so the same session can feed WebRTC, MP4, benchmark, or headless output. | | Metrics, artifacts, and profiling | Records timings, memory, quality data, logs, reports, traces, and optional NVTX ranges. | Shared observation layer for local runs, benchmarks, CI smoke, and hosted runs. | @@ -298,8 +298,7 @@ uses: - keyboard keydown/keyup events; - reset requests; -- prompt update requests; -- image update requests; +- prompt or image selection/update events; - future scalar controls such as throttle, brake, steer, or camera axes once an integration needs them. @@ -335,11 +334,11 @@ Examples of global conditioning include prompt, negative prompt, conditioning frame, input video, scene id, HD map asset, camera calibration, initial camera pose, seed, or model-specific fields. -Global conditioning is normally supplied when a session starts, but a non-empty -global slot on a mid-rollout input is an update request rather than a reset; -resetting rollout state is a separate `InferenceSession.reset()` call. Whether a -given value can be swapped mid-rollout is declared per field by -`InputField.update_policy`. +Global conditioning establishes session-global model state when a session +starts or resets. During an active rollout, a non-empty global-conditioning +payload passed to `InferenceSession.step()` asks the session to update that +state when the model supports it. Reset remains a separate explicit session +method. Examples of per-step conditioning include frame timestamps, pose segments, camera trajectory chunks, rendered HD map frames, conditioning video windows, @@ -349,16 +348,17 @@ Inference input payloads should use semantic names, not only modality names. For example, a first frame and an HD map frame should be distinct inputs even if both are image-like values. -Model input metadata may also include a lightweight lifecycle label, such as -runtime config, cache initialization, rollout binding, per-step input, or -session update. This should remain query metadata, not model-specific tensor -validation. +Model input names, payload kinds, semantic-type hints, and schema metadata +should be open-ended. Supported integrations such as SANA-WM, LingBot, +Omnidreams, and future external adapters may need different semantic fields. +Adding a new model should usually mean adding adapter-owned schema declarations +and mappings, not changing a central FlashDreams enum. -Model input names, payload kinds, lifecycle labels, and schema metadata should -be open-ended. Supported integrations such as SANA-WM, LingBot, Omnidreams, and -future external adapters may need different semantic fields. Adding a new model -should usually mean adding adapter-owned schema declarations and mappings, not -changing a central FlashDreams enum. +Consumption cadence is a separate hint from input scope. A field may be +provided through global conditioning because it is session-global state, while +the adapter consumes or slices it during every step. That can be recorded as +`frequency_consumed` metadata without changing whether the field belongs in +`global_conditioning_fields` or `step_fields`. For interactive runs, most `InferenceInput` values will be global conditioning plus per-step inputs produced by input mapping. For MP4 generation and benchmarking, the API @@ -374,7 +374,7 @@ These schemas are not meant to be a rich type system or a replacement for model-specific validation. They should be just enough to answer: - what can this app, transport, trace, or benchmark source provide? -- what does this model require before startup and at each step? +- what does this model require before session start and at each step? - can this event source drive this model with the selected mapping? The purpose is to fail early before expensive model initialization, produce @@ -386,7 +386,7 @@ coordinate frame, units, rough shape summary, accepted file suffixes, schema URI, model family, or source/transport details. Metadata should help humans and adapter selection code, but compatibility should still be based on the declared event capabilities, semantic model fields, payload representation hints, and -lifecycle labels. +schema phases. Consumption-cadence hints are descriptive and adapter-owned. For simple CLI text-to-video or image-to-video runs, `UserInputSchema` can be trivial or omitted because there may be no live controls. `InferenceInputSchema` is @@ -478,6 +478,14 @@ declare user inputs, declare model inputs, and provide a default mapping, while the runtime owns transport, event validation, timestamping, input queue/window selection, output delivery, and optional overrides. +`StepRequest` and `StepResult` are per-step runtime messages, not declarative +schemas. `InferenceSession.next_step_request()` returns a `StepRequest` to say +which step is next, which user-input time window to map, and whether this step +has any narrower `InferenceInputSchema` than the session default. The runner or +application then builds an `InferenceInput` and calls `InferenceSession.step()`, +which returns a `StepResult` carrying the generated output, output timing, +metrics, and step metadata. + Examples: - T2V mapping validates a prompt and creates no per-step control inputs. @@ -505,7 +513,7 @@ A run should: profiling, and optional scenario setup. 3. Validate that the event source and mapping can drive the selected model. 4. Initialize the runtime. -5. Start a session from initial model inputs. +5. Start a session from global conditioning inputs. 6. For each step, ask the session what it needs, gather live or fixed inputs, build step model inputs, run the session step, route outputs, and record metrics. @@ -553,8 +561,9 @@ generation, benchmarks, regression testing, and autotune. Two replay levels should be supported: -- user-event replay: records timestamped key events, prompt updates, image - updates, reset events, and timing, then runs normal input mapping; +- user-event replay: records timestamped key events, prompt or image + selection/update events, reset events, and timing, then runs normal input + mapping; - model-input replay: records or defines already-mapped per-step model inputs for stricter model-level regression tests. @@ -667,8 +676,10 @@ registry, standard loop, concrete output modes, or model migrations: - The model-specific integration boundary is named `ModelAdapter`. - Heavyweight lifecycle is split into `InferenceRuntime` and `InferenceSession`. -- Step data carriers are named `StepRequest` and `StepResult`; a session returns - `None` from `next_step_request()` when the rollout is complete. +- Step data carriers are named `StepRequest` and `StepResult`. They are runtime + messages around one call to `InferenceSession.step()`, not schema + declarations; a session returns `None` from `next_step_request()` when the + rollout is complete. - Raw inputs use `UserInputs`, canonicalized inputs use `CanonicalInputs`, and model-facing inputs use `InferenceInput`. Both remain lightweight payload envelopes with shallow read-only mappings. diff --git a/docs/inference_runtime_inputs_implementation.md b/docs/inference_runtime_inputs_implementation.md index 8d485768..763b9810 100644 --- a/docs/inference_runtime_inputs_implementation.md +++ b/docs/inference_runtime_inputs_implementation.md @@ -46,70 +46,60 @@ that touches no application, mapping, or model code. This path covers **live user control only**. Global conditioning is application-owned data and reaches `InferenceInput` directly, without passing -through canonicalization or a device converter. An application that wants a -trigger key to swap the prompt reads that as ordinary canonical control input -and updates its own global conditioning in response. +through canonicalization or a device converter. Session start/reset establishes +that global conditioning. During an active rollout, a non-empty +`global_conditioning` payload passed to `step()` requests an update of the +session-global state when the model supports it. ## Conditioning Slots -Both the canonical and encoded layers split into two slots, and the split means -the same thing at each: +The encoded layer splits model-facing inputs into two slots: -- **global conditioning** — conditions the whole rollout: prompt, conditioning - frame, scene. Normally supplied at session start. +- **global conditioning** — session-global model state: prompt, conditioning + frame, scene. - **per-step conditioning** — needed to generate the next chunk or frame: steering, HD map frames, camera trajectory. -`InputPhase` is `Literal["global", "step"]`. The axis names *which slot*, not -*when the value may arrive* — see the next section. +`InputPhase` is `Literal["global_conditioning", "step"]`. The phase names the +`InferenceInput` slot the caller provides. -## Global Conditioning Updates Are Not Resets +`InputField.frequency_consumed` is independent query metadata. It says how the +adapter consumes a field internally, such as `once` or `per_step`; it does not +decide whether the caller provides the field through `global_conditioning` or +`step`. -A non-empty global slot on a mid-rollout `InferenceInput` is an **update -request**. The session should apply it when the model supports doing so. -Resetting rollout state is a separate, explicit `InferenceSession.reset()` call. -The motivating case is changing prompt and conditioning frame mid-run to change -the weather in an Omnidreams rollout. +## Global Conditioning Is Session-Global State -```python -from flashdreams.runtime import InferenceInput - -steady_state = InferenceInput(step={"steering": 0.25}) -assert not steady_state.requests_global_update - -changed_weather = steady_state.with_global_update({"prompt": "heavy rain"}) -assert changed_weather.requests_global_update -``` - -Because `with_step()` carries the global slot through unchanged, use -`without_global_update()` for the steady-state case; otherwise every step looks -like an update request. - -Whether a value can actually be swapped mid-rollout is declared per field: +`InferenceInput.global_conditioning` carries session-scoped inputs. A runtime +passes those values to `InferenceRuntime.start_session()` or to +`InferenceSession.reset()` when the backend supports resetting a rollout. +During an active rollout, passing a non-empty `global_conditioning` payload to +`InferenceSession.step()` asks the session to update that session-global state. +The model/session owns whether that update is supported. ```python -from flashdreams.runtime import SESSION_START_ONLY, InferenceInputSchema, InputField +from flashdreams.runtime import InferenceInput, InferenceInputSchema, InputField schema = InferenceInputSchema( - global_fields=( - InputField(name="prompt", update_policy="step_boundary"), - InputField(name="scene_id", update_policy=SESSION_START_ONLY), + global_conditioning_fields=( + InputField(name="prompt"), + InputField(name="scene_id"), ) ) -schema.unsupported_global_updates( - InferenceInput(global_conditioning={"prompt": "heavy rain", "scene_id": "town_02"}) +schema.require_global_conditioning( + InferenceInput(global_conditioning={"prompt": "drive", "scene_id": "town_02"}) ) -# ("scene_id",) -``` -`SESSION_START_ONLY` is the one reserved `update_policy` token. Everything else -in that vocabulary, and all of `lifecycle`, is open and adapter-owned; this layer -only carries it as queryable metadata. +step_with_prompt_update = InferenceInput( + global_conditioning={"prompt": "heavy rain"}, + step={"steering": 0.0}, +) +``` -Steady-state steps must leave the global slot empty; otherwise every step reads -as an update request. Converters emit every window, because live control is -level-triggered: a key held across a step emits no events but still means full -throttle. +Per-step conditioning is different: those values are supplied through +`InferenceInput.step` for each generated chunk or frame. Converters still emit +every window, because live control is level-triggered: a key held across a step +emits no events but still means full throttle. ## Raw Inputs @@ -192,8 +182,9 @@ device does not resume from stale state. ## Mapping And Compatibility `InputMapping` is the canonical-to-encoded boundary. `InputMappingSchema` is its -declarative surface: `consumes` names canonical modalities; `produces_global` -and `produces_step` name the `InferenceInput` fields it can build. +declarative surface: `consumes` names canonical modalities; +`produces_global_conditioning` and `produces_step` name the `InferenceInput` +fields it can build. `InputMapping.validate()` raises, which fails a run late and cannot say *which* optional model input a source would enable or *which* missing modality makes a @@ -230,6 +221,13 @@ registered later, with no change to the mapping or the model schema. `undeclared_inference_inputs()` reports payload keys a mapping produced but did not declare, which keeps hand-written schemas honest as the code drifts. +`StepRequest` and `StepResult` sit around a single `InferenceSession.step()` +call. They are not schema declarations. A session returns `StepRequest` from +`next_step_request()` to name the next step, optionally provide a narrower +`InferenceInputSchema`, and request a `TimeWindow` of user inputs. The runner +then builds `InferenceInput` and calls `step()`, which returns a `StepResult` +for the output target and metrics recorder. + ## What This Does Not Validate The schemas intentionally avoid becoming a rich type system. These remain the @@ -237,8 +235,9 @@ responsibility of the model adapter, runtime, session, or mapping: - tensor shape and dtype, image decode details; - camera coordinate systems, pose and timestamp units; -- prompt-embedding swap mechanics; -- whether a model can actually apply a declared update policy at runtime; +- prompt-embedding mechanics; +- whether a model can actually apply a requested global-conditioning update; +- enforcing consumption-cadence metadata; - deep validation of scene, HD map, or actor-state data. The layer answers "can this source plausibly drive this model through this diff --git a/docs/inference_runtime_supported_inputs_inventory.md b/docs/inference_runtime_supported_inputs_inventory.md index ebe9d853..d56cf165 100644 --- a/docs/inference_runtime_supported_inputs_inventory.md +++ b/docs/inference_runtime_supported_inputs_inventory.md @@ -17,44 +17,44 @@ FastVideo Causal WAN 2.2 T2V, and Cosmos Predict2 T2V: - Source/app inputs: prompt text or prompt text file, pixel height/width, and fps or block count depending on runner. -- Model-facing initial inputs: prompt text plus latent/output height and width +- Model-facing global conditioning: prompt text plus latent/output height and width derived from run config. -- Model-facing step/update inputs: no live controls; AR loop steps with fixed +- Model-facing per-step inputs: no live controls; AR loop steps with fixed session state. WAN 2.1 I2V, Causal-Forcing I2V, and Cosmos Predict2 I2V: - Source/app inputs: prompt text or prompt file, first-frame image path or URL, and pixel height/width. -- Model-facing initial inputs: prompt text and decoded first-frame tensor. -- Model-facing step/update inputs: no live controls. +- Model-facing global conditioning: prompt text and decoded first-frame tensor. +- Model-facing per-step inputs: no live controls. FlashVSR: - Source/app inputs: input video path or URL, chunk size, crop region, sparse ratio, and optional output FPS. -- Model-facing initial inputs: no explicit prompt at runner time; the prompt +- Model-facing global conditioning: no explicit prompt at runner time; the prompt tensor is configured in the pipeline. Input video dimensions affect per-video runtime/pipeline setup. -- Model-facing step/update inputs: video chunks passed to +- Model-facing per-step inputs: video chunks passed to `pipeline.generate(input=clip)`. LingBot CLI: - Source/app inputs: prompt or prompt path, first-frame image path, pose path, intrinsics path, total blocks, dimensions, and fps. -- Model-facing initial inputs: prompt text and first-frame tensor. -- Model-facing step/update inputs: `CamCtrlInput` with intrinsics, camera poses, +- Model-facing global conditioning: prompt text and first-frame tensor. +- Model-facing per-step inputs: `CamCtrlInput` with intrinsics, camera poses, and world scale. LingBot WebRTC: - Source/app inputs: session prompt, uploaded/remote/default first-frame image, keyboard events, reset requests, text-event catalog, and trigger events. -- Model-facing initial inputs: prompt text, first-frame tensor, base text +- Model-facing global conditioning: prompt text, first-frame tensor, base text embeddings, precomputed text-event embeddings, base intrinsics, and world scale. -- Model-facing step/update inputs: keyboard event windows become pose segments +- Model-facing per-step inputs: keyboard event windows become pose segments and camera trajectories. Text-event triggers can replace rollout text embeddings when the model supports it. @@ -63,9 +63,9 @@ HY-WorldPlay WAN I2V: - Source/app inputs: prompt or prompt path, first-frame image path or example image, pose string or pose JSON, memory-selection settings, dimensions, fps, and seed. -- Model-facing initial inputs: prompt text and first-frame tensor for cache - initialization. -- Model-facing step/update inputs: pose data is bound for the rollout as action +- Model-facing global conditioning: prompt text and first-frame tensor for + session setup. +- Model-facing per-step inputs: pose data is bound for the rollout as action labels, view matrices, intrinsics, and memory-selection state before AR steps. Omnidreams CLI: @@ -73,18 +73,18 @@ Omnidreams CLI: - Source/app inputs: shared prompt or per-camera prompts, HDMap video paths, first-frame image/video paths, camera names, example-data UUID, and optional embedding save/load paths. -- Model-facing initial inputs: prompt list, first-frame tensor, view names; or +- Model-facing global conditioning: prompt list, first-frame tensor, view names; or precomputed text/image/negative-text embeddings. -- Model-facing step/update inputs: HDMap video chunks passed per AR step. +- Model-facing per-step inputs: HDMap video chunks passed per AR step. Omnidreams WebRTC: - Source/app inputs: scene directory or scene UUID, scene variant, camera name, prompt/first-frame assets resolved from the scene, keyboard events, reset requests, and optional postprocess preset. -- Model-facing initial inputs: scene data, renderer, first-frame tensor, prompt, +- Model-facing global conditioning: scene data, renderer, first-frame tensor, prompt, camera calibration/extrinsics, initial ego pose, and initial timestamp. -- Model-facing step/update inputs: keyboard event windows become ego poses, +- Model-facing per-step inputs: keyboard event windows become ego poses, camera poses per view, and frame timestamps. The wrapper renders HDMap conditioning internally for each step. @@ -92,26 +92,26 @@ Omnidreams interactive drive: - Source/app inputs: scene bundle, keyboard events or wheel/controller samples, view-mode/reset/scene-exit controls, and vehicle/chunk config. -- Model-facing initial inputs: scene bundle, selected camera, prompt, initial +- Model-facing global conditioning: scene bundle, selected camera, prompt, initial RGB frame, initial rig pose, and initial timestamp. -- Model-facing step/update inputs: `DriverCommand` samples become trajectory +- Model-facing per-step inputs: `DriverCommand` samples become trajectory chunks, rendered frames, and world-model conditioning. Template recipe: - Source/app inputs: synthetic runner config: batch size, height, width, context tokens, AR steps, and seed. -- Model-facing initial inputs: synthetic transformer context, optional negative +- Model-facing global conditioning: synthetic transformer context, optional negative context, height, and width. -- Model-facing step/update inputs: optional synthetic control tensor. +- Model-facing per-step inputs: optional synthetic control tensor. WAN 2.2 TI2V pipeline config: - Source/app inputs: downstream runners use this rather than a standalone runner in this tree. -- Model-facing initial inputs: prompt text and first-frame image for TI2V-style - cache initialization. -- Model-facing step/update inputs: downstream runners decide controls; +- Model-facing global conditioning: prompt text and first-frame image for + TI2V-style session setup. +- Model-facing per-step inputs: downstream runners decide controls; HY-WorldPlay currently binds action/camera state around it. SANA-WM bidirectional and streaming on `main`: @@ -120,10 +120,10 @@ SANA-WM bidirectional and streaming on `main`: negative prompt, camera trajectory path or action DSL, optional intrinsics path or derived intrinsics, frame count, fps, Stage-1 sampling knobs, seed, precision/refiner options, and streaming chunk/block settings. -- Model-facing initial inputs: decoder context such as prompt, fps, +- Model-facing global conditioning: decoder context such as prompt, fps, `save_stage1`, refiner seed, sink size, and streaming refiner window/block parameters. -- Model-facing step/update inputs: bidirectional passes one +- Model-facing per-step inputs: bidirectional passes one `SanaWMI2VConditioningRequest` into the single generation step. Streaming passes one `SanaWMStreamingI2VConditioningRequest` repeatedly; the conditioning encoder caches rollout-wide prompt, first-frame, camera, latent @@ -134,7 +134,7 @@ SANA-WM bidirectional and streaming on `main`: ## API Implications -The inventory changes the T2/T3 shape in four concrete ways. +The inventory changes the T2/T3 shape in five concrete ways. First, a selected mapping is often a composition. A LingBot-like run needs prompt mapping, first-frame mapping, and keyboard-to-camera mapping. Omnidreams may add @@ -142,25 +142,24 @@ scene selection, camera selection, and HDMap mapping. The implementation should support checking a set of mapping schemas as one compatibility surface, while still allowing a single mapping object when that is simpler. -Second, `InferenceInputSchema` needs a lightweight lifecycle tag in addition to the -`initial` versus `step` phase. The phase answers when the value is needed at the -standard-loop level. The lifecycle tag distinguishes where the model adapter -uses it, such as: - -- `runtime_config`: values that affect setup before model/runtime construction, - such as FlashVSR input-video dimensions; -- `cache_init`: values passed when initializing or resetting a rollout cache, - such as prompts, first frames, view names, and precomputed embeddings; -- `rollout_binding`: values bound after cache initialization but before AR - steps, such as HY-WorldPlay action labels, camera tensors, and memory state; -- `step_input`: values consumed for one generated chunk, such as HDMap frames, - camera trajectories, driver commands, video chunks, and timestamps; -- `session_update`: values that can update an active session when supported, - such as LingBot text-event embedding swaps. - -The lifecycle tag is metadata, not a new deep type system. If both a model field -and mapping output specify lifecycle, compatibility should require them to agree. -If either side omits it, matching stays permissive for simple schemas. +Second, `InferenceInputSchema` needs explicit global-conditioning and per-step +schema slots. `global_conditioning_fields` describe the session-global state +carried through `InferenceInput.global_conditioning`. Start/reset establishes +that state; a non-empty global-conditioning payload in a step context asks the +session to update it when the model supports that. `step_fields` arrive through +`InferenceInput.step` for one generated chunk or frame window. + +This distinction matters for rollout-wide values such as full camera +trajectories, action labels, intrinsics sequences, and memory-selection config. +Those can be supplied in the global-conditioning slot, even if the adapter later +slices them internally while executing steps. If the caller must supply a fresh +value for every generated chunk, that value belongs in `step_fields`. + +`frequency_consumed` is a separate optional hint for how the adapter uses a +field internally, such as `once` or `per_step`. It does not decide where the +caller provides the value. A field can live in `global_conditioning_fields` and +still have `frequency_consumed="per_step"` when the adapter slices or reads +rollout-wide state during step execution. Third, `semantic_type` should be treated as a representation hint rather than a universal semantic type. For example, `prompt` may arrive as inline text or a @@ -173,7 +172,7 @@ Fourth, schema objects need open-ended metadata for future adapters. This lets a SANA-WM-like adapter advertise that `camera_trajectory_c2w` uses an `[F,4,4]` OpenCV camera-to-world sequence, or lets another model advertise a schema URI, units, coordinate frame, accepted file suffixes, cardinality hints, -or update notes. Metadata should remain query information and should not become +or adapter notes. Metadata should remain query information and should not become the compatibility type system. Fifth, `UserInputSchema` describes raw source capabilities, `CanonicalModality` @@ -181,7 +180,7 @@ describes what an application consumes, and mapping schemas describe derived model-facing semantics. A browser may provide `key_down`, `key_up`, `prompt_set`, and `initial_frame_set` events. Those become canonical modalities such as `driver_command` or `conditioning_prompt`; whether they can then drive -`steering`, `camera_trajectory`, or text embedding updates depends on the +`steering`, `camera_trajectory`, or text embeddings depends on the selected mapping and model schema. ## Implemented T2/T3 Shape @@ -189,23 +188,24 @@ selected mapping and model schema. The implementation that came out of this inventory is: 1. Keep `UserInputEvent` and `UserInputs` as the raw event API, sliced by a - half-open `TimeWindow`. Static startup values remain timestamp-zero events. + half-open `TimeWindow`. Static session-start values remain timestamp-zero + events. 2. Keep `UserInputSchema` lightweight and source-facing. `event_types` declares that an event type exists; `UserInputCapability` additionally pins the payload fields it carries. 3. Add a canonical layer between raw and encoded. `CanonicalModality` names a - device-independent input and its conditioning phase; `InputCanonicalizer` + device-independent input and its payload fields; `InputCanonicalizer` registers per-device converters and produces `CanonicalInputs`. Applications and mappings consume canonical inputs and never read raw device events. 4. Split `InferenceInput` (formerly `ModelInputs`) into `global_conditioning` - and `step`. A non-empty global slot mid-rollout is an update request, not a - reset; `InputField.update_policy` declares whether the model can apply it. -5. Extend `InputField` with `update_policy`, `lifecycle`, and `metadata` so - models can distinguish runtime config, cache initialization, rollout binding, - per-step inputs, and supported active-session updates. + and `step`. Global conditioning is session-global state; `step` is the + payload for one generated chunk or frame window. +5. Keep `InputField.semantic_type` and `metadata` as lightweight query hints, + while leaving tensor shape, cadence, and model-specific validation to + adapters and sessions. 6. Keep `InputMappingSchema` as the canonical-to-encoded boundary, with mapping-set compatibility helpers for composed mappings. -7. Keep input names, semantic types, lifecycle labels, and metadata open-ended. +7. Keep input names, semantic types, and metadata open-ended. Adding a new model should usually mean adding adapter-owned schema declarations and mappings, not changing the core input dataclasses. 8. Leave deep validation to model adapters, sessions, and mappings. The schema @@ -227,11 +227,9 @@ Use these conventions when adding future model schemas: instead of `array`, or `hdmap_frames` instead of `image`. - Use `semantic_type` for a coarse representation hint, such as `path`, `decoded_tensor`, `c2w_sequence`, `intrinsics_vec4_sequence`, or `embedding`. -- Use `lifecycle` to say where the adapter consumes the value, such as - `runtime_config`, `cache_init`, `rollout_binding`, `step_input`, or - `session_update`. -- Use `update_policy` to say when a value may change. `SESSION_START_ONLY` is - the one reserved token, meaning the value cannot be swapped mid-rollout. +- Use `frequency_consumed` for adapter-consumption cadence, such as `once` or + `per_step`; keep it independent from whether the field is declared under + `global_conditioning_fields` or `step_fields`. - Use `metadata` for query hints: units, coordinate frame, shape summary, accepted suffixes, schema URI, model family, value ranges, or cardinality. - Keep deep validation in the adapter/mapping. The lightweight schemas answer @@ -247,18 +245,13 @@ can describe the supported input surfaces. All use ```python lingbot_model = InferenceInputSchema( description="lingbot-world", - global_fields=( - InputField(name="prompt", lifecycle="cache_init"), - InputField(name="global_conditioning_frame", lifecycle="cache_init"), + global_conditioning_fields=( + InputField(name="prompt", frequency_consumed="once"), + InputField(name="global_conditioning_frame", frequency_consumed="once"), + InputField(name="text_embeddings", required=False, frequency_consumed="once"), ), step_fields=( - InputField(name="camera_trajectory", lifecycle="step_input"), - InputField( - name="text_embeddings", - required=False, - update_policy="step_boundary", - lifecycle="session_update", - ), + InputField(name="camera_trajectory", frequency_consumed="per_step"), ), ) ``` @@ -266,27 +259,29 @@ lingbot_model = InferenceInputSchema( ```python omnidreams_model = InferenceInputSchema( description="omnidreams", - global_fields=( - InputField(name="prompts", lifecycle="cache_init"), - InputField(name="global_conditioning_frames", lifecycle="cache_init"), - InputField(name="view_names", lifecycle="cache_init"), - InputField(name="text_embeddings", required=False, lifecycle="cache_init"), - InputField(name="image_embeddings", required=False, lifecycle="cache_init"), + global_conditioning_fields=( + InputField(name="prompts", frequency_consumed="once"), + InputField(name="global_conditioning_frames", frequency_consumed="once"), + InputField(name="view_names", frequency_consumed="once"), + InputField(name="text_embeddings", required=False, frequency_consumed="once"), + InputField(name="image_embeddings", required=False, frequency_consumed="once"), + ), + step_fields=( + InputField(name="hdmap_frames", frequency_consumed="per_step"), ), - step_fields=(InputField(name="hdmap_frames", lifecycle="step_input"),), ) ``` ```python hy_worldplay_model = InferenceInputSchema( description="hy-worldplay", - global_fields=( - InputField(name="prompt", lifecycle="cache_init"), - InputField(name="global_conditioning_frame", lifecycle="cache_init"), - InputField(name="action_labels", lifecycle="rollout_binding"), - InputField(name="camera_viewmats", lifecycle="rollout_binding"), - InputField(name="camera_intrinsics", lifecycle="rollout_binding"), - InputField(name="memory_config", lifecycle="rollout_binding"), + global_conditioning_fields=( + InputField(name="prompt", frequency_consumed="once"), + InputField(name="global_conditioning_frame", frequency_consumed="once"), + InputField(name="action_labels", frequency_consumed="per_step"), + InputField(name="camera_viewmats", frequency_consumed="per_step"), + InputField(name="camera_intrinsics", frequency_consumed="per_step"), + InputField(name="memory_config", frequency_consumed="per_step"), ), ) ``` @@ -294,21 +289,21 @@ hy_worldplay_model = InferenceInputSchema( ```python sana_wm_model = InferenceInputSchema( description="sana-wm", - global_fields=( - InputField(name="prompt", lifecycle="cache_init"), - InputField(name="negative_prompt", required=False, lifecycle="cache_init"), - InputField(name="global_conditioning_frame", lifecycle="cache_init"), + global_conditioning_fields=( + InputField(name="prompt", frequency_consumed="once"), + InputField(name="negative_prompt", required=False, frequency_consumed="once"), + InputField(name="global_conditioning_frame", frequency_consumed="once"), InputField( name="camera_trajectory_c2w", semantic_type="c2w_sequence", - lifecycle="rollout_binding", + frequency_consumed="per_step", metadata={"shape": "[F,4,4]", "coordinates": "opencv_c2w"}, ), InputField( name="camera_intrinsics_vec4", required=False, semantic_type="intrinsics_vec4_sequence", - lifecycle="rollout_binding", + frequency_consumed="per_step", metadata={"shape": "[F,4]"}, ), ), diff --git a/flashdreams/flashdreams/runtime/__init__.py b/flashdreams/flashdreams/runtime/__init__.py index ab303c74..04f84ae5 100644 --- a/flashdreams/flashdreams/runtime/__init__.py +++ b/flashdreams/flashdreams/runtime/__init__.py @@ -19,7 +19,6 @@ from flashdreams.runtime.config import ExecutionBackend, InferenceConfig, Precision from flashdreams.runtime.inputs import ( INPUT_PHASES, - SESSION_START_ONLY, CanonicalInputs, CanonicalInputSchema, CanonicalModality, @@ -96,7 +95,6 @@ "Precision", "RuntimeMetricSample", "ScriptedModality", - "SESSION_START_ONLY", "StepRequest", "StepResult", "TimeWindow", diff --git a/flashdreams/flashdreams/runtime/inputs.py b/flashdreams/flashdreams/runtime/inputs.py index f0be31be..d88fbdb7 100644 --- a/flashdreams/flashdreams/runtime/inputs.py +++ b/flashdreams/flashdreams/runtime/inputs.py @@ -12,23 +12,17 @@ from flashdreams.runtime._utils import freeze_mapping -InputPhase = Literal["global", "step"] +InputPhase = Literal["global_conditioning", "step"] -INPUT_PHASES: tuple[InputPhase, ...] = ("global", "step") - -SESSION_START_ONLY = "session_start" -"""``InputField.update_policy`` value meaning "supply at session start only". - -``update_policy`` is otherwise an open, adapter-owned vocabulary. This is the -one reserved token, because the runtime needs to distinguish a conditioning -value that can be swapped mid-rollout from one that cannot. -""" +INPUT_PHASES: tuple[InputPhase, ...] = ("global_conditioning", "step") def validate_phase(value: str) -> InputPhase: """Return ``value`` as a validated :data:`InputPhase`.""" if value not in INPUT_PHASES: - raise ValueError(f"phase must be 'global' or 'step', got {value!r}.") + raise ValueError( + f"phase must be 'global_conditioning' or 'step', got {value!r}." + ) return cast(InputPhase, value) @@ -56,17 +50,15 @@ def contains(self, timestamp_s: float) -> bool: class InputField: """Lightweight schema field for user snapshots or model inputs. - ``update_policy`` and ``lifecycle`` are plain query metadata. They let a - model advertise facts such as "prompt updates land at step boundaries" or - "this value is consumed at cache init" without making this layer - responsible for implementing or deeply validating that behavior. + ``semantic_type``, ``frequency_consumed``, and ``metadata`` are query hints + only. Adapter-owned validation still decides concrete shape, dtype, units, + and tensor layout. """ name: str required: bool = True semantic_type: str | None = None - update_policy: str | None = None - lifecycle: str | None = None + frequency_consumed: str | None = None metadata: Mapping[str, Any] = field( default_factory=dict, compare=False, @@ -205,21 +197,21 @@ def require_snapshot(self, inputs: "UserInputs") -> None: @dataclass(frozen=True, kw_only=True, slots=True) class InferenceInputSchema: - """Minimal metadata for model-facing initial and per-step inputs.""" + """Minimal metadata for global conditioning and per-step inputs.""" - global_fields: tuple[InputField, ...] = () - """Model inputs required before starting the initial generation/session.""" + global_conditioning_fields: tuple[InputField, ...] = () + """Model inputs carried in the global conditioning slot.""" step_fields: tuple[InputField, ...] = () - """Per-step model inputs required after the session starts.""" + """Model inputs required for one session step.""" description: str = "" def fields_for(self, phase: InputPhase) -> tuple[InputField, ...]: """Return every declared field for ``phase``.""" return ( - self.global_fields - if validate_phase(phase) == "global" + self.global_conditioning_fields + if validate_phase(phase) == "global_conditioning" else self.step_fields ) @@ -258,32 +250,20 @@ def _select( if input_field.required is required ) - def unsupported_global_updates(self, inputs: "InferenceInput") -> tuple[str, ...]: - """Return requested conditioning updates this model cannot apply. - - A field whose ``update_policy`` is :data:`SESSION_START_ONLY` can be - supplied when the session starts but not changed mid-rollout. Any other - policy, including ``None``, is treated as permissive here; the adapter - still owns whether the swap actually succeeds. - """ - return tuple( - name - for name in inputs.global_conditioning - if (declared := self.field_for(name=name, phase="global")) is not None - and declared.update_policy == SESSION_START_ONLY + def missing_global_conditioning(self, inputs: "InferenceInput") -> tuple[str, ...]: + """Return required global conditioning fields absent from ``inputs``.""" + return _missing_required( + self.global_conditioning_fields, + inputs.global_conditioning, ) - def missing_global(self, inputs: "InferenceInput") -> tuple[str, ...]: - """Return required initial fields absent from ``inputs``.""" - return _missing_required(self.global_fields, inputs.global_conditioning) - def missing_step(self, inputs: "InferenceInput") -> tuple[str, ...]: """Return required per-step fields absent from ``inputs``.""" return _missing_required(self.step_fields, inputs.step) - def require_global(self, inputs: "InferenceInput") -> None: - """Raise if required initial fields are absent.""" - missing = self.missing_global(inputs) + def require_global_conditioning(self, inputs: "InferenceInput") -> None: + """Raise if required global conditioning fields are absent.""" + missing = self.missing_global_conditioning(inputs) if missing: raise ValueError( f"Missing required global conditioning input(s): {missing}" @@ -447,16 +427,10 @@ class InferenceInput: Two conditioning slots: - ``global_conditioning``: values that condition the whole rollout, such as - the conditioning frame or prompt. Normally supplied when the session - starts. + the conditioning frame or prompt. Session start/reset establishes this + state; a step call may carry a non-empty payload to request an update when + the model supports it. - ``step``: values needed to generate the next chunk or frame. - - A non-empty ``global_conditioning`` on a mid-rollout input is an *update - request*, not a reset. The session should apply it when the model supports - that; resetting rollout state is a separate, explicit - :meth:`InferenceSession.reset` call. Whether a given value can be updated - mid-rollout is declared per field by ``InputField.update_policy``; see - :meth:`InferenceInputSchema.unsupported_global_updates`. """ __hash__ = None @@ -472,42 +446,12 @@ def __post_init__(self) -> None: object.__setattr__(self, "step", freeze_mapping(self.step)) object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) - @property - def requests_global_update(self) -> bool: - """Return whether this input asks the session to update conditioning.""" - return bool(self.global_conditioning) - - def with_step(self, step: Mapping[str, Any]) -> "InferenceInput": - """Return a copy with replaced per-step payload. - - The global slot is carried through unchanged, so a mid-rollout input - built this way keeps whatever update request it already had. Use - :meth:`without_global_update` for the common steady-state case. - """ - return InferenceInput( - global_conditioning=self.global_conditioning, - step=step, - metadata=self.metadata, - ) - - def with_global_update( - self, global_conditioning: Mapping[str, Any] - ) -> "InferenceInput": - """Return a copy requesting a mid-rollout conditioning update.""" - return InferenceInput( - global_conditioning=global_conditioning, - step=self.step, - metadata=self.metadata, - ) - - def without_global_update(self) -> "InferenceInput": - """Return a copy that requests no conditioning update.""" - return InferenceInput(step=self.step, metadata=self.metadata) - def for_phase(self, phase: InputPhase) -> Mapping[str, Any]: """Return the payload mapping for ``phase``.""" return ( - self.global_conditioning if validate_phase(phase) == "global" else self.step + self.global_conditioning + if validate_phase(phase) == "global_conditioning" + else self.step ) diff --git a/flashdreams/flashdreams/runtime/interfaces.py b/flashdreams/flashdreams/runtime/interfaces.py index 852a77f1..5c5054dc 100644 --- a/flashdreams/flashdreams/runtime/interfaces.py +++ b/flashdreams/flashdreams/runtime/interfaces.py @@ -22,7 +22,7 @@ class InferenceSession(Protocol): """One rollout or stream with isolated model/cache state.""" def next_step_request(self) -> StepRequest | None: - """Describe the next step's inputs, or return ``None`` when complete.""" + """Return the next step's runtime request, or ``None`` when complete.""" ... def step(self, inputs: InferenceInput) -> StepResult: @@ -69,7 +69,7 @@ def model_id(self) -> str: @property def inference_input_schema(self) -> InferenceInputSchema: - """Model-facing initial and per-step input requirements.""" + """Model-facing global conditioning and per-step input requirements.""" ... @property diff --git a/flashdreams/flashdreams/runtime/mapping.py b/flashdreams/flashdreams/runtime/mapping.py index 94f48140..bbf11616 100644 --- a/flashdreams/flashdreams/runtime/mapping.py +++ b/flashdreams/flashdreams/runtime/mapping.py @@ -41,13 +41,13 @@ def validate( """Fail early for obvious app, event-source, and model mismatches.""" ... - def map_global_inputs( + def map_global_conditioning_inputs( self, *, canonical_inputs: CanonicalInputs, inference_input: InferenceInput, ) -> InferenceInput: - """Build global conditioning inputs before a session starts.""" + """Build global conditioning inputs for session start or reset.""" ... def map_step_inputs( @@ -72,7 +72,7 @@ def validate( ) -> None: del canonical_schema, inference_input_schema - def map_global_inputs( + def map_global_conditioning_inputs( self, *, canonical_inputs: CanonicalInputs, @@ -104,7 +104,7 @@ class InputMappingSchema: name: str = "input-mapping" consumes: tuple[CanonicalModality, ...] = () - produces_global: tuple[InputField, ...] = () + produces_global_conditioning: tuple[InputField, ...] = () produces_step: tuple[InputField, ...] = () metadata: Mapping[str, Any] = field( default_factory=dict, @@ -119,7 +119,11 @@ def __post_init__(self) -> None: def produces_for(self, phase: InputPhase) -> tuple[InputField, ...]: """Return the fields this mapping produces for ``phase``.""" - return self.produces_global if phase == "global" else self.produces_step + return ( + self.produces_global_conditioning + if phase == "global_conditioning" + else self.produces_step + ) def can_produce(self, phase: InputPhase, required: InputField) -> bool: """Return whether this mapping can produce ``required`` in ``phase``.""" @@ -136,12 +140,7 @@ def _field_matches(produced: InputField, required: InputField) -> bool: or required.semantic_type is None or produced.semantic_type == required.semantic_type ) - lifecycle_ok = ( - produced.lifecycle is None - or required.lifecycle is None - or produced.lifecycle == required.lifecycle - ) - return semantic_ok and lifecycle_ok + return semantic_ok @dataclass(frozen=True, kw_only=True, slots=True) @@ -223,7 +222,10 @@ def combine_mapping_schemas( first declaration winning on conflicting keys. """ consumes: list[CanonicalModality] = [] - produces: dict[InputPhase, list[InputField]] = {"global": [], "step": []} + produces: dict[InputPhase, list[InputField]] = { + "global_conditioning": [], + "step": [], + } def _merge(target: list[Any], value: Any) -> None: for index, existing in enumerate(target): @@ -248,7 +250,7 @@ def _merge(target: list[Any], value: Any) -> None: return InputMappingSchema( name=name, consumes=tuple(consumes), - produces_global=tuple(produces["global"]), + produces_global_conditioning=tuple(produces["global_conditioning"]), produces_step=tuple(produces["step"]), ) @@ -358,8 +360,9 @@ def undeclared_inference_inputs( """Return payload keys a mapping produced but did not declare. Mapping schemas are hand-written, so they drift from what - ``map_global_inputs``/``map_step_inputs`` actually return. Mapping tests - can use this to keep the declared compatibility surface honest. + ``map_global_conditioning_inputs``/``map_step_inputs`` actually return. + Mapping tests can use this to keep the declared compatibility surface + honest. """ return tuple( (phase, key) diff --git a/flashdreams/flashdreams/runtime/types.py b/flashdreams/flashdreams/runtime/types.py index 46775302..51d3846d 100644 --- a/flashdreams/flashdreams/runtime/types.py +++ b/flashdreams/flashdreams/runtime/types.py @@ -15,10 +15,11 @@ @dataclass(frozen=True, kw_only=True, slots=True) class StepRequest: - """Model-session request for the next step's inputs. + """Per-step runtime request emitted by an inference session. - ``user_input_window`` lets a runner drain or slice timestamped user events for - the current step before invoking the selected ``InputMapping``. + This is not a schema declaration. ``user_input_window`` lets a runner drain + or slice timestamped user events for the current step before invoking the + selected ``InputMapping``. """ __hash__ = None @@ -36,7 +37,7 @@ def __post_init__(self) -> None: @dataclass(frozen=True, kw_only=True, slots=True) class StepResult: - """Generated output and metadata for one inference step.""" + """Generated output and metadata returned by one inference step.""" __hash__ = None diff --git a/flashdreams/tests/test_inference_runtime_api.py b/flashdreams/tests/test_inference_runtime_api.py index edfafa63..d454f620 100644 --- a/flashdreams/tests/test_inference_runtime_api.py +++ b/flashdreams/tests/test_inference_runtime_api.py @@ -106,9 +106,9 @@ def test_runtime_metric_sample_rejects_bool_values() -> None: RuntimeMetricSample(name="sample", value=True) -def test_inference_input_schema_validates_initial_and_step_payloads() -> None: +def test_schema_validates_global_conditioning_and_step_payloads() -> None: schema = InferenceInputSchema( - global_fields=( + global_conditioning_fields=( InputField(name="prompt"), InputField(name="global_conditioning_frame"), ), @@ -118,7 +118,7 @@ def test_inference_input_schema_validates_initial_and_step_payloads() -> None: global_conditioning={"prompt": "drive", "global_conditioning_frame": object()} ) - schema.require_global(inputs) + schema.require_global_conditioning(inputs) assert schema.missing_step(inputs) == ("camera_poses",) with pytest.raises(ValueError, match="camera_poses"): @@ -190,7 +190,7 @@ def test_identity_input_mapping_leaves_inference_input_unchanged() -> None: request = StepRequest(step_index=0) assert ( - mapping.map_global_inputs( + mapping.map_global_conditioning_inputs( canonical_inputs=CanonicalInputs(), inference_input=inference_input, ) @@ -367,7 +367,7 @@ def _drive_two_step_session( canonical_schema=adapter.canonical_input_schema, inference_input_schema=adapter.inference_input_schema, ) - initial_inputs = mapping.map_global_inputs( + initial_inputs = mapping.map_global_conditioning_inputs( canonical_inputs=canonicalizer.canonicalize( user_inputs, window=TimeWindow(start_s=0.0, end_s=_SESSION_HORIZON_S), @@ -390,9 +390,8 @@ def _drive_two_step_session( or TimeWindow(start_s=0.0, end_s=_SESSION_HORIZON_S), source_schema=source_schema, ), - # The global slot stays empty in steady state. A mapping that - # sees ``canonical_inputs.has_global_change`` fills it via - # ``with_global_update`` to request a mid-rollout swap. + # Per-step calls carry only the step payload. A changed prompt + # or scene starts or resets a session outside this loop. inference_input=InferenceInput( step={"chunk_index": request.step_index}, ), @@ -417,7 +416,7 @@ def _drive_two_step_session( class _FakeAdapter: model_id = "fake-model" inference_input_schema = InferenceInputSchema( - global_fields=(InputField(name="prompt"),), + global_conditioning_fields=(InputField(name="prompt"),), step_fields=(InputField(name="chunk_index"),), ) canonical_input_schema = CanonicalInputSchema() @@ -440,7 +439,7 @@ def __init__(self, *, inference_input_schema: InferenceInputSchema) -> None: self.closed = False def start_session(self, inputs: InferenceInput) -> InferenceSession: - self._inference_input_schema.require_global(inputs) + self._inference_input_schema.require_global_conditioning(inputs) return _FakeSession(inference_input_schema=self._inference_input_schema) def close(self) -> None: diff --git a/flashdreams/tests/test_runtime_canonical.py b/flashdreams/tests/test_runtime_canonical.py index 1ad48d39..cfe2e364 100644 --- a/flashdreams/tests/test_runtime_canonical.py +++ b/flashdreams/tests/test_runtime_canonical.py @@ -217,13 +217,14 @@ def test_canonical_inputs_carry_live_control_only() -> None: def test_application_supplies_global_conditioning_directly() -> None: - """A prompt swap reaches the session without touching canonicalization.""" - update = InferenceInput(step={"steering": 0.0}).with_global_update( - {"prompt": "heavy rain"} + """A prompt reaches session start without touching canonicalization.""" + inputs = InferenceInput( + global_conditioning={"prompt": "heavy rain"}, + step={"steering": 0.0}, ) - assert update.requests_global_update - assert update.global_conditioning["prompt"] == "heavy rain" + assert inputs.global_conditioning["prompt"] == "heavy rain" + assert inputs.step["steering"] == 0.0 # --- device independence ------------------------------------------------ diff --git a/flashdreams/tests/test_runtime_input_mapping.py b/flashdreams/tests/test_runtime_input_mapping.py index 00cd9758..0750e733 100644 --- a/flashdreams/tests/test_runtime_input_mapping.py +++ b/flashdreams/tests/test_runtime_input_mapping.py @@ -4,9 +4,10 @@ """Tests for declarative input-mapping compatibility in the runtime API. These cover the T2/T3 contract: sources declare what user events they can -provide at payload granularity, models declare required and optional -initial/per-step inputs, and a mapping declares what it consumes and produces so -compatibility can be answered before expensive runtime initialization. +provide at payload granularity, models declare required and optional global +conditioning/per-step inputs, and a mapping declares what it consumes and +produces so compatibility can be answered before expensive runtime +initialization. """ from __future__ import annotations @@ -17,7 +18,6 @@ from flashdreams.runtime import ( DRIVER_COMMAND, - SESSION_START_ONLY, CanonicalInputs, CanonicalInputSchema, CanonicalModality, @@ -66,11 +66,13 @@ # modality, so this mapping consumes nothing and only declares what it produces. PROMPT_MAPPING = InputMappingSchema( name="prompt", - produces_global=(InputField(name="prompt", semantic_type="text"),), + produces_global_conditioning=(InputField(name="prompt", semantic_type="text"),), ) FRAME_MAPPING = InputMappingSchema( name="conditioning-frame", - produces_global=(InputField(name="global_conditioning_frame", required=False),), + produces_global_conditioning=( + InputField(name="global_conditioning_frame", required=False), + ), ) STEERING_MAPPING = InputMappingSchema( name="driver-command-to-steering", @@ -84,12 +86,10 @@ ) DRIVING_MODEL = InferenceInputSchema( - global_fields=( - InputField(name="prompt", semantic_type="text", lifecycle="cache_init"), - ), + global_conditioning_fields=(InputField(name="prompt", semantic_type="text"),), step_fields=( - InputField(name="steering", lifecycle="step_input"), - InputField(name="camera_delta", required=False, lifecycle="step_input"), + InputField(name="steering"), + InputField(name="camera_delta", required=False), ), ) @@ -97,7 +97,7 @@ # --- user input events and windowing ------------------------------------ -def test_startup_values_are_represented_as_events() -> None: +def test_session_start_values_are_represented_as_events() -> None: inputs = UserInputs( events=( UserInputEvent( @@ -198,7 +198,7 @@ def test_model_declares_required_and_optional_fields_per_phase() -> None: optional = DRIVING_MODEL.optional_fields() assert {(phase, f.name) for phase, f in required} == { - ("global", "prompt"), + ("global_conditioning", "prompt"), ("step", "steering"), } assert {(phase, f.name) for phase, f in optional} == {("step", "camera_delta")} @@ -211,7 +211,10 @@ def test_required_fields_can_be_filtered_by_phase() -> None: def test_field_lookup_is_phase_scoped() -> None: - assert DRIVING_MODEL.field_for(name="prompt", phase="global") is not None + assert ( + DRIVING_MODEL.field_for(name="prompt", phase="global_conditioning") + is not None + ) assert DRIVING_MODEL.field_for(name="prompt", phase="step") is None @@ -227,20 +230,28 @@ def test_inference_input_expose_payload_per_phase() -> None: global_conditioning={"prompt": "drive"}, step={"steering": 0.25} ) - assert inputs.for_phase("global")["prompt"] == "drive" + assert inputs.for_phase("global_conditioning")["prompt"] == "drive" assert inputs.for_phase("step")["steering"] == 0.25 -def test_lifecycle_and_update_policy_are_queryable_metadata() -> None: +def test_step_context_can_carry_global_conditioning_update_payload() -> None: + inputs = InferenceInput( + global_conditioning={"prompt": "heavy rain"}, + step={"steering": 0.25}, + ) + + assert inputs.global_conditioning["prompt"] == "heavy rain" + assert inputs.step["steering"] == 0.25 + + +def test_field_metadata_is_queryable() -> None: field = InputField( name="prompt", - update_policy="step_boundary", - lifecycle="cache_init", + frequency_consumed="once", metadata={"coordinates": "opencv_c2w"}, ) - assert field.update_policy == "step_boundary" - assert field.lifecycle == "cache_init" + assert field.frequency_consumed == "once" assert field.metadata["coordinates"] == "opencv_c2w" @@ -263,7 +274,7 @@ def test_compatible_source_model_and_mapping_can_drive() -> None: assert compatibility.can_drive assert {(p, f.name) for p, f in compatibility.satisfied_required_model_fields} == { - ("global", "prompt"), + ("global_conditioning", "prompt"), ("step", "steering"), } assert {(p, f.name) for p, f in compatibility.available_optional_model_fields} == { @@ -325,13 +336,17 @@ def test_optional_field_needs_mapping_support_to_be_available() -> None: assert compatibility.available_optional_model_fields == () -def test_lifecycle_disagreement_blocks_a_field_match() -> None: +def test_global_conditioning_mapping_matches_global_conditioning_field() -> None: model = InferenceInputSchema( - global_fields=(InputField(name="prompt", lifecycle="rollout_binding"),) + global_conditioning_fields=( + InputField(name="camera_trajectory", frequency_consumed="per_step"), + ) ) mapping = InputMappingSchema( - name="prompt", - produces_global=(InputField(name="prompt", lifecycle="cache_init"),), + name="trajectory", + produces_global_conditioning=( + InputField(name="camera_trajectory", frequency_consumed="once"), + ), ) compatibility = check_mapping_compatibility( @@ -340,11 +355,13 @@ def test_lifecycle_disagreement_blocks_a_field_match() -> None: mapping_schema=mapping, ) - assert not compatibility.can_drive + assert compatibility.can_drive -def test_unspecified_lifecycle_stays_permissive() -> None: - model = InferenceInputSchema(global_fields=(InputField(name="prompt"),)) +def test_unspecified_semantic_type_stays_permissive() -> None: + model = InferenceInputSchema( + global_conditioning_fields=(InputField(name="prompt"),) + ) compatibility = check_mapping_compatibility( canonical_schema=CANONICAL_ALL, @@ -398,26 +415,28 @@ def test_combining_mappings_unions_their_surfaces() -> None: combined = combine_mapping_schemas((PROMPT_MAPPING, STEERING_MAPPING)) assert {m.name for m in combined.consumes} == {"driver_command"} - assert [f.name for f in combined.produces_global] == ["prompt"] + assert [f.name for f in combined.produces_global_conditioning] == ["prompt"] assert [f.name for f in combined.produces_step] == ["steering"] def test_duplicate_declarations_collapse_and_merge_metadata() -> None: first = InputMappingSchema( name="a", - produces_global=(InputField(name="prompt", metadata={"source": "a"}),), + produces_global_conditioning=( + InputField(name="prompt", metadata={"source": "a"}), + ), ) second = InputMappingSchema( name="b", - produces_global=( + produces_global_conditioning=( InputField(name="prompt", metadata={"source": "b", "extra": "kept"}), ), ) combined = combine_mapping_schemas((first, second)) - assert len(combined.produces_global) == 1 - metadata = combined.produces_global[0].metadata + assert len(combined.produces_global_conditioning) == 1 + metadata = combined.produces_global_conditioning[0].metadata assert metadata["source"] == "a" assert metadata["extra"] == "kept" @@ -489,85 +508,3 @@ def test_model_with_no_requirements_is_always_drivable() -> None: ) assert compatibility.can_drive - - -# --- global conditioning updates vs reset ------------------------------- - - -def test_empty_global_slot_requests_no_update() -> None: - steady_state = InferenceInput(step={"steering": 0.25}) - - assert not steady_state.requests_global_update - - -def test_non_empty_global_slot_mid_rollout_is_an_update_request() -> None: - """Changing weather mid-run updates conditioning; it is not a reset.""" - updated = InferenceInput(step={"steering": 0.0}).with_global_update( - {"prompt": "heavy rain"} - ) - - assert updated.requests_global_update - assert updated.global_conditioning["prompt"] == "heavy rain" - assert updated.step["steering"] == 0.0 - - -def test_with_step_carries_the_global_slot_through() -> None: - started = InferenceInput(global_conditioning={"prompt": "drive"}) - - stepped = started.with_step({"steering": 0.5}) - - assert stepped.global_conditioning["prompt"] == "drive" - - -def test_without_global_update_clears_the_request() -> None: - started = InferenceInput( - global_conditioning={"prompt": "drive"}, step={"steering": 0.5} - ) - - steady_state = started.without_global_update() - - assert not steady_state.requests_global_update - assert steady_state.step["steering"] == 0.5 - - -def test_model_can_declare_conditioning_it_cannot_swap_mid_rollout() -> None: - schema = InferenceInputSchema( - global_fields=( - InputField(name="prompt", update_policy="step_boundary"), - InputField(name="scene_id", update_policy=SESSION_START_ONLY), - ) - ) - update = InferenceInput( - global_conditioning={"prompt": "heavy rain", "scene_id": "town_02"} - ) - - assert schema.unsupported_global_updates(update) == ("scene_id",) - - -def test_permissive_when_no_update_policy_is_declared() -> None: - schema = InferenceInputSchema(global_fields=(InputField(name="prompt"),)) - update = InferenceInput(global_conditioning={"prompt": "heavy rain"}) - - assert schema.unsupported_global_updates(update) == () - - -def test_undeclared_global_values_are_left_to_the_adapter() -> None: - schema = InferenceInputSchema(global_fields=(InputField(name="prompt"),)) - update = InferenceInput(global_conditioning={"mystery": 1}) - - assert schema.unsupported_global_updates(update) == () - - -def test_steady_state_steps_do_not_request_a_global_update() -> None: - """Carrying session-start conditioning forward would look like an update.""" - started = InferenceInput(global_conditioning={"prompt": "drive"}) - - steady_state = InferenceInput(step={"chunk_index": 1}) - - assert started.requests_global_update - assert not steady_state.requests_global_update - assert ( - not started.with_step({"chunk_index": 1}) - .without_global_update() - .requests_global_update - )